sendnrw/misskey
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

enhance(backend): migrate bull to bullmq (#10910)

Browse Source 
* wip

* wip

* Update QueueService.ts

* wip

* refactor

* ✌️

* fix

* Update QueueStatsService.ts

* refactor

* Update ApNoteService.ts

* Update mock-resolver.ts

* refactor

* Update mock-resolver.ts
This commit is contained in:
syuilo
2023-05-29 11:54:49 +09:00
committed by GitHub
parent 7cbd852fe5
commit fd7b77c542
53 changed files with 532 additions and 490 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
packages/backend/src/queue/processors/AggregateRetentionProcessorService.ts
View File

@@ -9,7 +9,7 @@ import { deepClone } from '@/misc/clone.js';
import { IdService } from '@/core/IdService.js';
import { isDuplicateKeyValueError } from '@/misc/is-duplicate-key-value-error.js';
import { QueueLoggerService } from '../QueueLoggerService.js';
import type Bull from 'bull';
import type * as Bull from 'bullmq';
@Injectable()
export class AggregateRetentionProcessorService {
@@ -32,7 +32,7 @@ export class AggregateRetentionProcessorService {
}
@bindThis
public async process(job: Bull.Job<Record<string, unknown>>, done: () => void): Promise<void> {
public async process(): Promise<void> {
this.logger.info('Aggregating retention...');
const now = new Date();
@@ -62,7 +62,6 @@ export class AggregateRetentionProcessorService {
} catch (err) {
if (isDuplicateKeyValueError(err)) {
this.logger.succ('Skip because it has already been processed by another worker.');
done();
return;
}
throw err;
@@ -88,6 +87,5 @@ export class AggregateRetentionProcessorService {
}
this.logger.succ('Retention aggregated.');
done();
}
}

5
packages/backend/src/queue/processors/CheckExpiredMutingsProcessorService.ts
View File

@@ -7,7 +7,7 @@ import type Logger from '@/logger.js';
import { bindThis } from '@/decorators.js';
import { UserMutingService } from '@/core/UserMutingService.js';
import { QueueLoggerService } from '../QueueLoggerService.js';
import type Bull from 'bull';
import type * as Bull from 'bullmq';
@Injectable()
export class CheckExpiredMutingsProcessorService {
@@ -27,7 +27,7 @@ export class CheckExpiredMutingsProcessorService {
}
@bindThis
public async process(job: Bull.Job<Record<string, unknown>>, done: () => void): Promise<void> {
public async process(): Promise<void> {
this.logger.info('Checking expired mutings...');
const expired = await this.mutingsRepository.createQueryBuilder('muting')
@@ -41,6 +41,5 @@ export class CheckExpiredMutingsProcessorService {
}
this.logger.succ('All expired mutings checked.');
done();
}
}

5
packages/backend/src/queue/processors/CleanChartsProcessorService.ts
View File

@@ -16,7 +16,7 @@ import PerUserDriveChart from '@/core/chart/charts/per-user-drive.js';
import ApRequestChart from '@/core/chart/charts/ap-request.js';
import { bindThis } from '@/decorators.js';
import { QueueLoggerService } from '../QueueLoggerService.js';
import type Bull from 'bull';
import type * as Bull from 'bullmq';
@Injectable()
export class CleanChartsProcessorService {
@@ -45,7 +45,7 @@ export class CleanChartsProcessorService {
}
@bindThis
public async process(job: Bull.Job<Record<string, unknown>>, done: () => void): Promise<void> {
public async process(): Promise<void> {
this.logger.info('Clean charts...');
await Promise.all([
@@ -64,6 +64,5 @@ export class CleanChartsProcessorService {
]);
this.logger.succ('All charts successfully cleaned.');
done();
}
}

5
packages/backend/src/queue/processors/CleanProcessorService.ts
View File

@@ -7,7 +7,7 @@ import type Logger from '@/logger.js';
import { bindThis } from '@/decorators.js';
import { IdService } from '@/core/IdService.js';
import { QueueLoggerService } from '../QueueLoggerService.js';
import type Bull from 'bull';
import type * as Bull from 'bullmq';
@Injectable()
export class CleanProcessorService {
@@ -36,7 +36,7 @@ export class CleanProcessorService {
}
@bindThis
public async process(job: Bull.Job<Record<string, unknown>>, done: () => void): Promise<void> {
public async process(): Promise<void> {
this.logger.info('Cleaning...');
this.userIpsRepository.delete({
@@ -72,6 +72,5 @@ export class CleanProcessorService {
}
this.logger.succ('Cleaned.');
done();
}
}

11
packages/backend/src/queue/processors/CleanRemoteFilesProcessorService.ts
View File

@@ -5,9 +5,9 @@ import type { DriveFilesRepository } from '@/models/index.js';
import type { Config } from '@/config.js';
import type Logger from '@/logger.js';
import { DriveService } from '@/core/DriveService.js';
import { QueueLoggerService } from '../QueueLoggerService.js';
import type Bull from 'bull';
import { bindThis } from '@/decorators.js';
import { QueueLoggerService } from '../QueueLoggerService.js';
import type * as Bull from 'bullmq';
@Injectable()
export class CleanRemoteFilesProcessorService {
@@ -27,7 +27,7 @@ export class CleanRemoteFilesProcessorService {
}
@bindThis
public async process(job: Bull.Job<Record<string, unknown>>, done: () => void): Promise<void> {
public async process(job: Bull.Job<Record<string, unknown>>): Promise<void> {
this.logger.info('Deleting cached remote files...');
let deletedCount = 0;
@@ -47,7 +47,7 @@ export class CleanRemoteFilesProcessorService {
});
if (files.length === 0) {
job.progress(100);
job.updateProgress(100);
break;
}
@@ -62,10 +62,9 @@ export class CleanRemoteFilesProcessorService {
isLink: false,
});
job.progress(deletedCount / total);
job.updateProgress(deletedCount / total);
}
this.logger.succ('All cached remote files has been deleted.');
done();
}
}

6
packages/backend/src/queue/processors/DeleteAccountProcessorService.ts
View File

@@ -8,10 +8,10 @@ import { DriveService } from '@/core/DriveService.js';
import type { DriveFile } from '@/models/entities/DriveFile.js';
import type { Note } from '@/models/entities/Note.js';
import { EmailService } from '@/core/EmailService.js';
import { QueueLoggerService } from '../QueueLoggerService.js';
import type Bull from 'bull';
import type { DbUserDeleteJobData } from '../types.js';
import { bindThis } from '@/decorators.js';
import { QueueLoggerService } from '../QueueLoggerService.js';
import type * as Bull from 'bullmq';
import type { DbUserDeleteJobData } from '../types.js';
@Injectable()
export class DeleteAccountProcessorService {

14
packages/backend/src/queue/processors/DeleteDriveFilesProcessorService.ts
View File

@@ -5,10 +5,10 @@ import type { UsersRepository, DriveFilesRepository } from '@/models/index.js';
import type { Config } from '@/config.js';
import type Logger from '@/logger.js';
import { DriveService } from '@/core/DriveService.js';
import { QueueLoggerService } from '../QueueLoggerService.js';
import type Bull from 'bull';
import type { DbJobDataWithUser } from '../types.js';
import { bindThis } from '@/decorators.js';
import { QueueLoggerService } from '../QueueLoggerService.js';
import type * as Bull from 'bullmq';
import type { DbJobDataWithUser } from '../types.js';
@Injectable()
export class DeleteDriveFilesProcessorService {
@@ -31,12 +31,11 @@ export class DeleteDriveFilesProcessorService {
}
@bindThis
public async process(job: Bull.Job<DbJobDataWithUser>, done: () => void): Promise<void> {
public async process(job: Bull.Job<DbJobDataWithUser>): Promise<void> {
this.logger.info(`Deleting drive files of ${job.data.user.id} ...`);
const user = await this.usersRepository.findOneBy({ id: job.data.user.id });
if (user == null) {
done();
return;
}
@@ -56,7 +55,7 @@ export class DeleteDriveFilesProcessorService {
});
if (files.length === 0) {
job.progress(100);
job.updateProgress(100);
break;
}
@@ -71,10 +70,9 @@ export class DeleteDriveFilesProcessorService {
userId: user.id,
});
job.progress(deletedCount / total);
job.updateProgress(deletedCount / total);
}
this.logger.succ(`All drive files (${deletedCount}) of ${user.id} has been deleted.`);
done();
}
}

6
packages/backend/src/queue/processors/DeleteFileProcessorService.ts
View File

@@ -3,10 +3,10 @@ import { DI } from '@/di-symbols.js';
import type { Config } from '@/config.js';
import type Logger from '@/logger.js';
import { DriveService } from '@/core/DriveService.js';
import { QueueLoggerService } from '../QueueLoggerService.js';
import type Bull from 'bull';
import type { ObjectStorageFileJobData } from '../types.js';
import { bindThis } from '@/decorators.js';
import { QueueLoggerService } from '../QueueLoggerService.js';
import type * as Bull from 'bullmq';
import type { ObjectStorageFileJobData } from '../types.js';
@Injectable()
export class DeleteFileProcessorService {

10
packages/backend/src/queue/processors/DeliverProcessorService.ts
View File

@@ -1,4 +1,5 @@
import { Inject, Injectable } from '@nestjs/common';
import * as Bull from 'bullmq';
import { DI } from '@/di-symbols.js';
import type { DriveFilesRepository, InstancesRepository } from '@/models/index.js';
import type { Config } from '@/config.js';
@@ -16,7 +17,6 @@ import { StatusError } from '@/misc/status-error.js';
import { UtilityService } from '@/core/UtilityService.js';
import { bindThis } from '@/decorators.js';
import { QueueLoggerService } from '../QueueLoggerService.js';
import type Bull from 'bull';
import type { DeliverJobData } from '../types.js';
@Injectable()
@@ -121,15 +121,13 @@ export class DeliverProcessorService {
isSuspended: true,
});
});
return `${host} is gone`;
throw new Bull.UnrecoverableError(`${host} is gone`);
}
// HTTPステータスコード4xxはクライアントエラーであり、それはつまり
// 何回再送しても成功することはないということなのでエラーにはしないでおく
return `${res.statusCode} ${res.statusMessage}`;
throw new Bull.UnrecoverableError(`${res.statusCode} ${res.statusMessage}`);
}
// 5xx etc.
throw `${res.statusCode} ${res.statusMessage}`;
throw new Error(`${res.statusCode} ${res.statusMessage}`);
} else {
// DNS error, socket error, timeout ...
throw res;

7
packages/backend/src/queue/processors/EndedPollNotificationProcessorService.ts
View File

@@ -6,7 +6,7 @@ import type Logger from '@/logger.js';
import { NotificationService } from '@/core/NotificationService.js';
import { bindThis } from '@/decorators.js';
import { QueueLoggerService } from '../QueueLoggerService.js';
import type Bull from 'bull';
import type * as Bull from 'bullmq';
import type { EndedPollNotificationJobData } from '../types.js';
@Injectable()
@@ -30,10 +30,9 @@ export class EndedPollNotificationProcessorService {
}
@bindThis
public async process(job: Bull.Job<EndedPollNotificationJobData>, done: () => void): Promise<void> {
public async process(job: Bull.Job<EndedPollNotificationJobData>): Promise<void> {
const note = await this.notesRepository.findOneBy({ id: job.data.noteId });
if (note == null || !note.hasPoll) {
done();
return;
}
@@ -51,7 +50,5 @@ export class EndedPollNotificationProcessorService {
noteId: note.id,
});
}
done();
}
}

6
packages/backend/src/queue/processors/ExportAntennasProcessorService.ts
View File

@@ -12,7 +12,7 @@ import { createTemp } from '@/misc/create-temp.js';
import { UtilityService } from '@/core/UtilityService.js';
import { QueueLoggerService } from '../QueueLoggerService.js';
import type { DBExportAntennasData } from '../types.js';
import type Bull from 'bull';
import type * as Bull from 'bullmq';
@Injectable()
export class ExportAntennasProcessorService {
@@ -39,10 +39,9 @@ export class ExportAntennasProcessorService {
}
@bindThis
public async process(job: Bull.Job<DBExportAntennasData>, done: () => void): Promise<void> {
public async process(job: Bull.Job<DBExportAntennasData>): Promise<void> {
const user = await this.usersRepository.findOneBy({ id: job.data.user.id });
if (user == null) {
done();
return;
}
const [path, cleanup] = await createTemp();
@@ -96,7 +95,6 @@ export class ExportAntennasProcessorService {
this.logger.succ('Exported to: ' + driveFile.id);
} finally {
cleanup();
done();
}
}
}

15
packages/backend/src/queue/processors/ExportBlockingProcessorService.ts
View File

@@ -9,10 +9,10 @@ import type Logger from '@/logger.js';
import { DriveService } from '@/core/DriveService.js';
import { createTemp } from '@/misc/create-temp.js';
import { UtilityService } from '@/core/UtilityService.js';
import { QueueLoggerService } from '../QueueLoggerService.js';
import type Bull from 'bull';
import type { DbJobDataWithUser } from '../types.js';
import { bindThis } from '@/decorators.js';
import { QueueLoggerService } from '../QueueLoggerService.js';
import type * as Bull from 'bullmq';
import type { DbJobDataWithUser } from '../types.js';
@Injectable()
export class ExportBlockingProcessorService {
@@ -36,12 +36,11 @@ export class ExportBlockingProcessorService {
}
@bindThis
public async process(job: Bull.Job<DbJobDataWithUser>, done: () => void): Promise<void> {
public async process(job: Bull.Job<DbJobDataWithUser>): Promise<void> {
this.logger.info(`Exporting blocking of ${job.data.user.id} ...`);
const user = await this.usersRepository.findOneBy({ id: job.data.user.id });
if (user == null) {
done();
return;
}
@@ -69,7 +68,7 @@ export class ExportBlockingProcessorService {
});
if (blockings.length === 0) {
job.progress(100);
job.updateProgress(100);
break;
}
@@ -99,7 +98,7 @@ export class ExportBlockingProcessorService {
blockerId: user.id,
});
job.progress(exportedCount / total);
job.updateProgress(exportedCount / total);
}
stream.end();
@@ -112,7 +111,5 @@ export class ExportBlockingProcessorService {
} finally {
cleanup();
}
done();
}
}

39
packages/backend/src/queue/processors/ExportCustomEmojisProcessorService.ts
View File

@@ -13,7 +13,7 @@ import { createTemp, createTempDir } from '@/misc/create-temp.js';
import { DownloadService } from '@/core/DownloadService.js';
import { bindThis } from '@/decorators.js';
import { QueueLoggerService } from '../QueueLoggerService.js';
import type Bull from 'bull';
import type * as Bull from 'bullmq';
@Injectable()
export class ExportCustomEmojisProcessorService {
@@ -37,12 +37,11 @@ export class ExportCustomEmojisProcessorService {
}
@bindThis
public async process(job: Bull.Job, done: () => void): Promise<void> {
public async process(job: Bull.Job): Promise<void> {
this.logger.info('Exporting custom emojis ...');
const user = await this.usersRepository.findOneBy({ id: job.data.user.id });
if (user == null) {
done();
return;
}
@@ -117,24 +116,26 @@ export class ExportCustomEmojisProcessorService {
metaStream.end();
// Create archive
const [archivePath, archiveCleanup] = await createTemp();
const archiveStream = fs.createWriteStream(archivePath);
const archive = archiver('zip', {
zlib: { level: 0 },
});
archiveStream.on('close', async () => {
this.logger.succ(`Exported to: ${archivePath}`);
await new Promise<void>(async (resolve) => {
const [archivePath, archiveCleanup] = await createTemp();
const archiveStream = fs.createWriteStream(archivePath);
const archive = archiver('zip', {
zlib: { level: 0 },
});
archiveStream.on('close', async () => {
this.logger.succ(`Exported to: ${archivePath}`);
const fileName = 'custom-emojis-' + dateFormat(new Date(), 'yyyy-MM-dd-HH-mm-ss') + '.zip';
const driveFile = await this.driveService.addFile({ user, path: archivePath, name: fileName, force: true });
const fileName = 'custom-emojis-' + dateFormat(new Date(), 'yyyy-MM-dd-HH-mm-ss') + '.zip';
const driveFile = await this.driveService.addFile({ user, path: archivePath, name: fileName, force: true });
this.logger.succ(`Exported to: ${driveFile.id}`);
cleanup();
archiveCleanup();
done();
this.logger.succ(`Exported to: ${driveFile.id}`);
cleanup();
archiveCleanup();
resolve();
});
archive.pipe(archiveStream);
archive.directory(path, false);
archive.finalize();
});
archive.pipe(archiveStream);
archive.directory(path, false);
archive.finalize();
}
}

11
packages/backend/src/queue/processors/ExportFavoritesProcessorService.ts
View File

@@ -12,7 +12,7 @@ import type { Poll } from '@/models/entities/Poll.js';
import type { Note } from '@/models/entities/Note.js';
import { bindThis } from '@/decorators.js';
import { QueueLoggerService } from '../QueueLoggerService.js';
import type Bull from 'bull';
import type * as Bull from 'bullmq';
import type { DbJobDataWithUser } from '../types.js';
@Injectable()
@@ -42,12 +42,11 @@ export class ExportFavoritesProcessorService {
}
@bindThis
public async process(job: Bull.Job<DbJobDataWithUser>, done: () => void): Promise<void> {
public async process(job: Bull.Job<DbJobDataWithUser>): Promise<void> {
this.logger.info(`Exporting favorites of ${job.data.user.id} ...`);
const user = await this.usersRepository.findOneBy({ id: job.data.user.id });
if (user == null) {
done();
return;
}
@@ -91,7 +90,7 @@ export class ExportFavoritesProcessorService {
}) as (NoteFavorite & { note: Note & { user: User } })[];
if (favorites.length === 0) {
job.progress(100);
job.updateProgress(100);
break;
}
@@ -112,7 +111,7 @@ export class ExportFavoritesProcessorService {
userId: user.id,
});
job.progress(exportedFavoritesCount / total);
job.updateProgress(exportedFavoritesCount / total);
}
await write(']');
@@ -127,8 +126,6 @@ export class ExportFavoritesProcessorService {
} finally {
cleanup();
}
done();
}
}

11
packages/backend/src/queue/processors/ExportFollowingProcessorService.ts
View File

@@ -10,10 +10,10 @@ import { DriveService } from '@/core/DriveService.js';
import { createTemp } from '@/misc/create-temp.js';
import type { Following } from '@/models/entities/Following.js';
import { UtilityService } from '@/core/UtilityService.js';
import { QueueLoggerService } from '../QueueLoggerService.js';
import type Bull from 'bull';
import type { DbExportFollowingData } from '../types.js';
import { bindThis } from '@/decorators.js';
import { QueueLoggerService } from '../QueueLoggerService.js';
import type * as Bull from 'bullmq';
import type { DbExportFollowingData } from '../types.js';
@Injectable()
export class ExportFollowingProcessorService {
@@ -40,12 +40,11 @@ export class ExportFollowingProcessorService {
}
@bindThis
public async process(job: Bull.Job<DbExportFollowingData>, done: () => void): Promise<void> {
public async process(job: Bull.Job<DbExportFollowingData>): Promise<void> {
this.logger.info(`Exporting following of ${job.data.user.id} ...`);
const user = await this.usersRepository.findOneBy({ id: job.data.user.id });
if (user == null) {
done();
return;
}
@@ -116,7 +115,5 @@ export class ExportFollowingProcessorService {
} finally {
cleanup();
}
done();
}
}

15
packages/backend/src/queue/processors/ExportMutingProcessorService.ts
View File

@@ -9,10 +9,10 @@ import type Logger from '@/logger.js';
import { DriveService } from '@/core/DriveService.js';
import { createTemp } from '@/misc/create-temp.js';
import { UtilityService } from '@/core/UtilityService.js';
import { QueueLoggerService } from '../QueueLoggerService.js';
import type Bull from 'bull';
import type { DbJobDataWithUser } from '../types.js';
import { bindThis } from '@/decorators.js';
import { QueueLoggerService } from '../QueueLoggerService.js';
import type * as Bull from 'bullmq';
import type { DbJobDataWithUser } from '../types.js';
@Injectable()
export class ExportMutingProcessorService {
@@ -39,12 +39,11 @@ export class ExportMutingProcessorService {
}
@bindThis
public async process(job: Bull.Job<DbJobDataWithUser>, done: () => void): Promise<void> {
public async process(job: Bull.Job<DbJobDataWithUser>): Promise<void> {
this.logger.info(`Exporting muting of ${job.data.user.id} ...`);
const user = await this.usersRepository.findOneBy({ id: job.data.user.id });
if (user == null) {
done();
return;
}
@@ -73,7 +72,7 @@ export class ExportMutingProcessorService {
});
if (mutes.length === 0) {
job.progress(100);
job.updateProgress(100);
break;
}
@@ -103,7 +102,7 @@ export class ExportMutingProcessorService {
muterId: user.id,
});
job.progress(exportedCount / total);
job.updateProgress(exportedCount / total);
}
stream.end();
@@ -116,7 +115,5 @@ export class ExportMutingProcessorService {
} finally {
cleanup();
}
done();
}
}

11
packages/backend/src/queue/processors/ExportNotesProcessorService.ts
View File

@@ -12,7 +12,7 @@ import type { Poll } from '@/models/entities/Poll.js';
import type { Note } from '@/models/entities/Note.js';
import { bindThis } from '@/decorators.js';
import { QueueLoggerService } from '../QueueLoggerService.js';
import type Bull from 'bull';
import type * as Bull from 'bullmq';
import type { DbJobDataWithUser } from '../types.js';
@Injectable()
@@ -39,12 +39,11 @@ export class ExportNotesProcessorService {
}
@bindThis
public async process(job: Bull.Job<DbJobDataWithUser>, done: () => void): Promise<void> {
public async process(job: Bull.Job<DbJobDataWithUser>): Promise<void> {
this.logger.info(`Exporting notes of ${job.data.user.id} ...`);
const user = await this.usersRepository.findOneBy({ id: job.data.user.id });
if (user == null) {
done();
return;
}
@@ -87,7 +86,7 @@ export class ExportNotesProcessorService {
}) as Note[];
if (notes.length === 0) {
job.progress(100);
job.updateProgress(100);
break;
}
@@ -108,7 +107,7 @@ export class ExportNotesProcessorService {
userId: user.id,
});
job.progress(exportedNotesCount / total);
job.updateProgress(exportedNotesCount / total);
}
await write(']');
@@ -123,8 +122,6 @@ export class ExportNotesProcessorService {
} finally {
cleanup();
}
done();
}
}

11
packages/backend/src/queue/processors/ExportUserListsProcessorService.ts
View File

@@ -9,10 +9,10 @@ import type Logger from '@/logger.js';
import { DriveService } from '@/core/DriveService.js';
import { createTemp } from '@/misc/create-temp.js';
import { UtilityService } from '@/core/UtilityService.js';
import { QueueLoggerService } from '../QueueLoggerService.js';
import type Bull from 'bull';
import type { DbJobDataWithUser } from '../types.js';
import { bindThis } from '@/decorators.js';
import { QueueLoggerService } from '../QueueLoggerService.js';
import type * as Bull from 'bullmq';
import type { DbJobDataWithUser } from '../types.js';
@Injectable()
export class ExportUserListsProcessorService {
@@ -39,12 +39,11 @@ export class ExportUserListsProcessorService {
}
@bindThis
public async process(job: Bull.Job<DbJobDataWithUser>, done: () => void): Promise<void> {
public async process(job: Bull.Job<DbJobDataWithUser>): Promise<void> {
this.logger.info(`Exporting user lists of ${job.data.user.id} ...`);
const user = await this.usersRepository.findOneBy({ id: job.data.user.id });
if (user == null) {
done();
return;
}
@@ -92,7 +91,5 @@ export class ExportUserListsProcessorService {
} finally {
cleanup();
}
done();
}
}

6
packages/backend/src/queue/processors/ImportAntennasProcessorService.ts
View File

@@ -8,7 +8,7 @@ import { DI } from '@/di-symbols.js';
import { bindThis } from '@/decorators.js';
import { QueueLoggerService } from '../QueueLoggerService.js';
import { DBAntennaImportJobData } from '../types.js';
import type Bull from 'bull';
import type * as Bull from 'bullmq';
const validate = new Ajv().compile({
type: 'object',
@@ -59,7 +59,7 @@ export class ImportAntennasProcessorService {
}
@bindThis
public async process(job: Bull.Job<DBAntennaImportJobData>, done: () => void): Promise<void> {
public async process(job: Bull.Job<DBAntennaImportJobData>): Promise<void> {
const now = new Date();
try {
for (const antenna of job.data.antenna) {
@@ -89,8 +89,6 @@ export class ImportAntennasProcessorService {
}
} catch (err: any) {
this.logger.error(err);
} finally {
done();
}
}
}

13
packages/backend/src/queue/processors/ImportBlockingProcessorService.ts
View File

@@ -7,11 +7,11 @@ import * as Acct from '@/misc/acct.js';
import { RemoteUserResolveService } from '@/core/RemoteUserResolveService.js';
import { DownloadService } from '@/core/DownloadService.js';
import { UtilityService } from '@/core/UtilityService.js';
import { QueueLoggerService } from '../QueueLoggerService.js';
import type Bull from 'bull';
import type { DbUserImportJobData, DbUserImportToDbJobData } from '../types.js';
import { bindThis } from '@/decorators.js';
import { QueueService } from '@/core/QueueService.js';
import { QueueLoggerService } from '../QueueLoggerService.js';
import type * as Bull from 'bullmq';
import type { DbUserImportJobData, DbUserImportToDbJobData } from '../types.js';
@Injectable()
export class ImportBlockingProcessorService {
@@ -34,12 +34,11 @@ export class ImportBlockingProcessorService {
}
@bindThis
public async process(job: Bull.Job<DbUserImportJobData>, done: () => void): Promise<void> {
public async process(job: Bull.Job<DbUserImportJobData>): Promise<void> {
this.logger.info(`Importing blocking of ${job.data.user.id} ...`);
const user = await this.usersRepository.findOneBy({ id: job.data.user.id });
if (user == null) {
done();
return;
}
@@ -47,7 +46,6 @@ export class ImportBlockingProcessorService {
id: job.data.fileId,
});
if (file == null) {
done();
return;
}
@@ -56,7 +54,6 @@ export class ImportBlockingProcessorService {
this.queueService.createImportBlockingToDbJob({ id: user.id }, targets);
this.logger.succ('Import jobs created');
done();
}
@bindThis
@@ -85,7 +82,7 @@ export class ImportBlockingProcessorService {
}
if (target == null) {
throw `Unable to resolve user: @${username}@${host}`;
throw new Error(`Unable to resolve user: @${username}@${host}`);
}
// skip myself

6
packages/backend/src/queue/processors/ImportCustomEmojisProcessorService.ts
View File

@@ -12,7 +12,7 @@ import { DriveService } from '@/core/DriveService.js';
import { DownloadService } from '@/core/DownloadService.js';
import { bindThis } from '@/decorators.js';
import { QueueLoggerService } from '../QueueLoggerService.js';
import type Bull from 'bull';
import type * as Bull from 'bullmq';
import type { DbUserImportJobData } from '../types.js';
// TODO: 名前衝突時の動作を選べるようにする
@@ -45,14 +45,13 @@ export class ImportCustomEmojisProcessorService {
}
@bindThis
public async process(job: Bull.Job<DbUserImportJobData>, done: () => void): Promise<void> {
public async process(job: Bull.Job<DbUserImportJobData>): Promise<void> {
this.logger.info('Importing custom emojis ...');
const file = await this.driveFilesRepository.findOneBy({
id: job.data.fileId,
});
if (file == null) {
done();
return;
}
@@ -116,7 +115,6 @@ export class ImportCustomEmojisProcessorService {
cleanup();
this.logger.succ('Imported');
done();
});
unzipStream.pipe(extractor);
this.logger.succ(`Unzipping to ${outputPath}`);

13
packages/backend/src/queue/processors/ImportFollowingProcessorService.ts
View File

@@ -7,11 +7,11 @@ import * as Acct from '@/misc/acct.js';
import { RemoteUserResolveService } from '@/core/RemoteUserResolveService.js';
import { DownloadService } from '@/core/DownloadService.js';
import { UtilityService } from '@/core/UtilityService.js';
import { QueueLoggerService } from '../QueueLoggerService.js';
import type Bull from 'bull';
import type { DbUserImportJobData, DbUserImportToDbJobData } from '../types.js';
import { bindThis } from '@/decorators.js';
import { QueueService } from '@/core/QueueService.js';
import { QueueLoggerService } from '../QueueLoggerService.js';
import type * as Bull from 'bullmq';
import type { DbUserImportJobData, DbUserImportToDbJobData } from '../types.js';
@Injectable()
export class ImportFollowingProcessorService {
@@ -34,12 +34,11 @@ export class ImportFollowingProcessorService {
}
@bindThis
public async process(job: Bull.Job<DbUserImportJobData>, done: () => void): Promise<void> {
public async process(job: Bull.Job<DbUserImportJobData>): Promise<void> {
this.logger.info(`Importing following of ${job.data.user.id} ...`);
const user = await this.usersRepository.findOneBy({ id: job.data.user.id });
if (user == null) {
done();
return;
}
@@ -47,7 +46,6 @@ export class ImportFollowingProcessorService {
id: job.data.fileId,
});
if (file == null) {
done();
return;
}
@@ -56,7 +54,6 @@ export class ImportFollowingProcessorService {
this.queueService.createImportFollowingToDbJob({ id: user.id }, targets);
this.logger.succ('Import jobs created');
done();
}
@bindThis
@@ -85,7 +82,7 @@ export class ImportFollowingProcessorService {
}
if (target == null) {
throw `Unable to resolve user: @${username}@${host}`;
throw new Error(`Unable to resolve user: @${username}@${host}`);
}
// skip myself

13
packages/backend/src/queue/processors/ImportMutingProcessorService.ts
View File

@@ -9,10 +9,10 @@ import { RemoteUserResolveService } from '@/core/RemoteUserResolveService.js';
import { DownloadService } from '@/core/DownloadService.js';
import { UserMutingService } from '@/core/UserMutingService.js';
import { UtilityService } from '@/core/UtilityService.js';
import { QueueLoggerService } from '../QueueLoggerService.js';
import type Bull from 'bull';
import type { DbUserImportJobData } from '../types.js';
import { bindThis } from '@/decorators.js';
import { QueueLoggerService } from '../QueueLoggerService.js';
import type * as Bull from 'bullmq';
import type { DbUserImportJobData } from '../types.js';
@Injectable()
export class ImportMutingProcessorService {
@@ -38,12 +38,11 @@ export class ImportMutingProcessorService {
}
@bindThis
public async process(job: Bull.Job<DbUserImportJobData>, done: () => void): Promise<void> {
public async process(job: Bull.Job<DbUserImportJobData>): Promise<void> {
this.logger.info(`Importing muting of ${job.data.user.id} ...`);
const user = await this.usersRepository.findOneBy({ id: job.data.user.id });
if (user == null) {
done();
return;
}
@@ -51,7 +50,6 @@ export class ImportMutingProcessorService {
id: job.data.fileId,
});
if (file == null) {
done();
return;
}
@@ -83,7 +81,7 @@ export class ImportMutingProcessorService {
}
if (target == null) {
throw `cannot resolve user: @${username}@${host}`;
throw new Error(`cannot resolve user: @${username}@${host}`);
}
// skip myself
@@ -98,6 +96,5 @@ export class ImportMutingProcessorService {
}
this.logger.succ('Imported');
done();
}
}

7
packages/backend/src/queue/processors/ImportUserListsProcessorService.ts
View File

@@ -12,7 +12,7 @@ import { IdService } from '@/core/IdService.js';
import { UtilityService } from '@/core/UtilityService.js';
import { bindThis } from '@/decorators.js';
import { QueueLoggerService } from '../QueueLoggerService.js';
import type Bull from 'bull';
import type * as Bull from 'bullmq';
import type { DbUserImportJobData } from '../types.js';
@Injectable()
@@ -46,12 +46,11 @@ export class ImportUserListsProcessorService {
}
@bindThis
public async process(job: Bull.Job<DbUserImportJobData>, done: () => void): Promise<void> {
public async process(job: Bull.Job<DbUserImportJobData>): Promise<void> {
this.logger.info(`Importing user lists of ${job.data.user.id} ...`);
const user = await this.usersRepository.findOneBy({ id: job.data.user.id });
if (user == null) {
done();
return;
}
@@ -59,7 +58,6 @@ export class ImportUserListsProcessorService {
id: job.data.fileId,
});
if (file == null) {
done();
return;
}
@@ -109,6 +107,5 @@ export class ImportUserListsProcessorService {
}
this.logger.succ('Imported');
done();
}
}

38
packages/backend/src/queue/processors/InboxProcessorService.ts
View File

@@ -1,8 +1,8 @@
import { URL } from 'node:url';
import { Inject, Injectable } from '@nestjs/common';
import httpSignature from '@peertube/http-signature';
import * as Bull from 'bullmq';
import { DI } from '@/di-symbols.js';
import type { InstancesRepository, DriveFilesRepository } from '@/models/index.js';
import type { Config } from '@/config.js';
import type Logger from '@/logger.js';
import { MetaService } from '@/core/MetaService.js';
@@ -23,10 +23,8 @@ import { LdSignatureService } from '@/core/activitypub/LdSignatureService.js';
import { ApInboxService } from '@/core/activitypub/ApInboxService.js';
import { bindThis } from '@/decorators.js';
import { QueueLoggerService } from '../QueueLoggerService.js';
import type Bull from 'bull';
import type { InboxJobData } from '../types.js';
// ユーザーのinboxにアクティビティが届いた時の処理
@Injectable()
export class InboxProcessorService {
private logger: Logger;
@@ -35,12 +33,6 @@ export class InboxProcessorService {
@Inject(DI.config)
private config: Config,
@Inject(DI.instancesRepository)
private instancesRepository: InstancesRepository,
@Inject(DI.driveFilesRepository)
private driveFilesRepository: DriveFilesRepository,
private utilityService: UtilityService,
private metaService: MetaService,
private apInboxService: ApInboxService,
@@ -93,24 +85,24 @@ export class InboxProcessorService {
try {
authUser = await this.apDbResolverService.getAuthUserFromApId(getApId(activity.actor));
} catch (err) {
// 対象が4xxならスキップ
// 対象が4xxならスキップ
if (err instanceof StatusError) {
if (err.isClientError) {
return `skip: Ignored deleted actors on both ends ${activity.actor} - ${err.statusCode}`;
throw new Bull.UnrecoverableError(`skip: Ignored deleted actors on both ends ${activity.actor} - ${err.statusCode}`);
}
throw `Error in actor ${activity.actor} - ${err.statusCode ?? err}`;
throw new Error(`Error in actor ${activity.actor} - ${err.statusCode ?? err}`);
}
}
}
// それでもわからなければ終了
if (authUser == null) {
return 'skip: failed to resolve user';
throw new Bull.UnrecoverableError('skip: failed to resolve user');
}
// publicKey がなくても終了
if (authUser.key == null) {
return 'skip: failed to resolve user publicKey';
throw new Bull.UnrecoverableError('skip: failed to resolve user publicKey');
}
// HTTP-Signatureの検証
@@ -118,10 +110,10 @@ export class InboxProcessorService {
// また、signatureのsignerは、activity.actorと一致する必要がある
if (!httpSignatureValidated || authUser.user.uri !== activity.actor) {
// 一致しなくても、でもLD-Signatureがありそうならそっちも見る
// 一致しなくても、でもLD-Signatureがありそうならそっちも見る
if (activity.signature) {
if (activity.signature.type !== 'RsaSignature2017') {
return `skip: unsupported LD-signature type ${activity.signature.type}`;
throw new Bull.UnrecoverableError(`skip: unsupported LD-signature type ${activity.signature.type}`);
}
// activity.signature.creator: https://example.oom/users/user#main-key
@@ -134,32 +126,32 @@ export class InboxProcessorService {
// keyIdからLD-Signatureのユーザーを取得
authUser = await this.apDbResolverService.getAuthUserFromKeyId(activity.signature.creator);
if (authUser == null) {
return 'skip: LD-Signatureのユーザーが取得できませんでした';
throw new Bull.UnrecoverableError('skip: LD-Signatureのユーザーが取得できませんでした');
}
if (authUser.key == null) {
return 'skip: LD-SignatureのユーザーはpublicKeyを持っていませんでした';
throw new Bull.UnrecoverableError('skip: LD-SignatureのユーザーはpublicKeyを持っていませんでした');
}
// LD-Signature検証
const ldSignature = this.ldSignatureService.use();
const verified = await ldSignature.verifyRsaSignature2017(activity, authUser.key.keyPem).catch(() => false);
if (!verified) {
return 'skip: LD-Signatureの検証に失敗しました';
throw new Bull.UnrecoverableError('skip: LD-Signatureの検証に失敗しました');
}
// もう一度actorチェック
if (authUser.user.uri !== activity.actor) {
return `skip: LD-Signature user(${authUser.user.uri}) !== activity.actor(${activity.actor})`;
throw new Bull.UnrecoverableError(`skip: LD-Signature user(${authUser.user.uri}) !== activity.actor(${activity.actor})`);
}
// ブロックしてたら中断
const ldHost = this.utilityService.extractDbHost(authUser.user.uri);
if (this.utilityService.isBlockedHost(meta.blockedHosts, ldHost)) {
return `Blocked request: ${ldHost}`;
throw new Bull.UnrecoverableError(`Blocked request: ${ldHost}`);
}
} else {
return `skip: http-signature verification failed and no LD-Signature. keyId=${signature.keyId}`;
throw new Bull.UnrecoverableError(`skip: http-signature verification failed and no LD-Signature. keyId=${signature.keyId}`);
}
}
@@ -168,7 +160,7 @@ export class InboxProcessorService {
const signerHost = this.utilityService.extractDbHost(authUser.user.uri!);
const activityIdHost = this.utilityService.extractDbHost(activity.id);
if (signerHost !== activityIdHost) {
return `skip: signerHost(${signerHost}) !== activity.id host(${activityIdHost}`;
throw new Bull.UnrecoverableError(`skip: signerHost(${signerHost}) !== activity.id host(${activityIdHost}`);
}
}

2
packages/backend/src/queue/processors/RelationshipProcessorService.ts
View File

@@ -1,5 +1,5 @@
import { Inject, Injectable } from '@nestjs/common';
import type Bull from 'bull';
import type * as Bull from 'bullmq';
import { UserFollowingService } from '@/core/UserFollowingService.js';
import { UserBlockingService } from '@/core/UserBlockingService.js';

5
packages/backend/src/queue/processors/ResyncChartsProcessorService.ts
View File

@@ -15,7 +15,7 @@ import PerUserDriveChart from '@/core/chart/charts/per-user-drive.js';
import ApRequestChart from '@/core/chart/charts/ap-request.js';
import { bindThis } from '@/decorators.js';
import { QueueLoggerService } from '../QueueLoggerService.js';
import type Bull from 'bull';
import type * as Bull from 'bullmq';
@Injectable()
export class ResyncChartsProcessorService {
@@ -43,7 +43,7 @@ export class ResyncChartsProcessorService {
}
@bindThis
public async process(job: Bull.Job<Record<string, unknown>>, done: () => void): Promise<void> {
public async process(): Promise<void> {
this.logger.info('Resync charts...');
// TODO: ユーザーごとのチャートも更新する
@@ -55,6 +55,5 @@ export class ResyncChartsProcessorService {
]);
this.logger.succ('All charts successfully resynced.');
done();
}
}

5
packages/backend/src/queue/processors/TickChartsProcessorService.ts
View File

@@ -16,7 +16,7 @@ import PerUserDriveChart from '@/core/chart/charts/per-user-drive.js';
import ApRequestChart from '@/core/chart/charts/ap-request.js';
import { bindThis } from '@/decorators.js';
import { QueueLoggerService } from '../QueueLoggerService.js';
import type Bull from 'bull';
import type * as Bull from 'bullmq';
@Injectable()
export class TickChartsProcessorService {
@@ -45,7 +45,7 @@ export class TickChartsProcessorService {
}
@bindThis
public async process(job: Bull.Job<Record<string, unknown>>, done: () => void): Promise<void> {
public async process(): Promise<void> {
this.logger.info('Tick charts...');
await Promise.all([
@@ -64,6 +64,5 @@ export class TickChartsProcessorService {
]);
this.logger.succ('All charts successfully ticked.');
done();
}
}

6
packages/backend/src/queue/processors/WebhookDeliverProcessorService.ts
View File

@@ -1,4 +1,5 @@
import { Inject, Injectable } from '@nestjs/common';
import * as Bull from 'bullmq';
import { DI } from '@/di-symbols.js';
import type { WebhooksRepository } from '@/models/index.js';
import type { Config } from '@/config.js';
@@ -7,7 +8,6 @@ import { HttpRequestService } from '@/core/HttpRequestService.js';
import { StatusError } from '@/misc/status-error.js';
import { bindThis } from '@/decorators.js';
import { QueueLoggerService } from '../QueueLoggerService.js';
import type Bull from 'bull';
import type { WebhookDeliverJobData } from '../types.js';
@Injectable()
@@ -66,11 +66,11 @@ export class WebhookDeliverProcessorService {
if (res instanceof StatusError) {
// 4xx
if (res.isClientError) {
return `${res.statusCode} ${res.statusMessage}`;
throw new Bull.UnrecoverableError(`${res.statusCode} ${res.statusMessage}`);
}
// 5xx etc.
throw `${res.statusCode} ${res.statusMessage}`;
throw new Error(`${res.statusCode} ${res.statusMessage}`);
} else {
// DNS error, socket error, timeout ...
throw res;