sendnrw/misskey
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: queueing bulk follow/unfollow and block/unblock (#10544)

Browse Source 
* wrap follow/unfollow and block/unblock as job queue

* create import job to follow in each iteration

* make relationship jobs concurrent

* replace to job queue if called repeatedly

* use addBulk to import

* omit stream when importing

* fix job caller

* use ThinUser instead of User to reduce redis memory consumption

* createImportFollowingToDbJobの呼び出し方を変える, 型補強

* Force ThinUser

* オブジェクト操作のみのメソッド名はgenerate...Data

* Force ThinUser in generateRelationshipJobData

* silent bulk unfollow at admin api endpoint

---------

Co-authored-by: tamaina <tamaina@hotmail.co.jp>
This commit is contained in:
Namekuji
2023-04-11 20:13:58 -04:00
committed by GitHub
parent b463490d9f
commit da83322200
23 changed files with 418 additions and 186 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
packages/backend/src/queue/processors/DeleteDriveFilesProcessorService.ts
View File

@@ -7,7 +7,7 @@ import type Logger from '@/logger.js';
import { DriveService } from '@/core/DriveService.js';
import { QueueLoggerService } from '../QueueLoggerService.js';
import type Bull from 'bull';
import type { DbUserJobData } from '../types.js';
import type { DbJobDataWithUser } from '../types.js';
import { bindThis } from '@/decorators.js';
@Injectable()
@@ -31,7 +31,7 @@ export class DeleteDriveFilesProcessorService {
}
@bindThis
public async process(job: Bull.Job<DbUserJobData>, done: () => void): Promise<void> {
public async process(job: Bull.Job<DbJobDataWithUser>, done: () => void): Promise<void> {
this.logger.info(`Deleting drive files of ${job.data.user.id} ...`);
const user = await this.usersRepository.findOneBy({ id: job.data.user.id });

4
packages/backend/src/queue/processors/ExportBlockingProcessorService.ts
View File

@@ -11,7 +11,7 @@ import { createTemp } from '@/misc/create-temp.js';
import { UtilityService } from '@/core/UtilityService.js';
import { QueueLoggerService } from '../QueueLoggerService.js';
import type Bull from 'bull';
import type { DbUserJobData } from '../types.js';
import type { DbJobDataWithUser } from '../types.js';
import { bindThis } from '@/decorators.js';
@Injectable()
@@ -36,7 +36,7 @@ export class ExportBlockingProcessorService {
}
@bindThis
public async process(job: Bull.Job<DbUserJobData>, done: () => void): Promise<void> {
public async process(job: Bull.Job<DbJobDataWithUser>, done: () => void): Promise<void> {
this.logger.info(`Exporting blocking of ${job.data.user.id} ...`);
const user = await this.usersRepository.findOneBy({ id: job.data.user.id });

4
packages/backend/src/queue/processors/ExportFavoritesProcessorService.ts
View File

@@ -13,7 +13,7 @@ import type { Note } from '@/models/entities/Note.js';
import { bindThis } from '@/decorators.js';
import { QueueLoggerService } from '../QueueLoggerService.js';
import type Bull from 'bull';
import type { DbUserJobData } from '../types.js';
import type { DbJobDataWithUser } from '../types.js';
@Injectable()
export class ExportFavoritesProcessorService {
@@ -42,7 +42,7 @@ export class ExportFavoritesProcessorService {
}
@bindThis
public async process(job: Bull.Job<DbUserJobData>, done: () => void): Promise<void> {
public async process(job: Bull.Job<DbJobDataWithUser>, done: () => void): Promise<void> {
this.logger.info(`Exporting favorites of ${job.data.user.id} ...`);
const user = await this.usersRepository.findOneBy({ id: job.data.user.id });

4
packages/backend/src/queue/processors/ExportFollowingProcessorService.ts
View File

@@ -12,7 +12,7 @@ import type { Following } from '@/models/entities/Following.js';
import { UtilityService } from '@/core/UtilityService.js';
import { QueueLoggerService } from '../QueueLoggerService.js';
import type Bull from 'bull';
import type { DbUserJobData } from '../types.js';
import type { DbExportFollowingData } from '../types.js';
import { bindThis } from '@/decorators.js';
@Injectable()
@@ -40,7 +40,7 @@ export class ExportFollowingProcessorService {
}
@bindThis
public async process(job: Bull.Job<DbUserJobData>, done: () => void): Promise<void> {
public async process(job: Bull.Job<DbExportFollowingData>, done: () => void): Promise<void> {
this.logger.info(`Exporting following of ${job.data.user.id} ...`);
const user = await this.usersRepository.findOneBy({ id: job.data.user.id });

4
packages/backend/src/queue/processors/ExportMutingProcessorService.ts
View File

@@ -11,7 +11,7 @@ import { createTemp } from '@/misc/create-temp.js';
import { UtilityService } from '@/core/UtilityService.js';
import { QueueLoggerService } from '../QueueLoggerService.js';
import type Bull from 'bull';
import type { DbUserJobData } from '../types.js';
import type { DbJobDataWithUser } from '../types.js';
import { bindThis } from '@/decorators.js';
@Injectable()
@@ -39,7 +39,7 @@ export class ExportMutingProcessorService {
}
@bindThis
public async process(job: Bull.Job<DbUserJobData>, done: () => void): Promise<void> {
public async process(job: Bull.Job<DbJobDataWithUser>, done: () => void): Promise<void> {
this.logger.info(`Exporting muting of ${job.data.user.id} ...`);
const user = await this.usersRepository.findOneBy({ id: job.data.user.id });

4
packages/backend/src/queue/processors/ExportNotesProcessorService.ts
View File

@@ -13,7 +13,7 @@ import type { Note } from '@/models/entities/Note.js';
import { bindThis } from '@/decorators.js';
import { QueueLoggerService } from '../QueueLoggerService.js';
import type Bull from 'bull';
import type { DbUserJobData } from '../types.js';
import type { DbJobDataWithUser } from '../types.js';
@Injectable()
export class ExportNotesProcessorService {
@@ -39,7 +39,7 @@ export class ExportNotesProcessorService {
}
@bindThis
public async process(job: Bull.Job<DbUserJobData>, done: () => void): Promise<void> {
public async process(job: Bull.Job<DbJobDataWithUser>, done: () => void): Promise<void> {
this.logger.info(`Exporting notes of ${job.data.user.id} ...`);
const user = await this.usersRepository.findOneBy({ id: job.data.user.id });

4
packages/backend/src/queue/processors/ExportUserListsProcessorService.ts
View File

@@ -11,7 +11,7 @@ import { createTemp } from '@/misc/create-temp.js';
import { UtilityService } from '@/core/UtilityService.js';
import { QueueLoggerService } from '../QueueLoggerService.js';
import type Bull from 'bull';
import type { DbUserJobData } from '../types.js';
import type { DbJobDataWithUser } from '../types.js';
import { bindThis } from '@/decorators.js';
@Injectable()
@@ -39,7 +39,7 @@ export class ExportUserListsProcessorService {
}
@bindThis
public async process(job: Bull.Job<DbUserJobData>, done: () => void): Promise<void> {
public async process(job: Bull.Job<DbJobDataWithUser>, done: () => void): Promise<void> {
this.logger.info(`Exporting user lists of ${job.data.user.id} ...`);
const user = await this.usersRepository.findOneBy({ id: job.data.user.id });

97
packages/backend/src/queue/processors/ImportBlockingProcessorService.ts
View File

@@ -1,38 +1,31 @@
import { Inject, Injectable } from '@nestjs/common';
import { IsNull } from 'typeorm';
import { DI } from '@/di-symbols.js';
import type { UsersRepository, BlockingsRepository, DriveFilesRepository } from '@/models/index.js';
import type { Config } from '@/config.js';
import type { UsersRepository, DriveFilesRepository } from '@/models/index.js';
import type Logger from '@/logger.js';
import * as Acct from '@/misc/acct.js';
import { RemoteUserResolveService } from '@/core/RemoteUserResolveService.js';
import { UserBlockingService } from '@/core/UserBlockingService.js';
import { DownloadService } from '@/core/DownloadService.js';
import { UtilityService } from '@/core/UtilityService.js';
import { QueueLoggerService } from '../QueueLoggerService.js';
import type Bull from 'bull';
import type { DbUserImportJobData } from '../types.js';
import type { DbUserImportJobData, DbUserImportToDbJobData } from '../types.js';
import { bindThis } from '@/decorators.js';
import { QueueService } from '@/core/QueueService.js';
@Injectable()
export class ImportBlockingProcessorService {
private logger: Logger;
constructor(
@Inject(DI.config)
private config: Config,
@Inject(DI.usersRepository)
private usersRepository: UsersRepository,
@Inject(DI.blockingsRepository)
private blockingsRepository: BlockingsRepository,
@Inject(DI.driveFilesRepository)
private driveFilesRepository: DriveFilesRepository,
private queueService: QueueService,
private utilityService: UtilityService,
private userBlockingService: UserBlockingService,
private remoteUserResolveService: RemoteUserResolveService,
private downloadService: DownloadService,
private queueLoggerService: QueueLoggerService,
@@ -59,46 +52,50 @@ export class ImportBlockingProcessorService {
}
const csv = await this.downloadService.downloadTextFile(file.url);
const targets = csv.trim().split('\n');
this.queueService.createImportBlockingToDbJob({ id: user.id }, targets);
let linenum = 0;
for (const line of csv.trim().split('\n')) {
linenum++;
try {
const acct = line.split(',')[0].trim();
const { username, host } = Acct.parse(acct);
let target = this.utilityService.isSelfHost(host!) ? await this.usersRepository.findOneBy({
host: IsNull(),
usernameLower: username.toLowerCase(),
}) : await this.usersRepository.findOneBy({
host: this.utilityService.toPuny(host!),
usernameLower: username.toLowerCase(),
});
if (host == null && target == null) continue;
if (target == null) {
target = await this.remoteUserResolveService.resolveUser(username, host);
}
if (target == null) {
throw `cannot resolve user: @${username}@${host}`;
}
// skip myself
if (target.id === job.data.user.id) continue;
this.logger.info(`Block[${linenum}] ${target.id} ...`);
await this.userBlockingService.block(user, target);
} catch (e) {
this.logger.warn(`Error in line:${linenum} ${e}`);
}
}
this.logger.succ('Imported');
this.logger.succ('Import jobs created');
done();
}
@bindThis
public async processDb(job: Bull.Job<DbUserImportToDbJobData>): Promise<void> {
const line = job.data.target;
const user = job.data.user;
try {
const acct = line.split(',')[0].trim();
const { username, host } = Acct.parse(acct);
if (!host) return;
let target = this.utilityService.isSelfHost(host) ? await this.usersRepository.findOneBy({
host: IsNull(),
usernameLower: username.toLowerCase(),
}) : await this.usersRepository.findOneBy({
host: this.utilityService.toPuny(host),
usernameLower: username.toLowerCase(),
});
if (host == null && target == null) return;
if (target == null) {
target = await this.remoteUserResolveService.resolveUser(username, host);
}
if (target == null) {
throw `Unable to resolve user: @${username}@${host}`;
}
// skip myself
if (target.id === job.data.user.id) return;
this.logger.info(`Block ${target.id} ...`);
this.queueService.createBlockJob([{ from: { id: user.id }, to: { id: target.id }, silent: true }]);
} catch (e) {
this.logger.warn(`Error: ${e}`);
}
}
}

92
packages/backend/src/queue/processors/ImportFollowingProcessorService.ts
View File

@@ -2,34 +2,30 @@ import { Inject, Injectable } from '@nestjs/common';
import { IsNull } from 'typeorm';
import { DI } from '@/di-symbols.js';
import type { UsersRepository, DriveFilesRepository } from '@/models/index.js';
import type { Config } from '@/config.js';
import type Logger from '@/logger.js';
import * as Acct from '@/misc/acct.js';
import { RemoteUserResolveService } from '@/core/RemoteUserResolveService.js';
import { DownloadService } from '@/core/DownloadService.js';
import { UserFollowingService } from '@/core/UserFollowingService.js';
import { UtilityService } from '@/core/UtilityService.js';
import { QueueLoggerService } from '../QueueLoggerService.js';
import type Bull from 'bull';
import type { DbUserImportJobData } from '../types.js';
import type { DbUserImportJobData, DbUserImportToDbJobData } from '../types.js';
import { bindThis } from '@/decorators.js';
import { QueueService } from '@/core/QueueService.js';
@Injectable()
export class ImportFollowingProcessorService {
private logger: Logger;
constructor(
@Inject(DI.config)
private config: Config,
@Inject(DI.usersRepository)
private usersRepository: UsersRepository,
@Inject(DI.driveFilesRepository)
private driveFilesRepository: DriveFilesRepository,
private queueService: QueueService,
private utilityService: UtilityService,
private userFollowingService: UserFollowingService,
private remoteUserResolveService: RemoteUserResolveService,
private downloadService: DownloadService,
private queueLoggerService: QueueLoggerService,
@@ -56,46 +52,50 @@ export class ImportFollowingProcessorService {
}
const csv = await this.downloadService.downloadTextFile(file.url);
const targets = csv.trim().split('\n');
this.queueService.createImportFollowingToDbJob({ id: user.id }, targets);
let linenum = 0;
for (const line of csv.trim().split('\n')) {
linenum++;
try {
const acct = line.split(',')[0].trim();
const { username, host } = Acct.parse(acct);
let target = this.utilityService.isSelfHost(host!) ? await this.usersRepository.findOneBy({
host: IsNull(),
usernameLower: username.toLowerCase(),
}) : await this.usersRepository.findOneBy({
host: this.utilityService.toPuny(host!),
usernameLower: username.toLowerCase(),
});
if (host == null && target == null) continue;
if (target == null) {
target = await this.remoteUserResolveService.resolveUser(username, host);
}
if (target == null) {
throw `cannot resolve user: @${username}@${host}`;
}
// skip myself
if (target.id === job.data.user.id) continue;
this.logger.info(`Follow[${linenum}] ${target.id} ...`);
this.userFollowingService.follow(user, target);
} catch (e) {
this.logger.warn(`Error in line:${linenum} ${e}`);
}
}
this.logger.succ('Imported');
this.logger.succ('Import jobs created');
done();
}
@bindThis
public async processDb(job: Bull.Job<DbUserImportToDbJobData>): Promise<void> {
const line = job.data.target;
const user = job.data.user;
try {
const acct = line.split(',')[0].trim();
const { username, host } = Acct.parse(acct);
if (!host) return;
let target = this.utilityService.isSelfHost(host) ? await this.usersRepository.findOneBy({
host: IsNull(),
usernameLower: username.toLowerCase(),
}) : await this.usersRepository.findOneBy({
host: this.utilityService.toPuny(host),
usernameLower: username.toLowerCase(),
});
if (host == null && target == null) return;
if (target == null) {
target = await this.remoteUserResolveService.resolveUser(username, host);
}
if (target == null) {
throw `Unable to resolve user: @${username}@${host}`;
}
// skip myself
if (target.id === job.data.user.id) return;
this.logger.info(`Follow ${target.id} ...`);
this.queueService.createFollowJob([{ from: user, to: { id: target.id }, silent: true }]);
} catch (e) {
this.logger.warn(`Error: ${e}`);
}
}
}

6
packages/backend/src/queue/processors/ImportMutingProcessorService.ts
View File

@@ -66,11 +66,13 @@ export class ImportMutingProcessorService {
const acct = line.split(',')[0].trim();
const { username, host } = Acct.parse(acct);
let target = this.utilityService.isSelfHost(host!) ? await this.usersRepository.findOneBy({
if (!host) continue;
let target = this.utilityService.isSelfHost(host) ? await this.usersRepository.findOneBy({
host: IsNull(),
usernameLower: username.toLowerCase(),
}) : await this.usersRepository.findOneBy({
host: this.utilityService.toPuny(host!),
host: this.utilityService.toPuny(host),
usernameLower: username.toLowerCase(),
});

68
packages/backend/src/queue/processors/RelationshipProcessorService.ts Normal file
View File

@@ -0,0 +1,68 @@
import { Inject, Injectable } from '@nestjs/common';
import type Bull from 'bull';
import { UserFollowingService } from '@/core/UserFollowingService.js';
import { UserBlockingService } from '@/core/UserBlockingService.js';
import { bindThis } from '@/decorators.js';
import type Logger from '@/logger.js';
import { QueueLoggerService } from '../QueueLoggerService.js';
import { RelationshipJobData } from '../types.js';
import type { UsersRepository } from '@/models/index.js';
import { DI } from '@/di-symbols.js';
@Injectable()
export class RelationshipProcessorService {
private logger: Logger;
constructor(
@Inject(DI.usersRepository)
private usersRepository: UsersRepository,
private queueLoggerService: QueueLoggerService,
private userFollowingService: UserFollowingService,
private userBlockingService: UserBlockingService,
) {
this.logger = this.queueLoggerService.logger.createSubLogger('follow-block');
}
@bindThis
public async processFollow(job: Bull.Job<RelationshipJobData>): Promise<string> {
this.logger.info(`${job.data.from.id} is trying to follow ${job.data.to.id}`);
await this.userFollowingService.follow(job.data.from, job.data.to, job.data.requestId, job.data.silent);
return 'ok';
}
@bindThis
public async processUnfollow(job: Bull.Job<RelationshipJobData>): Promise<string> {
this.logger.info(`${job.data.from.id} is trying to unfollow ${job.data.to.id}`);
const [follower, followee] = await Promise.all([
this.usersRepository.findOneByOrFail({ id: job.data.from.id }),
this.usersRepository.findOneByOrFail({ id: job.data.to.id }),
]);
await this.userFollowingService.unfollow(follower, followee, job.data.silent);
return 'ok';
}
@bindThis
public async processBlock(job: Bull.Job<RelationshipJobData>): Promise<string> {
this.logger.info(`${job.data.from.id} is trying to block ${job.data.to.id}`);
const [blockee, blocker] = await Promise.all([
this.usersRepository.findOneByOrFail({ id: job.data.from.id }),
this.usersRepository.findOneByOrFail({ id: job.data.to.id }),
]);
await this.userBlockingService.block(blockee, blocker, job.data.silent);
return 'ok';
}
@bindThis
public async processUnblock(job: Bull.Job<RelationshipJobData>): Promise<string> {
this.logger.info(`${job.data.from.id} is trying to unblock ${job.data.to.id}`);
const [blockee, blocker] = await Promise.all([
this.usersRepository.findOneByOrFail({ id: job.data.from.id }),
this.usersRepository.findOneByOrFail({ id: job.data.to.id }),
]);
await this.userBlockingService.unblock(blockee, blocker);
return 'ok';
}
}