sendnrw/misskey
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'upstream/develop' into merge-upstream

Browse Source
This commit is contained in:
riku6460
2023-05-10 18:48:15 +09:00
parent cda7d4ee9c 6d4d9f417b
commit f3fd0cd93e
42 changed files with 639 additions and 671 deletions
Download Patch File Download Diff File Expand all files Collapse all files

11
packages/backend/migration/1683682889948-prevent-ai-larning.js Normal file
View File

@@ -0,0 +1,11 @@
export class PreventAiLarning1683682889948 {
name = 'PreventAiLarning1683682889948'
async up(queryRunner) {
await queryRunner.query(`ALTER TABLE "user_profile" ADD "preventAiLarning" boolean NOT NULL DEFAULT true`);
}
async down(queryRunner) {
await queryRunner.query(`ALTER TABLE "user_profile" DROP COLUMN "preventAiLarning"`);
}
}

11
packages/backend/migration/1683683083083-public-reactions-default-true.js Normal file
View File

@@ -0,0 +1,11 @@
export class PublicReactionsDefaultTrue1683683083083 {
name = 'PublicReactionsDefaultTrue1683683083083'
async up(queryRunner) {
await queryRunner.query(`ALTER TABLE "user_profile" ALTER COLUMN "publicReactions" SET DEFAULT true`);
}
async down(queryRunner) {
await queryRunner.query(`ALTER TABLE "user_profile" ALTER COLUMN "publicReactions" SET DEFAULT false`);
}
}

10
packages/backend/src/boot/common.ts
View File

@@ -18,10 +18,12 @@ export async function server() {
const serverService = app.get(ServerService);
await serverService.launch();
app.get(ChartManagementService).start();
app.get(JanitorService).start();
app.get(QueueStatsService).start();
app.get(ServerStatsService).start();
if (process.env.NODE_ENV !== 'test') {
app.get(ChartManagementService).start();
app.get(JanitorService).start();
app.get(QueueStatsService).start();
app.get(ServerStatsService).start();
}
return app;
}

29
packages/backend/src/core/NoteCreateService.ts
View File

@@ -3,6 +3,7 @@ import * as mfm from 'mfm-js';
import { In, DataSource } from 'typeorm';
import * as Redis from 'ioredis';
import { Inject, Injectable, OnApplicationShutdown } from '@nestjs/common';
import RE2 from 're2';
import { extractMentions } from '@/misc/extract-mentions.js';
import { extractCustomEmojisFromMfm } from '@/misc/extract-custom-emojis-from-mfm.js';
import { extractHashtags } from '@/misc/extract-hashtags.js';
@@ -238,7 +239,8 @@ export class NoteCreateService implements OnApplicationShutdown {
if (data.channel != null) data.localOnly = true;
if (data.visibility === 'public' && data.channel == null) {
if ((data.text != null) && (await this.metaService.fetch()).sensitiveWords.some(w => data.text!.includes(w))) {
const sensitiveWords = (await this.metaService.fetch()).sensitiveWords;
if (this.isSensitive(data, sensitiveWords)) {
data.visibility = 'home';
} else if ((await this.roleService.getUserPolicies(user.id)).canPublicNote === false) {
data.visibility = 'home';
@@ -670,6 +672,31 @@ export class NoteCreateService implements OnApplicationShutdown {
// Register to search database
this.index(note);
}
@bindThis
private isSensitive(note: Option, sensitiveWord: string[]): boolean {
if (sensitiveWord.length > 0) {
const text = note.cw ?? note.text ?? '';
if (text === '') return false;
const matched = sensitiveWord.some(filter => {
// represents RegExp
const regexp = filter.match(/^\/(.+)\/(.*)$/);
// This should never happen due to input sanitisation.
if (!regexp) {
const words = filter.split(' ');
return words.every(keyword => text.includes(keyword));
}
try {
return new RE2(regexp[1], regexp[2]).test(text);
} catch (err) {
// This should never happen due to input sanitisation.
return false;
}
});
if (matched) return true;
}
return false;
}
@bindThis
private incRenoteCount(renote: Note) {

41
packages/backend/src/core/QueueModule.ts
View File

@@ -1,4 +1,5 @@
import { Module } from '@nestjs/common';
import { setTimeout } from 'node:timers/promises';
import { Inject, Module, OnApplicationShutdown } from '@nestjs/common';
import Bull from 'bull';
import { DI } from '@/di-symbols.js';
import type { Config } from '@/config.js';
@@ -41,9 +42,9 @@ export type SystemQueue = Bull.Queue<Record<string, unknown>>;
export type EndedPollNotificationQueue = Bull.Queue<EndedPollNotificationJobData>;
export type DeliverQueue = Bull.Queue<DeliverJobData>;
export type InboxQueue = Bull.Queue<InboxJobData>;
export type DbQueue = Bull.Queue<DbJobData<keyof DbJobMap>>;
export type DbQueue = Bull.Queue;
export type RelationshipQueue = Bull.Queue<RelationshipJobData>;
export type ObjectStorageQueue = Bull.Queue<ObjectStorageJobData>;
export type ObjectStorageQueue = Bull.Queue;
export type WebhookDeliverQueue = Bull.Queue<WebhookDeliverJobData>;
const $system: Provider = {
@@ -118,4 +119,36 @@ const $webhookDeliver: Provider = {
$webhookDeliver,
],
})
export class QueueModule {}
export class QueueModule implements OnApplicationShutdown {
constructor(
@Inject('queue:system') public systemQueue: SystemQueue,
@Inject('queue:endedPollNotification') public endedPollNotificationQueue: EndedPollNotificationQueue,
@Inject('queue:deliver') public deliverQueue: DeliverQueue,
@Inject('queue:inbox') public inboxQueue: InboxQueue,
@Inject('queue:db') public dbQueue: DbQueue,
@Inject('queue:relationship') public relationshipQueue: RelationshipQueue,
@Inject('queue:objectStorage') public objectStorageQueue: ObjectStorageQueue,
@Inject('queue:webhookDeliver') public webhookDeliverQueue: WebhookDeliverQueue,
) {}
async onApplicationShutdown(signal: string): Promise<void> {
if (process.env.NODE_ENV === 'test') {
// XXX:
// Shutting down the existing connections causes errors on Jest as
// Misskey has asynchronous postgres/redis connections that are not
// awaited.
// Let's wait for some random time for them to finish.
await setTimeout(5000);
}
await Promise.all([
this.systemQueue.close(),
this.endedPollNotificationQueue.close(),
this.deliverQueue.close(),
this.inboxQueue.close(),
this.dbQueue.close(),
this.relationshipQueue.close(),
this.objectStorageQueue.close(),
this.webhookDeliverQueue.close(),
]);
}
}

1
packages/backend/src/core/entities/UserEntityService.ts
View File

@@ -445,6 +445,7 @@ export class UserEntityService implements OnModuleInit {
carefulBot: profile!.carefulBot,
autoAcceptFollowed: profile!.autoAcceptFollowed,
noCrawle: profile!.noCrawle,
preventAiLarning: profile!.preventAiLarning,
isExplorable: user.isExplorable,
isDeleted: user.isDeleted,
hideOnlineStatus: user.hideOnlineStatus,

7
packages/backend/src/models/entities/UserProfile.ts
View File

@@ -76,7 +76,7 @@ export class UserProfile {
public emailNotificationTypes: string[];
@Column('boolean', {
default: false,
default: true,
})
public publicReactions: boolean;
@@ -147,6 +147,11 @@ export class UserProfile {
})
public noCrawle: boolean;
@Column('boolean', {
default: true,
})
public preventAiLarning: boolean;
@Column('boolean', {
default: false,
})

6
packages/backend/src/models/json-schema/user.ts
View File

@@ -302,7 +302,11 @@ export const packedMeDetailedOnlySchema = {
},
noCrawle: {
type: 'boolean',
nullable: true, optional: false,
nullable: false, optional: false,
},
preventAiLarning: {
type: 'boolean',
nullable: false, optional: false,
},
isExplorable: {
type: 'boolean',

69
packages/backend/src/queue/DbQueueProcessorsService.ts
View File

@@ -1,69 +0,0 @@
import { Inject, Injectable } from '@nestjs/common';
import { DI } from '@/di-symbols.js';
import type { Config } from '@/config.js';
import { bindThis } from '@/decorators.js';
import { DeleteDriveFilesProcessorService } from './processors/DeleteDriveFilesProcessorService.js';
import { ExportCustomEmojisProcessorService } from './processors/ExportCustomEmojisProcessorService.js';
import { ExportNotesProcessorService } from './processors/ExportNotesProcessorService.js';
import { ExportFollowingProcessorService } from './processors/ExportFollowingProcessorService.js';
import { ExportMutingProcessorService } from './processors/ExportMutingProcessorService.js';
import { ExportBlockingProcessorService } from './processors/ExportBlockingProcessorService.js';
import { ExportUserListsProcessorService } from './processors/ExportUserListsProcessorService.js';
import { ExportAntennasProcessorService } from './processors/ExportAntennasProcessorService.js';
import { ImportFollowingProcessorService } from './processors/ImportFollowingProcessorService.js';
import { ImportMutingProcessorService } from './processors/ImportMutingProcessorService.js';
import { ImportBlockingProcessorService } from './processors/ImportBlockingProcessorService.js';
import { ImportUserListsProcessorService } from './processors/ImportUserListsProcessorService.js';
import { ImportCustomEmojisProcessorService } from './processors/ImportCustomEmojisProcessorService.js';
import { ImportAntennasProcessorService } from './processors/ImportAntennasProcessorService.js';
import { DeleteAccountProcessorService } from './processors/DeleteAccountProcessorService.js';
import { ExportFavoritesProcessorService } from './processors/ExportFavoritesProcessorService.js';
import type Bull from 'bull';
@Injectable()
export class DbQueueProcessorsService {
constructor(
@Inject(DI.config)
private config: Config,
private deleteDriveFilesProcessorService: DeleteDriveFilesProcessorService,
private exportCustomEmojisProcessorService: ExportCustomEmojisProcessorService,
private exportNotesProcessorService: ExportNotesProcessorService,
private exportFavoritesProcessorService: ExportFavoritesProcessorService,
private exportFollowingProcessorService: ExportFollowingProcessorService,
private exportMutingProcessorService: ExportMutingProcessorService,
private exportBlockingProcessorService: ExportBlockingProcessorService,
private exportUserListsProcessorService: ExportUserListsProcessorService,
private exportAntennasProcessorService: ExportAntennasProcessorService,
private importFollowingProcessorService: ImportFollowingProcessorService,
private importMutingProcessorService: ImportMutingProcessorService,
private importBlockingProcessorService: ImportBlockingProcessorService,
private importUserListsProcessorService: ImportUserListsProcessorService,
private importCustomEmojisProcessorService: ImportCustomEmojisProcessorService,
private importAntennasProcessorService: ImportAntennasProcessorService,
private deleteAccountProcessorService: DeleteAccountProcessorService,
) {
}
@bindThis
public start(q: Bull.Queue): void {
q.process('deleteDriveFiles', (job, done) => this.deleteDriveFilesProcessorService.process(job, done));
q.process('exportCustomEmojis', (job, done) => this.exportCustomEmojisProcessorService.process(job, done));
q.process('exportNotes', (job, done) => this.exportNotesProcessorService.process(job, done));
q.process('exportFavorites', (job, done) => this.exportFavoritesProcessorService.process(job, done));
q.process('exportFollowing', (job, done) => this.exportFollowingProcessorService.process(job, done));
q.process('exportMuting', (job, done) => this.exportMutingProcessorService.process(job, done));
q.process('exportBlocking', (job, done) => this.exportBlockingProcessorService.process(job, done));
q.process('exportUserLists', (job, done) => this.exportUserListsProcessorService.process(job, done));
q.process('exportAntennas', (job, done) => this.exportAntennasProcessorService.process(job, done));
q.process('importFollowing', (job, done) => this.importFollowingProcessorService.process(job, done));
q.process('importFollowingToDb', (job) => this.importFollowingProcessorService.processDb(job));
q.process('importMuting', (job, done) => this.importMutingProcessorService.process(job, done));
q.process('importBlocking', (job, done) => this.importBlockingProcessorService.process(job, done));
q.process('importBlockingToDb', (job) => this.importBlockingProcessorService.processDb(job));
q.process('importUserLists', (job, done) => this.importUserListsProcessorService.process(job, done));
q.process('importCustomEmojis', (job, done) => this.importCustomEmojisProcessorService.process(job, done));
q.process('importAntennas', (job, done) => this.importAntennasProcessorService.process(job, done));
q.process('deleteAccount', (job) => this.deleteAccountProcessorService.process(job));
}
}

25
packages/backend/src/queue/ObjectStorageQueueProcessorsService.ts
View File

@@ -1,25 +0,0 @@
import { Inject, Injectable } from '@nestjs/common';
import { DI } from '@/di-symbols.js';
import type { Config } from '@/config.js';
import { CleanRemoteFilesProcessorService } from './processors/CleanRemoteFilesProcessorService.js';
import { DeleteFileProcessorService } from './processors/DeleteFileProcessorService.js';
import type Bull from 'bull';
import { bindThis } from '@/decorators.js';
@Injectable()
export class ObjectStorageQueueProcessorsService {
constructor(
@Inject(DI.config)
private config: Config,
private deleteFileProcessorService: DeleteFileProcessorService,
private cleanRemoteFilesProcessorService: CleanRemoteFilesProcessorService,
) {
}
@bindThis
public start(q: Bull.Queue): void {
q.process('deleteFile', 16, (job) => this.deleteFileProcessorService.process(job));
q.process('cleanRemoteFiles', 16, (job, done) => this.cleanRemoteFilesProcessorService.process(job, done));
}
}

8
packages/backend/src/queue/QueueProcessorModule.ts
View File

@@ -3,14 +3,10 @@ import { CoreModule } from '@/core/CoreModule.js';
import { GlobalModule } from '@/GlobalModule.js';
import { QueueLoggerService } from './QueueLoggerService.js';
import { QueueProcessorService } from './QueueProcessorService.js';
import { DbQueueProcessorsService } from './DbQueueProcessorsService.js';
import { RelationshipQueueProcessorsService } from './RelationshipQueueProcessorsService.js';
import { ObjectStorageQueueProcessorsService } from './ObjectStorageQueueProcessorsService.js';
import { DeliverProcessorService } from './processors/DeliverProcessorService.js';
import { EndedPollNotificationProcessorService } from './processors/EndedPollNotificationProcessorService.js';
import { InboxProcessorService } from './processors/InboxProcessorService.js';
import { WebhookDeliverProcessorService } from './processors/WebhookDeliverProcessorService.js';
import { SystemQueueProcessorsService } from './SystemQueueProcessorsService.js';
import { CheckExpiredMutingsProcessorService } from './processors/CheckExpiredMutingsProcessorService.js';
import { CleanChartsProcessorService } from './processors/CleanChartsProcessorService.js';
import { CleanProcessorService } from './processors/CleanProcessorService.js';
@@ -68,10 +64,6 @@ import { RelationshipProcessorService } from './processors/RelationshipProcessor
DeleteFileProcessorService,
CleanRemoteFilesProcessorService,
RelationshipProcessorService,
SystemQueueProcessorsService,
ObjectStorageQueueProcessorsService,
DbQueueProcessorsService,
RelationshipQueueProcessorsService,
WebhookDeliverProcessorService,
EndedPollNotificationProcessorService,
DeliverProcessorService,

108
packages/backend/src/queue/QueueProcessorService.ts
View File

@@ -5,15 +5,36 @@ import type Logger from '@/logger.js';
import { QueueService } from '@/core/QueueService.js';
import { bindThis } from '@/decorators.js';
import { getJobInfo } from './get-job-info.js';
import { SystemQueueProcessorsService } from './SystemQueueProcessorsService.js';
import { ObjectStorageQueueProcessorsService } from './ObjectStorageQueueProcessorsService.js';
import { DbQueueProcessorsService } from './DbQueueProcessorsService.js';
import { WebhookDeliverProcessorService } from './processors/WebhookDeliverProcessorService.js';
import { EndedPollNotificationProcessorService } from './processors/EndedPollNotificationProcessorService.js';
import { DeliverProcessorService } from './processors/DeliverProcessorService.js';
import { InboxProcessorService } from './processors/InboxProcessorService.js';
import { DeleteDriveFilesProcessorService } from './processors/DeleteDriveFilesProcessorService.js';
import { ExportCustomEmojisProcessorService } from './processors/ExportCustomEmojisProcessorService.js';
import { ExportNotesProcessorService } from './processors/ExportNotesProcessorService.js';
import { ExportFollowingProcessorService } from './processors/ExportFollowingProcessorService.js';
import { ExportMutingProcessorService } from './processors/ExportMutingProcessorService.js';
import { ExportBlockingProcessorService } from './processors/ExportBlockingProcessorService.js';
import { ExportUserListsProcessorService } from './processors/ExportUserListsProcessorService.js';
import { ExportAntennasProcessorService } from './processors/ExportAntennasProcessorService.js';
import { ImportFollowingProcessorService } from './processors/ImportFollowingProcessorService.js';
import { ImportMutingProcessorService } from './processors/ImportMutingProcessorService.js';
import { ImportBlockingProcessorService } from './processors/ImportBlockingProcessorService.js';
import { ImportUserListsProcessorService } from './processors/ImportUserListsProcessorService.js';
import { ImportCustomEmojisProcessorService } from './processors/ImportCustomEmojisProcessorService.js';
import { ImportAntennasProcessorService } from './processors/ImportAntennasProcessorService.js';
import { DeleteAccountProcessorService } from './processors/DeleteAccountProcessorService.js';
import { ExportFavoritesProcessorService } from './processors/ExportFavoritesProcessorService.js';
import { CleanRemoteFilesProcessorService } from './processors/CleanRemoteFilesProcessorService.js';
import { DeleteFileProcessorService } from './processors/DeleteFileProcessorService.js';
import { RelationshipProcessorService } from './processors/RelationshipProcessorService.js';
import { TickChartsProcessorService } from './processors/TickChartsProcessorService.js';
import { ResyncChartsProcessorService } from './processors/ResyncChartsProcessorService.js';
import { CleanChartsProcessorService } from './processors/CleanChartsProcessorService.js';
import { CheckExpiredMutingsProcessorService } from './processors/CheckExpiredMutingsProcessorService.js';
import { CleanProcessorService } from './processors/CleanProcessorService.js';
import { AggregateRetentionProcessorService } from './processors/AggregateRetentionProcessorService.js';
import { QueueLoggerService } from './QueueLoggerService.js';
import { RelationshipQueueProcessorsService } from './RelationshipQueueProcessorsService.js';
@Injectable()
export class QueueProcessorService {
@@ -25,14 +46,35 @@ export class QueueProcessorService {
private queueLoggerService: QueueLoggerService,
private queueService: QueueService,
private systemQueueProcessorsService: SystemQueueProcessorsService,
private objectStorageQueueProcessorsService: ObjectStorageQueueProcessorsService,
private dbQueueProcessorsService: DbQueueProcessorsService,
private relationshipQueueProcessorsService: RelationshipQueueProcessorsService,
private webhookDeliverProcessorService: WebhookDeliverProcessorService,
private endedPollNotificationProcessorService: EndedPollNotificationProcessorService,
private deliverProcessorService: DeliverProcessorService,
private inboxProcessorService: InboxProcessorService,
private deleteDriveFilesProcessorService: DeleteDriveFilesProcessorService,
private exportCustomEmojisProcessorService: ExportCustomEmojisProcessorService,
private exportNotesProcessorService: ExportNotesProcessorService,
private exportFavoritesProcessorService: ExportFavoritesProcessorService,
private exportFollowingProcessorService: ExportFollowingProcessorService,
private exportMutingProcessorService: ExportMutingProcessorService,
private exportBlockingProcessorService: ExportBlockingProcessorService,
private exportUserListsProcessorService: ExportUserListsProcessorService,
private exportAntennasProcessorService: ExportAntennasProcessorService,
private importFollowingProcessorService: ImportFollowingProcessorService,
private importMutingProcessorService: ImportMutingProcessorService,
private importBlockingProcessorService: ImportBlockingProcessorService,
private importUserListsProcessorService: ImportUserListsProcessorService,
private importCustomEmojisProcessorService: ImportCustomEmojisProcessorService,
private importAntennasProcessorService: ImportAntennasProcessorService,
private deleteAccountProcessorService: DeleteAccountProcessorService,
private deleteFileProcessorService: DeleteFileProcessorService,
private cleanRemoteFilesProcessorService: CleanRemoteFilesProcessorService,
private relationshipProcessorService: RelationshipProcessorService,
private tickChartsProcessorService: TickChartsProcessorService,
private resyncChartsProcessorService: ResyncChartsProcessorService,
private cleanChartsProcessorService: CleanChartsProcessorService,
private aggregateRetentionProcessorService: AggregateRetentionProcessorService,
private checkExpiredMutingsProcessorService: CheckExpiredMutingsProcessorService,
private cleanProcessorService: CleanProcessorService,
) {
this.logger = this.queueLoggerService.logger;
}
@@ -119,14 +161,6 @@ export class QueueProcessorService {
.on('error', (job: any, err: Error) => webhookLogger.error(`error ${err}`, { job, e: renderError(err) }))
.on('stalled', (job) => webhookLogger.warn(`stalled ${getJobInfo(job)} to=${job.data.to}`));
this.queueService.deliverQueue.process(this.config.deliverJobConcurrency ?? 128, (job) => this.deliverProcessorService.process(job));
this.queueService.inboxQueue.process(this.config.inboxJobConcurrency ?? 16, (job) => this.inboxProcessorService.process(job));
this.queueService.endedPollNotificationQueue.process((job, done) => this.endedPollNotificationProcessorService.process(job, done));
this.queueService.webhookDeliverQueue.process(64, (job) => this.webhookDeliverProcessorService.process(job));
this.dbQueueProcessorsService.start(this.queueService.dbQueue);
this.relationshipQueueProcessorsService.start(this.queueService.relationshipQueue);
this.objectStorageQueueProcessorsService.start(this.queueService.objectStorageQueue);
this.queueService.systemQueue.add('tickCharts', {
}, {
repeat: { cron: '55 * * * *' },
@@ -163,6 +197,46 @@ export class QueueProcessorService {
removeOnComplete: true,
});
this.systemQueueProcessorsService.start(this.queueService.systemQueue);
this.queueService.deliverQueue.process(this.config.deliverJobConcurrency ?? 128, (job) => this.deliverProcessorService.process(job));
this.queueService.inboxQueue.process(this.config.inboxJobConcurrency ?? 16, (job) => this.inboxProcessorService.process(job));
this.queueService.endedPollNotificationQueue.process((job, done) => this.endedPollNotificationProcessorService.process(job, done));
this.queueService.webhookDeliverQueue.process(64, (job) => this.webhookDeliverProcessorService.process(job));
this.queueService.dbQueue.process('deleteDriveFiles', (job, done) => this.deleteDriveFilesProcessorService.process(job, done));
this.queueService.dbQueue.process('exportCustomEmojis', (job, done) => this.exportCustomEmojisProcessorService.process(job, done));
this.queueService.dbQueue.process('exportNotes', (job, done) => this.exportNotesProcessorService.process(job, done));
this.queueService.dbQueue.process('exportFavorites', (job, done) => this.exportFavoritesProcessorService.process(job, done));
this.queueService.dbQueue.process('exportFollowing', (job, done) => this.exportFollowingProcessorService.process(job, done));
this.queueService.dbQueue.process('exportMuting', (job, done) => this.exportMutingProcessorService.process(job, done));
this.queueService.dbQueue.process('exportBlocking', (job, done) => this.exportBlockingProcessorService.process(job, done));
this.queueService.dbQueue.process('exportUserLists', (job, done) => this.exportUserListsProcessorService.process(job, done));
this.queueService.dbQueue.process('exportAntennas', (job, done) => this.exportAntennasProcessorService.process(job, done));
this.queueService.dbQueue.process('importFollowing', (job, done) => this.importFollowingProcessorService.process(job, done));
this.queueService.dbQueue.process('importFollowingToDb', (job) => this.importFollowingProcessorService.processDb(job));
this.queueService.dbQueue.process('importMuting', (job, done) => this.importMutingProcessorService.process(job, done));
this.queueService.dbQueue.process('importBlocking', (job, done) => this.importBlockingProcessorService.process(job, done));
this.queueService.dbQueue.process('importBlockingToDb', (job) => this.importBlockingProcessorService.processDb(job));
this.queueService.dbQueue.process('importUserLists', (job, done) => this.importUserListsProcessorService.process(job, done));
this.queueService.dbQueue.process('importCustomEmojis', (job, done) => this.importCustomEmojisProcessorService.process(job, done));
this.queueService.dbQueue.process('importAntennas', (job, done) => this.importAntennasProcessorService.process(job, done));
this.queueService.dbQueue.process('deleteAccount', (job) => this.deleteAccountProcessorService.process(job));
this.queueService.objectStorageQueue.process('deleteFile', 16, (job) => this.deleteFileProcessorService.process(job));
this.queueService.objectStorageQueue.process('cleanRemoteFiles', 16, (job, done) => this.cleanRemoteFilesProcessorService.process(job, done));
{
const maxJobs = this.config.relashionshipJobConcurrency ?? 16;
this.queueService.relationshipQueue.process('follow', maxJobs, (job) => this.relationshipProcessorService.processFollow(job));
this.queueService.relationshipQueue.process('unfollow', maxJobs, (job) => this.relationshipProcessorService.processUnfollow(job));
this.queueService.relationshipQueue.process('block', maxJobs, (job) => this.relationshipProcessorService.processBlock(job));
this.queueService.relationshipQueue.process('unblock', maxJobs, (job) => this.relationshipProcessorService.processUnblock(job));
}
this.queueService.systemQueue.process('tickCharts', (job, done) => this.tickChartsProcessorService.process(job, done));
this.queueService.systemQueue.process('resyncCharts', (job, done) => this.resyncChartsProcessorService.process(job, done));
this.queueService.systemQueue.process('cleanCharts', (job, done) => this.cleanChartsProcessorService.process(job, done));
this.queueService.systemQueue.process('aggregateRetention', (job, done) => this.aggregateRetentionProcessorService.process(job, done));
this.queueService.systemQueue.process('checkExpiredMutings', (job, done) => this.checkExpiredMutingsProcessorService.process(job, done));
this.queueService.systemQueue.process('clean', (job, done) => this.cleanProcessorService.process(job, done));
}
}

26
packages/backend/src/queue/RelationshipQueueProcessorsService.ts
View File

@@ -1,26 +0,0 @@
import { Inject, Injectable } from '@nestjs/common';
import { bindThis } from '@/decorators.js';
import { RelationshipProcessorService } from './processors/RelationshipProcessorService.js';
import type Bull from 'bull';
import { DI } from '@/di-symbols.js';
import type { Config } from '@/config.js';
@Injectable()
export class RelationshipQueueProcessorsService {
constructor(
@Inject(DI.config)
private config: Config,
private relationshipProcessorService: RelationshipProcessorService,
) {
}
@bindThis
public start(q: Bull.Queue): void {
const maxJobs = this.config.relashionshipJobConcurrency ?? 16;
q.process('follow', maxJobs, (job) => this.relationshipProcessorService.processFollow(job));
q.process('unfollow', maxJobs, (job) => this.relationshipProcessorService.processUnfollow(job));
q.process('block', maxJobs, (job) => this.relationshipProcessorService.processBlock(job));
q.process('unblock', maxJobs, (job) => this.relationshipProcessorService.processUnblock(job));
}
}

37
packages/backend/src/queue/SystemQueueProcessorsService.ts
View File

@@ -1,37 +0,0 @@
import { Inject, Injectable } from '@nestjs/common';
import { DI } from '@/di-symbols.js';
import type { Config } from '@/config.js';
import { bindThis } from '@/decorators.js';
import { TickChartsProcessorService } from './processors/TickChartsProcessorService.js';
import { ResyncChartsProcessorService } from './processors/ResyncChartsProcessorService.js';
import { CleanChartsProcessorService } from './processors/CleanChartsProcessorService.js';
import { CheckExpiredMutingsProcessorService } from './processors/CheckExpiredMutingsProcessorService.js';
import { CleanProcessorService } from './processors/CleanProcessorService.js';
import { AggregateRetentionProcessorService } from './processors/AggregateRetentionProcessorService.js';
import type Bull from 'bull';
@Injectable()
export class SystemQueueProcessorsService {
constructor(
@Inject(DI.config)
private config: Config,
private tickChartsProcessorService: TickChartsProcessorService,
private resyncChartsProcessorService: ResyncChartsProcessorService,
private cleanChartsProcessorService: CleanChartsProcessorService,
private aggregateRetentionProcessorService: AggregateRetentionProcessorService,
private checkExpiredMutingsProcessorService: CheckExpiredMutingsProcessorService,
private cleanProcessorService: CleanProcessorService,
) {
}
@bindThis
public start(q: Bull.Queue): void {
q.process('tickCharts', (job, done) => this.tickChartsProcessorService.process(job, done));
q.process('resyncCharts', (job, done) => this.resyncChartsProcessorService.process(job, done));
q.process('cleanCharts', (job, done) => this.cleanChartsProcessorService.process(job, done));
q.process('aggregateRetention', (job, done) => this.aggregateRetentionProcessorService.process(job, done));
q.process('checkExpiredMutings', (job, done) => this.checkExpiredMutingsProcessorService.process(job, done));
q.process('clean', (job, done) => this.cleanProcessorService.process(job, done));
}
}

1
packages/backend/src/server/api/endpoints/admin/show-user.ts
View File

@@ -68,6 +68,7 @@ export default class extends Endpoint<typeof meta, typeof paramDef> {
emailVerified: profile.emailVerified,
autoAcceptFollowed: profile.autoAcceptFollowed,
noCrawle: profile.noCrawle,
preventAiLarning: profile.preventAiLarning,
alwaysMarkNsfw: profile.alwaysMarkNsfw,
autoSensitive: profile.autoSensitive,
carefulBot: profile.carefulBot,

4
packages/backend/src/server/api/endpoints/i/update.ts
View File

@@ -98,7 +98,7 @@ export const meta = {
message: 'This feature is restricted by your role.',
code: 'RESTRICTED_BY_ROLE',
id: '8feff0ba-5ab5-585b-31f4-4df816663fad',
}
},
},
res: {
@@ -138,6 +138,7 @@ export const paramDef = {
carefulBot: { type: 'boolean' },
autoAcceptFollowed: { type: 'boolean' },
noCrawle: { type: 'boolean' },
preventAiLarning: { type: 'boolean' },
isBot: { type: 'boolean' },
isCat: { type: 'boolean' },
showTimelineReplies: { type: 'boolean' },
@@ -242,6 +243,7 @@ export default class extends Endpoint<typeof meta, typeof paramDef> {
if (typeof ps.carefulBot === 'boolean') profileUpdates.carefulBot = ps.carefulBot;
if (typeof ps.autoAcceptFollowed === 'boolean') profileUpdates.autoAcceptFollowed = ps.autoAcceptFollowed;
if (typeof ps.noCrawle === 'boolean') profileUpdates.noCrawle = ps.noCrawle;
if (typeof ps.preventAiLarning === 'boolean') profileUpdates.preventAiLarning = ps.preventAiLarning;
if (typeof ps.isCat === 'boolean') updates.isCat = ps.isCat;
if (typeof ps.injectFeaturedNote === 'boolean') profileUpdates.injectFeaturedNote = ps.injectFeaturedNote;
if (typeof ps.receiveAnnouncementEmail === 'boolean') profileUpdates.receiveAnnouncementEmail = ps.receiveAnnouncementEmail;

26
packages/backend/src/server/web/ClientServerService.ts
View File

@@ -36,8 +36,8 @@ import { RoleService } from '@/core/RoleService.js';
import manifest from './manifest.json' assert { type: 'json' };
import { FeedService } from './FeedService.js';
import { UrlPreviewService } from './UrlPreviewService.js';
import type { FastifyInstance, FastifyPluginOptions, FastifyReply } from 'fastify';
import { ClientLoggerService } from './ClientLoggerService.js';
import type { FastifyInstance, FastifyPluginOptions, FastifyReply } from 'fastify';
const _filename = fileURLToPath(import.meta.url);
const _dirname = dirname(_filename);
@@ -437,6 +437,10 @@ export class ClientServerService {
: [];
reply.header('Cache-Control', 'public, max-age=15');
if (profile.preventAiLarning) {
reply.header('X-Robots-Tag', 'noimageai');
reply.header('X-Robots-Tag', 'noai');
}
return await reply.view('user', {
user, profile, me,
avatarUrl: user.avatarUrl ?? this.userEntityService.getIdenticonUrl(user),
@@ -481,6 +485,10 @@ export class ClientServerService {
const profile = await this.userProfilesRepository.findOneByOrFail({ userId: note.userId });
const meta = await this.metaService.fetch();
reply.header('Cache-Control', 'public, max-age=15');
if (profile.preventAiLarning) {
reply.header('X-Robots-Tag', 'noimageai');
reply.header('X-Robots-Tag', 'noai');
}
return await reply.view('note', {
note: _note,
profile,
@@ -520,6 +528,10 @@ export class ClientServerService {
} else {
reply.header('Cache-Control', 'private, max-age=0, must-revalidate');
}
if (profile.preventAiLarning) {
reply.header('X-Robots-Tag', 'noimageai');
reply.header('X-Robots-Tag', 'noai');
}
return await reply.view('page', {
page: _page,
profile,
@@ -544,6 +556,10 @@ export class ClientServerService {
const profile = await this.userProfilesRepository.findOneByOrFail({ userId: flash.userId });
const meta = await this.metaService.fetch();
reply.header('Cache-Control', 'public, max-age=15');
if (profile.preventAiLarning) {
reply.header('X-Robots-Tag', 'noimageai');
reply.header('X-Robots-Tag', 'noai');
}
return await reply.view('flash', {
flash: _flash,
profile,
@@ -568,6 +584,10 @@ export class ClientServerService {
const profile = await this.userProfilesRepository.findOneByOrFail({ userId: clip.userId });
const meta = await this.metaService.fetch();
reply.header('Cache-Control', 'public, max-age=15');
if (profile.preventAiLarning) {
reply.header('X-Robots-Tag', 'noimageai');
reply.header('X-Robots-Tag', 'noai');
}
return await reply.view('clip', {
clip: _clip,
profile,
@@ -590,6 +610,10 @@ export class ClientServerService {
const profile = await this.userProfilesRepository.findOneByOrFail({ userId: post.userId });
const meta = await this.metaService.fetch();
reply.header('Cache-Control', 'public, max-age=15');
if (profile.preventAiLarning) {
reply.header('X-Robots-Tag', 'noimageai');
reply.header('X-Robots-Tag', 'noai');
}
return await reply.view('gallery-post', {
post: _post,
profile,

3
packages/backend/src/server/web/views/clip.pug
View File

@@ -21,6 +21,9 @@ block og
block meta
if profile.noCrawle
meta(name='robots' content='noindex')
if profile.preventAiLarning
meta(name='robots' content='noimageai')
meta(name='robots' content='noai')
meta(name='misskey:user-username' content=user.username)
meta(name='misskey:user-id' content=user.id)

3
packages/backend/src/server/web/views/flash.pug
View File

@@ -21,6 +21,9 @@ block og
block meta
if profile.noCrawle
meta(name='robots' content='noindex')
if profile.preventAiLarning
meta(name='robots' content='noimageai')
meta(name='robots' content='noai')
meta(name='misskey:user-username' content=user.username)
meta(name='misskey:user-id' content=user.id)

3
packages/backend/src/server/web/views/gallery-post.pug
View File

@@ -21,6 +21,9 @@ block og
block meta
if user.host || profile.noCrawle
meta(name='robots' content='noindex')
if profile.preventAiLarning
meta(name='robots' content='noimageai')
meta(name='robots' content='noai')
meta(name='misskey:user-username' content=user.username)
meta(name='misskey:user-id' content=user.id)

3
packages/backend/src/server/web/views/note.pug
View File

@@ -22,6 +22,9 @@ block og
block meta
if user.host || isRenote || profile.noCrawle
meta(name='robots' content='noindex')
if profile.preventAiLarning
meta(name='robots' content='noimageai')
meta(name='robots' content='noai')
meta(name='misskey:user-username' content=user.username)
meta(name='misskey:user-id' content=user.id)

3
packages/backend/src/server/web/views/page.pug
View File

@@ -21,6 +21,9 @@ block og
block meta
if profile.noCrawle
meta(name='robots' content='noindex')
if profile.preventAiLarning
meta(name='robots' content='noimageai')
meta(name='robots' content='noai')
meta(name='misskey:user-username' content=user.username)
meta(name='misskey:user-id' content=user.id)

3
packages/backend/src/server/web/views/user.pug
View File

@@ -20,6 +20,9 @@ block og
block meta
if user.host || profile.noCrawle
meta(name='robots' content='noindex')
if profile.preventAiLarning
meta(name='robots' content='noimageai')
meta(name='robots' content='noai')
meta(name='misskey:user-username' content=user.username)
meta(name='misskey:user-id' content=user.id)

55
packages/backend/test/e2e/note.ts
View File

@@ -541,6 +541,61 @@ describe('Note', () => {
assert.strictEqual(res.status, 400);
});
test('センシティブな投稿はhomeになる (単語指定)', async () => {
const sensitive = await api('admin/update-meta', {
sensitiveWords: [
"test",
]
}, alice);
assert.strictEqual(sensitive.status, 204);
await new Promise(x => setTimeout(x, 2));
const note1 = await api('/notes/create', {
text: 'hogetesthuge',
}, alice);
assert.strictEqual(note1.status, 200);
assert.strictEqual(note1.body.createdNote.visibility, 'home');
});
test('センシティブな投稿はhomeになる (正規表現)', async () => {
const sensitive = await api('admin/update-meta', {
sensitiveWords: [
"/Test/i",
]
}, alice);
assert.strictEqual(sensitive.status, 204);
const note2 = await api('/notes/create', {
text: 'hogetesthuge',
}, alice);
assert.strictEqual(note2.status, 200);
assert.strictEqual(note2.body.createdNote.visibility, 'home');
});
test('センシティブな投稿はhomeになる (スペースアンド)', async () => {
const sensitive = await api('admin/update-meta', {
sensitiveWords: [
"Test hoge"
]
}, alice);
assert.strictEqual(sensitive.status, 204);
const note2 = await api('/notes/create', {
text: 'hogeTesthuge',
}, alice);
assert.strictEqual(note2.status, 200);
assert.strictEqual(note2.body.createdNote.visibility, 'home');
});
});
describe('notes/delete', () => {

6
packages/backend/test/e2e/users.ts
View File

@@ -145,6 +145,7 @@ describe('ユーザー', () => {
carefulBot: user.carefulBot,
autoAcceptFollowed: user.autoAcceptFollowed,
noCrawle: user.noCrawle,
preventAiLarning: user.preventAiLarning,
isExplorable: user.isExplorable,
isDeleted: user.isDeleted,
hideOnlineStatus: user.hideOnlineStatus,
@@ -370,7 +371,7 @@ describe('ユーザー', () => {
assert.deepStrictEqual(response.pinnedNotes, []);
assert.strictEqual(response.pinnedPageId, null);
assert.strictEqual(response.pinnedPage, null);
assert.strictEqual(response.publicReactions, false);
assert.strictEqual(response.publicReactions, true);
assert.strictEqual(response.ffVisibility, 'public');
assert.strictEqual(response.twoFactorEnabled, false);
assert.strictEqual(response.usePasswordLessLogin, false);
@@ -390,6 +391,7 @@ describe('ユーザー', () => {
assert.strictEqual(response.carefulBot, false);
assert.strictEqual(response.autoAcceptFollowed, true);
assert.strictEqual(response.noCrawle, false);
assert.strictEqual(response.preventAiLarning, true);
assert.strictEqual(response.isExplorable, true);
assert.strictEqual(response.isDeleted, false);
assert.strictEqual(response.hideOnlineStatus, false);
@@ -462,6 +464,8 @@ describe('ユーザー', () => {
{ parameters: (): object => ({ autoAcceptFollowed: false }) },
{ parameters: (): object => ({ noCrawle: true }) },
{ parameters: (): object => ({ noCrawle: false }) },
{ parameters: (): object => ({ preventAiLarning: false }) },
{ parameters: (): object => ({ preventAiLarning: true }) },
{ parameters: (): object => ({ isBot: true }) },
{ parameters: (): object => ({ isBot: false }) },
{ parameters: (): object => ({ isCat: true }) },