Merge branch 'develop' of misskey-dev into merge-upstream

packages/backend/src/GlobalModule.ts

@@ -3,7 +3,6 @@
  * SPDX-License-Identifier: AGPL-3.0-only
  */
 
-import { setTimeout } from 'node:timers/promises';
 import process from 'node:process';
 import { Global, Inject, Module } from '@nestjs/common';
 import * as Redis from 'ioredis';
@@ -13,6 +12,7 @@ import { DI } from './di-symbols.js';
 import { Config, loadConfig } from './config.js';
 import { createPostgresDataSource } from './postgres.js';
 import { RepositoryModule } from './models/RepositoryModule.js';
+import { allSettled } from './misc/promise-tracker.js';
 import type { Provider, OnApplicationShutdown } from '@nestjs/common';
 
 const $config: Provider = {
@@ -34,7 +34,7 @@ const $meilisearch: Provider = {
 	useFactory: (config: Config) => {
 		if (config.meilisearch) {
 			return new MeiliSearch({
-				host: `${config.meilisearch.ssl ? 'https' : 'http' }://${config.meilisearch.host}:${config.meilisearch.port}`,
+				host: `${config.meilisearch.ssl ? 'https' : 'http'}://${config.meilisearch.host}:${config.meilisearch.port}`,
 				apiKey: config.meilisearch.apiKey,
 			});
 		} else {
@@ -128,17 +128,12 @@ export class GlobalModule implements OnApplicationShutdown {
 		@Inject(DI.redisForPub) private redisForPub: Redis.Redis,
 		@Inject(DI.redisForSub) private redisForSub: Redis.Redis,
 		@Inject(DI.redisForTimelines) private redisForTimelines: Redis.Redis,
-	) {}
+	) { }
 
 	public async dispose(): Promise<void> {
-		if (process.env.NODE_ENV === 'test') {
-			// XXX:
-			// Shutting down the existing connections causes errors on Jest as
-			// Misskey has asynchronous postgres/redis connections that are not
-			// awaited.
-			// Let's wait for some random time for them to finish.
-			await setTimeout(5000);
-		}
+		// Wait for all potential DB queries
+		await allSettled();
+		// And then disconnect from DB
 		await Promise.all([
 			this.db.destroy(),
 			this.redisClient.disconnect(),

packages/backend/src/core/AchievementService.ts

@@ -87,6 +87,8 @@ export const ACHIEVEMENT_TYPES = [
 	'brainDiver',
 	'smashTestNotificationButton',
 	'tutorialCompleted',
+	'bubbleGameExplodingHead',
+	'bubbleGameDoubleExplodingHead',
 ] as const;
 
 @Injectable()

packages/backend/src/core/CaptchaService.ts

@@ -73,6 +73,37 @@ export class CaptchaService {
 		}
 	}
 
+	// https://codeberg.org/Gusted/mCaptcha/src/branch/main/mcaptcha.go
+	@bindThis
+	public async verifyMcaptcha(secret: string, siteKey: string, instanceHost: string, response: string | null | undefined): Promise<void> {
+		if (response == null) {
+			throw new Error('mcaptcha-failed: no response provided');
+		}
+
+		const endpointUrl = new URL('/api/v1/pow/siteverify', instanceHost);
+		const result = await this.httpRequestService.send(endpointUrl.toString(), {
+			method: 'POST',
+			body: JSON.stringify({
+				key: siteKey,
+				secret: secret,
+				token: response,
+			}),
+			headers: {
+				'Content-Type': 'application/json',
+			},
+		});
+
+		if (result.status !== 200) {
+			throw new Error('mcaptcha-failed: mcaptcha didn\'t return 200 OK');
+		}
+
+		const resp = (await result.json()) as { valid: boolean };
+
+		if (!resp.valid) {
+			throw new Error('mcaptcha-request-failed');
+		}
+	}
+
 	@bindThis
 	public async verifyTurnstile(secret: string, response: string | null | undefined): Promise<void> {
 		if (response == null) {

packages/backend/src/core/DriveService.ts

@@ -7,7 +7,7 @@ import { randomUUID } from 'node:crypto';
 import * as fs from 'node:fs';
 import { Inject, Injectable } from '@nestjs/common';
 import sharp from 'sharp';
-import { sharpBmp } from 'sharp-read-bmp';
+import { sharpBmp } from '@misskey-dev/sharp-read-bmp';
 import { IsNull } from 'typeorm';
 import { DeleteObjectCommandInput, PutObjectCommandInput, NoSuchKey } from '@aws-sdk/client-s3';
 import { DI } from '@/di-symbols.js';
@@ -655,7 +655,7 @@ export class DriveService {
 	public async updateFile(file: MiDriveFile, values: Partial<MiDriveFile>, updater: MiUser) {
 		const alwaysMarkNsfw = (await this.roleService.getUserPolicies(file.userId)).alwaysMarkNsfw;
 
-		if (values.name && !this.driveFileEntityService.validateFileName(file.name)) {
+		if (values.name != null && !this.driveFileEntityService.validateFileName(values.name)) {
 			throw new DriveService.InvalidFileNameError();
 		}
 

packages/backend/src/core/EmailService.ts

@@ -156,7 +156,7 @@ export class EmailService {
 	@bindThis
 	public async validateEmailForAccount(emailAddress: string): Promise<{
 		available: boolean;
-		reason: null | 'used' | 'format' | 'disposable' | 'mx' | 'smtp' | 'banned';
+		reason: null | 'used' | 'format' | 'disposable' | 'mx' | 'smtp' | 'banned' | 'network' | 'blacklist';
 	}> {
 		const meta = await this.metaService.fetch();
 
@@ -183,6 +183,10 @@ export class EmailService {
 			if (validated.valid && meta.enableVerifymailApi && meta.verifymailAuthKey != null) {
 				validated = await this.verifyMail(emailAddress, meta.verifymailAuthKey);
 			}
+
+			if (validated.valid && meta.enableTruemailApi && meta.truemailInstance && meta.truemailAuthKey != null) {
+				validated = await this.trueMail(meta.truemailInstance, emailAddress, meta.truemailAuthKey);
+			}
 		} else {
 			validated = { valid: true, reason: null };
 		}
@@ -201,6 +205,8 @@ export class EmailService {
 			validated.reason === 'disposable' ? 'disposable' :
 			validated.reason === 'mx' ? 'mx' :
 			validated.reason === 'smtp' ? 'smtp' :
+			validated.reason === 'network' ? 'network' :
+			validated.reason === 'blacklist' ? 'blacklist' :
 			null,
 		};
 	}
@@ -265,4 +271,67 @@ export class EmailService {
 			reason: null,
 		};
 	}
+
+	private async trueMail<T>(truemailInstance: string, emailAddress: string, truemailAuthKey: string): Promise<{
+		valid: boolean;
+		reason: 'used' | 'format' | 'blacklist' | 'mx' | 'smtp' | 'network' | T | null;
+	}> {
+		const endpoint = truemailInstance + '?email=' + emailAddress;
+		try {
+			const res = await this.httpRequestService.send(endpoint, {
+				method: 'POST',
+				headers: {
+					'Content-Type': 'application/json',
+					Accept: 'application/json',
+					Authorization: truemailAuthKey
+				},
+			});
+			
+			const json = (await res.json()) as {
+				email: string;
+				success: boolean;
+				errors?: { 
+					list_match?: string;
+					regex?: string;
+					mx?: string;
+					smtp?: string;
+				} | null;
+			};
+			
+			if (json.email === undefined || (json.email !== undefined && json.errors?.regex)) {
+				return {
+						valid: false,
+						reason: 'format',
+				};
+			}
+			if (json.errors?.smtp) {
+				return {
+					valid: false,
+					reason: 'smtp',
+				};
+			}
+			if (json.errors?.mx) {
+				return {
+					valid: false,
+					reason: 'mx',
+				};
+			}
+			if (!json.success) {
+				return {
+					valid: false,
+					reason: json.errors?.list_match as T || 'blacklist',
+				};
+			}
+			
+			return {
+				valid: true,
+				reason: null,
+			};
+		} catch (error) {
+			return {
+				valid: false,
+				reason: 'network',
+			};
+		}
+	}
 }

packages/backend/src/core/NoteCreateService.ts

@@ -58,6 +58,7 @@ import { FanoutTimelineService } from '@/core/FanoutTimelineService.js';
 import { UtilityService } from '@/core/UtilityService.js';
 import { UserBlockingService } from '@/core/UserBlockingService.js';
 import { isReply } from '@/misc/is-reply.js';
+import { trackPromise } from '@/misc/promise-tracker.js';
 
 type NotificationType = 'reply' | 'renote' | 'quote' | 'mention';
 
@@ -677,7 +678,7 @@ export class NoteCreateService implements OnApplicationShutdown {
 						this.relayService.deliverToRelays(user, noteActivity);
 					}
 
-					dm.execute();
+					trackPromise(dm.execute());
 				})();
 			}
 			//#endregion

packages/backend/src/core/NoteReadService.ts

@@ -14,6 +14,7 @@ import { IdService } from '@/core/IdService.js';
 import { GlobalEventService } from '@/core/GlobalEventService.js';
 import type { NoteUnreadsRepository, MutingsRepository, NoteThreadMutingsRepository } from '@/models/_.js';
 import { bindThis } from '@/decorators.js';
+import { trackPromise } from '@/misc/promise-tracker.js';
 
 @Injectable()
 export class NoteReadService implements OnApplicationShutdown {
@@ -107,7 +108,7 @@ export class NoteReadService implements OnApplicationShutdown {
 
 			// TODO: ↓まとめてクエリしたい
 
-			this.noteUnreadsRepository.countBy({
+			trackPromise(this.noteUnreadsRepository.countBy({
 				userId: userId,
 				isMentioned: true,
 			}).then(mentionsCount => {
@@ -115,9 +116,9 @@ export class NoteReadService implements OnApplicationShutdown {
 					// 全て既読になったイベントを発行
 					this.globalEventService.publishMainStream(userId, 'readAllUnreadMentions');
 				}
-			});
+			}));
 
-			this.noteUnreadsRepository.countBy({
+			trackPromise(this.noteUnreadsRepository.countBy({
 				userId: userId,
 				isSpecified: true,
 			}).then(specifiedCount => {
@@ -125,7 +126,7 @@ export class NoteReadService implements OnApplicationShutdown {
 					// 全て既読になったイベントを発行
 					this.globalEventService.publishMainStream(userId, 'readAllUnreadSpecifiedNotes');
 				}
-			});
+			}));
 		}
 	}
 

packages/backend/src/core/NotificationService.ts

@@ -20,6 +20,7 @@ import { CacheService } from '@/core/CacheService.js';
 import type { Config } from '@/config.js';
 import { UserListService } from '@/core/UserListService.js';
 import type { FilterUnionByProperty } from '@/types.js';
+import { trackPromise } from '@/misc/promise-tracker.js';
 
 @Injectable()
 export class NotificationService implements OnApplicationShutdown {
@@ -74,7 +75,18 @@ export class NotificationService implements OnApplicationShutdown {
 	}
 
 	@bindThis
-	public async createNotification<T extends MiNotification['type']>(
+	public createNotification<T extends MiNotification['type']>(
+		notifieeId: MiUser['id'],
+		type: T,
+		data: Omit<FilterUnionByProperty<MiNotification, 'type', T>, 'type' | 'id' | 'createdAt' | 'notifierId'>,
+		notifierId?: MiUser['id'] | null,
+	) {
+		trackPromise(
+			this.#createNotificationInternal(notifieeId, type, data, notifierId),
+		);
+	}
+
+	async #createNotificationInternal<T extends MiNotification['type']>(
 		notifieeId: MiUser['id'],
 		type: T,
 		data: Omit<FilterUnionByProperty<MiNotification, 'type', T>, 'type' | 'id' | 'createdAt' | 'notifierId'>,

packages/backend/src/core/QueueModule.ts

@@ -3,12 +3,12 @@
  * SPDX-License-Identifier: AGPL-3.0-only
  */
 
-import { setTimeout } from 'node:timers/promises';
 import { Inject, Module, OnApplicationShutdown } from '@nestjs/common';
 import * as Bull from 'bullmq';
 import { DI } from '@/di-symbols.js';
 import type { Config } from '@/config.js';
 import { QUEUE, baseQueueOptions } from '@/queue/const.js';
+import { allSettled } from '@/misc/promise-tracker.js';
 import type { Provider } from '@nestjs/common';
 import type { DeliverJobData, InboxJobData, EndedPollNotificationJobData, WebhookDeliverJobData, RelationshipJobData } from '../queue/types.js';
 
@@ -106,14 +106,9 @@ export class QueueModule implements OnApplicationShutdown {
 	) {}
 
 	public async dispose(): Promise<void> {
-		if (process.env.NODE_ENV === 'test') {
-			// XXX:
-			// Shutting down the existing connections causes errors on Jest as
-			// Misskey has asynchronous postgres/redis connections that are not
-			// awaited.
-			// Let's wait for some random time for them to finish.
-			await setTimeout(5000);
-		}
+		// Wait for all potential queue jobs
+		await allSettled();
+		// And then close all queues
 		await Promise.all([
 			this.systemQueue.close(),
 			this.endedPollNotificationQueue.close(),

packages/backend/src/core/QueueService.ts

@@ -183,6 +183,16 @@ export class QueueService {
 		});
 	}
 
+	@bindThis
+	public createExportClipsJob(user: ThinUser) {
+		return this.dbQueue.add('exportClips', {
+			user: { id: user.id },
+		}, {
+			removeOnComplete: true,
+			removeOnFail: true,
+		});
+	}
+
 	@bindThis
 	public createExportFavoritesJob(user: ThinUser) {
 		return this.dbQueue.add('exportFavorites', {

packages/backend/src/core/ReactionService.ts

@@ -28,6 +28,7 @@ import { UserBlockingService } from '@/core/UserBlockingService.js';
 import { CustomEmojiService } from '@/core/CustomEmojiService.js';
 import { RoleService } from '@/core/RoleService.js';
 import { FeaturedService } from '@/core/FeaturedService.js';
+import { trackPromise } from '@/misc/promise-tracker.js';
 
 const FALLBACK = '❤';
 const PER_NOTE_REACTION_USER_PAIR_CACHE_MAX = 16;
@@ -273,7 +274,7 @@ export class ReactionService {
 				}
 			}
 
-			dm.execute();
+			trackPromise(dm.execute());
 		}
 		//#endregion
 	}
@@ -321,7 +322,7 @@ export class ReactionService {
 				dm.addDirectRecipe(reactee as MiRemoteUser);
 			}
 			dm.addFollowersRecipe();
-			dm.execute();
+			trackPromise(dm.execute());
 		}
 		//#endregion
 	}

packages/backend/src/core/activitypub/ApDeliverManagerService.ts

@@ -144,7 +144,7 @@ class DeliverManager {
 		}
 
 		// deliver
-		this.queueService.deliverMany(this.actor, this.activity, inboxes);
+		await this.queueService.deliverMany(this.actor, this.activity, inboxes);
 	}
 }
 

packages/backend/src/core/entities/NoteEntityService.ts

@@ -356,6 +356,7 @@ export class NoteEntityService implements OnModuleInit {
 				color: channel.color,
 				isSensitive: channel.isSensitive,
 				allowRenoteToExternal: channel.allowRenoteToExternal,
+				userId: channel.userId,
 			} : undefined,
 			mentions: note.mentions.length > 0 ? note.mentions : undefined,
 			uri: note.uri ?? undefined,

packages/backend/src/daemons/ServerStatsService.ts

@@ -37,7 +37,7 @@ export class ServerStatsService implements OnApplicationShutdown {
 		const log = [] as any[];
 
 		ev.on('requestServerStatsLog', x => {
-			ev.emit(`serverStatsLog:${x.id}`, log.slice(0, x.length ?? 50));
+			ev.emit(`serverStatsLog:${x.id}`, log.slice(0, x.length));
 		});
 
 		const tick = async () => {

packages/backend/src/logger.ts

@@ -84,8 +84,11 @@ export default class Logger {
 		let log = `${l} ${worker}\t[${contexts.join(' ')}]\t${m}`;
 		if (envOption.withLogTime) log = chalk.gray(time) + ' ' + log;
 
-		console.log(important ? chalk.bold(log) : log);
-		if (level === 'error' && data) console.log(data);
+		const args: unknown[] = [important ? chalk.bold(log) : log];
+		if (data != null) {
+			args.push(data);
+		}
+		console.log(...args);
 	}
 
 	@bindThis

packages/backend/src/misc/promise-tracker.ts

@@ -0,0 +1,23 @@
+/*
+ * SPDX-FileCopyrightText: syuilo and other misskey contributors
+ * SPDX-License-Identifier: AGPL-3.0-only
+ */
+
+const promiseRefs: Set<WeakRef<Promise<unknown>>> = new Set();
+
+/**
+ * This tracks promises that other modules decided not to wait for,
+ * and makes sure they are all settled before fully closing down the server.
+ */
+export function trackPromise(promise: Promise<unknown>) {
+	if (process.env.NODE_ENV !== 'test') {
+		return;
+	}
+	const ref = new WeakRef(promise);
+	promiseRefs.add(ref);
+	promise.finally(() => promiseRefs.delete(ref));
+}
+
+export async function allSettled(): Promise<void> {
+	await Promise.allSettled([...promiseRefs].map(r => r.deref()));
+}

packages/backend/src/models/Meta.ts

@@ -196,6 +196,29 @@ export class MiMeta {
 	})
 	public hcaptchaSecretKey: string | null;
 
+	@Column('boolean', {
+		default: false,
+	})
+	public enableMcaptcha: boolean;
+
+	@Column('varchar', {
+		length: 1024,
+		nullable: true,
+	})
+	public mcaptchaSitekey: string | null;
+
+	@Column('varchar', {
+		length: 1024,
+		nullable: true,
+	})
+	public mcaptchaSecretKey: string | null;
+
+	@Column('varchar', {
+		length: 1024,
+		nullable: true,
+	})
+	public mcaptchaInstanceUrl: string | null;
+
 	@Column('boolean', {
 		default: false,
 	})
@@ -462,6 +485,23 @@ export class MiMeta {
 	})
 	public verifymailAuthKey: string | null;
 
+	@Column('boolean', {
+		default: false,
+	})
+	public enableTruemailApi: boolean;
+
+	@Column('varchar', {
+		length: 1024,
+		nullable: true,
+	})
+	public truemailInstance: string | null;
+
+	@Column('varchar', {
+		length: 1024,
+		nullable: true,
+	})
+	public truemailAuthKey: string | null;
+
 	@Column('boolean', {
 		default: true,
 	})

packages/backend/src/models/json-schema/note.ts

@@ -148,6 +148,10 @@ export const packedNoteSchema = {
 					type: 'boolean',
 					optional: false, nullable: false,
 				},
+				userId: {
+					type: 'string',
+					optional: false, nullable: true,
+				},
 			},
 		},
 		localOnly: {

packages/backend/src/queue/QueueProcessorModule.ts

@@ -24,6 +24,7 @@ import { ExportCustomEmojisProcessorService } from './processors/ExportCustomEmo
 import { ExportFollowingProcessorService } from './processors/ExportFollowingProcessorService.js';
 import { ExportMutingProcessorService } from './processors/ExportMutingProcessorService.js';
 import { ExportNotesProcessorService } from './processors/ExportNotesProcessorService.js';
+import { ExportClipsProcessorService } from './processors/ExportClipsProcessorService.js';
 import { ExportUserListsProcessorService } from './processors/ExportUserListsProcessorService.js';
 import { ExportAntennasProcessorService } from './processors/ExportAntennasProcessorService.js';
 import { ImportBlockingProcessorService } from './processors/ImportBlockingProcessorService.js';
@@ -54,6 +55,7 @@ import { RelationshipProcessorService } from './processors/RelationshipProcessor
 		DeleteDriveFilesProcessorService,
 		ExportCustomEmojisProcessorService,
 		ExportNotesProcessorService,
+		ExportClipsProcessorService,
 		ExportFavoritesProcessorService,
 		ExportFollowingProcessorService,
 		ExportMutingProcessorService,

packages/backend/src/queue/QueueProcessorService.ts

@@ -16,6 +16,7 @@ import { InboxProcessorService } from './processors/InboxProcessorService.js';
 import { DeleteDriveFilesProcessorService } from './processors/DeleteDriveFilesProcessorService.js';
 import { ExportCustomEmojisProcessorService } from './processors/ExportCustomEmojisProcessorService.js';
 import { ExportNotesProcessorService } from './processors/ExportNotesProcessorService.js';
+import { ExportClipsProcessorService } from './processors/ExportClipsProcessorService.js';
 import { ExportFollowingProcessorService } from './processors/ExportFollowingProcessorService.js';
 import { ExportMutingProcessorService } from './processors/ExportMutingProcessorService.js';
 import { ExportBlockingProcessorService } from './processors/ExportBlockingProcessorService.js';
@@ -92,6 +93,7 @@ export class QueueProcessorService implements OnApplicationShutdown {
 		private deleteDriveFilesProcessorService: DeleteDriveFilesProcessorService,
 		private exportCustomEmojisProcessorService: ExportCustomEmojisProcessorService,
 		private exportNotesProcessorService: ExportNotesProcessorService,
+		private exportClipsProcessorService: ExportClipsProcessorService,
 		private exportFavoritesProcessorService: ExportFavoritesProcessorService,
 		private exportFollowingProcessorService: ExportFollowingProcessorService,
 		private exportMutingProcessorService: ExportMutingProcessorService,
@@ -166,6 +168,7 @@ export class QueueProcessorService implements OnApplicationShutdown {
 				case 'deleteDriveFiles': return this.deleteDriveFilesProcessorService.process(job);
 				case 'exportCustomEmojis': return this.exportCustomEmojisProcessorService.process(job);
 				case 'exportNotes': return this.exportNotesProcessorService.process(job);
+				case 'exportClips': return this.exportClipsProcessorService.process(job);
 				case 'exportFavorites': return this.exportFavoritesProcessorService.process(job);
 				case 'exportFollowing': return this.exportFollowingProcessorService.process(job);
 				case 'exportMuting': return this.exportMutingProcessorService.process(job);

packages/backend/src/queue/processors/ExportClipsProcessorService.ts

@@ -0,0 +1,206 @@
+/*
+ * SPDX-FileCopyrightText: syuilo and other misskey contributors
+ * SPDX-License-Identifier: AGPL-3.0-only
+ */
+
+import * as fs from 'node:fs';
+import { Writable } from 'node:stream';
+import { Inject, Injectable, StreamableFile } from '@nestjs/common';
+import { MoreThan } from 'typeorm';
+import { format as dateFormat } from 'date-fns';
+import { DI } from '@/di-symbols.js';
+import type { ClipNotesRepository, ClipsRepository, MiClip, MiClipNote, MiUser, NotesRepository, PollsRepository, UsersRepository } from '@/models/_.js';
+import type Logger from '@/logger.js';
+import { DriveService } from '@/core/DriveService.js';
+import { createTemp } from '@/misc/create-temp.js';
+import type { MiPoll } from '@/models/Poll.js';
+import type { MiNote } from '@/models/Note.js';
+import { bindThis } from '@/decorators.js';
+import { DriveFileEntityService } from '@/core/entities/DriveFileEntityService.js';
+import { Packed } from '@/misc/json-schema.js';
+import { IdService } from '@/core/IdService.js';
+import { QueueLoggerService } from '../QueueLoggerService.js';
+import type * as Bull from 'bullmq';
+import type { DbJobDataWithUser } from '../types.js';
+
+@Injectable()
+export class ExportClipsProcessorService {
+	private logger: Logger;
+
+	constructor(
+		@Inject(DI.usersRepository)
+		private usersRepository: UsersRepository,
+
+		@Inject(DI.pollsRepository)
+		private pollsRepository: PollsRepository,
+
+		@Inject(DI.clipsRepository)
+		private clipsRepository: ClipsRepository,
+
+		@Inject(DI.clipNotesRepository)
+		private clipNotesRepository: ClipNotesRepository,
+
+		private driveService: DriveService,
+		private queueLoggerService: QueueLoggerService,
+		private idService: IdService,
+	) {
+		this.logger = this.queueLoggerService.logger.createSubLogger('export-clips');
+	}
+
+	@bindThis
+	public async process(job: Bull.Job<DbJobDataWithUser>): Promise<void> {
+		this.logger.info(`Exporting clips of ${job.data.user.id} ...`);
+
+		const user = await this.usersRepository.findOneBy({ id: job.data.user.id });
+		if (user == null) {
+			return;
+		}
+
+		// Create temp file
+		const [path, cleanup] = await createTemp();
+
+		this.logger.info(`Temp file is ${path}`);
+
+		try {
+			const stream = Writable.toWeb(fs.createWriteStream(path, { flags: 'a' }));
+			const writer = stream.getWriter();
+			writer.closed.catch(this.logger.error);
+
+			await writer.write('[');
+
+			await this.processClips(writer, user, job);
+
+			await writer.write(']');
+			await writer.close();
+
+			this.logger.succ(`Exported to: ${path}`);
+
+			const fileName = 'clips-' + dateFormat(new Date(), 'yyyy-MM-dd-HH-mm-ss') + '.json';
+			const driveFile = await this.driveService.addFile({ user, path, name: fileName, force: true, ext: 'json' });
+
+			this.logger.succ(`Exported to: ${driveFile.id}`);
+		} finally {
+			cleanup();
+		}
+	}
+
+	async processClips(writer: WritableStreamDefaultWriter, user: MiUser, job: Bull.Job<DbJobDataWithUser>) {
+		let exportedClipsCount = 0;
+		let cursor: MiClip['id'] | null = null;
+
+		while (true) {
+			const clips = await this.clipsRepository.find({
+				where: {
+					userId: user.id,
+					...(cursor ? { id: MoreThan(cursor) } : {}),
+				},
+				take: 100,
+				order: {
+					id: 1,
+				},
+			});
+
+			if (clips.length === 0) {
+				job.updateProgress(100);
+				break;
+			}
+
+			cursor = clips.at(-1)?.id ?? null;
+
+			for (const clip of clips) {
+				// Stringify but remove the last `]}`
+				const content = JSON.stringify(this.serializeClip(clip)).slice(0, -2);
+				const isFirst = exportedClipsCount === 0;
+				await writer.write(isFirst ? content : ',\n' + content);
+
+				await this.processClipNotes(writer, clip.id);
+
+				await writer.write(']}');
+				exportedClipsCount++;
+			}
+
+			const total = await this.clipsRepository.countBy({
+				userId: user.id,
+			});
+
+			job.updateProgress(exportedClipsCount / total);
+		}
+	}
+
+	async processClipNotes(writer: WritableStreamDefaultWriter, clipId: string): Promise<void> {
+		let exportedClipNotesCount = 0;
+		let cursor: MiClipNote['id'] | null = null;
+
+		while (true) {
+			const clipNotes = await this.clipNotesRepository.find({
+				where: {
+					clipId,
+					...(cursor ? { id: MoreThan(cursor) } : {}),
+				},
+				take: 100,
+				order: {
+					id: 1,
+				},
+				relations: ['note', 'note.user'],
+			}) as (MiClipNote & { note: MiNote & { user: MiUser } })[];
+
+			if (clipNotes.length === 0) {
+				break;
+			}
+
+			cursor = clipNotes.at(-1)?.id ?? null;
+
+			for (const clipNote of clipNotes) {
+				let poll: MiPoll | undefined;
+				if (clipNote.note.hasPoll) {
+					poll = await this.pollsRepository.findOneByOrFail({ noteId: clipNote.note.id });
+				}
+				const content = JSON.stringify(this.serializeClipNote(clipNote, poll));
+				const isFirst = exportedClipNotesCount === 0;
+				await writer.write(isFirst ? content : ',\n' + content);
+
+				exportedClipNotesCount++;
+			}
+		}
+	}
+
+	private serializeClip(clip: MiClip): Record<string, unknown> {
+		return {
+			id: clip.id,
+			name: clip.name,
+			description: clip.description,
+			lastClippedAt: clip.lastClippedAt?.toISOString(),
+			clipNotes: [],
+		};
+	}
+
+	private serializeClipNote(clip: MiClipNote & { note: MiNote & { user: MiUser } }, poll: MiPoll | undefined): Record<string, unknown> {
+		return {
+			id: clip.id,
+			createdAt: this.idService.parse(clip.id).date.toISOString(),
+			note: {
+				id: clip.note.id,
+				text: clip.note.text,
+				createdAt: this.idService.parse(clip.note.id).date.toISOString(),
+				fileIds: clip.note.fileIds,
+				replyId: clip.note.replyId,
+				renoteId: clip.note.renoteId,
+				poll: poll,
+				cw: clip.note.cw,
+				visibility: clip.note.visibility,
+				visibleUserIds: clip.note.visibleUserIds,
+				localOnly: clip.note.localOnly,
+				reactionAcceptance: clip.note.reactionAcceptance,
+				uri: clip.note.uri,
+				url: clip.note.url,
+				user: {
+					id: clip.note.user.id,
+					name: clip.note.user.name,
+					username: clip.note.user.username,
+					host: clip.note.user.host,
+					uri: clip.note.user.uri,
+				},
+			},
+		};
+	}
+}

packages/backend/src/server/FileServerService.ts

@@ -9,7 +9,7 @@ import { dirname } from 'node:path';
 import { Inject, Injectable } from '@nestjs/common';
 import rename from 'rename';
 import sharp from 'sharp';
-import { sharpBmp } from 'sharp-read-bmp';
+import { sharpBmp } from '@misskey-dev/sharp-read-bmp';
 import type { Config } from '@/config.js';
 import type { MiDriveFile, DriveFilesRepository } from '@/models/_.js';
 import { DI } from '@/di-symbols.js';

packages/backend/src/server/api/EndpointsModule.ts

@@ -212,6 +212,7 @@ import * as ep___i_exportBlocking from './endpoints/i/export-blocking.js';
 import * as ep___i_exportFollowing from './endpoints/i/export-following.js';
 import * as ep___i_exportMute from './endpoints/i/export-mute.js';
 import * as ep___i_exportNotes from './endpoints/i/export-notes.js';
+import * as ep___i_exportClips from './endpoints/i/export-clips.js';
 import * as ep___i_exportFavorites from './endpoints/i/export-favorites.js';
 import * as ep___i_exportUserLists from './endpoints/i/export-user-lists.js';
 import * as ep___i_exportAntennas from './endpoints/i/export-antennas.js';
@@ -578,6 +579,7 @@ const $i_exportBlocking: Provider = { provide: 'ep:i/export-blocking', useClass:
 const $i_exportFollowing: Provider = { provide: 'ep:i/export-following', useClass: ep___i_exportFollowing.default };
 const $i_exportMute: Provider = { provide: 'ep:i/export-mute', useClass: ep___i_exportMute.default };
 const $i_exportNotes: Provider = { provide: 'ep:i/export-notes', useClass: ep___i_exportNotes.default };
+const $i_exportClips: Provider = { provide: 'ep:i/export-clips', useClass: ep___i_exportClips.default };
 const $i_exportFavorites: Provider = { provide: 'ep:i/export-favorites', useClass: ep___i_exportFavorites.default };
 const $i_exportUserLists: Provider = { provide: 'ep:i/export-user-lists', useClass: ep___i_exportUserLists.default };
 const $i_exportAntennas: Provider = { provide: 'ep:i/export-antennas', useClass: ep___i_exportAntennas.default };
@@ -948,6 +950,7 @@ const $retention: Provider = { provide: 'ep:retention', useClass: ep___retention
 		$i_exportFollowing,
 		$i_exportMute,
 		$i_exportNotes,
+		$i_exportClips,
 		$i_exportFavorites,
 		$i_exportUserLists,
 		$i_exportAntennas,
@@ -1312,6 +1315,7 @@ const $retention: Provider = { provide: 'ep:retention', useClass: ep___retention
 		$i_exportFollowing,
 		$i_exportMute,
 		$i_exportNotes,
+		$i_exportClips,
 		$i_exportFavorites,
 		$i_exportUserLists,
 		$i_exportAntennas,

packages/backend/src/server/api/SignupApiService.ts

@@ -65,6 +65,7 @@ export class SignupApiService {
 				'hcaptcha-response'?: string;
 				'g-recaptcha-response'?: string;
 				'turnstile-response'?: string;
+				'm-captcha-response'?: string;
 			}
 		}>,
 		reply: FastifyReply,
@@ -82,6 +83,12 @@ export class SignupApiService {
 				});
 			}
 
+			if (instance.enableMcaptcha && instance.mcaptchaSecretKey && instance.mcaptchaSitekey && instance.mcaptchaInstanceUrl) {
+				await this.captchaService.verifyMcaptcha(instance.mcaptchaSecretKey, instance.mcaptchaSitekey, instance.mcaptchaInstanceUrl, body['m-captcha-response']).catch(err => {
+					throw new FastifyReplyError(400, err);
+				});
+			}
+
 			if (instance.enableRecaptcha && instance.recaptchaSecretKey) {
 				await this.captchaService.verifyRecaptcha(instance.recaptchaSecretKey, body['g-recaptcha-response']).catch(err => {
 					throw new FastifyReplyError(400, err);

packages/backend/src/server/api/endpoints.ts

@@ -3,8 +3,8 @@
  * SPDX-License-Identifier: AGPL-3.0-only
  */
 
-import type { Schema } from '@/misc/json-schema.js';
 import { permissions } from 'misskey-js';
+import type { Schema } from '@/misc/json-schema.js';
 import { RolePolicies } from '@/core/RoleService.js';
 
 import * as ep___admin_meta from './endpoints/admin/meta.js';
@@ -213,6 +213,7 @@ import * as ep___i_exportBlocking from './endpoints/i/export-blocking.js';
 import * as ep___i_exportFollowing from './endpoints/i/export-following.js';
 import * as ep___i_exportMute from './endpoints/i/export-mute.js';
 import * as ep___i_exportNotes from './endpoints/i/export-notes.js';
+import * as ep___i_exportClips from './endpoints/i/export-clips.js';
 import * as ep___i_exportFavorites from './endpoints/i/export-favorites.js';
 import * as ep___i_exportUserLists from './endpoints/i/export-user-lists.js';
 import * as ep___i_exportAntennas from './endpoints/i/export-antennas.js';
@@ -577,6 +578,7 @@ const eps = [
 	['i/export-following', ep___i_exportFollowing],
 	['i/export-mute', ep___i_exportMute],
 	['i/export-notes', ep___i_exportNotes],
+	['i/export-clips', ep___i_exportClips],
 	['i/export-favorites', ep___i_exportFavorites],
 	['i/export-user-lists', ep___i_exportUserLists],
 	['i/export-antennas', ep___i_exportAntennas],

packages/backend/src/server/api/endpoints/admin/meta.ts

@@ -41,6 +41,18 @@ export const meta = {
 				type: 'string',
 				optional: false, nullable: true,
 			},
+			enableMcaptcha: {
+				type: 'boolean',
+				optional: false, nullable: false,
+			},
+			mcaptchaSiteKey: {
+				type: 'string',
+				optional: false, nullable: true,
+			},
+			mcaptchaInstanceUrl: {
+				type: 'string',
+				optional: false, nullable: true,
+			},
 			enableRecaptcha: {
 				type: 'boolean',
 				optional: false, nullable: false,
@@ -173,6 +185,10 @@ export const meta = {
 				type: 'string',
 				optional: false, nullable: true,
 			},
+			mcaptchaSecretKey: {
+				type: 'string',
+				optional: false, nullable: true,
+			},
 			recaptchaSecretKey: {
 				type: 'string',
 				optional: false, nullable: true,
@@ -294,6 +310,18 @@ export const meta = {
 				type: 'string',
 				optional: false, nullable: true,
 			},
+			enableTruemailApi: {
+				type: 'boolean',
+				optional: false, nullable: false,
+			},
+			truemailInstance: {
+				type: 'string',
+				optional: false, nullable: true,
+			},
+			truemailAuthKey: {
+				type: 'string',
+				optional: false, nullable: true,
+			},
 			enableChartsForRemoteUser: {
 				type: 'boolean',
 				optional: false, nullable: false,
@@ -474,6 +502,9 @@ export default class extends Endpoint<typeof meta, typeof paramDef> { // eslint-
 				emailRequiredForSignup: instance.emailRequiredForSignup,
 				enableHcaptcha: instance.enableHcaptcha,
 				hcaptchaSiteKey: instance.hcaptchaSiteKey,
+				enableMcaptcha: instance.enableMcaptcha,
+				mcaptchaSiteKey: instance.mcaptchaSitekey,
+				mcaptchaInstanceUrl: instance.mcaptchaInstanceUrl,
 				enableRecaptcha: instance.enableRecaptcha,
 				recaptchaSiteKey: instance.recaptchaSiteKey,
 				enableTurnstile: instance.enableTurnstile,
@@ -505,6 +536,7 @@ export default class extends Endpoint<typeof meta, typeof paramDef> { // eslint-
 				sensitiveWords: instance.sensitiveWords,
 				preservedUsernames: instance.preservedUsernames,
 				hcaptchaSecretKey: instance.hcaptchaSecretKey,
+				mcaptchaSecretKey: instance.mcaptchaSecretKey,
 				recaptchaSecretKey: instance.recaptchaSecretKey,
 				turnstileSecretKey: instance.turnstileSecretKey,
 				sensitiveMediaDetection: instance.sensitiveMediaDetection,
@@ -539,6 +571,9 @@ export default class extends Endpoint<typeof meta, typeof paramDef> { // eslint-
 				enableActiveEmailValidation: instance.enableActiveEmailValidation,
 				enableVerifymailApi: instance.enableVerifymailApi,
 				verifymailAuthKey: instance.verifymailAuthKey,
+				enableTruemailApi: instance.enableTruemailApi,
+				truemailInstance: instance.truemailInstance,
+				truemailAuthKey: instance.truemailAuthKey,
 				enableChartsForRemoteUser: instance.enableChartsForRemoteUser,
 				enableChartsForFederatedInstances: instance.enableChartsForFederatedInstances,
 				enableServerMachineStats: instance.enableServerMachineStats,

packages/backend/src/server/api/endpoints/admin/update-meta.ts

@@ -63,6 +63,10 @@ export const paramDef = {
 		enableHcaptcha: { type: 'boolean' },
 		hcaptchaSiteKey: { type: 'string', nullable: true },
 		hcaptchaSecretKey: { type: 'string', nullable: true },
+		enableMcaptcha: { type: 'boolean' },
+		mcaptchaSiteKey: { type: 'string', nullable: true },
+		mcaptchaInstanceUrl: { type: 'string', nullable: true },
+		mcaptchaSecretKey: { type: 'string', nullable: true },
 		enableRecaptcha: { type: 'boolean' },
 		recaptchaSiteKey: { type: 'string', nullable: true },
 		recaptchaSecretKey: { type: 'string', nullable: true },
@@ -116,6 +120,9 @@ export const paramDef = {
 		enableActiveEmailValidation: { type: 'boolean' },
 		enableVerifymailApi: { type: 'boolean' },
 		verifymailAuthKey: { type: 'string', nullable: true },
+		enableTruemailApi: { type: 'boolean' },
+		truemailInstance: { type: 'string', nullable: true },
+		truemailAuthKey: { type: 'string', nullable: true },
 		enableChartsForRemoteUser: { type: 'boolean' },
 		enableChartsForFederatedInstances: { type: 'boolean' },
 		enableServerMachineStats: { type: 'boolean' },
@@ -289,6 +296,22 @@ export default class extends Endpoint<typeof meta, typeof paramDef> { // eslint-
 				set.hcaptchaSecretKey = ps.hcaptchaSecretKey;
 			}
 
+			if (ps.enableMcaptcha !== undefined) {
+				set.enableMcaptcha = ps.enableMcaptcha;
+			}
+
+			if (ps.mcaptchaSiteKey !== undefined) {
+				set.mcaptchaSitekey = ps.mcaptchaSiteKey;
+			}
+
+			if (ps.mcaptchaInstanceUrl !== undefined) {
+				set.mcaptchaInstanceUrl = ps.mcaptchaInstanceUrl;
+			}
+
+			if (ps.mcaptchaSecretKey !== undefined) {
+				set.mcaptchaSecretKey = ps.mcaptchaSecretKey;
+			}
+
 			if (ps.enableRecaptcha !== undefined) {
 				set.enableRecaptcha = ps.enableRecaptcha;
 			}
@@ -493,6 +516,26 @@ export default class extends Endpoint<typeof meta, typeof paramDef> { // eslint-
 				}
 			}
 
+			if (ps.enableTruemailApi !== undefined) {
+				set.enableTruemailApi = ps.enableTruemailApi;
+			}
+
+			if (ps.truemailInstance !== undefined) {
+				if (ps.truemailInstance === '') {
+					set.truemailInstance = null;
+				} else {
+					set.truemailInstance = ps.truemailInstance;
+				}
+			}
+
+			if (ps.truemailAuthKey !== undefined) {
+				if (ps.truemailAuthKey === '') {
+					set.truemailAuthKey = null;
+				} else {
+					set.truemailAuthKey = ps.truemailAuthKey;
+				}
+			}
+
 			if (ps.enableChartsForRemoteUser !== undefined) {
 				set.enableChartsForRemoteUser = ps.enableChartsForRemoteUser;
 			}

packages/backend/src/server/api/endpoints/antennas/notes.ts

@@ -14,6 +14,7 @@ import { NoteEntityService } from '@/core/entities/NoteEntityService.js';
 import { IdService } from '@/core/IdService.js';
 import { FanoutTimelineService } from '@/core/FanoutTimelineService.js';
 import { GlobalEventService } from '@/core/GlobalEventService.js';
+import { trackPromise } from '@/misc/promise-tracker.js';
 import { ApiError } from '../../error.js';
 
 export const meta = {
@@ -92,7 +93,7 @@ export default class extends Endpoint<typeof meta, typeof paramDef> { // eslint-
 
 			antenna.isActive = true;
 			antenna.lastUsedAt = new Date();
-			this.antennasRepository.update(antenna.id, antenna);
+			trackPromise(this.antennasRepository.update(antenna.id, antenna));
 
 			if (needPublishEvent) {
 				this.globalEventService.publishInternalEvent('antennaUpdated', antenna);

packages/backend/src/server/api/endpoints/i/export-clips.ts

@@ -0,0 +1,35 @@
+/*
+ * SPDX-FileCopyrightText: syuilo and other misskey contributors
+ * SPDX-License-Identifier: AGPL-3.0-only
+ */
+
+import { Injectable } from '@nestjs/common';
+import ms from 'ms';
+import { Endpoint } from '@/server/api/endpoint-base.js';
+import { QueueService } from '@/core/QueueService.js';
+
+export const meta = {
+	secure: true,
+	requireCredential: true,
+	limit: {
+		duration: ms('1day'),
+		max: 1,
+	},
+} as const;
+
+export const paramDef = {
+	type: 'object',
+	properties: {},
+	required: [],
+} as const;
+
+@Injectable()
+export default class extends Endpoint<typeof meta, typeof paramDef> { // eslint-disable-line import/no-default-export
+	constructor(
+		private queueService: QueueService,
+	) {
+		super(meta, paramDef, async (ps, me) => {
+			this.queueService.createExportClipsJob(me);
+		});
+	}
+}

packages/backend/src/server/api/endpoints/meta.ts

@@ -108,6 +108,18 @@ export const meta = {
 				type: 'string',
 				optional: false, nullable: true,
 			},
+			enableMcaptcha: {
+				type: 'boolean',
+				optional: false, nullable: false,
+			},
+			mcaptchaSiteKey: {
+				type: 'string',
+				optional: false, nullable: true,
+			},
+			mcaptchaInstanceUrl: {
+				type: 'string',
+				optional: false, nullable: true,
+			},
 			enableRecaptcha: {
 				type: 'boolean',
 				optional: false, nullable: false,
@@ -352,6 +364,9 @@ export default class extends Endpoint<typeof meta, typeof paramDef> { // eslint-
 				emailRequiredForSignup: instance.emailRequiredForSignup,
 				enableHcaptcha: instance.enableHcaptcha,
 				hcaptchaSiteKey: instance.hcaptchaSiteKey,
+				enableMcaptcha: instance.enableMcaptcha,
+				mcaptchaSiteKey: instance.mcaptchaSitekey,
+				mcaptchaInstanceUrl: instance.mcaptchaInstanceUrl,
 				enableRecaptcha: instance.enableRecaptcha,
 				recaptchaSiteKey: instance.recaptchaSiteKey,
 				enableTurnstile: instance.enableTurnstile,

packages/backend/src/server/web/UrlPreviewService.ts

@@ -4,7 +4,7 @@
  */
 
 import { Inject, Injectable } from '@nestjs/common';
-import { summaly } from 'summaly';
+import { summaly } from '@misskey-dev/summaly';
 import RE2 from 're2';
 import { DI } from '@/di-symbols.js';
 import type { Config } from '@/config.js';