Merge remote-tracking branch 'upstream/develop' into refactor-frontend-types

packages/backend/src/core/QueueService.ts

@@ -16,6 +16,7 @@ import type { DbQueue, DeliverQueue, EndedPollNotificationQueue, InboxQueue, Obj
 import type { DbJobData, DeliverJobData, RelationshipJobData, ThinUser } from '../queue/types.js';
 import type httpSignature from '@peertube/http-signature';
 import type * as Bull from 'bullmq';
+import { ApRequestCreator } from '@/core/activitypub/ApRequestService.js';
 
 @Injectable()
 export class QueueService {
@@ -74,11 +75,15 @@ export class QueueService {
 		if (content == null) return null;
 		if (to == null) return null;
 
+		const contentBody = JSON.stringify(content);
+		const digest = ApRequestCreator.createDigest(contentBody);
+
 		const data: DeliverJobData = {
 			user: {
 				id: user.id,
 			},
-			content,
+			content: contentBody,
+			digest,
 			to,
 			isSharedInbox,
 		};
@@ -103,6 +108,8 @@ export class QueueService {
 	@bindThis
 	public async deliverMany(user: ThinUser, content: IActivity | null, inboxes: Map<string, boolean>) {
 		if (content == null) return null;
+		const contentBody = JSON.stringify(content);
+		const digest = ApRequestCreator.createDigest(contentBody);
 
 		const opts = {
 			attempts: this.config.deliverJobMaxAttempts ?? 12,
@@ -117,7 +124,8 @@ export class QueueService {
 			name: d[0],
 			data: {
 				user,
-				content,
+				content: contentBody,
+				digest,
 				to: d[0],
 				isSharedInbox: d[1],
 			} as DeliverJobData,

packages/backend/src/core/activitypub/ApInboxService.ts

@@ -97,6 +97,8 @@ export class ApInboxService {
 				} catch (err) {
 					if (err instanceof Error || typeof err === 'string') {
 						this.logger.error(err);
+					} else {
+						throw err;
 					}
 				}
 			}
@@ -256,7 +258,7 @@ export class ApInboxService {
 
 		const targetUri = getApId(activity.object);
 
-		this.announceNote(actor, activity, targetUri);
+		await this.announceNote(actor, activity, targetUri);
 	}
 
 	@bindThis
@@ -288,7 +290,7 @@ export class ApInboxService {
 			} catch (err) {
 				// 対象が4xxならスキップ
 				if (err instanceof StatusError) {
-					if (err.isClientError) {
+					if (!err.isRetryable) {
 						this.logger.warn(`Ignored announce target ${targetUri} - ${err.statusCode}`);
 						return;
 					}
@@ -373,7 +375,7 @@ export class ApInboxService {
 		});
 
 		if (isPost(object)) {
-			this.createNote(resolver, actor, object, false, activity);
+			await this.createNote(resolver, actor, object, false, activity);
 		} else {
 			this.logger.warn(`Unknown type: ${getApType(object)}`);
 		}
@@ -404,7 +406,7 @@ export class ApInboxService {
 			await this.apNoteService.createNote(note, resolver, silent);
 			return 'ok';
 		} catch (err) {
-			if (err instanceof StatusError && err.isClientError) {
+			if (err instanceof StatusError && !err.isRetryable) {
 				return `skip ${err.statusCode}`;
 			} else {
 				throw err;

packages/backend/src/core/activitypub/ApRequestService.ts

@@ -34,9 +34,9 @@ type PrivateKey = {
 };
 
 export class ApRequestCreator {
-	static createSignedPost(args: { key: PrivateKey, url: string, body: string, additionalHeaders: Record<string, string> }): Signed {
+	static createSignedPost(args: { key: PrivateKey, url: string, body: string, digest?: string, additionalHeaders: Record<string, string> }): Signed {
 		const u = new URL(args.url);
-		const digestHeader = `SHA-256=${crypto.createHash('sha256').update(args.body).digest('base64')}`;
+		const digestHeader = args.digest ?? this.createDigest(args.body);
 
 		const request: Request = {
 			url: u.href,
@@ -59,6 +59,10 @@ export class ApRequestCreator {
 		};
 	}
 
+	static createDigest(body: string) {
+		return `SHA-256=${crypto.createHash('sha256').update(body).digest('base64')}`;
+	}
+
 	static createSignedGet(args: { key: PrivateKey, url: string, additionalHeaders: Record<string, string> }): Signed {
 		const u = new URL(args.url);
 
@@ -145,8 +149,8 @@ export class ApRequestService {
 	}
 
 	@bindThis
-	public async signedPost(user: { id: MiUser['id'] }, url: string, object: unknown): Promise<void> {
-		const body = JSON.stringify(object);
+	public async signedPost(user: { id: MiUser['id'] }, url: string, object: unknown, digest?: string): Promise<void> {
+		const body = typeof object === 'string' ? object : JSON.stringify(object);
 
 		const keypair = await this.userKeypairService.getUserKeypair(user.id);
 
@@ -157,6 +161,7 @@ export class ApRequestService {
 			},
 			url,
 			body,
+			digest,
 			additionalHeaders: {
 			},
 		});

packages/backend/src/core/activitypub/models/ApNoteService.ts

@@ -216,7 +216,7 @@ export class ApNoteService {
 					return { status: 'ok', res };
 				} catch (e) {
 					return {
-						status: (e instanceof StatusError && e.isClientError) ? 'permerror' : 'temperror',
+						status: (e instanceof StatusError && !e.isRetryable) ? 'permerror' : 'temperror',
 					};
 				}
 			};

packages/backend/src/core/entities/NoteEntityService.ts

@@ -351,6 +351,7 @@ export class NoteEntityService implements OnModuleInit {
 				color: channel.color,
 				isSensitive: channel.isSensitive,
 				allowRenoteToExternal: channel.allowRenoteToExternal,
+				userId: channel.userId,
 			} : undefined,
 			mentions: note.mentions.length > 0 ? note.mentions : undefined,
 			uri: note.uri ?? undefined,

packages/backend/src/logger.ts

@@ -71,8 +71,11 @@ export default class Logger {
 		let log = `${l} ${worker}\t[${contexts.join(' ')}]\t${m}`;
 		if (envOption.withLogTime) log = chalk.gray(time) + ' ' + log;
 
-		console.log(important ? chalk.bold(log) : log);
-		if (level === 'error' && data) console.log(data);
+		const args: unknown[] = [important ? chalk.bold(log) : log];
+		if (data != null) {
+			args.push(data);
+		}
+		console.log(...args);
 	}
 
 	@bindThis

packages/backend/src/misc/status-error.ts

@@ -7,6 +7,7 @@ export class StatusError extends Error {
 	public statusCode: number;
 	public statusMessage?: string;
 	public isClientError: boolean;
+	public isRetryable: boolean;
 
 	constructor(message: string, statusCode: number, statusMessage?: string) {
 		super(message);
@@ -14,5 +15,6 @@ export class StatusError extends Error {
 		this.statusCode = statusCode;
 		this.statusMessage = statusMessage;
 		this.isClientError = typeof this.statusCode === 'number' && this.statusCode >= 400 && this.statusCode < 500;
+		this.isRetryable = !this.isClientError || this.statusCode === 429;
 	}
 }

packages/backend/src/models/json-schema/note.ts

@@ -152,6 +152,10 @@ export const packedNoteSchema = {
 					type: 'boolean',
 					optional: false, nullable: false,
 				},
+				userId: {
+					type: 'string',
+					optional: false, nullable: true,
+				},
 			},
 		},
 		localOnly: {

packages/backend/src/queue/processors/DeliverProcessorService.ts

@@ -72,7 +72,7 @@ export class DeliverProcessorService {
 		}
 
 		try {
-			await this.apRequestService.signedPost(job.data.user, job.data.to, job.data.content);
+			await this.apRequestService.signedPost(job.data.user, job.data.to, job.data.content, job.data.digest);
 
 			// Update stats
 			this.federatedInstanceService.fetch(host).then(i => {
@@ -111,7 +111,7 @@ export class DeliverProcessorService {
 
 			if (res instanceof StatusError) {
 				// 4xx
-				if (res.isClientError) {
+				if (!res.isRetryable) {
 					// 相手が閉鎖していることを明示しているため、配送停止する
 					if (job.data.isSharedInbox && res.statusCode === 410) {
 						this.federatedInstanceService.fetch(host).then(i => {

packages/backend/src/queue/processors/InboxProcessorService.ts

@@ -85,7 +85,7 @@ export class InboxProcessorService {
 			} catch (err) {
 				// 対象が4xxならスキップ
 				if (err instanceof StatusError) {
-					if (err.isClientError) {
+					if (!err.isRetryable) {
 						throw new Bull.UnrecoverableError(`skip: Ignored deleted actors on both ends ${activity.actor} - ${err.statusCode}`);
 					}
 					throw new Error(`Error in actor ${activity.actor} - ${err.statusCode}`);

packages/backend/src/queue/processors/WebhookDeliverProcessorService.ts

@@ -71,7 +71,7 @@ export class WebhookDeliverProcessorService {
 
 			if (res instanceof StatusError) {
 				// 4xx
-				if (res.isClientError) {
+				if (!res.isRetryable) {
 					throw new Bull.UnrecoverableError(`${res.statusCode} ${res.statusMessage}`);
 				}
 

packages/backend/src/queue/types.ts

@@ -15,7 +15,9 @@ export type DeliverJobData = {
 	/** Actor */
 	user: ThinUser;
 	/** Activity */
-	content: unknown;
+	content: string;
+	/** Digest header */
+	digest: string;
 	/** inbox URL to deliver */
 	to: string;
 	/** whether it is sharedInbox */

packages/backend/test/e2e/2fa.ts

@@ -24,7 +24,7 @@ import type * as misskey from 'misskey-js';
 
 describe('2要素認証', () => {
 	let app: INestApplicationContext;
-	let alice: misskey.entities.MeSignup;
+	let alice: misskey.entities.SignupResponse;
 
 	const config = loadConfig();
 	const password = 'test';

packages/backend/test/e2e/antennas.ts

@@ -37,7 +37,7 @@ describe('アンテナ', () => {
 	// - srcのenumにgroupが残っている
 	// - userGroupIdが残っている, isActiveがない
 	type Antenna = misskey.entities.Antenna | Packed<'Antenna'>;
-	type User = misskey.entities.MeSignup;
+	type User = misskey.entities.SignupResponse;
 	type Note = misskey.entities.Note;
 
 	// アンテナを作成できる最小のパラメタ

packages/backend/test/e2e/api-visibility.ts

@@ -24,15 +24,15 @@ describe('API visibility', () => {
 	describe('Note visibility', () => {
 		//#region vars
 		/** ヒロイン */
-		let alice: misskey.entities.MeSignup;
+		let alice: misskey.entities.SignupResponse;
 		/** フォロワー */
-		let follower: misskey.entities.MeSignup;
+		let follower: misskey.entities.SignupResponse;
 		/** 非フォロワー */
-		let other: misskey.entities.MeSignup;
+		let other: misskey.entities.SignupResponse;
 		/** 非フォロワーでもリプライやメンションをされた人 */
-		let target: misskey.entities.MeSignup;
+		let target: misskey.entities.SignupResponse;
 		/** specified mentionでmentionを飛ばされる人 */
-		let target2: misskey.entities.MeSignup;
+		let target2: misskey.entities.SignupResponse;
 
 		/** public-post */
 		let pub: any;

packages/backend/test/e2e/api.ts

@@ -13,9 +13,9 @@ import type * as misskey from 'misskey-js';
 
 describe('API', () => {
 	let app: INestApplicationContext;
-	let alice: misskey.entities.MeSignup;
-	let bob: misskey.entities.MeSignup;
-	let carol: misskey.entities.MeSignup;
+	let alice: misskey.entities.SignupResponse;
+	let bob: misskey.entities.SignupResponse;
+	let carol: misskey.entities.SignupResponse;
 
 	beforeAll(async () => {
 		app = await startServer();

packages/backend/test/e2e/block.ts

@@ -14,9 +14,9 @@ describe('Block', () => {
 	let app: INestApplicationContext;
 
 	// alice blocks bob
-	let alice: misskey.entities.MeSignup;
-	let bob: misskey.entities.MeSignup;
-	let carol: misskey.entities.MeSignup;
+	let alice: misskey.entities.SignupResponse;
+	let bob: misskey.entities.SignupResponse;
+	let carol: misskey.entities.SignupResponse;
 
 	beforeAll(async () => {
 		app = await startServer();

packages/backend/test/e2e/endpoints.ts

@@ -17,10 +17,10 @@ import type * as misskey from 'misskey-js';
 describe('Endpoints', () => {
 	let app: INestApplicationContext;
 
-	let alice: misskey.entities.MeSignup;
-	let bob: misskey.entities.MeSignup;
-	let carol: misskey.entities.MeSignup;
-	let dave: misskey.entities.MeSignup;
+	let alice: misskey.entities.SignupResponse;
+	let bob: misskey.entities.SignupResponse;
+	let carol: misskey.entities.SignupResponse;
+	let dave: misskey.entities.SignupResponse;
 
 	beforeAll(async () => {
 		app = await startServer();

packages/backend/test/e2e/fetch-resource.ts

@@ -25,7 +25,7 @@ const JSON_UTF8 = 'application/json; charset=utf-8';
 describe('Webリソース', () => {
 	let app: INestApplicationContext;
 
-	let alice: misskey.entities.MeSignup;
+	let alice: misskey.entities.SignupResponse;
 	let aliceUploadedFile: any;
 	let alicesPost: any;
 	let alicePage: any;
@@ -34,7 +34,7 @@ describe('Webリソース', () => {
 	let aliceGalleryPost: any;
 	let aliceChannel: any;
 
-	let bob: misskey.entities.MeSignup;
+	let bob: misskey.entities.SignupResponse;
 
 	type Request = {
 		path: string,

packages/backend/test/e2e/ff-visibility.ts

@@ -13,8 +13,8 @@ import type * as misskey from 'misskey-js';
 describe('FF visibility', () => {
 	let app: INestApplicationContext;
 
-	let alice: misskey.entities.MeSignup;
-	let bob: misskey.entities.MeSignup;
+	let alice: misskey.entities.SignupResponse;
+	let bob: misskey.entities.SignupResponse;
 
 	beforeAll(async () => {
 		app = await startServer();

packages/backend/test/e2e/move.ts

@@ -20,12 +20,12 @@ describe('Account Move', () => {
 	let url: URL;
 
 	let root: any;
-	let alice: misskey.entities.MeSignup;
-	let bob: misskey.entities.MeSignup;
-	let carol: misskey.entities.MeSignup;
-	let dave: misskey.entities.MeSignup;
-	let eve: misskey.entities.MeSignup;
-	let frank: misskey.entities.MeSignup;
+	let alice: misskey.entities.SignupResponse;
+	let bob: misskey.entities.SignupResponse;
+	let carol: misskey.entities.SignupResponse;
+	let dave: misskey.entities.SignupResponse;
+	let eve: misskey.entities.SignupResponse;
+	let frank: misskey.entities.SignupResponse;
 
 	let Users: UsersRepository;
 

packages/backend/test/e2e/mute.ts

@@ -14,9 +14,9 @@ describe('Mute', () => {
 	let app: INestApplicationContext;
 
 	// alice mutes carol
-	let alice: misskey.entities.MeSignup;
-	let bob: misskey.entities.MeSignup;
-	let carol: misskey.entities.MeSignup;
+	let alice: misskey.entities.SignupResponse;
+	let bob: misskey.entities.SignupResponse;
+	let carol: misskey.entities.SignupResponse;
 
 	beforeAll(async () => {
 		app = await startServer();

packages/backend/test/e2e/note.ts

@@ -16,8 +16,8 @@ describe('Note', () => {
 	let app: INestApplicationContext;
 	let Notes: any;
 
-	let alice: misskey.entities.MeSignup;
-	let bob: misskey.entities.MeSignup;
+	let alice: misskey.entities.SignupResponse;
+	let bob: misskey.entities.SignupResponse;
 
 	beforeAll(async () => {
 		app = await startServer();

packages/backend/test/e2e/oauth.ts

@@ -75,7 +75,7 @@ function getMeta(html: string): { transactionId: string | undefined, clientName:
 	};
 }
 
-function fetchDecision(transactionId: string, user: misskey.entities.MeSignup, { cancel }: { cancel?: boolean } = {}): Promise<Response> {
+function fetchDecision(transactionId: string, user: misskey.entities.SignupResponse, { cancel }: { cancel?: boolean } = {}): Promise<Response> {
 	return fetch(new URL('/oauth/decision', host), {
 		method: 'post',
 		body: new URLSearchParams({
@@ -90,14 +90,14 @@ function fetchDecision(transactionId: string, user: misskey.entities.MeSignup, {
 	});
 }
 
-async function fetchDecisionFromResponse(response: Response, user: misskey.entities.MeSignup, { cancel }: { cancel?: boolean } = {}): Promise<Response> {
+async function fetchDecisionFromResponse(response: Response, user: misskey.entities.SignupResponse, { cancel }: { cancel?: boolean } = {}): Promise<Response> {
 	const { transactionId } = getMeta(await response.text());
 	assert.ok(transactionId);
 
 	return await fetchDecision(transactionId, user, { cancel });
 }
 
-async function fetchAuthorizationCode(user: misskey.entities.MeSignup, scope: string, code_challenge: string): Promise<{ client: AuthorizationCode, code: string }> {
+async function fetchAuthorizationCode(user: misskey.entities.SignupResponse, scope: string, code_challenge: string): Promise<{ client: AuthorizationCode, code: string }> {
 	const client = new AuthorizationCode(clientConfig);
 
 	const response = await fetch(client.authorizeURL({
@@ -150,8 +150,8 @@ describe('OAuth', () => {
 	let app: INestApplicationContext;
 	let fastify: FastifyInstance;
 
-	let alice: misskey.entities.MeSignup;
-	let bob: misskey.entities.MeSignup;
+	let alice: misskey.entities.SignupResponse;
+	let bob: misskey.entities.SignupResponse;
 
 	let sender: (reply: FastifyReply) => void;
 

packages/backend/test/e2e/renote-mute.ts

@@ -14,9 +14,9 @@ describe('Renote Mute', () => {
 	let app: INestApplicationContext;
 
 	// alice mutes carol
-	let alice: misskey.entities.MeSignup;
-	let bob: misskey.entities.MeSignup;
-	let carol: misskey.entities.MeSignup;
+	let alice: misskey.entities.SignupResponse;
+	let bob: misskey.entities.SignupResponse;
+	let carol: misskey.entities.SignupResponse;
 
 	beforeAll(async () => {
 		app = await startServer();

packages/backend/test/e2e/streaming.ts

@@ -32,15 +32,15 @@ describe('Streaming', () => {
 
 	describe('Streaming', () => {
 		// Local users
-		let ayano: misskey.entities.MeSignup;
-		let kyoko: misskey.entities.MeSignup;
-		let chitose: misskey.entities.MeSignup;
-		let kanako: misskey.entities.MeSignup;
+		let ayano: misskey.entities.SignupResponse;
+		let kyoko: misskey.entities.SignupResponse;
+		let chitose: misskey.entities.SignupResponse;
+		let kanako: misskey.entities.SignupResponse;
 
 		// Remote users
-		let akari: misskey.entities.MeSignup;
-		let chinatsu: misskey.entities.MeSignup;
-		let takumi: misskey.entities.MeSignup;
+		let akari: misskey.entities.SignupResponse;
+		let chinatsu: misskey.entities.SignupResponse;
+		let takumi: misskey.entities.SignupResponse;
 
 		let kyokoNote: any;
 		let kanakoNote: any;

packages/backend/test/e2e/thread-mute.ts

@@ -13,9 +13,9 @@ import type * as misskey from 'misskey-js';
 describe('Note thread mute', () => {
 	let app: INestApplicationContext;
 
-	let alice: misskey.entities.MeSignup;
-	let bob: misskey.entities.MeSignup;
-	let carol: misskey.entities.MeSignup;
+	let alice: misskey.entities.SignupResponse;
+	let bob: misskey.entities.SignupResponse;
+	let carol: misskey.entities.SignupResponse;
 
 	beforeAll(async () => {
 		app = await startServer();

packages/backend/test/e2e/user-notes.ts

@@ -13,7 +13,7 @@ import type * as misskey from 'misskey-js';
 describe('users/notes', () => {
 	let app: INestApplicationContext;
 
-	let alice: misskey.entities.MeSignup;
+	let alice: misskey.entities.SignupResponse;
 	let jpgNote: any;
 	let pngNote: any;
 	let jpgPngNote: any;