feat: improve tl performance (#11946)

* wip * wip * wip * wip * wip * wip * Update NoteCreateService.ts * wip * wip * wip * wip * Update NoteCreateService.ts * wip * Update NoteCreateService.ts * wip * Update user-notes.ts * wip * wip * wip * Update NoteCreateService.ts * wip * Update timeline.ts * Update timeline.ts * Update timeline.ts * Update timeline.ts * Update timeline.ts * wip * Update timelines.ts * Update timelines.ts * Update timelines.ts * wip * wip * wip * Update timelines.ts * Update misskey-js.api.md * Update timelines.ts * Update timelines.ts * wip * wip * wip * Update timelines.ts * wip * Update timelines.ts * wip * test * Update activitypub.ts * refactor: UserListJoining -> UserListMembership * Update NoteCreateService.ts * wip
2023-10-03 20:26:11 +09:00
parent 5ee93dc4a2
commit 6277a5545c
84 changed files with 1898 additions and 947 deletions
--- a/packages/backend/src/server/api/endpoints/users/lists/create-from-public.ts
+++ b/packages/backend/src/server/api/endpoints/users/lists/create-from-public.ts
@@ -4,7 +4,7 @@
 */

 import { Inject, Injectable } from '@nestjs/common';
-import type { UserListsRepository, UserListJoiningsRepository, BlockingsRepository } from '@/models/_.js';
+import type { UserListsRepository, UserListMembershipsRepository, BlockingsRepository } from '@/models/_.js';
 import { IdService } from '@/core/IdService.js';
 import type { MiUserList } from '@/models/UserList.js';
 import { Endpoint } from '@/server/api/endpoint-base.js';
@@ -76,8 +76,8 @@ export default class extends Endpoint<typeof meta, typeof paramDef> { // eslint-
 		@Inject(DI.userListsRepository)
 		private userListsRepository: UserListsRepository,

-		@Inject(DI.userListJoiningsRepository)
-		private userListJoiningsRepository: UserListJoiningsRepository,
+		@Inject(DI.userListMembershipsRepository)
+		private userListMembershipsRepository: UserListMembershipsRepository,

 		@Inject(DI.blockingsRepository)
 		private blockingsRepository: BlockingsRepository,
@@ -110,7 +110,7 @@ export default class extends Endpoint<typeof meta, typeof paramDef> { // eslint-
 				name: ps.name,
 			} as MiUserList).then(x => this.userListsRepository.findOneByOrFail(x.identifiers[0]));

-			const users = (await this.userListJoiningsRepository.findBy({
+			const users = (await this.userListMembershipsRepository.findBy({
 				userListId: ps.listId,
 			})).map(x => x.userId);

@@ -132,7 +132,7 @@ export default class extends Endpoint<typeof meta, typeof paramDef> { // eslint-
 					}
 				}

-				const exist = await this.userListJoiningsRepository.exist({
+				const exist = await this.userListMembershipsRepository.exist({
 					where: {
 						userListId: userList.id,
 						userId: currentUser.id,
--- a/packages/backend/src/server/api/endpoints/users/lists/get-memberships.ts
+++ b/packages/backend/src/server/api/endpoints/users/lists/get-memberships.ts
@@ -0,0 +1,79 @@
+/*
+ * SPDX-FileCopyrightText: syuilo and other misskey contributors
+ * SPDX-License-Identifier: AGPL-3.0-only
+ */
+
+import { Inject, Injectable } from '@nestjs/common';
+import type { UserListsRepository, UserListFavoritesRepository, UserListMembershipsRepository } from '@/models/_.js';
+import { Endpoint } from '@/server/api/endpoint-base.js';
+import { UserListEntityService } from '@/core/entities/UserListEntityService.js';
+import { DI } from '@/di-symbols.js';
+import { QueryService } from '@/core/QueryService.js';
+import { ApiError } from '../../../error.js';
+
+export const meta = {
+	tags: ['lists', 'account'],
+
+	requireCredential: false,
+
+	kind: 'read:account',
+
+	errors: {
+		noSuchList: {
+			message: 'No such list.',
+			code: 'NO_SUCH_LIST',
+			id: '7bc05c21-1d7a-41ae-88f1-66820f4dc686',
+		},
+	},
+} as const;
+
+export const paramDef = {
+	type: 'object',
+	properties: {
+		listId: { type: 'string', format: 'misskey:id' },
+		forPublic: { type: 'boolean', default: false },
+		limit: { type: 'integer', minimum: 1, maximum: 100, default: 30 },
+		sinceId: { type: 'string', format: 'misskey:id' },
+		untilId: { type: 'string', format: 'misskey:id' },
+	},
+	required: ['listId'],
+} as const;
+
+@Injectable() // eslint-disable-next-line import/no-default-export
+export default class extends Endpoint<typeof meta, typeof paramDef> {
+	constructor(
+		@Inject(DI.userListsRepository)
+		private userListsRepository: UserListsRepository,
+
+		@Inject(DI.userListMembershipsRepository)
+		private userListMembershipsRepository: UserListMembershipsRepository,
+
+		private userListEntityService: UserListEntityService,
+		private queryService: QueryService,
+	) {
+		super(meta, paramDef, async (ps, me) => {
+			// Fetch the list
+			const userList = await this.userListsRepository.findOneBy(!ps.forPublic && me !== null ? {
+				id: ps.listId,
+				userId: me.id,
+			} : {
+				id: ps.listId,
+				isPublic: true,
+			});
+
+			if (userList == null) {
+				throw new ApiError(meta.errors.noSuchList);
+			}
+
+			const query = this.queryService.makePaginationQuery(this.userListMembershipsRepository.createQueryBuilder('membership'), ps.sinceId, ps.untilId)
+				.andWhere('membership.userListId = :userListId', { userListId: userList.id })
+				.innerJoinAndSelect('membership.user', 'user');
+
+			const memberships = await query
+				.limit(ps.limit)
+				.getMany();
+
+			return this.userListEntityService.packMembershipsMany(memberships);
+		});
+	}
+}
--- a/packages/backend/src/server/api/endpoints/users/lists/push.ts
+++ b/packages/backend/src/server/api/endpoints/users/lists/push.ts
@@ -5,7 +5,7 @@

 import { Inject, Injectable } from '@nestjs/common';
 import ms from 'ms';
-import type { UserListsRepository, UserListJoiningsRepository, BlockingsRepository } from '@/models/_.js';
+import type { UserListsRepository, UserListMembershipsRepository, BlockingsRepository } from '@/models/_.js';
 import { Endpoint } from '@/server/api/endpoint-base.js';
 import { GetterService } from '@/server/api/GetterService.js';
 import { UserListService } from '@/core/UserListService.js';
@@ -76,8 +76,8 @@ export default class extends Endpoint<typeof meta, typeof paramDef> { // eslint-
 		@Inject(DI.userListsRepository)
 		private userListsRepository: UserListsRepository,

-		@Inject(DI.userListJoiningsRepository)
-		private userListJoiningsRepository: UserListJoiningsRepository,
+		@Inject(DI.userListMembershipsRepository)
+		private userListMembershipsRepository: UserListMembershipsRepository,

 		@Inject(DI.blockingsRepository)
 		private blockingsRepository: BlockingsRepository,
@@ -115,7 +115,7 @@ export default class extends Endpoint<typeof meta, typeof paramDef> { // eslint-
 				}
 			}

-			const exist = await this.userListJoiningsRepository.exist({
+			const exist = await this.userListMembershipsRepository.exist({
 				where: {
 					userListId: userList.id,
 					userId: user.id,
--- a/packages/backend/src/server/api/endpoints/users/lists/update-membership.ts
+++ b/packages/backend/src/server/api/endpoints/users/lists/update-membership.ts
@@ -0,0 +1,79 @@
+/*
+ * SPDX-FileCopyrightText: syuilo and other misskey contributors
+ * SPDX-License-Identifier: AGPL-3.0-only
+ */
+
+import { Inject, Injectable } from '@nestjs/common';
+import type { UserListsRepository } from '@/models/_.js';
+import { Endpoint } from '@/server/api/endpoint-base.js';
+import { GetterService } from '@/server/api/GetterService.js';
+import { DI } from '@/di-symbols.js';
+import { UserListService } from '@/core/UserListService.js';
+import { ApiError } from '../../../error.js';
+
+export const meta = {
+	tags: ['lists', 'users'],
+
+	requireCredential: true,
+
+	prohibitMoved: true,
+
+	kind: 'write:account',
+
+	errors: {
+		noSuchList: {
+			message: 'No such list.',
+			code: 'NO_SUCH_LIST',
+			id: '7f44670e-ab16-43b8-b4c1-ccd2ee89cc02',
+		},
+
+		noSuchUser: {
+			message: 'No such user.',
+			code: 'NO_SUCH_USER',
+			id: '588e7f72-c744-4a61-b180-d354e912bda2',
+		},
+	},
+} as const;
+
+export const paramDef = {
+	type: 'object',
+	properties: {
+		listId: { type: 'string', format: 'misskey:id' },
+		userId: { type: 'string', format: 'misskey:id' },
+		withReplies: { type: 'boolean' },
+	},
+	required: ['listId', 'userId'],
+} as const;
+
+@Injectable()
+export default class extends Endpoint<typeof meta, typeof paramDef> { // eslint-disable-line import/no-default-export
+	constructor(
+		@Inject(DI.userListsRepository)
+		private userListsRepository: UserListsRepository,
+
+		private userListService: UserListService,
+		private getterService: GetterService,
+	) {
+		super(meta, paramDef, async (ps, me) => {
+			// Fetch the list
+			const userList = await this.userListsRepository.findOneBy({
+				id: ps.listId,
+				userId: me.id,
+			});
+
+			if (userList == null) {
+				throw new ApiError(meta.errors.noSuchList);
+			}
+
+			// Fetch the user
+			const user = await this.getterService.getUser(ps.userId).catch(err => {
+				if (err.id === '15348ddd-432d-49c2-8a5a-8069753becff') throw new ApiError(meta.errors.noSuchUser);
+				throw err;
+			});
+
+			await this.userListService.updateMembership(user, userList, {
+				withReplies: ps.withReplies,
+			});
+		});
+	}
+}
--- a/packages/backend/src/server/api/endpoints/users/notes.ts
+++ b/packages/backend/src/server/api/endpoints/users/notes.ts
@@ -5,12 +5,14 @@

 import { Brackets } from 'typeorm';
 import { Inject, Injectable } from '@nestjs/common';
-import type { NotesRepository } from '@/models/_.js';
+import * as Redis from 'ioredis';
+import type { MiNote, NotesRepository } from '@/models/_.js';
 import { Endpoint } from '@/server/api/endpoint-base.js';
-import { QueryService } from '@/core/QueryService.js';
 import { NoteEntityService } from '@/core/entities/NoteEntityService.js';
 import { DI } from '@/di-symbols.js';
 import { GetterService } from '@/server/api/GetterService.js';
+import { CacheService } from '@/core/CacheService.js';
+import { IdService } from '@/core/IdService.js';
 import { ApiError } from '../../error.js';

 export const meta = {
@@ -50,9 +52,6 @@ export const paramDef = {
 		untilDate: { type: 'integer' },
 		includeMyRenotes: { type: 'boolean', default: true },
 		withFiles: { type: 'boolean', default: false },
-		fileType: { type: 'array', items: {
-			type: 'string',
-		} },
 		excludeNsfw: { type: 'boolean', default: false },
 	},
 	required: ['userId'],
@@ -61,87 +60,63 @@ export const paramDef = {
@Injectable()
 export default class extends Endpoint<typeof meta, typeof paramDef> { // eslint-disable-line import/no-default-export
 	constructor(
+		@Inject(DI.redisForTimelines)
+		private redisForTimelines: Redis.Redis,
+
 		@Inject(DI.notesRepository)
 		private notesRepository: NotesRepository,

 		private noteEntityService: NoteEntityService,
-		private queryService: QueryService,
 		private getterService: GetterService,
+		private cacheService: CacheService,
+		private idService: IdService,
 	) {
 		super(meta, paramDef, async (ps, me) => {
-			// Lookup user
-			const user = await this.getterService.getUser(ps.userId).catch(err => {
-				if (err.id === '15348ddd-432d-49c2-8a5a-8069753becff') throw new ApiError(meta.errors.noSuchUser);
-				throw err;
-			});
+			let timeline: MiNote[] = [];

-			//#region Construct query
-			const query = this.queryService.makePaginationQuery(this.notesRepository.createQueryBuilder('note'), ps.sinceId, ps.untilId, ps.sinceDate, ps.untilDate)
-				.andWhere('note.userId = :userId', { userId: user.id })
+			const limit = ps.limit + (ps.untilId ? 1 : 0); // untilIdに指定したものも含まれるため+1
+			let noteIdsRes: [string, string[]][] = [];
+
+			if (!ps.sinceId && !ps.sinceDate) {
+				noteIdsRes = await this.redisForTimelines.xrevrange(
+					ps.withFiles ? `userTimelineWithFiles:${ps.userId}` : ps.withReplies ? `userTimelineWithReplies:${ps.userId}` : `userTimeline:${ps.userId}`,
+					ps.untilId ? this.idService.parse(ps.untilId).date.getTime() : ps.untilDate ?? '+',
+					'-',
+					'COUNT', limit);
+			}
+
+			const noteIds = noteIdsRes.map(x => x[1][1]).filter(x => x !== ps.untilId);
+
+			if (noteIds.length === 0) {
+				return [];
+			}
+
+			const isFollowing = me ? Object.hasOwn(await this.cacheService.userFollowingsCache.fetch(me.id), ps.userId) : false;
+
+			const query = this.notesRepository.createQueryBuilder('note')
+				.where('note.id IN (:...noteIds)', { noteIds: noteIds })
 				.innerJoinAndSelect('note.user', 'user')
 				.leftJoinAndSelect('note.reply', 'reply')
 				.leftJoinAndSelect('note.renote', 'renote')
-				.leftJoinAndSelect('note.channel', 'channel')
 				.leftJoinAndSelect('reply.user', 'replyUser')
-				.leftJoinAndSelect('renote.user', 'renoteUser');
+				.leftJoinAndSelect('renote.user', 'renoteUser')
+				.leftJoinAndSelect('note.channel', 'channel');

-			query.andWhere(new Brackets(qb => {
-				qb.orWhere('note.channelId IS NULL');
-				qb.orWhere('channel.isSensitive = false');
-			}));
+			timeline = await query.getMany();

-			this.queryService.generateVisibilityQuery(query, me);
-			if (me) {
-				this.queryService.generateMutedUserQuery(query, me, user);
-				this.queryService.generateBlockedUserQuery(query, me);
-			}
-
-			if (ps.withFiles) {
-				query.andWhere('note.fileIds != \'{}\'');
-			}
-
-			if (ps.fileType != null) {
-				query.andWhere('note.fileIds != \'{}\'');
-				query.andWhere(new Brackets(qb => {
-					for (const type of ps.fileType!) {
-						const i = ps.fileType!.indexOf(type);
-						qb.orWhere(`:type${i} = ANY(note.attachedFileTypes)`, { [`type${i}`]: type });
+			timeline = timeline.filter(note => {
+				if (note.renoteId) {
+					if (note.text == null && note.fileIds.length === 0 && !note.hasPoll) {
+						if (ps.withRenotes === false) return false;
 					}
-				}));
-
-				if (ps.excludeNsfw) {
-					query.andWhere('note.cw IS NULL');
-					query.andWhere('0 = (SELECT COUNT(*) FROM drive_file df WHERE df.id = ANY(note."fileIds") AND df."isSensitive" = TRUE)');
 				}
-			}

-			if (!ps.withReplies) {
-				query.andWhere('note.replyId IS NULL');
-			}
+				if (note.visibility === 'followers' && !isFollowing) return false;

-			if (ps.withRenotes === false) {
-				query.andWhere(new Brackets(qb => {
-					qb.orWhere('note.renoteId IS NULL');
-					qb.orWhere(new Brackets(qb => {
-						qb.orWhere('note.text IS NOT NULL');
-						qb.orWhere('note.fileIds != \'{}\'');
-					}));
-				}));
-			}
+				return true;
+			});

-			if (ps.includeMyRenotes === false) {
-				query.andWhere(new Brackets(qb => {
-					qb.orWhere('note.userId != :userId', { userId: user.id });
-					qb.orWhere('note.renoteId IS NULL');
-					qb.orWhere('note.text IS NOT NULL');
-					qb.orWhere('note.fileIds != \'{}\'');
-					qb.orWhere('0 < (SELECT COUNT(*) FROM poll WHERE poll."noteId" = note.id)');
-				}));
-			}
-
-			//#endregion
-
-			const timeline = await query.limit(ps.limit).getMany();
+			timeline.sort((a, b) => a.id > b.id ? -1 : 1);

 			return await this.noteEntityService.packMany(timeline, me);
 		});