sendnrw/misskey
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch 'develop' into ed25519

Browse Source
This commit is contained in:
tamaina
2024-02-28 14:21:25 +00:00
parent aaacfabc1b 5f43c2faa2
commit 2bc4221f40
43 changed files with 995 additions and 520 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
packages/backend/src/core/NotificationService.ts
View File

@@ -122,6 +122,14 @@ export class NotificationService implements OnApplicationShutdown {
return null;
}
} else if (recieveConfig?.type === 'mutualFollow') {
const [isFollowing, isFollower] = await Promise.all([
this.cacheService.userFollowingsCache.fetch(notifieeId).then(followings => Object.hasOwn(followings, notifierId)),
this.cacheService.userFollowingsCache.fetch(notifierId).then(followings => Object.hasOwn(followings, notifieeId)),
]);
if (!(isFollowing && isFollower)) {
return null;
}
} else if (recieveConfig?.type === 'followingOrFollower') {
const [isFollowing, isFollower] = await Promise.all([
this.cacheService.userFollowingsCache.fetch(notifieeId).then(followings => Object.hasOwn(followings, notifierId)),
this.cacheService.userFollowingsCache.fetch(notifierId).then(followings => Object.hasOwn(followings, notifieeId)),
@@ -155,6 +163,8 @@ export class NotificationService implements OnApplicationShutdown {
const packed = await this.notificationEntityService.pack(notification, notifieeId, {});
if (packed == null) return null;
// Publish notification event
this.globalEventService.publishMainStream(notifieeId, 'notification', packed);

19
packages/backend/src/core/RoleService.ts
View File

@@ -200,17 +200,20 @@ export class RoleService implements OnApplicationShutdown, OnModuleInit {
}
@bindThis
private evalCond(user: MiUser, value: RoleCondFormulaValue): boolean {
private evalCond(user: MiUser, roles: MiRole[], value: RoleCondFormulaValue): boolean {
try {
switch (value.type) {
case 'and': {
return value.values.every(v => this.evalCond(user, v));
return value.values.every(v => this.evalCond(user, roles, v));
}
case 'or': {
return value.values.some(v => this.evalCond(user, v));
return value.values.some(v => this.evalCond(user, roles, v));
}
case 'not': {
return !this.evalCond(user, value.value);
return !this.evalCond(user, roles, value.value);
}
case 'roleAssignedTo': {
return roles.some(r => r.id === value.roleId);
}
case 'isLocal': {
return this.userEntityService.isLocalUser(user);
@@ -272,7 +275,7 @@ export class RoleService implements OnApplicationShutdown, OnModuleInit {
const assigns = await this.getUserAssigns(userId);
const assignedRoles = roles.filter(r => assigns.map(x => x.roleId).includes(r.id));
const user = roles.some(r => r.target === 'conditional') ? await this.cacheService.findUserById(userId) : null;
const matchedCondRoles = roles.filter(r => r.target === 'conditional' && this.evalCond(user!, r.condFormula));
const matchedCondRoles = roles.filter(r => r.target === 'conditional' && this.evalCond(user!, assignedRoles, r.condFormula));
return [...assignedRoles, ...matchedCondRoles];
}
@@ -285,13 +288,13 @@ export class RoleService implements OnApplicationShutdown, OnModuleInit {
let assigns = await this.roleAssignmentByUserIdCache.fetch(userId, () => this.roleAssignmentsRepository.findBy({ userId }));
// 期限切れのロールを除外
assigns = assigns.filter(a => a.expiresAt == null || (a.expiresAt.getTime() > now));
const assignedRoleIds = assigns.map(x => x.roleId);
const roles = await this.rolesCache.fetch(() => this.rolesRepository.findBy({}));
const assignedBadgeRoles = roles.filter(r => r.asBadge && assignedRoleIds.includes(r.id));
const assignedRoles = roles.filter(r => assigns.map(x => x.roleId).includes(r.id));
const assignedBadgeRoles = assignedRoles.filter(r => r.asBadge);
const badgeCondRoles = roles.filter(r => r.asBadge && (r.target === 'conditional'));
if (badgeCondRoles.length > 0) {
const user = roles.some(r => r.target === 'conditional') ? await this.cacheService.findUserById(userId) : null;
const matchedBadgeCondRoles = badgeCondRoles.filter(r => this.evalCond(user!, r.condFormula));
const matchedBadgeCondRoles = badgeCondRoles.filter(r => this.evalCond(user!, assignedRoles, r.condFormula));
return [...assignedBadgeRoles, ...matchedBadgeCondRoles];
} else {
return assignedBadgeRoles;

273
packages/backend/src/core/entities/NotificationEntityService.ts
View File

@@ -14,14 +14,14 @@ import type { MiNote } from '@/models/Note.js';
import type { Packed } from '@/misc/json-schema.js';
import { bindThis } from '@/decorators.js';
import { isNotNull } from '@/misc/is-not-null.js';
import { FilterUnionByProperty, notificationTypes } from '@/types.js';
import { FilterUnionByProperty, groupedNotificationTypes } from '@/types.js';
import { CacheService } from '@/core/CacheService.js';
import { RoleEntityService } from './RoleEntityService.js';
import type { OnModuleInit } from '@nestjs/common';
import type { UserEntityService } from './UserEntityService.js';
import type { NoteEntityService } from './NoteEntityService.js';
const NOTE_REQUIRED_NOTIFICATION_TYPES = new Set(['note', 'mention', 'reply', 'renote', 'quote', 'reaction', 'pollEnded'] as (typeof notificationTypes[number])[]);
const NOTE_REQUIRED_GROUPED_NOTIFICATION_TYPES = new Set(['note', 'mention', 'reply', 'renote', 'renote:grouped', 'quote', 'reaction', 'reaction:grouped', 'pollEnded']);
const NOTE_REQUIRED_NOTIFICATION_TYPES = new Set(['note', 'mention', 'reply', 'renote', 'renote:grouped', 'quote', 'reaction', 'reaction:grouped', 'pollEnded'] as (typeof groupedNotificationTypes[number])[]);
@Injectable()
export class NotificationEntityService implements OnModuleInit {
@@ -41,6 +41,8 @@ export class NotificationEntityService implements OnModuleInit {
@Inject(DI.followRequestsRepository)
private followRequestsRepository: FollowRequestsRepository,
private cacheService: CacheService,
//private userEntityService: UserEntityService,
//private noteEntityService: NoteEntityService,
) {
@@ -52,130 +54,48 @@ export class NotificationEntityService implements OnModuleInit {
this.roleEntityService = this.moduleRef.get('RoleEntityService');
}
@bindThis
public async pack(
src: MiNotification,
/**
* 通知をパックする共通処理
*/
async #packInternal <T extends MiNotification | MiGroupedNotification> (
src: T,
meId: MiUser['id'],
// eslint-disable-next-line @typescript-eslint/ban-types
options: {
checkValidNotifier?: boolean;
},
hint?: {
packedNotes: Map<MiNote['id'], Packed<'Note'>>;
packedUsers: Map<MiUser['id'], Packed<'UserLite'>>;
},
): Promise<Packed<'Notification'>> {
): Promise<Packed<'Notification'> | null> {
const notification = src;
const noteIfNeed = NOTE_REQUIRED_NOTIFICATION_TYPES.has(notification.type) && 'noteId' in notification ? (
if (options.checkValidNotifier !== false && !(await this.#isValidNotifier(notification, meId))) return null;
const needsNote = NOTE_REQUIRED_NOTIFICATION_TYPES.has(notification.type) && 'noteId' in notification;
const noteIfNeed = needsNote ? (
hint?.packedNotes != null
? hint.packedNotes.get(notification.noteId)
: this.noteEntityService.pack(notification.noteId, { id: meId }, {
detail: true,
})
) : undefined;
const userIfNeed = 'notifierId' in notification ? (
hint?.packedUsers != null
? hint.packedUsers.get(notification.notifierId)
: this.userEntityService.pack(notification.notifierId, { id: meId })
) : undefined;
const role = notification.type === 'roleAssigned' ? await this.roleEntityService.pack(notification.roleId) : undefined;
return await awaitAll({
id: notification.id,
createdAt: new Date(notification.createdAt).toISOString(),
type: notification.type,
userId: 'notifierId' in notification ? notification.notifierId : undefined,
...(userIfNeed != null ? { user: userIfNeed } : {}),
...(noteIfNeed != null ? { note: noteIfNeed } : {}),
...(notification.type === 'reaction' ? {
reaction: notification.reaction,
} : {}),
...(notification.type === 'roleAssigned' ? {
role: role,
} : {}),
...(notification.type === 'achievementEarned' ? {
achievement: notification.achievement,
} : {}),
...(notification.type === 'app' ? {
body: notification.customBody,
header: notification.customHeader,
icon: notification.customIcon,
} : {}),
});
}
@bindThis
public async packMany(
notifications: MiNotification[],
meId: MiUser['id'],
) {
if (notifications.length === 0) return [];
let validNotifications = notifications;
const noteIds = validNotifications.map(x => 'noteId' in x ? x.noteId : null).filter(isNotNull);
const notes = noteIds.length > 0 ? await this.notesRepository.find({
where: { id: In(noteIds) },
relations: ['user', 'reply', 'reply.user', 'renote', 'renote.user'],
}) : [];
const packedNotesArray = await this.noteEntityService.packMany(notes, { id: meId }, {
detail: true,
});
const packedNotes = new Map(packedNotesArray.map(p => [p.id, p]));
validNotifications = validNotifications.filter(x => !('noteId' in x) || packedNotes.has(x.noteId));
const userIds = validNotifications.map(x => 'notifierId' in x ? x.notifierId : null).filter(isNotNull);
const users = userIds.length > 0 ? await this.usersRepository.find({
where: { id: In(userIds) },
}) : [];
const packedUsersArray = await this.userEntityService.packMany(users, { id: meId });
const packedUsers = new Map(packedUsersArray.map(p => [p.id, p]));
// 既に解決されたフォローリクエストの通知を除外
const followRequestNotifications = validNotifications.filter((x): x is FilterUnionByProperty<MiGroupedNotification, 'type', 'receiveFollowRequest'> => x.type === 'receiveFollowRequest');
if (followRequestNotifications.length > 0) {
const reqs = await this.followRequestsRepository.find({
where: { followerId: In(followRequestNotifications.map(x => x.notifierId)) },
});
validNotifications = validNotifications.filter(x => (x.type !== 'receiveFollowRequest') || reqs.some(r => r.followerId === x.notifierId));
}
return await Promise.all(validNotifications.map(x => this.pack(x, meId, {}, {
packedNotes,
packedUsers,
})));
}
@bindThis
public async packGrouped(
src: MiGroupedNotification,
meId: MiUser['id'],
// eslint-disable-next-line @typescript-eslint/ban-types
options: {
},
hint?: {
packedNotes: Map<MiNote['id'], Packed<'Note'>>;
packedUsers: Map<MiUser['id'], Packed<'UserLite'>>;
},
): Promise<Packed<'Notification'>> {
const notification = src;
const noteIfNeed = NOTE_REQUIRED_GROUPED_NOTIFICATION_TYPES.has(notification.type) && 'noteId' in notification ? (
hint?.packedNotes != null
? hint.packedNotes.get(notification.noteId)
: this.noteEntityService.pack(notification.noteId, { id: meId }, {
detail: true,
})
) : undefined;
const userIfNeed = 'notifierId' in notification ? (
// if the note has been deleted, don't show this notification
if (needsNote && !noteIfNeed) return null;
const needsUser = 'notifierId' in notification;
const userIfNeed = needsUser ? (
hint?.packedUsers != null
? hint.packedUsers.get(notification.notifierId)
: this.userEntityService.pack(notification.notifierId, { id: meId })
) : undefined;
// if the user has been deleted, don't show this notification
if (needsUser && !userIfNeed) return null;
// #region Grouped notifications
if (notification.type === 'reaction:grouped') {
const reactions = await Promise.all(notification.reactions.map(async reaction => {
const reactions = (await Promise.all(notification.reactions.map(async reaction => {
const user = hint?.packedUsers != null
? hint.packedUsers.get(reaction.userId)!
: await this.userEntityService.pack(reaction.userId, { id: meId });
@@ -183,7 +103,12 @@ export class NotificationEntityService implements OnModuleInit {
user,
reaction: reaction.reaction,
};
}));
}))).filter(r => isNotNull(r.user));
// if all users have been deleted, don't show this notification
if (reactions.length === 0) {
return null;
}
return await awaitAll({
id: notification.id,
createdAt: new Date(notification.createdAt).toISOString(),
@@ -192,14 +117,19 @@ export class NotificationEntityService implements OnModuleInit {
reactions,
});
} else if (notification.type === 'renote:grouped') {
const users = await Promise.all(notification.userIds.map(userId => {
const users = (await Promise.all(notification.userIds.map(userId => {
const packedUser = hint?.packedUsers != null ? hint.packedUsers.get(userId) : null;
if (packedUser) {
return packedUser;
}
return this.userEntityService.pack(userId, { id: meId });
}));
}))).filter(isNotNull);
// if all users have been deleted, don't show this notification
if (users.length === 0) {
return null;
}
return await awaitAll({
id: notification.id,
createdAt: new Date(notification.createdAt).toISOString(),
@@ -208,8 +138,14 @@ export class NotificationEntityService implements OnModuleInit {
users,
});
}
// #endregion
const role = notification.type === 'roleAssigned' ? await this.roleEntityService.pack(notification.roleId) : undefined;
const needsRole = notification.type === 'roleAssigned';
const role = needsRole ? await this.roleEntityService.pack(notification.roleId) : undefined;
// if the role has been deleted, don't show this notification
if (needsRole && !role) {
return null;
}
return await awaitAll({
id: notification.id,
@@ -235,15 +171,16 @@ export class NotificationEntityService implements OnModuleInit {
});
}
@bindThis
public async packGroupedMany(
notifications: MiGroupedNotification[],
async #packManyInternal <T extends MiNotification | MiGroupedNotification> (
notifications: T[],
meId: MiUser['id'],
) {
): Promise<T[]> {
if (notifications.length === 0) return [];
let validNotifications = notifications;
validNotifications = await this.#filterValidNotifier(validNotifications, meId);
const noteIds = validNotifications.map(x => 'noteId' in x ? x.noteId : null).filter(isNotNull);
const notes = noteIds.length > 0 ? await this.notesRepository.find({
where: { id: In(noteIds) },
@@ -269,7 +206,7 @@ export class NotificationEntityService implements OnModuleInit {
const packedUsers = new Map(packedUsersArray.map(p => [p.id, p]));
// 既に解決されたフォローリクエストの通知を除外
const followRequestNotifications = validNotifications.filter((x): x is FilterUnionByProperty<MiGroupedNotification, 'type', 'receiveFollowRequest'> => x.type === 'receiveFollowRequest');
const followRequestNotifications = validNotifications.filter((x): x is FilterUnionByProperty<T, 'type', 'receiveFollowRequest'> => x.type === 'receiveFollowRequest');
if (followRequestNotifications.length > 0) {
const reqs = await this.followRequestsRepository.find({
where: { followerId: In(followRequestNotifications.map(x => x.notifierId)) },
@@ -277,9 +214,107 @@ export class NotificationEntityService implements OnModuleInit {
validNotifications = validNotifications.filter(x => (x.type !== 'receiveFollowRequest') || reqs.some(r => r.followerId === x.notifierId));
}
return await Promise.all(validNotifications.map(x => this.packGrouped(x, meId, {}, {
packedNotes,
packedUsers,
})));
const packPromises = validNotifications.map(x => {
return this.pack(
x,
meId,
{ checkValidNotifier: false },
{ packedNotes, packedUsers },
);
});
return (await Promise.all(packPromises)).filter(isNotNull);
}
@bindThis
public async pack(
src: MiNotification | MiGroupedNotification,
meId: MiUser['id'],
// eslint-disable-next-line @typescript-eslint/ban-types
options: {
checkValidNotifier?: boolean;
},
hint?: {
packedNotes: Map<MiNote['id'], Packed<'Note'>>;
packedUsers: Map<MiUser['id'], Packed<'UserLite'>>;
},
): Promise<Packed<'Notification'> | null> {
return await this.#packInternal(src, meId, options, hint);
}
@bindThis
public async packMany(
notifications: MiNotification[],
meId: MiUser['id'],
): Promise<MiNotification[]> {
return await this.#packManyInternal(notifications, meId);
}
@bindThis
public async packGroupedMany(
notifications: MiGroupedNotification[],
meId: MiUser['id'],
): Promise<MiGroupedNotification[]> {
return await this.#packManyInternal(notifications, meId);
}
/**
* notifierが存在するか、ミュートされていないか、サスペンドされていないかを確認するvalidator
*/
#validateNotifier <T extends MiNotification | MiGroupedNotification> (
notification: T,
userIdsWhoMeMuting: Set<MiUser['id']>,
userMutedInstances: Set<string>,
notifiers: MiUser[],
): boolean {
if (!('notifierId' in notification)) return true;
if (userIdsWhoMeMuting.has(notification.notifierId)) return false;
const notifier = notifiers.find(x => x.id === notification.notifierId) ?? null;
if (notifier == null) return false;
if (notifier.host && userMutedInstances.has(notifier.host)) return false;
if (notifier.isSuspended) return false;
return true;
}
/**
* notifierが存在するか、ミュートされていないか、サスペンドされていないかを実際に確認する
*/
async #isValidNotifier(
notification: MiNotification | MiGroupedNotification,
meId: MiUser['id'],
): Promise<boolean> {
return (await this.#filterValidNotifier([notification], meId)).length === 1;
}
/**
* notifierが存在するか、ミュートされていないか、サスペンドされていないかを実際に複数確認する
*/
async #filterValidNotifier <T extends MiNotification | MiGroupedNotification> (
notifications: T[],
meId: MiUser['id'],
): Promise<T[]> {
const [
userIdsWhoMeMuting,
userMutedInstances,
] = await Promise.all([
this.cacheService.userMutingsCache.fetch(meId),
this.cacheService.userProfileCache.fetch(meId).then(p => new Set(p.mutedInstances)),
]);
const notifierIds = notifications.map(notification => 'notifierId' in notification ? notification.notifierId : null).filter(isNotNull);
const notifiers = notifierIds.length > 0 ? await this.usersRepository.find({
where: { id: In(notifierIds) },
}) : [];
const filteredNotifications = ((await Promise.all(notifications.map(async (notification) => {
const isValid = this.#validateNotifier(notification, userIdsWhoMeMuting, userMutedInstances, notifiers);
return isValid ? notification : null;
}))) as [T | null] ).filter(isNotNull);
return filteredNotifications;
}
}

31
packages/backend/src/misc/FileWriterStream.ts Normal file
View File

@@ -0,0 +1,31 @@
import * as fs from 'node:fs/promises';
import type { PathLike } from 'node:fs';
/**
* `fs.createWriteStream()`相当のことを行う`WritableStream` (Web標準)
*/
export class FileWriterStream extends WritableStream<Uint8Array> {
constructor(path: PathLike) {
let file: fs.FileHandle | null = null;
super({
start: async () => {
file = await fs.open(path, 'a');
},
write: async (chunk, controller) => {
if (file === null) {
controller.error();
throw new Error();
}
await file.write(chunk);
},
close: async () => {
await file?.close();
},
abort: async () => {
await file?.close();
},
});
}
}

30
packages/backend/src/misc/JsonArrayStream.ts Normal file
View File

@@ -0,0 +1,30 @@
import { TransformStream } from 'node:stream/web';
/**
* ストリームに流れてきた各データについて`JSON.stringify()`した上で、それらを一つの配列にまとめる
*/
export class JsonArrayStream extends TransformStream<unknown, string> {
constructor() {
/** 最初の要素かどうかを変数に記録 */
let isFirst = true;
super({
start(controller) {
controller.enqueue('[');
},
flush(controller) {
controller.enqueue(']');
},
transform(chunk, controller) {
if (isFirst) {
isFirst = false;
} else {
// 妥当なJSON配列にするためには最初以外の要素の前に`,`を挿入しなければならない
controller.enqueue(',\n');
}
controller.enqueue(JSON.stringify(chunk));
},
});
}
}

2
packages/backend/src/misc/json-schema.ts
View File

@@ -44,6 +44,7 @@ import {
packedRoleCondFormulaLogicsSchema,
packedRoleCondFormulaValueNot,
packedRoleCondFormulaValueIsLocalOrRemoteSchema,
packedRoleCondFormulaValueAssignedRoleSchema,
packedRoleCondFormulaValueCreatedSchema,
packedRoleCondFormulaFollowersOrFollowingOrNotesSchema,
packedRoleCondFormulaValueSchema,
@@ -96,6 +97,7 @@ export const refs = {
RoleCondFormulaLogics: packedRoleCondFormulaLogicsSchema,
RoleCondFormulaValueNot: packedRoleCondFormulaValueNot,
RoleCondFormulaValueIsLocalOrRemote: packedRoleCondFormulaValueIsLocalOrRemoteSchema,
RoleCondFormulaValueAssignedRole: packedRoleCondFormulaValueAssignedRoleSchema,
RoleCondFormulaValueCreated: packedRoleCondFormulaValueCreatedSchema,
RoleCondFormulaFollowersOrFollowingOrNotes: packedRoleCondFormulaFollowersOrFollowingOrNotesSchema,
RoleCondFormulaValue: packedRoleCondFormulaValueSchema,

6
packages/backend/src/models/Role.ts
View File

@@ -29,6 +29,11 @@ type CondFormulaValueIsRemote = {
type: 'isRemote';
};
type CondFormulaValueRoleAssignedTo = {
type: 'roleAssignedTo';
roleId: string;
};
type CondFormulaValueCreatedLessThan = {
type: 'createdLessThan';
sec: number;
@@ -75,6 +80,7 @@ export type RoleCondFormulaValue = { id: string } & (
CondFormulaValueNot |
CondFormulaValueIsLocal |
CondFormulaValueIsRemote |
CondFormulaValueRoleAssignedTo |
CondFormulaValueCreatedLessThan |
CondFormulaValueCreatedMoreThan |
CondFormulaValueFollowersLessThanOrEq |

2
packages/backend/src/models/UserProfile.ts
View File

@@ -249,6 +249,8 @@ export class MiUserProfile {
type: 'follower';
} | {
type: 'mutualFollow';
} | {
type: 'followingOrFollower';
} | {
type: 'list';
userListId: MiUserList['id'];

20
packages/backend/src/models/json-schema/role.ts
View File

@@ -57,6 +57,23 @@ export const packedRoleCondFormulaValueIsLocalOrRemoteSchema = {
},
} as const;
export const packedRoleCondFormulaValueAssignedRoleSchema = {
type: 'object',
properties: {
type: {
type: 'string',
nullable: false, optional: false,
enum: ['roleAssignedTo'],
},
roleId: {
type: 'string',
nullable: false, optional: false,
format: 'id',
example: 'xxxxxxxxxx',
},
},
} as const;
export const packedRoleCondFormulaValueCreatedSchema = {
type: 'object',
properties: {
@@ -115,6 +132,9 @@ export const packedRoleCondFormulaValueSchema = {
{
ref: 'RoleCondFormulaValueIsLocalOrRemote',
},
{
ref: 'RoleCondFormulaValueAssignedRole',
},
{
ref: 'RoleCondFormulaValueCreated',
},

2
packages/backend/src/models/json-schema/user.ts
View File

@@ -13,7 +13,7 @@ export const notificationRecieveConfig = {
type: {
type: 'string',
nullable: false,
enum: ['all', 'following', 'follower', 'mutualFollow', 'never'],
enum: ['all', 'following', 'follower', 'mutualFollow', 'followingOrFollower', 'never'],
},
},
required: ['type'],

164
packages/backend/src/queue/processors/ExportNotesProcessorService.ts
View File

@@ -3,7 +3,7 @@
* SPDX-License-Identifier: AGPL-3.0-only
*/
import * as fs from 'node:fs';
import { ReadableStream, TextEncoderStream } from 'node:stream/web';
import { Inject, Injectable } from '@nestjs/common';
import { MoreThan } from 'typeorm';
import { format as dateFormat } from 'date-fns';
@@ -18,10 +18,82 @@ import { bindThis } from '@/decorators.js';
import { DriveFileEntityService } from '@/core/entities/DriveFileEntityService.js';
import { Packed } from '@/misc/json-schema.js';
import { IdService } from '@/core/IdService.js';
import { JsonArrayStream } from '@/misc/JsonArrayStream.js';
import { FileWriterStream } from '@/misc/FileWriterStream.js';
import { QueueLoggerService } from '../QueueLoggerService.js';
import type * as Bull from 'bullmq';
import type { DbJobDataWithUser } from '../types.js';
class NoteStream extends ReadableStream<Record<string, unknown>> {
constructor(
job: Bull.Job,
notesRepository: NotesRepository,
pollsRepository: PollsRepository,
driveFileEntityService: DriveFileEntityService,
idService: IdService,
userId: string,
) {
let exportedNotesCount = 0;
let cursor: MiNote['id'] | null = null;
const serialize = (
note: MiNote,
poll: MiPoll | null,
files: Packed<'DriveFile'>[],
): Record<string, unknown> => {
return {
id: note.id,
text: note.text,
createdAt: idService.parse(note.id).date.toISOString(),
fileIds: note.fileIds,
files: files,
replyId: note.replyId,
renoteId: note.renoteId,
poll: poll,
cw: note.cw,
visibility: note.visibility,
visibleUserIds: note.visibleUserIds,
localOnly: note.localOnly,
reactionAcceptance: note.reactionAcceptance,
};
};
super({
async pull(controller): Promise<void> {
const notes = await notesRepository.find({
where: {
userId,
...(cursor !== null ? { id: MoreThan(cursor) } : {}),
},
take: 100, // 100件ずつ取得
order: { id: 1 },
});
if (notes.length === 0) {
job.updateProgress(100);
controller.close();
}
cursor = notes.at(-1)?.id ?? null;
for (const note of notes) {
const poll = note.hasPoll
? await pollsRepository.findOneByOrFail({ noteId: note.id }) // N+1
: null;
const files = await driveFileEntityService.packManyByIds(note.fileIds); // N+1
const content = serialize(note, poll, files);
controller.enqueue(content);
exportedNotesCount++;
}
const total = await notesRepository.countBy({ userId });
job.updateProgress(exportedNotesCount / total);
},
});
}
}
@Injectable()
export class ExportNotesProcessorService {
private logger: Logger;
@@ -59,67 +131,19 @@ export class ExportNotesProcessorService {
this.logger.info(`Temp file is ${path}`);
try {
const stream = fs.createWriteStream(path, { flags: 'a' });
// メモリが足りなくならないようにストリームで処理する
await new NoteStream(
job,
this.notesRepository,
this.pollsRepository,
this.driveFileEntityService,
this.idService,
user.id,
)
.pipeThrough(new JsonArrayStream())
.pipeThrough(new TextEncoderStream())
.pipeTo(new FileWriterStream(path));
const write = (text: string): Promise<void> => {
return new Promise<void>((res, rej) => {
stream.write(text, err => {
if (err) {
this.logger.error(err);
rej(err);
} else {
res();
}
});
});
};
await write('[');
let exportedNotesCount = 0;
let cursor: MiNote['id'] | null = null;
while (true) {
const notes = await this.notesRepository.find({
where: {
userId: user.id,
...(cursor ? { id: MoreThan(cursor) } : {}),
},
take: 100,
order: {
id: 1,
},
}) as MiNote[];
if (notes.length === 0) {
job.updateProgress(100);
break;
}
cursor = notes.at(-1)?.id ?? null;
for (const note of notes) {
let poll: MiPoll | undefined;
if (note.hasPoll) {
poll = await this.pollsRepository.findOneByOrFail({ noteId: note.id });
}
const files = await this.driveFileEntityService.packManyByIds(note.fileIds);
const content = JSON.stringify(this.serialize(note, poll, files));
const isFirst = exportedNotesCount === 0;
await write(isFirst ? content : ',\n' + content);
exportedNotesCount++;
}
const total = await this.notesRepository.countBy({
userId: user.id,
});
job.updateProgress(exportedNotesCount / total);
}
await write(']');
stream.end();
this.logger.succ(`Exported to: ${path}`);
const fileName = 'notes-' + dateFormat(new Date(), 'yyyy-MM-dd-HH-mm-ss') + '.json';
@@ -130,22 +154,4 @@ export class ExportNotesProcessorService {
cleanup();
}
}
private serialize(note: MiNote, poll: MiPoll | null = null, files: Packed<'DriveFile'>[]): Record<string, unknown> {
return {
id: note.id,
text: note.text,
createdAt: this.idService.parse(note.id).date.toISOString(),
fileIds: note.fileIds,
files: files,
replyId: note.replyId,
renoteId: note.renoteId,
poll: poll,
cw: note.cw,
visibility: note.visibility,
visibleUserIds: note.visibleUserIds,
localOnly: note.localOnly,
reactionAcceptance: note.reactionAcceptance,
};
}
}

15
packages/backend/src/server/api/endpoints/i/notifications-grouped.ts
View File

@@ -3,11 +3,11 @@
* SPDX-License-Identifier: AGPL-3.0-only
*/
import { Brackets, In } from 'typeorm';
import { In } from 'typeorm';
import * as Redis from 'ioredis';
import { Inject, Injectable } from '@nestjs/common';
import type { NotesRepository } from '@/models/_.js';
import { obsoleteNotificationTypes, notificationTypes, FilterUnionByProperty } from '@/types.js';
import { obsoleteNotificationTypes, groupedNotificationTypes, FilterUnionByProperty } from '@/types.js';
import { Endpoint } from '@/server/api/endpoint-base.js';
import { NoteReadService } from '@/core/NoteReadService.js';
import { NotificationEntityService } from '@/core/entities/NotificationEntityService.js';
@@ -48,10 +48,10 @@ export const paramDef = {
markAsRead: { type: 'boolean', default: true },
// 後方互換のため、廃止された通知タイプも受け付ける
includeTypes: { type: 'array', items: {
type: 'string', enum: [...notificationTypes, ...obsoleteNotificationTypes],
type: 'string', enum: [...groupedNotificationTypes, ...obsoleteNotificationTypes],
} },
excludeTypes: { type: 'array', items: {
type: 'string', enum: [...notificationTypes, ...obsoleteNotificationTypes],
type: 'string', enum: [...groupedNotificationTypes, ...obsoleteNotificationTypes],
} },
},
required: [],
@@ -79,12 +79,12 @@ export default class extends Endpoint<typeof meta, typeof paramDef> { // eslint-
return [];
}
// excludeTypes に全指定されている場合はクエリしない
if (notificationTypes.every(type => ps.excludeTypes?.includes(type))) {
if (groupedNotificationTypes.every(type => ps.excludeTypes?.includes(type))) {
return [];
}
const includeTypes = ps.includeTypes && ps.includeTypes.filter(type => !(obsoleteNotificationTypes).includes(type as any)) as typeof notificationTypes[number][];
const excludeTypes = ps.excludeTypes && ps.excludeTypes.filter(type => !(obsoleteNotificationTypes).includes(type as any)) as typeof notificationTypes[number][];
const includeTypes = ps.includeTypes && ps.includeTypes.filter(type => !(obsoleteNotificationTypes).includes(type as any)) as typeof groupedNotificationTypes[number][];
const excludeTypes = ps.excludeTypes && ps.excludeTypes.filter(type => !(obsoleteNotificationTypes).includes(type as any)) as typeof groupedNotificationTypes[number][];
const limit = (ps.limit + EXTRA_LIMIT) + (ps.untilId ? 1 : 0) + (ps.sinceId ? 1 : 0); // untilIdに指定したものも含まれるため+1
const notificationsRes = await this.redisClient.xrevrange(
@@ -162,7 +162,6 @@ export default class extends Endpoint<typeof meta, typeof paramDef> { // eslint-
}
groupedNotifications = groupedNotifications.slice(0, ps.limit);
const noteIds = groupedNotifications
.filter((notification): notification is FilterUnionByProperty<MiNotification, 'type', 'mention' | 'reply' | 'quote'> => ['mention', 'reply', 'quote'].includes(notification.type))
.map(notification => notification.noteId!);

2
packages/backend/src/server/api/endpoints/i/notifications.ts
View File

@@ -3,7 +3,7 @@
* SPDX-License-Identifier: AGPL-3.0-only
*/
import { Brackets, In } from 'typeorm';
import { In } from 'typeorm';
import * as Redis from 'ioredis';
import { Inject, Injectable } from '@nestjs/common';
import type { NotesRepository } from '@/models/_.js';

10
packages/backend/src/server/api/stream/channels/home-timeline.ts
View File

@@ -71,7 +71,15 @@ class HomeTimelineChannel extends Channel {
}
}
if (note.renote && note.text == null && (note.fileIds == null || note.fileIds.length === 0) && !this.withRenotes) return;
// 純粋なリノート（引用リノートでないリノート）の場合
if (note.renote && note.text == null && (note.fileIds == null || note.fileIds.length === 0) && note.poll == null) {
if (!this.withRenotes) return;
if (note.renote.reply) {
const reply = note.renote.reply;
// 自分のフォローしていないユーザーの visibility: followers な投稿への返信のリノートは弾く
if (reply.visibility === 'followers' && !Object.hasOwn(this.following, reply.userId)) return;
}
}
// 流れてきたNoteがミュートしているユーザーが関わるものだったら無視する
if (isUserRelated(note, this.userIdsWhoMeMuting)) return;

11
packages/backend/src/types.ts
View File

@@ -18,6 +18,7 @@
* achievementEarned - 実績を獲得
* app - アプリ通知
* test - テスト通知（サーバー側）
*
*/
export const notificationTypes = [
'note',
@@ -33,7 +34,15 @@ export const notificationTypes = [
'roleAssigned',
'achievementEarned',
'app',
'test'] as const;
'test',
] as const;
export const groupedNotificationTypes = [
...notificationTypes,
'reaction:grouped',
'renote:grouped',
] as const;
export const obsoleteNotificationTypes = ['pollVote', 'groupInvited'] as const;
export const noteVisibilities = ['public', 'home', 'followers', 'specified'] as const;

179
packages/backend/test/e2e/mute.ts
View File

@@ -117,5 +117,184 @@ describe('Mute', () => {
assert.strictEqual(res.body.some((notification: any) => notification.userId === bob.id), true);
assert.strictEqual(res.body.some((notification: any) => notification.userId === carol.id), false);
});
test('通知にミュートしているユーザーからのリプライが含まれない', async () => {
const aliceNote = await post(alice, { text: 'hi' });
await post(bob, { text: '@alice hi', replyId: aliceNote.id });
await post(carol, { text: '@alice hi', replyId: aliceNote.id });
const res = await api('/i/notifications', {}, alice);
assert.strictEqual(res.status, 200);
assert.strictEqual(Array.isArray(res.body), true);
assert.strictEqual(res.body.some((notification: any) => notification.userId === bob.id), true);
assert.strictEqual(res.body.some((notification: any) => notification.userId === carol.id), false);
});
test('通知にミュートしているユーザーからのリプライが含まれない', async () => {
await post(alice, { text: 'hi' });
await post(bob, { text: '@alice hi' });
await post(carol, { text: '@alice hi' });
const res = await api('/i/notifications', {}, alice);
assert.strictEqual(res.status, 200);
assert.strictEqual(Array.isArray(res.body), true);
assert.strictEqual(res.body.some((notification: any) => notification.userId === bob.id), true);
assert.strictEqual(res.body.some((notification: any) => notification.userId === carol.id), false);
});
test('通知にミュートしているユーザーからの引用リノートが含まれない', async () => {
const aliceNote = await post(alice, { text: 'hi' });
await post(bob, { text: 'hi', renoteId: aliceNote.id });
await post(carol, { text: 'hi', renoteId: aliceNote.id });
const res = await api('/i/notifications', {}, alice);
assert.strictEqual(res.status, 200);
assert.strictEqual(Array.isArray(res.body), true);
assert.strictEqual(res.body.some((notification: any) => notification.userId === bob.id), true);
assert.strictEqual(res.body.some((notification: any) => notification.userId === carol.id), false);
});
test('通知にミュートしているユーザーからのリノートが含まれない', async () => {
const aliceNote = await post(alice, { text: 'hi' });
await post(bob, { renoteId: aliceNote.id });
await post(carol, { renoteId: aliceNote.id });
const res = await api('/i/notifications', {}, alice);
assert.strictEqual(res.status, 200);
assert.strictEqual(Array.isArray(res.body), true);
assert.strictEqual(res.body.some((notification: any) => notification.userId === bob.id), true);
assert.strictEqual(res.body.some((notification: any) => notification.userId === carol.id), false);
});
test('通知にミュートしているユーザーからのフォロー通知が含まれない', async () => {
await api('/i/follow', { userId: alice.id }, bob);
await api('/i/follow', { userId: alice.id }, carol);
const res = await api('/i/notifications', {}, alice);
assert.strictEqual(res.status, 200);
assert.strictEqual(Array.isArray(res.body), true);
assert.strictEqual(res.body.some((notification: any) => notification.userId === bob.id), true);
assert.strictEqual(res.body.some((notification: any) => notification.userId === carol.id), false);
});
test('通知にミュートしているユーザーからのフォローリクエストが含まれない', async () => {
await api('/i/update/', { isLocked: true }, alice);
await api('/following/create', { userId: alice.id }, bob);
await api('/following/create', { userId: alice.id }, carol);
const res = await api('/i/notifications', {}, alice);
assert.strictEqual(res.status, 200);
assert.strictEqual(Array.isArray(res.body), true);
assert.strictEqual(res.body.some((notification: any) => notification.userId === bob.id), true);
assert.strictEqual(res.body.some((notification: any) => notification.userId === carol.id), false);
});
});
describe('Notification (Grouped)', () => {
test('通知にミュートしているユーザーの通知が含まれない(リアクション)', async () => {
const aliceNote = await post(alice, { text: 'hi' });
await react(bob, aliceNote, 'like');
await react(carol, aliceNote, 'like');
const res = await api('/i/notifications-grouped', {}, alice);
assert.strictEqual(res.status, 200);
assert.strictEqual(Array.isArray(res.body), true);
assert.strictEqual(res.body.some((notification: any) => notification.userId === bob.id), true);
assert.strictEqual(res.body.some((notification: any) => notification.userId === carol.id), false);
});
test('通知にミュートしているユーザーからのリプライが含まれない', async () => {
const aliceNote = await post(alice, { text: 'hi' });
await post(bob, { text: '@alice hi', replyId: aliceNote.id });
await post(carol, { text: '@alice hi', replyId: aliceNote.id });
const res = await api('/i/notifications-grouped', {}, alice);
assert.strictEqual(res.status, 200);
assert.strictEqual(Array.isArray(res.body), true);
assert.strictEqual(res.body.some((notification: any) => notification.userId === bob.id), true);
assert.strictEqual(res.body.some((notification: any) => notification.userId === carol.id), false);
});
test('通知にミュートしているユーザーからのリプライが含まれない', async () => {
await post(alice, { text: 'hi' });
await post(bob, { text: '@alice hi' });
await post(carol, { text: '@alice hi' });
const res = await api('/i/notifications-grouped', {}, alice);
assert.strictEqual(res.status, 200);
assert.strictEqual(Array.isArray(res.body), true);
assert.strictEqual(res.body.some((notification: any) => notification.userId === bob.id), true);
assert.strictEqual(res.body.some((notification: any) => notification.userId === carol.id), false);
});
test('通知にミュートしているユーザーからの引用リノートが含まれない', async () => {
const aliceNote = await post(alice, { text: 'hi' });
await post(bob, { text: 'hi', renoteId: aliceNote.id });
await post(carol, { text: 'hi', renoteId: aliceNote.id });
const res = await api('/i/notifications-grouped', {}, alice);
assert.strictEqual(res.status, 200);
assert.strictEqual(Array.isArray(res.body), true);
assert.strictEqual(res.body.some((notification: any) => notification.userId === bob.id), true);
assert.strictEqual(res.body.some((notification: any) => notification.userId === carol.id), false);
});
test('通知にミュートしているユーザーからのリノートが含まれない', async () => {
const aliceNote = await post(alice, { text: 'hi' });
await post(bob, { renoteId: aliceNote.id });
await post(carol, { renoteId: aliceNote.id });
const res = await api('/i/notifications-grouped', {}, alice);
assert.strictEqual(res.status, 200);
assert.strictEqual(Array.isArray(res.body), true);
assert.strictEqual(res.body.some((notification: any) => notification.userId === bob.id), true);
assert.strictEqual(res.body.some((notification: any) => notification.userId === carol.id), false);
});
test('通知にミュートしているユーザーからのフォロー通知が含まれない', async () => {
await api('/i/follow', { userId: alice.id }, bob);
await api('/i/follow', { userId: alice.id }, carol);
const res = await api('/i/notifications-grouped', {}, alice);
assert.strictEqual(res.status, 200);
assert.strictEqual(Array.isArray(res.body), true);
assert.strictEqual(res.body.some((notification: any) => notification.userId === bob.id), true);
assert.strictEqual(res.body.some((notification: any) => notification.userId === carol.id), false);
});
test('通知にミュートしているユーザーからのフォローリクエストが含まれない', async () => {
await api('/i/update/', { isLocked: true }, alice);
await api('/following/create', { userId: alice.id }, bob);
await api('/following/create', { userId: alice.id }, carol);
const res = await api('/i/notifications-grouped', {}, alice);
assert.strictEqual(res.status, 200);
assert.strictEqual(Array.isArray(res.body), true);
assert.strictEqual(res.body.some((notification: any) => notification.userId === bob.id), true);
assert.strictEqual(res.body.some((notification: any) => notification.userId === carol.id), false);
});
});
});

65
packages/backend/test/e2e/streaming.ts
View File

@@ -40,9 +40,9 @@ describe('Streaming', () => {
let chinatsu: misskey.entities.SignupResponse;
let takumi: misskey.entities.SignupResponse;
let kyokoNote: any;
let kanakoNote: any;
let takumiNote: any;
let kyokoNote: misskey.entities.Note;
let kanakoNote: misskey.entities.Note;
let takumiNote: misskey.entities.Note;
let list: any;
beforeAll(async () => {
@@ -68,6 +68,9 @@ describe('Streaming', () => {
// Follow: ayano => akari
await follow(ayano, akari);
// Follow: kyoko => chitose
await api('following/create', { userId: chitose.id }, kyoko);
// Mute: chitose => kanako
await api('mute/create', { userId: kanako.id }, chitose);
@@ -170,7 +173,28 @@ describe('Streaming', () => {
*/
test('フォローしているユーザーのフォローしていないユーザーの visibility: followers な投稿への返信が流れない', async () => {
// TODO
const chitoseNote = await post(chitose, { text: 'followers-only post', visibility: 'followers' });
const fired = await waitFire(
ayano, 'homeTimeline', // ayano:home
() => api('notes/create', { text: 'reply to chitose\'s followers-only post', replyId: chitoseNote.id }, kyoko), // kyoko's reply to chitose's followers-only post
msg => msg.type === 'note' && msg.body.userId === kyoko.id, // wait kyoko
);
assert.strictEqual(fired, false);
});
test('フォローしているユーザーのフォローしていないユーザーの visibility: followers な投稿への返信のリノートが流れない', async () => {
const chitoseNote = await post(chitose, { text: 'followers-only post', visibility: 'followers' });
const kyokoReply = await post(kyoko, { text: 'reply to followers-only post', replyId: chitoseNote.id });
const fired = await waitFire(
ayano, 'homeTimeline', // ayano:home
() => api('notes/create', { renoteId: kyokoReply.id }, kyoko), // kyoko's renote of kyoko's reply to chitose's followers-only post
msg => msg.type === 'note' && msg.body.userId === kyoko.id, // wait kyoko
);
assert.strictEqual(fired, false);
});
test('フォローしていないユーザーの投稿は流れない', async () => {
@@ -202,6 +226,39 @@ describe('Streaming', () => {
assert.strictEqual(fired, false);
});
test('withRenotes: false のときリノートが流れない', async () => {
const fired = await waitFire(
ayano, 'homeTimeline', // ayano:home
() => api('notes/create', { renoteId: kyokoNote.id }, kyoko), // kyoko renote
msg => msg.type === 'note' && msg.body.userId === kyoko.id, // wait kyoko
{ withRenotes: false },
);
assert.strictEqual(fired, false);
});
test('withRenotes: false のとき引用リノートが流れる', async () => {
const fired = await waitFire(
ayano, 'homeTimeline', // ayano:home
() => api('notes/create', { text: 'quote', renoteId: kyokoNote.id }, kyoko), // kyoko quote
msg => msg.type === 'note' && msg.body.userId === kyoko.id, // wait kyoko
{ withRenotes: false },
);
assert.strictEqual(fired, true);
});
test('withRenotes: false のとき投票のみのリノートが流れる', async () => {
const fired = await waitFire(
ayano, 'homeTimeline', // ayano:home
() => api('notes/create', { poll: { choices: ['kinoko', 'takenoko'] }, renoteId: kyokoNote.id }, kyoko), // kyoko renote with poll
msg => msg.type === 'note' && msg.body.userId === kyoko.id, // wait kyoko
{ withRenotes: false },
);
assert.strictEqual(fired, true);
});
}); // Home
describe('Local Timeline', () => {

28
packages/backend/test/unit/RoleService.ts
View File

@@ -251,6 +251,34 @@ describe('RoleService', () => {
expect(user2Policies.canManageCustomEmojis).toBe(true);
});
test('コンディショナルロール: マニュアルロールにアサイン済み', async () => {
const [user1, user2, role1] = await Promise.all([
createUser(),
createUser(),
createRole({
name: 'manual role',
}),
]);
const role2 = await createRole({
name: 'conditional role',
target: 'conditional',
condFormula: {
// idはバックエンドのロジックに必要ない？
id: 'bdc612bd-9d54-4675-ae83-0499c82ea670',
type: 'roleAssignedTo',
roleId: role1.id,
},
});
await roleService.assign(user2.id, role1.id);
const [u1role, u2role] = await Promise.all([
roleService.getUserRoles(user1.id),
roleService.getUserRoles(user2.id),
]);
expect(u1role.some(r => r.id === role2.id)).toBe(false);
expect(u2role.some(r => r.id === role2.id)).toBe(true);
});
test('expired role', async () => {
const user = await createUser();
const role = await createRole({

6
packages/backend/test/utils.ts
View File

@@ -355,7 +355,7 @@ export const uploadUrl = async (user: UserToken, url: string): Promise<Packed<'D
return catcher;
};
export function connectStream(user: UserToken, channel: string, listener: (message: Record<string, any>) => any, params?: any): Promise<WebSocket> {
export function connectStream<C extends keyof misskey.Channels>(user: UserToken, channel: C, listener: (message: Record<string, any>) => any, params?: misskey.Channels[C]['params']): Promise<WebSocket> {
return new Promise((res, rej) => {
const url = new URL(`ws://127.0.0.1:${port}/streaming`);
const options: ClientOptions = {};
@@ -390,7 +390,7 @@ export function connectStream(user: UserToken, channel: string, listener: (messa
});
}
export const waitFire = async (user: UserToken, channel: string, trgr: () => any, cond: (msg: Record<string, any>) => boolean, params?: any) => {
export const waitFire = async <C extends keyof misskey.Channels>(user: UserToken, channel: C, trgr: () => any, cond: (msg: Record<string, any>) => boolean, params?: misskey.Channels[C]['params']) => {
return new Promise<boolean>(async (res, rej) => {
let timer: NodeJS.Timeout | null = null;
@@ -435,7 +435,7 @@ export const waitFire = async (user: UserToken, channel: string, trgr: () => any
*/
export function makeStreamCatcher<T>(
user: UserToken,
channel: string,
channel: keyof misskey.Channels,
cond: (message: Record<string, any>) => boolean,
extractor: (message: Record<string, any>) => T,
timeout = 60 * 1000): Promise<T> {