sendnrw/misskey
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch 'notification-read-api' into swn

Browse Source
This commit is contained in:
tamaina
2022-03-23 23:49:01 +09:00
parent 9c4f5a7435 fccd461252
commit 74a3f388f8
51 changed files with 1382 additions and 1277 deletions
Download Patch File Download Diff File Expand all files Collapse all files

12
packages/backend/src/boot/master.ts
View File

@@ -6,6 +6,7 @@ import cluster from 'node:cluster';
import chalk from 'chalk';
import chalkTemplate from 'chalk-template';
import * as portscanner from 'portscanner';
import semver from 'semver';
import { getConnection } from 'typeorm';
import Logger from '@/services/logger.js';
@@ -88,10 +89,6 @@ export async function masterMain() {
}
}
const runningNodejsVersion = process.version.slice(1).split('.').map(x => parseInt(x, 10));
const requiredNodejsVersion = [11, 7, 0];
const satisfyNodejsVersion = !lessThan(runningNodejsVersion, requiredNodejsVersion);
function showEnvironment(): void {
const env = process.env.NODE_ENV;
const logger = bootLogger.createSubLogger('env');
@@ -108,10 +105,11 @@ function showEnvironment(): void {
function showNodejsVersion(): void {
const nodejsLogger = bootLogger.createSubLogger('nodejs');
nodejsLogger.info(`Version ${runningNodejsVersion.join('.')}`);
nodejsLogger.info(`Version ${process.version} detected.`);
if (!satisfyNodejsVersion) {
nodejsLogger.error(`Node.js version is less than ${requiredNodejsVersion.join('.')}. Please upgrade it.`, null, true);
const minVersion = fs.readFileSync(`${_dirname}/../../../../.node-version`, 'utf-8').trim();
if (semver.lt(process.version, minVersion)) {
nodejsLogger.error(`At least Node.js ${minVersion} required!`);
process.exit(1);
}
}

2
packages/backend/src/db/postgre.ts
View File

@@ -85,7 +85,7 @@ class MyCustomLogger implements Logger {
public logQuery(query: string, parameters?: any[]) {
if (envOption.verbose) {
sqlLogger.info(this.highlight(query));
sqlLogger.info(this.highlight(query).substring(0, 100));
}
}

17
packages/backend/src/misc/cache.ts
View File

@@ -28,11 +28,22 @@ export class Cache<T> {
this.cache.delete(key);
}
public async fetch(key: string | null, fetcher: () => Promise<T>): Promise<T> {
/**
* キャッシュがあればそれを返し、無ければfetcherを呼び出して結果をキャッシュ&返します
* optional: キャッシュが存在してもvalidatorでfalseを返すとキャッシュ無効扱いにします
*/
public async fetch(key: string | null, fetcher: () => Promise<T>, validator?: (cachedValue: T) => boolean): Promise<T> {
const cachedValue = this.get(key);
if (cachedValue !== undefined) {
// Cache HIT
return cachedValue;
if (validator) {
if (validator(cachedValue)) {
// Cache HIT
return cachedValue;
}
} else {
// Cache HIT
return cachedValue;
}
}
// Cache MISS

13
packages/backend/src/misc/check-hit-antenna.ts
View File

@@ -1,17 +1,26 @@
import { Antenna } from '@/models/entities/antenna.js';
import { Note } from '@/models/entities/note.js';
import { User } from '@/models/entities/user.js';
import { UserListJoinings, UserGroupJoinings } from '@/models/index.js';
import { UserListJoinings, UserGroupJoinings, Blockings } from '@/models/index.js';
import { getFullApAccount } from './convert-host.js';
import * as Acct from '@/misc/acct.js';
import { Packed } from './schema.js';
import { Cache } from './cache.js';
const blockingCache = new Cache<User['id'][]>(1000 * 60 * 5);
// NOTE: フォローしているユーザーのノート、リストのユーザーのノート、グループのユーザーのノート指定はパフォーマンス上の理由で無効になっている
/**
* noteUserFollowers / antennaUserFollowing はどちらか一方が指定されていればよい
*/
export async function checkHitAntenna(antenna: Antenna, note: (Note | Packed<'Note'>), noteUser: { username: string; host: string | null; }, noteUserFollowers?: User['id'][], antennaUserFollowing?: User['id'][]): Promise<boolean> {
export async function checkHitAntenna(antenna: Antenna, note: (Note | Packed<'Note'>), noteUser: { id: User['id']; username: string; host: string | null; }, noteUserFollowers?: User['id'][], antennaUserFollowing?: User['id'][]): Promise<boolean> {
if (note.visibility === 'specified') return false;
// アンテナ作成者がノート作成者にブロックされていたらスキップ
const blockings = await blockingCache.fetch(noteUser.id, () => Blockings.find({ blockerId: noteUser.id }).then(res => res.map(x => x.blockeeId)));
if (blockings.some(blocking => blocking === antenna.userId)) return false;
if (note.visibility === 'followers') {
if (noteUserFollowers && !noteUserFollowers.includes(antenna.userId)) return false;
if (antennaUserFollowing && !antennaUserFollowing.includes(note.userId)) return false;

12
packages/backend/src/models/repositories/user.ts
View File

@@ -8,6 +8,10 @@ import { awaitAll, Promiseable } from '@/prelude/await-all.js';
import { populateEmojis } from '@/misc/populate-emojis.js';
import { getAntennas } from '@/misc/antenna-cache.js';
import { USER_ACTIVE_THRESHOLD, USER_ONLINE_THRESHOLD } from '@/const.js';
import { Cache } from '@/misc/cache.js';
import { Instance } from '../entities/instance.js';
const userInstanceCache = new Cache<Instance | null>(1000 * 60 * 60 * 3);
type IsUserDetailed<Detailed extends boolean> = Detailed extends true ? Packed<'UserDetailed'> : Packed<'UserLite'>;
type IsMeAndIsUserDetailed<ExpectsMe extends boolean | null, Detailed extends boolean> =
@@ -254,8 +258,11 @@ export class UserRepository extends Repository<User> {
isModerator: user.isModerator || falsy,
isBot: user.isBot || falsy,
isCat: user.isCat || falsy,
showTimelineReplies: user.showTimelineReplies || falsy,
instance: user.host ? Instances.findOne({ host: user.host }).then(instance => instance ? {
// TODO: typeorm 3.0にしたら .then(x => x || null) は消せる
instance: user.host ? userInstanceCache.fetch(user.host,
() => Instances.findOne({ host: user.host }).then(x => x || null),
v => v != null
).then(instance => instance ? {
name: instance.name,
softwareName: instance.softwareName,
softwareVersion: instance.softwareVersion,
@@ -334,6 +341,7 @@ export class UserRepository extends Repository<User> {
mutedInstances: profile!.mutedInstances,
mutingNotificationTypes: profile!.mutingNotificationTypes,
emailNotificationTypes: profile!.emailNotificationTypes,
showTimelineReplies: user.showTimelineReplies || falsy,
} : {}),
...(opts.includeSecrets ? {

2
packages/backend/src/queue/index.ts
View File

@@ -263,6 +263,7 @@ export default function() {
systemQueue.add('tickCharts', {
}, {
repeat: { cron: '55 * * * *' },
removeOnComplete: true,
});
systemQueue.add('resyncCharts', {
@@ -278,6 +279,7 @@ export default function() {
systemQueue.add('checkExpiredMutings', {
}, {
repeat: { cron: '*/5 * * * *' },
removeOnComplete: true,
});
processSystemQueue(systemQueue);

9
packages/backend/src/queue/queues.ts
View File

@@ -8,3 +8,12 @@ export const deliverQueue = initializeQueue<DeliverJobData>('deliver', config.de
export const inboxQueue = initializeQueue<InboxJobData>('inbox', config.inboxJobPerSec || 16);
export const dbQueue = initializeQueue<DbJobData>('db');
export const objectStorageQueue = initializeQueue<ObjectStorageJobData>('objectStorage');
export const queues = [
systemQueue,
endedPollNotificationQueue,
deliverQueue,
inboxQueue,
dbQueue,
objectStorageQueue,
];

6
packages/backend/src/remote/activitypub/db-resolver.ts
View File

@@ -78,14 +78,14 @@ export default class DbResolver {
public async getAuthUserFromKeyId(keyId: string): Promise<AuthUser | null> {
const key = await UserPublickeys.findOne({
keyId,
}, {
relations: ['user'],
});
if (key == null) return null;
const user = await Users.findOne(key.userId) as IRemoteUser;
return {
user,
user: key.user as IRemoteUser,
key,
};
}

2
packages/backend/src/remote/activitypub/models/note.ts
View File

@@ -197,7 +197,7 @@ export async function createNote(value: string | IObject, resolver?: Resolver, s
const cw = note.summary === '' ? null : note.summary;
// テキストのパース
const text = note._misskey_content || (note.content ? htmlToMfm(note.content, note.tag) : null);
const text = typeof note._misskey_content !== 'undefined' ? note._misskey_content : (note.content ? htmlToMfm(note.content, note.tag) : null);
// vote
if (reply && reply.hasPoll) {

4
packages/backend/src/remote/activitypub/renderer/note.ts
View File

@@ -53,9 +53,7 @@ export default async function renderNote(note: Note, dive = true, isTalk = false
}
}
const user = await Users.findOneOrFail(note.userId);
const attributedTo = `${config.url}/users/${user.id}`;
const attributedTo = `${config.url}/users/${note.userId}`;
const mentions = (JSON.parse(note.mentionedRemoteUsers) as IMentionedRemoteUsers).map(x => x.uri);

2
packages/backend/src/remote/activitypub/renderer/undo.ts
View File

@@ -3,9 +3,11 @@ import { ILocalUser, User } from '@/models/entities/user.js';
export default (object: any, user: { id: User['id'] }) => {
if (object == null) return null;
const id = typeof object.id === 'string' && object.id.startsWith(config.url) ? `${object.id}/undo` : undefined;
return {
type: 'Undo',
...(id ? { id } : {}),
actor: `${config.url}/users/${user.id}`,
object,
published: new Date().toISOString(),

15
packages/backend/src/server/activitypub.ts
View File

@@ -18,6 +18,7 @@ import { ILocalUser, User } from '@/models/entities/user.js';
import { In } from 'typeorm';
import { renderLike } from '@/remote/activitypub/renderer/like.js';
import { getUserKeypair } from '@/misc/keypair-store.js';
import { noteCache, userCache } from './activitypub/cache.js';
// Init router
const router = new Router();
@@ -65,11 +66,13 @@ router.post('/users/:user/inbox', json(), inbox);
router.get('/notes/:note', async (ctx, next) => {
if (!isActivityPubReq(ctx)) return await next();
const note = await Notes.findOne({
// TODO: typeorm 3.0にしたら .then(x => x || null) は消せる
// nginxとかでキャッシュしてくれそうだからそもそもnode側でのキャッシュ不要かも？
const note = await noteCache.fetch(ctx.params.note, () => Notes.findOne({
id: ctx.params.note,
visibility: In(['public' as const, 'home' as const]),
localOnly: false,
});
}).then(x => x || null));
if (note == null) {
ctx.status = 404;
@@ -148,7 +151,7 @@ router.get('/users/:user/publickey', async ctx => {
});
// user
async function userInfo(ctx: Router.RouterContext, user: User | undefined) {
async function userInfo(ctx: Router.RouterContext, user: User | undefined | null) {
if (user == null) {
ctx.status = 404;
return;
@@ -164,11 +167,13 @@ router.get('/users/:user', async (ctx, next) => {
const userId = ctx.params.user;
const user = await Users.findOne({
// TODO: typeorm 3.0にしたら .then(x => x || null) は消せる
// nginxとかでキャッシュしてくれそうだからそもそもnode側でのキャッシュ不要かも？
const user = await userCache.fetch(userId, () => Users.findOne({
id: userId,
host: null,
isSuspended: false,
});
}).then(x => x || null));
await userInfo(ctx, user);
});

6
packages/backend/src/server/activitypub/cache.ts Normal file
View File

@@ -0,0 +1,6 @@
import { Cache } from "@/misc/cache.js";
import { Note } from "@/models/entities/note.js";
import { User } from "@/models/entities/user.js";
export const userCache = new Cache<User | null>(1000 * 60 * 30);
export const noteCache = new Cache<Note | null>(1000 * 60 * 30);

7
packages/backend/src/server/activitypub/featured.ts
View File

@@ -5,15 +5,16 @@ import renderOrderedCollection from '@/remote/activitypub/renderer/ordered-colle
import { setResponseType } from '../activitypub.js';
import renderNote from '@/remote/activitypub/renderer/note.js';
import { Users, Notes, UserNotePinings } from '@/models/index.js';
import { userCache } from './cache.js';
export default async (ctx: Router.RouterContext) => {
const userId = ctx.params.user;
// Verify user
const user = await Users.findOne({
// TODO: typeorm 3.0にしたら .then(x => x || null) は消せる
const user = await userCache.fetch(userId, () => Users.findOne({
id: userId,
host: null,
});
}).then(x => x || null));
if (user == null) {
ctx.status = 404;

7
packages/backend/src/server/activitypub/followers.ts
View File

@@ -10,6 +10,7 @@ import renderFollowUser from '@/remote/activitypub/renderer/follow-user.js';
import { setResponseType } from '../activitypub.js';
import { Users, Followings, UserProfiles } from '@/models/index.js';
import { LessThan } from 'typeorm';
import { userCache } from './cache.js';
export default async (ctx: Router.RouterContext) => {
const userId = ctx.params.user;
@@ -27,11 +28,11 @@ export default async (ctx: Router.RouterContext) => {
return;
}
// Verify user
const user = await Users.findOne({
// TODO: typeorm 3.0にしたら .then(x => x || null) は消せる
const user = await userCache.fetch(userId, () => Users.findOne({
id: userId,
host: null,
});
}).then(x => x || null));
if (user == null) {
ctx.status = 404;

7
packages/backend/src/server/activitypub/following.ts
View File

@@ -11,6 +11,7 @@ import { setResponseType } from '../activitypub.js';
import { Users, Followings, UserProfiles } from '@/models/index.js';
import { LessThan, FindConditions } from 'typeorm';
import { Following } from '@/models/entities/following.js';
import { userCache } from './cache.js';
export default async (ctx: Router.RouterContext) => {
const userId = ctx.params.user;
@@ -28,11 +29,11 @@ export default async (ctx: Router.RouterContext) => {
return;
}
// Verify user
const user = await Users.findOne({
// TODO: typeorm 3.0にしたら .then(x => x || null) は消せる
const user = await userCache.fetch(userId, () => Users.findOne({
id: userId,
host: null,
});
}).then(x => x || null));
if (user == null) {
ctx.status = 404;

7
packages/backend/src/server/activitypub/outbox.ts
View File

@@ -15,6 +15,7 @@ import { Users, Notes } from '@/models/index.js';
import { makePaginationQuery } from '../api/common/make-pagination-query.js';
import { Brackets } from 'typeorm';
import { Note } from '@/models/entities/note.js';
import { userCache } from './cache.js';
export default async (ctx: Router.RouterContext) => {
const userId = ctx.params.user;
@@ -35,11 +36,11 @@ export default async (ctx: Router.RouterContext) => {
return;
}
// Verify user
const user = await Users.findOne({
// TODO: typeorm 3.0にしたら .then(x => x || null) は消せる
const user = await userCache.fetch(userId, () => Users.findOne({
id: userId,
host: null,
});
}).then(x => x || null));
if (user == null) {
ctx.status = 404;

4
packages/backend/src/server/api/endpoints/endpoint.ts
View File

@@ -20,9 +20,9 @@ export default define(meta, paramDef, async (ps) => {
const ep = endpoints.find(x => x.name === ps.endpoint);
if (ep == null) return null;
return {
params: Object.entries(ep.meta.params || {}).map(([k, v]) => ({
params: Object.entries(ep.params.properties || {}).map(([k, v]) => ({
name: k,
type: v.validator.name === 'ID' ? 'String' : v.validator.name,
type: v.type.charAt(0).toUpperCase() + v.type.slice(1),
})),
};
});

7
packages/backend/src/server/nodeinfo.ts
View File

@@ -4,6 +4,7 @@ import { fetchMeta } from '@/misc/fetch-meta.js';
import { Users, Notes } from '@/models/index.js';
import { MoreThan } from 'typeorm';
import { MAX_NOTE_TEXT_LENGTH } from '@/const.js';
import { Cache } from '@/misc/cache.js';
const router = new Router();
@@ -81,15 +82,17 @@ const nodeinfo2 = async () => {
};
};
const cache = new Cache<Awaited<ReturnType<typeof nodeinfo2>>>(1000 * 60 * 10);
router.get(nodeinfo2_1path, async ctx => {
const base = await nodeinfo2();
const base = await cache.fetch(null, () => nodeinfo2());
ctx.body = { version: '2.1', ...base };
ctx.set('Cache-Control', 'public, max-age=600');
});
router.get(nodeinfo2_0path, async ctx => {
const base = await nodeinfo2();
const base = await cache.fetch(null, () => nodeinfo2());
delete base.software.repository;

35
packages/backend/src/server/web/index.ts
View File

@@ -10,6 +10,9 @@ import Router from '@koa/router';
import send from 'koa-send';
import favicon from 'koa-favicon';
import views from 'koa-views';
import { createBullBoard } from '@bull-board/api';
import { BullAdapter } from '@bull-board/api/bullAdapter.js';
import { KoaAdapter } from '@bull-board/koa';
import packFeed from './feed.js';
import { fetchMeta } from '@/misc/fetch-meta.js';
@@ -20,6 +23,7 @@ import * as Acct from '@/misc/acct.js';
import { getNoteSummary } from '@/misc/get-note-summary.js';
import { urlPreviewHandler } from './url-preview.js';
import { manifestHandler } from './manifest.js';
import { queues } from '@/queue/queues.js';
const _filename = fileURLToPath(import.meta.url);
const _dirname = dirname(_filename);
@@ -32,6 +36,37 @@ const swAssets = `${_dirname}/../../../../../built/_sw_dist_/`;
// Init app
const app = new Koa();
//#region Bull Dashboard
const bullBoardPath = '/queue';
// Authenticate
app.use(async (ctx, next) => {
if (ctx.path === bullBoardPath || ctx.path.startsWith(bullBoardPath + '/')) {
const token = ctx.cookies.get('token');
if (token == null) {
ctx.status = 401;
return;
}
const user = await Users.findOne({ token });
if (user == null || !(user.isAdmin || user.isModerator)) {
ctx.status = 403;
return;
}
}
await next();
});
const serverAdapter = new KoaAdapter();
createBullBoard({
queues: queues.map(q => new BullAdapter(q)),
serverAdapter,
});
serverAdapter.setBasePath(bullBoardPath);
app.use(serverAdapter.registerPlugin());
//#endregion
// Init renderer
app.use(views(_dirname + '/views', {
extension: 'pug',

53
packages/backend/src/services/note/create.ts
View File

@@ -35,6 +35,12 @@ import { Channel } from '@/models/entities/channel.js';
import { normalizeForSearch } from '@/misc/normalize-for-search.js';
import { getAntennas } from '@/misc/antenna-cache.js';
import { endedPollNotificationQueue } from '@/queue/queues.js';
import { Cache } from '@/misc/cache.js';
import { UserProfile } from '@/models/entities/user-profile.js';
const usersCache = new Cache<MinimumUser>(Infinity);
const mutedWordsCache = new Cache<{ userId: UserProfile['userId']; mutedWords: UserProfile['mutedWords']; }[]>(1000 * 60 * 5);
type NotificationType = 'reply' | 'renote' | 'quote' | 'mention';
@@ -91,6 +97,13 @@ class NotificationManager {
}
}
type MinimumUser = {
id: User['id'];
host: User['host'];
username: User['username'];
uri: User['uri'];
};
type Option = {
createdAt?: Date | null;
name?: string | null;
@@ -102,9 +115,9 @@ type Option = {
localOnly?: boolean | null;
cw?: string | null;
visibility?: string;
visibleUsers?: User[] | null;
visibleUsers?: MinimumUser[] | null;
channel?: Channel | null;
apMentions?: User[] | null;
apMentions?: MinimumUser[] | null;
apHashtags?: string[] | null;
apEmojis?: string[] | null;
uri?: string | null;
@@ -199,7 +212,7 @@ export default async (user: { id: User['id']; username: User['username']; host:
tags = tags.filter(tag => Array.from(tag || '').length <= 128).splice(0, 32);
if (data.reply && (user.id !== data.reply.userId) && !mentionedUsers.some(u => u.id === data.reply!.userId)) {
mentionedUsers.push(await Users.findOneOrFail(data.reply.userId));
mentionedUsers.push(await usersCache.fetch(data.reply.userId, () => Users.findOneOrFail(data.reply!.userId)));
}
if (data.visibility === 'specified') {
@@ -212,7 +225,7 @@ export default async (user: { id: User['id']; username: User['username']; host:
}
if (data.reply && !data.visibleUsers.some(x => x.id === data.reply!.userId)) {
data.visibleUsers.push(await Users.findOneOrFail(data.reply.userId));
data.visibleUsers.push(await usersCache.fetch(data.reply.userId, () => Users.findOneOrFail(data.reply!.userId)));
}
}
@@ -241,10 +254,12 @@ export default async (user: { id: User['id']; username: User['username']; host:
incNotesCountOfUser(user);
// Word mute
// TODO: cache
UserProfiles.find({
enableWordMute: true,
}).then(us => {
mutedWordsCache.fetch(null, () => UserProfiles.find({
where: {
enableWordMute: true,
},
select: ['userId', 'mutedWords'],
})).then(us => {
for (const u of us) {
checkWordMute(note, { id: u.userId }, u.mutedWords).then(shouldMute => {
if (shouldMute) {
@@ -260,21 +275,13 @@ export default async (user: { id: User['id']; username: User['username']; host:
});
// Antenna
Followings.createQueryBuilder('following')
.andWhere(`following.followeeId = :userId`, { userId: note.userId })
.getMany()
.then(async followings => {
const blockings = await Blockings.find({ blockerId: user.id }); // TODO: キャッシュしたい
const followers = followings.map(f => f.followerId);
for (const antenna of (await getAntennas())) {
if (blockings.some(blocking => blocking.blockeeId === antenna.userId)) continue; // この処理は checkHitAntenna 内でやるようにしてもいいかも
checkHitAntenna(antenna, note, user, followers).then(hit => {
if (hit) {
addNoteToAntenna(antenna, note, user);
}
});
for (const antenna of (await getAntennas())) {
checkHitAntenna(antenna, note, user).then(hit => {
if (hit) {
addNoteToAntenna(antenna, note, user);
}
});
}
// Channel
if (note.channelId) {
@@ -465,7 +472,7 @@ function incRenoteCount(renote: Note) {
.execute();
}
async function insertNote(user: { id: User['id']; host: User['host']; }, data: Option, tags: string[], emojis: string[], mentionedUsers: User[]) {
async function insertNote(user: { id: User['id']; host: User['host']; }, data: Option, tags: string[], emojis: string[], mentionedUsers: MinimumUser[]) {
const insert = new Note({
id: genId(data.createdAt!),
createdAt: data.createdAt!,
@@ -597,7 +604,7 @@ async function notifyToWatchersOfReplyee(reply: Note, user: { id: User['id']; },
}
}
async function createMentionedEvents(mentionedUsers: User[], note: Note, nm: NotificationManager) {
async function createMentionedEvents(mentionedUsers: MinimumUser[], note: Note, nm: NotificationManager) {
for (const u of mentionedUsers.filter(u => Users.isLocalUser(u))) {
const threadMuted = await NoteThreadMutings.findOne({
userId: u.id,

4
packages/backend/src/services/note/delete.ts
View File

@@ -29,6 +29,10 @@ export default async function(user: User, note: Note, quiet = false) {
Notes.decrement({ id: note.renoteId }, 'score', 1);
}
if (note.replyId) {
await Notes.decrement({ id: note.replyId }, 'repliesCount', 1);
}
if (!quiet) {
publishNoteStream(note.id, 'deleted', {
deletedAt: deletedAt,