sendnrw/misskey
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch 'io' into merge-upstream

Browse Source
This commit is contained in:
riku6460
2023-11-09 17:43:42 +09:00
parent 2bc887a6c5 85c15bfee0
commit 50e811e862
59 changed files with 534 additions and 410 deletions
Download Patch File Download Diff File Expand all files Collapse all files

16
packages/backend/migration/1699284486293-urlPreviewDenyList.js Normal file
View File

@@ -0,0 +1,16 @@
/*
* SPDX-FileCopyrightText: syuilo and other misskey contributors
* SPDX-License-Identifier: AGPL-3.0-only
*/
export class UrlPreviewDenyList1699284486293 {
name = 'UrlPreviewDenyList1699284486293'
async up(queryRunner) {
await queryRunner.query(`ALTER TABLE "meta" ADD "urlPreviewDenyList" character varying(3072) array NOT NULL DEFAULT '{}'`);
}
async down(queryRunner) {
await queryRunner.query(`ALTER TABLE "meta" DROP COLUMN "urlPreviewDenyList"`);
}
}

10
packages/backend/src/core/FeaturedService.ts
View File

@@ -38,7 +38,7 @@ export class FeaturedService {
redisTransaction.expire(
`${name}:${currentWindow}`,
(windowRange * 3) / 1000,
'NX'); // "NX -- Set expiry only when the key has no expiry" = 有効期限がないときだけ設定
);
await redisTransaction.exec();
}
@@ -48,10 +48,10 @@ export class FeaturedService {
const previousWindow = currentWindow - 1;
const redisPipeline = this.redisClient.pipeline();
redisPipeline.zrange(
`${name}:${currentWindow}`, 0, threshold, 'REV', 'WITHSCORES');
redisPipeline.zrange(
`${name}:${previousWindow}`, 0, threshold, 'REV', 'WITHSCORES');
redisPipeline.zrevrange(
`${name}:${currentWindow}`, 0, threshold, 'WITHSCORES');
redisPipeline.zrevrange(
`${name}:${previousWindow}`, 0, threshold, 'WITHSCORES');
const [currentRankingResult, previousRankingResult] = await redisPipeline.exec().then(result => result ? result.map(r => (r[1] ?? []) as string[]) : [[], []]);
const ranking = new Map<string, number>();

8
packages/backend/src/core/FetchInstanceMetadataService.ts
View File

@@ -52,20 +52,20 @@ export class FetchInstanceMetadataService {
@bindThis
public async tryLock(host: string): Promise<boolean> {
const mutex = await this.redisClient.set(`fetchInstanceMetadata:mutex:${host}`, '1', 'EX', 60 * 5, 'NX', 'GET');
return mutex !== '1';
const mutex = await this.redisClient.set(`fetchInstanceMetadata:mutex:${host}`, Date.now(), 'EX', 60 * 5, 'NX');
return mutex !== null;
}
@bindThis
public unlock(host: string): Promise<number> {
return this.redisClient.del(`fetchInstanceMetadata:mutex:${host}`);
return this.redisClient.unlink(`fetchInstanceMetadata:mutex:${host}`);
}
@bindThis
public async fetchInstanceMetadata(instance: MiInstance, force = false): Promise<void> {
const host = instance.host;
// Acquire mutex to ensure no parallel runs
if (!await this.tryLock(host)) return;
if (!await this.tryLock(host) && !force) return;
try {
if (!force) {
const _instance = await this.federatedInstanceService.fetch(host);

3
packages/backend/src/core/WebAuthnService.ts
View File

@@ -7,7 +7,8 @@ import { Inject, Injectable } from '@nestjs/common';
import * as Redis from 'ioredis';
import {
generateAuthenticationOptions,
generateRegistrationOptions, verifyAuthenticationResponse,
generateRegistrationOptions,
verifyAuthenticationResponse,
verifyRegistrationResponse,
} from '@simplewebauthn/server';
import { AttestationFormat, isoCBOR } from '@simplewebauthn/server/helpers';

5
packages/backend/src/models/Meta.ts
View File

@@ -518,4 +518,9 @@ export class MiMeta {
default: 0,
})
public notesPerOneAd: number;
@Column('varchar', {
length: 3072, array: true, default: '{}',
})
public urlPreviewDenyList: string[];
}

8
packages/backend/src/server/api/EndpointsModule.ts
View File

@@ -27,6 +27,8 @@ import * as ep___admin_avatarDecorations_delete from './endpoints/admin/avatar-d
import * as ep___admin_avatarDecorations_list from './endpoints/admin/avatar-decorations/list.js';
import * as ep___admin_avatarDecorations_update from './endpoints/admin/avatar-decorations/update.js';
import * as ep___admin_deleteAllFilesOfAUser from './endpoints/admin/delete-all-files-of-a-user.js';
import * as ep___admin_deleteUserAvatar from './endpoints/admin/delete-user-avatar.js';
import * as ep___admin_deleteUserBanner from './endpoints/admin/delete-user-banner.js';
import * as ep___admin_drive_cleanRemoteFiles from './endpoints/admin/drive/clean-remote-files.js';
import * as ep___admin_drive_cleanup from './endpoints/admin/drive/cleanup.js';
import * as ep___admin_drive_files from './endpoints/admin/drive/files.js';
@@ -389,6 +391,8 @@ const $admin_avatarDecorations_delete: Provider = { provide: 'ep:admin/avatar-de
const $admin_avatarDecorations_list: Provider = { provide: 'ep:admin/avatar-decorations/list', useClass: ep___admin_avatarDecorations_list.default };
const $admin_avatarDecorations_update: Provider = { provide: 'ep:admin/avatar-decorations/update', useClass: ep___admin_avatarDecorations_update.default };
const $admin_deleteAllFilesOfAUser: Provider = { provide: 'ep:admin/delete-all-files-of-a-user', useClass: ep___admin_deleteAllFilesOfAUser.default };
const $admin_deleteUserAvatar: Provider = { provide: 'ep:admin/delete-user-avatar', useClass: ep___admin_deleteUserAvatar.default };
const $admin_deleteUserBanner: Provider = { provide: 'ep:admin/delete-user-banner', useClass: ep___admin_deleteUserBanner.default };
const $admin_drive_cleanRemoteFiles: Provider = { provide: 'ep:admin/drive/clean-remote-files', useClass: ep___admin_drive_cleanRemoteFiles.default };
const $admin_drive_cleanup: Provider = { provide: 'ep:admin/drive/cleanup', useClass: ep___admin_drive_cleanup.default };
const $admin_drive_files: Provider = { provide: 'ep:admin/drive/files', useClass: ep___admin_drive_files.default };
@@ -755,6 +759,8 @@ const $retention: Provider = { provide: 'ep:retention', useClass: ep___retention
$admin_avatarDecorations_list,
$admin_avatarDecorations_update,
$admin_deleteAllFilesOfAUser,
$admin_deleteUserAvatar,
$admin_deleteUserBanner,
$admin_drive_cleanRemoteFiles,
$admin_drive_cleanup,
$admin_drive_files,
@@ -1115,6 +1121,8 @@ const $retention: Provider = { provide: 'ep:retention', useClass: ep___retention
$admin_avatarDecorations_list,
$admin_avatarDecorations_update,
$admin_deleteAllFilesOfAUser,
$admin_deleteUserAvatar,
$admin_deleteUserBanner,
$admin_drive_cleanRemoteFiles,
$admin_drive_cleanup,
$admin_drive_files,

4
packages/backend/src/server/api/endpoints.ts
View File

@@ -27,6 +27,8 @@ import * as ep___admin_avatarDecorations_delete from './endpoints/admin/avatar-d
import * as ep___admin_avatarDecorations_list from './endpoints/admin/avatar-decorations/list.js';
import * as ep___admin_avatarDecorations_update from './endpoints/admin/avatar-decorations/update.js';
import * as ep___admin_deleteAllFilesOfAUser from './endpoints/admin/delete-all-files-of-a-user.js';
import * as ep___admin_deleteUserAvatar from './endpoints/admin/delete-user-avatar.js';
import * as ep___admin_deleteUserBanner from './endpoints/admin/delete-user-banner.js';
import * as ep___admin_drive_cleanRemoteFiles from './endpoints/admin/drive/clean-remote-files.js';
import * as ep___admin_drive_cleanup from './endpoints/admin/drive/cleanup.js';
import * as ep___admin_drive_files from './endpoints/admin/drive/files.js';
@@ -387,6 +389,8 @@ const eps = [
['admin/avatar-decorations/list', ep___admin_avatarDecorations_list],
['admin/avatar-decorations/update', ep___admin_avatarDecorations_update],
['admin/delete-all-files-of-a-user', ep___admin_deleteAllFilesOfAUser],
['admin/delete-user-avatar', ep___admin_deleteUserAvatar],
['admin/delete-user-banner', ep___admin_deleteUserBanner],
['admin/drive/clean-remote-files', ep___admin_drive_cleanRemoteFiles],
['admin/drive/cleanup', ep___admin_drive_cleanup],
['admin/drive/files', ep___admin_drive_files],

48
packages/backend/src/server/api/endpoints/admin/delete-user-avatar.ts Normal file
View File

@@ -0,0 +1,48 @@
/*
* SPDX-FileCopyrightText: syuilo and other misskey contributors
* SPDX-License-Identifier: AGPL-3.0-only
*/
import { Inject, Injectable } from '@nestjs/common';
import type { UsersRepository } from '@/models/index.js';
import { Endpoint } from '@/server/api/endpoint-base.js';
import { DI } from '@/di-symbols.js';
export const meta = {
tags: ['admin'],
requireCredential: true,
requireModerator: true,
} as const;
export const paramDef = {
type: 'object',
properties: {
userId: { type: 'string', format: 'misskey:id' },
},
required: ['userId'],
} as const;
// eslint-disable-next-line import/no-default-export
@Injectable()
export default class extends Endpoint<typeof meta, typeof paramDef> {
constructor(
@Inject(DI.usersRepository)
private usersRepository: UsersRepository,
) {
super(meta, paramDef, async (ps, me) => {
const user = await this.usersRepository.findOneBy({ id: ps.userId });
if (user == null) {
throw new Error('user not found');
}
await this.usersRepository.update(user.id, {
avatar: null,
avatarId: null,
avatarUrl: null,
avatarBlurhash: null,
});
});
}
}

48
packages/backend/src/server/api/endpoints/admin/delete-user-banner.ts Normal file
View File

@@ -0,0 +1,48 @@
/*
* SPDX-FileCopyrightText: syuilo and other misskey contributors
* SPDX-License-Identifier: AGPL-3.0-only
*/
import { Inject, Injectable } from '@nestjs/common';
import type { UsersRepository } from '@/models/index.js';
import { Endpoint } from '@/server/api/endpoint-base.js';
import { DI } from '@/di-symbols.js';
export const meta = {
tags: ['admin'],
requireCredential: true,
requireModerator: true,
} as const;
export const paramDef = {
type: 'object',
properties: {
userId: { type: 'string', format: 'misskey:id' },
},
required: ['userId'],
} as const;
// eslint-disable-next-line import/no-default-export
@Injectable()
export default class extends Endpoint<typeof meta, typeof paramDef> {
constructor(
@Inject(DI.usersRepository)
private usersRepository: UsersRepository,
) {
super(meta, paramDef, async (ps, me) => {
const user = await this.usersRepository.findOneBy({ id: ps.userId });
if (user == null) {
throw new Error('user not found');
}
await this.usersRepository.update(user.id, {
banner: null,
bannerId: null,
bannerUrl: null,
bannerBlurhash: null,
});
});
}
}

9
packages/backend/src/server/api/endpoints/admin/meta.ts
View File

@@ -315,6 +315,14 @@ export const meta = {
type: 'number',
optional: false, nullable: false,
},
urlPreviewDenyList: {
type: 'array',
optional: true, nullable: false,
items: {
type: 'string',
optional: false, nullable: false,
},
},
},
},
} as const;
@@ -429,6 +437,7 @@ export default class extends Endpoint<typeof meta, typeof paramDef> { // eslint-
perUserHomeTimelineCacheMax: instance.perUserHomeTimelineCacheMax,
perUserListTimelineCacheMax: instance.perUserListTimelineCacheMax,
notesPerOneAd: instance.notesPerOneAd,
urlPreviewDenyList: instance.urlPreviewDenyList,
};
});
}

8
packages/backend/src/server/api/endpoints/admin/update-meta.ts
View File

@@ -133,6 +133,9 @@ export const paramDef = {
type: 'string',
},
},
urlPreviewDenyList: { type: 'array', nullable: true, items: {
type: 'string',
} },
},
required: [],
} as const;
@@ -173,6 +176,11 @@ export default class extends Endpoint<typeof meta, typeof paramDef> { // eslint-
return h !== '' && h !== lv && !set.blockedHosts?.includes(h);
});
}
if (Array.isArray(ps.urlPreviewDenyList)) {
set.urlPreviewDenyList = ps.urlPreviewDenyList.filter(Boolean);
}
if (ps.themeColor !== undefined) {
set.themeColor = ps.themeColor;
}

18
packages/backend/src/server/web/UrlPreviewService.ts
View File

@@ -5,6 +5,7 @@
import { Inject, Injectable } from '@nestjs/common';
import { summaly } from 'summaly';
import RE2 from 're2';
import { DI } from '@/di-symbols.js';
import type { Config } from '@/config.js';
import { MetaService } from '@/core/MetaService.js';
@@ -94,6 +95,23 @@ export class UrlPreviewService {
summary.icon = this.wrap(summary.icon);
summary.thumbnail = this.wrap(summary.thumbnail);
const includeDenyList = meta.urlPreviewDenyList.some(filter => {
// represents RegExp
const regexp = /^\/(.+)\/(.*)$/.exec(filter);
// This should never happen due to input sanitisation.
if (!regexp) {
const words = filter.split(' ');
return words.every(keyword => summary.url.includes(keyword));
}
try {
return new RE2(regexp[1], regexp[2]).test(summary.url);
} catch (err) {
// This should never happen due to input sanitisation.
return false;
}
});
if (includeDenyList) summary.sensitive = true;
// Cache 7days
reply.header('Cache-Control', 'max-age=604800, immutable');

6
packages/backend/test/docker-compose.yml
View File

@@ -1,13 +1,13 @@
version: "3"
services:
redistest:
image: redis:7
keydbtest:
image: eqalpha/keydb:latest
ports:
- "127.0.0.1:56312:6379"
dbtest:
image: postgres:13
image: postgres:15
ports:
- "127.0.0.1:54312:5432"
environment:

5
packages/backend/test/unit/FetchInstanceMetadataService.ts
View File

@@ -21,9 +21,10 @@ import type { TestingModule } from '@nestjs/testing';
function mockRedis() {
const hash = {};
const set = jest.fn((key, value) => {
const ret = hash[key];
// このテストで呼び出すSETにはNXオプションが付いてる
if (hash[key]) return null;
hash[key] = value;
return ret;
return 'OK';
});
return set;
}