sendnrw/misskey
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'misskey-dev/develop' into io

Browse Source
This commit is contained in:
まっちゃとーにゅ
2024-01-16 05:42:56 +09:00
parent 411a8bd8e7 8b0fdfcd69
commit 47eaf0b446
18 changed files with 1393 additions and 98 deletions
Download Patch File Download Diff File Expand all files Collapse all files

24
packages/backend/migration/1705222772858-optimize-note-index-for-array-column.js Normal file
View File

@@ -0,0 +1,24 @@
/*
* SPDX-FileCopyrightText: syuilo and other misskey contributors
* SPDX-License-Identifier: AGPL-3.0-only
*/
export class OptimizeNoteIndexForArrayColumns1705222772858 {
name = 'OptimizeNoteIndexForArrayColumns1705222772858'
async up(queryRunner) {
await queryRunner.query(`DROP INDEX "public"."IDX_796a8c03959361f97dc2be1d5c"`);
await queryRunner.query(`DROP INDEX "public"."IDX_54ebcb6d27222913b908d56fd8"`);
await queryRunner.query(`DROP INDEX "public"."IDX_88937d94d7443d9a99a76fa5c0"`);
await queryRunner.query(`DROP INDEX "public"."IDX_51c063b6a133a9cb87145450f5"`);
await queryRunner.query(`CREATE INDEX "IDX_NOTE_FILE_IDS" ON "note" using gin ("fileIds")`)
}
async down(queryRunner) {
await queryRunner.query(`DROP INDEX "IDX_NOTE_FILE_IDS"`)
await queryRunner.query(`CREATE INDEX "IDX_51c063b6a133a9cb87145450f5" ON "note" ("fileIds") `);
await queryRunner.query(`CREATE INDEX "IDX_88937d94d7443d9a99a76fa5c0" ON "note" ("tags") `);
await queryRunner.query(`CREATE INDEX "IDX_54ebcb6d27222913b908d56fd8" ON "note" ("mentions") `);
await queryRunner.query(`CREATE INDEX "IDX_796a8c03959361f97dc2be1d5c" ON "note" ("visibleUserIds") `);
}
}

11
packages/backend/src/models/Note.ts
View File

@@ -11,9 +11,6 @@ import { MiChannel } from './Channel.js';
import type { MiDriveFile } from './DriveFile.js';
@Entity('note')
@Index('IDX_NOTE_TAGS', { synchronize: false })
@Index('IDX_NOTE_MENTIONS', { synchronize: false })
@Index('IDX_NOTE_VISIBLE_USER_IDS', { synchronize: false })
export class MiNote {
@PrimaryColumn(id())
public id: string;
@@ -140,7 +137,7 @@ export class MiNote {
})
public url: string | null;
@Index()
@Index('IDX_NOTE_FILE_IDS', { synchronize: false })
@Column({
...id(),
array: true, default: '{}',
@@ -152,14 +149,14 @@ export class MiNote {
})
public attachedFileTypes: string[];
@Index()
@Index('IDX_NOTE_VISIBLE_USER_IDS', { synchronize: false })
@Column({
...id(),
array: true, default: '{}',
})
public visibleUserIds: MiUser['id'][];
@Index()
@Index('IDX_NOTE_MENTIONS', { synchronize: false })
@Column({
...id(),
array: true, default: '{}',
@@ -181,7 +178,7 @@ export class MiNote {
})
public emojis: string[];
@Index()
@Index('IDX_NOTE_TAGS', { synchronize: false })
@Column('varchar', {
length: 128, array: true, default: '{}',
})

111
packages/backend/src/server/FileServerService.ts
View File

@@ -168,11 +168,35 @@ export class FileServerService {
}
if (!image) {
image = {
data: fs.createReadStream(file.path),
ext: file.ext,
type: file.mime,
};
if (request.headers.range && file.file.size > 0) {
const range = request.headers.range as string;
const parts = range.replace(/bytes=/, '').split('-');
const start = parseInt(parts[0], 10);
let end = parts[1] ? parseInt(parts[1], 10) : file.file.size - 1;
if (end > file.file.size) {
end = file.file.size - 1;
}
const chunksize = end - start + 1;
image = {
data: fs.createReadStream(file.path, {
start,
end,
}),
ext: file.ext,
type: file.mime,
};
reply.header('Content-Range', `bytes ${start}-${end}/${file.file.size}`);
reply.header('Accept-Ranges', 'bytes');
reply.header('Content-Length', chunksize);
} else {
image = {
data: fs.createReadStream(file.path),
ext: file.ext,
type: file.mime,
};
}
}
if ('pipe' in image.data && typeof image.data.pipe === 'function') {
@@ -203,11 +227,54 @@ export class FileServerService {
reply.header('Content-Type', FILE_TYPE_BROWSERSAFE.includes(file.mime) ? file.mime : 'application/octet-stream');
reply.header('Cache-Control', 'max-age=31536000, immutable');
reply.header('Content-Disposition', contentDisposition('inline', filename));
if (request.headers.range && file.file.size > 0) {
const range = request.headers.range as string;
const parts = range.replace(/bytes=/, '').split('-');
const start = parseInt(parts[0], 10);
let end = parts[1] ? parseInt(parts[1], 10) : file.file.size - 1;
if (end > file.file.size) {
end = file.file.size - 1;
}
const chunksize = end - start + 1;
const fileStream = fs.createReadStream(file.path, {
start,
end,
});
reply.header('Content-Range', `bytes ${start}-${end}/${file.file.size}`);
reply.header('Accept-Ranges', 'bytes');
reply.header('Content-Length', chunksize);
reply.code(206);
return fileStream;
}
return fs.createReadStream(file.path);
} else {
reply.header('Content-Type', FILE_TYPE_BROWSERSAFE.includes(file.file.type) ? file.file.type : 'application/octet-stream');
reply.header('Cache-Control', 'max-age=31536000, immutable');
reply.header('Content-Disposition', contentDisposition('inline', file.filename));
if (request.headers.range && file.file.size > 0) {
const range = request.headers.range as string;
const parts = range.replace(/bytes=/, '').split('-');
const start = parseInt(parts[0], 10);
let end = parts[1] ? parseInt(parts[1], 10) : file.file.size - 1;
console.log(end);
if (end > file.file.size) {
end = file.file.size - 1;
}
const chunksize = end - start + 1;
const fileStream = fs.createReadStream(file.path, {
start,
end,
});
reply.header('Content-Range', `bytes ${start}-${end}/${file.file.size}`);
reply.header('Accept-Ranges', 'bytes');
reply.header('Content-Length', chunksize);
reply.code(206);
return fileStream;
}
return fs.createReadStream(file.path);
}
} catch (e) {
@@ -340,11 +407,35 @@ export class FileServerService {
}
if (!image) {
image = {
data: fs.createReadStream(file.path),
ext: file.ext,
type: file.mime,
};
if (request.headers.range && file.file && file.file.size > 0) {
const range = request.headers.range as string;
const parts = range.replace(/bytes=/, '').split('-');
const start = parseInt(parts[0], 10);
let end = parts[1] ? parseInt(parts[1], 10) : file.file.size - 1;
if (end > file.file.size) {
end = file.file.size - 1;
}
const chunksize = end - start + 1;
image = {
data: fs.createReadStream(file.path, {
start,
end,
}),
ext: file.ext,
type: file.mime,
};
reply.header('Content-Range', `bytes ${start}-${end}/${file.file.size}`);
reply.header('Accept-Ranges', 'bytes');
reply.header('Content-Length', chunksize);
} else {
image = {
data: fs.createReadStream(file.path),
ext: file.ext,
type: file.mime,
};
}
}
if ('cleanup' in file) {

95
packages/backend/test/e2e/drive.ts Normal file
View File

@@ -0,0 +1,95 @@
/*
* SPDX-FileCopyrightText: syuilo and other misskey contributors
* SPDX-License-Identifier: AGPL-3.0-only
*/
process.env.NODE_ENV = 'test';
import * as assert from 'assert';
import { MiNote } from '@/models/Note.js';
import { api, initTestDb, makeStreamCatcher, post, signup, uploadFile } from '../utils.js';
import type * as misskey from 'misskey-js';
import type{ Repository } from 'typeorm'
import type { Packed } from '@/misc/json-schema.js';
describe('Drive', () => {
let Notes: Repository<MiNote>;
let alice: misskey.entities.SignupResponse;
let bob: misskey.entities.SignupResponse;
beforeAll(async () => {
const connection = await initTestDb(true);
Notes = connection.getRepository(MiNote);
alice = await signup({ username: 'alice' });
bob = await signup({ username: 'bob' });
}, 1000 * 60 * 2);
test('ファイルURLからアップロードできる', async () => {
// utils.js uploadUrl の処理だがAPIレスポンスも見るためここで同様の処理を書いている
const marker = Math.random().toString();
const url = 'https://raw.githubusercontent.com/misskey-dev/misskey/develop/packages/backend/test/resources/Lenna.jpg'
const catcher = makeStreamCatcher(
alice,
'main',
(msg) => msg.type === 'urlUploadFinished' && msg.body.marker === marker,
(msg) => msg.body.file as Packed<'DriveFile'>,
10 * 1000);
const res = await api('drive/files/upload-from-url', {
url,
marker,
force: true,
}, alice);
const file = await catcher;
assert.strictEqual(res.status, 204);
assert.strictEqual(file.name, 'Lenna.jpg');
assert.strictEqual(file.type, 'image/jpeg');
})
test('ローカルからアップロードできる', async () => {
// APIレスポンスを直接使用するので utils.js uploadFile が通過することで成功とする
const res = await uploadFile(alice, { path: 'Lenna.jpg', name: 'テスト画像' });
assert.strictEqual(res.body?.name, 'テスト画像.jpg');
assert.strictEqual(res.body?.type, 'image/jpeg');
})
test('添付ノート一覧を取得できる', async () => {
const ids = (await Promise.all([uploadFile(alice), uploadFile(alice), uploadFile(alice)])).map(elm => elm.body!.id)
const note0 = await post(alice, { fileIds: [ids[0]] });
const note1 = await post(alice, { fileIds: [ids[0], ids[1]] });
const attached0 = await api('drive/files/attached-notes', { fileId: ids[0] }, alice);
assert.strictEqual(attached0.body.length, 2);
assert.strictEqual(attached0.body[0].id, note1.id)
assert.strictEqual(attached0.body[1].id, note0.id)
const attached1 = await api('drive/files/attached-notes', { fileId: ids[1] }, alice);
assert.strictEqual(attached1.body.length, 1);
assert.strictEqual(attached1.body[0].id, note1.id)
const attached2 = await api('drive/files/attached-notes', { fileId: ids[2] }, alice);
assert.strictEqual(attached2.body.length, 0)
})
test('添付ノート一覧は他の人から見えない', async () => {
const file = await uploadFile(alice);
await post(alice, { fileIds: [file.body!.id] });
const res = await api('drive/files/attached-notes', { fileId: file.body!.id }, bob);
assert.strictEqual(res.status, 400);
assert.strictEqual('error' in res.body, true);
})
});

63
packages/backend/test/utils.ts
View File

@@ -16,6 +16,7 @@ import { DEFAULT_POLICIES } from '@/core/RoleService.js';
import { entities } from '../src/postgres.js';
import { loadConfig } from '../src/config.js';
import type * as misskey from 'misskey-js';
import { Packed } from '@/misc/json-schema.js';
export { server as startServer, jobQueue as startJobQueue } from '@/boot/common.js';
@@ -114,6 +115,20 @@ export function randomString(chars = 'abcdefghijklmnopqrstuvwxyz0123456789', len
return randomString;
}
/**
* @brief プロミスにタイムアウト追加
* @param p 待ち対象プロミス
* @param timeout 待機ミリ秒
*/
function timeoutPromise<T>(p: Promise<T>, timeout: number): Promise<T> {
return Promise.race([
p,
new Promise((reject) =>{
setTimeout(() => { reject(new Error('timed out')); }, timeout)
}) as never
]);
}
export const signup = async (params?: Partial<misskey.Endpoints['signup']['req']>): Promise<NonNullable<misskey.Endpoints['signup']['res']>> => {
const q = Object.assign({
username: randomString(),
@@ -320,17 +335,16 @@ export const uploadFile = async (user?: UserToken, { path, name, blob }: UploadO
};
};
export const uploadUrl = async (user: UserToken, url: string) => {
let resolve: unknown;
const file = new Promise(ok => resolve = ok);
export const uploadUrl = async (user: UserToken, url: string): Promise<Packed<'DriveFile'>> => {
const marker = Math.random().toString();
const ws = await connectStream(user, 'main', (msg) => {
if (msg.type === 'urlUploadFinished' && msg.body.marker === marker) {
ws.close();
resolve(msg.body.file);
}
});
const catcher = makeStreamCatcher(
user,
'main',
(msg) => msg.type === 'urlUploadFinished' && msg.body.marker === marker,
(msg) => msg.body.file as Packed<'DriveFile'>,
60 * 1000
);
await api('drive/files/upload-from-url', {
url,
@@ -338,7 +352,7 @@ export const uploadUrl = async (user: UserToken, url: string) => {
force: true,
}, user);
return file;
return catcher;
};
export function connectStream(user: UserToken, channel: string, listener: (message: Record<string, any>) => any, params?: any): Promise<WebSocket> {
@@ -410,6 +424,35 @@ export const waitFire = async (user: UserToken, channel: string, trgr: () => any
});
};
/**
* @brief WebSocketストリームから特定条件の通知を拾うプロミスを生成
* @param user ユーザー認証情報
* @param channel チャンネル
* @param cond 条件
* @param extractor 取り出し処理
* @param timeout ミリ秒タイムアウト
* @returns 時間内に正常に処理できた場合に通知からextractorを通した値を得る
*/
export function makeStreamCatcher<T>(
user: UserToken,
channel: string,
cond: (message: Record<string, any>) => boolean,
extractor: (message: Record<string, any>) => T,
timeout = 60 * 1000): Promise<T> {
let ws: WebSocket
const p = new Promise<T>(async (resolve) => {
ws = await connectStream(user, channel, (msg) => {
if (cond(msg)) {
resolve(extractor(msg))
}
});
}).finally(() => {
ws?.close();
});
return timeoutPromise(p, timeout);
}
export type SimpleGetResponse = {
status: number,
body: any | JSDOM | null,