10.9.0

docs/setup.ja.md

@@ -27,9 +27,14 @@ adduser --disabled-password --disabled-login misskey
 
 ##### オプション
 * [Redis](https://redis.io/)
-  * Redisはオプションですが、インストールすることを強く推奨します。
-	* インストールしなくていいのは、あなたのインスタンスが自分専用のときだけです
-* [Elasticsearch](https://www.elastic.co/) - 検索機能を向上させるために用います。
+	* Redisはオプションですが、インストールすることを強く推奨します。
+	* インストールしなくていいのは、あなたのインスタンスが自分専用のときだけとお考えください。
+	* 具体的には、Redisをインストールしないと、次の事が出来なくなります:
+		* Misskeyプロセスを複数起動しての負荷分散
+		* レートリミット
+		* Twitter連携
+* [Elasticsearch](https://www.elastic.co/)
+	* 検索機能を有効にするためにはインストールが必要です。
 
 *3.* MongoDBの設定
 ----------------------------------------------------------------

package.json

@@ -1,8 +1,8 @@
 {
 	"name": "misskey",
 	"author": "syuilo <i@syuilo.com>",
-	"version": "10.7.1",
-	"clientVersion": "1.0.10417",
+	"version": "10.9.0",
+	"clientVersion": "1.0.10443",
 	"codename": "nighthike",
 	"main": "./built/index.js",
 	"private": true,
@@ -169,6 +169,7 @@
 		"parse5": "5.1.0",
 		"portscanner": "2.2.0",
 		"progress-bar-webpack-plugin": "1.11.0",
+		"promise-limit": "2.7.0",
 		"promise-sequential": "1.1.1",
 		"pug": "2.0.3",
 		"punycode": "2.1.1",

src/client/app/common/scripts/stream.ts

@@ -10,7 +10,6 @@ import MiOS from '../../mios';
 export default class Stream extends EventEmitter {
 	private stream: ReconnectingWebsocket;
 	private state: string;
-	private buffer: any[];
 	private sharedConnectionPools: Pool[] = [];
 	private sharedConnections: SharedConnection[] = [];
 	private nonSharedConnections: NonSharedConnection[] = [];
@@ -19,7 +18,6 @@ export default class Stream extends EventEmitter {
 		super();
 
 		this.state = 'initializing';
-		this.buffer = [];
 
 		const user = os.store.state.i;
 
@@ -70,17 +68,12 @@ export default class Stream extends EventEmitter {
 		this.state = 'connected';
 		this.emit('_connected_');
 
-		// バッファーを処理
-		const _buffer = [].concat(this.buffer); // Shallow copy
-		this.buffer = []; // Clear buffer
-		_buffer.forEach(data => {
-			this.send(data); // Resend each buffered messages
-		});
-
 		// チャンネル再接続
 		if (isReconnect) {
 			this.sharedConnectionPools.forEach(p => {
-				p.connect();
+				if (p.users > 0) {
+					p.connect();
+				}
 			});
 			this.nonSharedConnections.forEach(c => {
 				c.connect();
@@ -133,12 +126,6 @@ export default class Stream extends EventEmitter {
 			body: payload
 		};
 
-		// まだ接続が確立されていなかったらバッファリングして次に接続した時に送信する
-		if (this.state != 'connected') {
-			this.buffer.push(data);
-			return;
-		}
-
 		this.stream.send(JSON.stringify(data));
 	}
 
@@ -156,7 +143,7 @@ class Pool {
 	public channel: string;
 	public id: string;
 	protected stream: Stream;
-	private users = 0;
+	public users = 0;
 	private disposeTimerId: any;
 	private isConnected = false;
 

src/server/api/stream/index.ts

@@ -1,6 +1,5 @@
 import autobind from 'autobind-decorator';
 import * as websocket from 'websocket';
-import Xev from 'xev';
 import * as debug from 'debug';
 
 import User, { IUser } from '../../../models/user';
@@ -11,6 +10,7 @@ import readNote from '../../../services/note/read';
 
 import Channel from './channel';
 import channels from './channels';
+import { EventEmitter } from 'events';
 
 const log = debug('misskey');
 
@@ -21,14 +21,14 @@ export default class Connection {
 	public user?: IUser;
 	public app: IApp;
 	private wsConnection: websocket.connection;
-	public subscriber: Xev;
+	public subscriber: EventEmitter;
 	private channels: Channel[] = [];
 	private subscribingNotes: any = {};
 	public sendMessageToWsOverride: any = null; // 後方互換性のため
 
 	constructor(
 		wsConnection: websocket.connection,
-		subscriber: Xev,
+		subscriber: EventEmitter,
 		user: IUser,
 		app: IApp
 	) {

src/server/api/streaming.ts

@@ -1,11 +1,14 @@
 import * as http from 'http';
 import * as websocket from 'websocket';
+import * as redis from 'redis';
 import Xev from 'xev';
 
 import MainStreamConnection from './stream';
 import { ParsedUrlQuery } from 'querystring';
 import authenticate from './authenticate';
 import channels from './stream/channels';
+import { EventEmitter } from 'events';
+import config from '../../config';
 
 module.exports = (server: http.Server) => {
 	// Init websocket server
@@ -16,11 +19,34 @@ module.exports = (server: http.Server) => {
 	ws.on('request', async (request) => {
 		const connection = request.accept();
 
-		const ev = new Xev();
-
 		const q = request.resourceURL.query as ParsedUrlQuery;
 		const [user, app] = await authenticate(q.i as string);
 
+		let ev: EventEmitter;
+
+		if (config.redis) {
+			// Connect to Redis
+			const subscriber = redis.createClient(
+				config.redis.port, config.redis.host);
+
+			subscriber.subscribe('misskey');
+
+			ev = new EventEmitter();
+
+			subscriber.on('message', async (_, data) => {
+				const obj = JSON.parse(data);
+
+				ev.emit(obj.channel, obj.message);
+			});
+
+			connection.once('close', () => {
+				subscriber.unsubscribe();
+				subscriber.quit();
+			});
+		} else {
+			ev = new Xev();
+		}
+
 		const main = new MainStreamConnection(connection, ev, user, app);
 
 		// 後方互換性のため

src/stream.ts

@@ -1,4 +1,5 @@
 import * as mongo from 'mongodb';
+import redis from './db/redis';
 import Xev from 'xev';
 import Meta, { IMeta } from './models/meta';
 
@@ -9,7 +10,10 @@ class Publisher {
 	private meta: IMeta;
 
 	constructor() {
-		this.ev = new Xev();
+		// Redisがインストールされてないときはプロセス間通信を使う
+		if (redis == null) {
+			this.ev = new Xev();
+		}
 
 		setInterval(async () => {
 			this.meta = await Meta.findOne({});
@@ -28,7 +32,14 @@ class Publisher {
 			{ type: type, body: null } :
 			{ type: type, body: value };
 
-		this.ev.emit(channel, message);
+		if (this.ev) {
+			this.ev.emit(channel, message);
+		} else {
+			redis.publish('misskey', JSON.stringify({
+				channel: channel,
+				message: message
+			}));
+		}
 	}
 
 	public publishMainStream = (userId: ID, type: string, value?: any): void => {

src/tools/move-drive-files.ts

@@ -1,16 +1,19 @@
 import * as Minio from 'minio';
 import * as uuid from 'uuid';
-const sequential = require('promise-sequential');
-import DriveFile, { DriveFileChunk, getDriveFileBucket } from '../models/drive-file';
+import * as promiseLimit from 'promise-limit';
+import DriveFile, { DriveFileChunk, getDriveFileBucket, IDriveFile } from '../models/drive-file';
 import DriveFileThumbnail, { DriveFileThumbnailChunk } from '../models/drive-file-thumbnail';
 import config from '../config';
 
+const limit = promiseLimit(16);
+
 DriveFile.find({
 	$or: [{
 		withoutChunks: { $exists: false }
 	}, {
 		withoutChunks: false
-	}]
+	}],
+	'metadata.deletedAt': { $exists: false }
 }, {
 	fields: {
 		_id: true
@@ -18,58 +21,63 @@ DriveFile.find({
 }).then(async files => {
 	console.log(`there is ${files.length} files`);
 
-	await sequential(files.map(file => async () => {
-		file = await DriveFile.findOne({ _id: file._id });
+	await Promise.all(files.map(file => limit(() => job(file))));
 
-		const minio = new Minio.Client(config.drive.config);
-
-		const keyDir = `${config.drive.prefix}/${uuid.v4()}`;
-		const key = `${keyDir}/${name}`;
-		const thumbnailKeyDir = `${config.drive.prefix}/${uuid.v4()}`;
-		const thumbnailKey = `${thumbnailKeyDir}/${name}.thumbnail.jpg`;
-
-		const baseUrl = config.drive.baseUrl
-			|| `${ config.drive.config.useSSL ? 'https' : 'http' }://${ config.drive.config.endPoint }${ config.drive.config.port ? `:${config.drive.config.port}` : '' }/${ config.drive.bucket }`;
-
-		const bucket = await getDriveFileBucket();
-		const readable = bucket.openDownloadStream(file._id);
-
-		await minio.putObject(config.drive.bucket, key, readable, file.length, {
-			'Content-Type': file.contentType,
-			'Cache-Control': 'max-age=31536000, immutable'
-		});
-
-		await DriveFile.findOneAndUpdate({ _id: file._id }, {
-			$set: {
-				'metadata.withoutChunks': true,
-				'metadata.storage': 'minio',
-				'metadata.storageProps': {
-					key: key,
-					thumbnailKey: thumbnailKey
-				},
-				'metadata.url': `${ baseUrl }/${ keyDir }/${ encodeURIComponent(name) }`,
-			}
-		});
-
-		// チャンクをすべて削除
-		await DriveFileChunk.remove({
-			files_id: file._id
-		});
-
-		//#region サムネイルもあれば削除
-		const thumbnail = await DriveFileThumbnail.findOne({
-			'metadata.originalId': file._id
-		});
-
-		if (thumbnail) {
-			await DriveFileThumbnailChunk.remove({
-				files_id: thumbnail._id
-			});
-
-			await DriveFileThumbnail.remove({ _id: thumbnail._id });
-		}
-		//#endregion
-
-		console.log('done', file._id);
-	}));
+	console.log('ALL DONE');
 });
+
+async function job(file: IDriveFile): Promise<any> {
+	file = await DriveFile.findOne({ _id: file._id });
+
+	const minio = new Minio.Client(config.drive.config);
+
+	const name = file.filename;
+	const keyDir = `${config.drive.prefix}/${uuid.v4()}`;
+	const key = `${keyDir}/${name}`;
+	const thumbnailKeyDir = `${config.drive.prefix}/${uuid.v4()}`;
+	const thumbnailKey = `${thumbnailKeyDir}/${name}.thumbnail.jpg`;
+
+	const baseUrl = config.drive.baseUrl
+		|| `${ config.drive.config.useSSL ? 'https' : 'http' }://${ config.drive.config.endPoint }${ config.drive.config.port ? `:${config.drive.config.port}` : '' }/${ config.drive.bucket }`;
+
+	const bucket = await getDriveFileBucket();
+	const readable = bucket.openDownloadStream(file._id);
+
+	await minio.putObject(config.drive.bucket, key, readable, file.length, {
+		'Content-Type': file.contentType,
+		'Cache-Control': 'max-age=31536000, immutable'
+	});
+
+	await DriveFile.findOneAndUpdate({ _id: file._id }, {
+		$set: {
+			'metadata.withoutChunks': true,
+			'metadata.storage': 'minio',
+			'metadata.storageProps': {
+				key: key,
+				thumbnailKey: thumbnailKey
+			},
+			'metadata.url': `${ baseUrl }/${ keyDir }/${ encodeURIComponent(name) }`,
+		}
+	});
+
+	// チャンクをすべて削除
+	await DriveFileChunk.remove({
+		files_id: file._id
+	});
+
+	//#region サムネイルもあれば削除
+	const thumbnail = await DriveFileThumbnail.findOne({
+		'metadata.originalId': file._id
+	});
+
+	if (thumbnail) {
+		await DriveFileThumbnailChunk.remove({
+			files_id: thumbnail._id
+		});
+
+		await DriveFileThumbnail.remove({ _id: thumbnail._id });
+	}
+	//#endregion
+
+	console.log('done', file._id);
+}