Use PostgreSQL instead of MongoDB (#4572)

* wip

* Update note.ts

* Update timeline.ts

* Update core.ts

* wip

* Update generate-visibility-query.ts

* wip

* wip

* wip

* wip

* wip

* Update global-timeline.ts

* wip

* wip

* wip

* Update vote.ts

* wip

* wip

* Update create.ts

* wip

* wip

* wip

* wip

* wip

* wip

* wip

* wip

* wip

* wip

* wip

* wip

* Update files.ts

* wip

* wip

* Update CONTRIBUTING.md

* wip

* wip

* wip

* wip

* wip

* wip

* wip

* wip

* Update read-notification.ts

* wip

* wip

* wip

* wip

* wip

* wip

* wip

* Update cancel.ts

* wip

* wip

* wip

* Update show.ts

* wip

* wip

* Update gen-id.ts

* Update create.ts

* Update id.ts

* wip

* wip

* wip

* wip

* wip

* wip

* wip

* Docker: Update files about Docker (#4599)

* Docker: Use cache if files used by `yarn install` was not updated

This patch reduces the number of times to installing node_modules.
For example, `yarn install` step will be skipped when only ".config/default.yml" is updated.

* Docker: Migrate MongoDB to Postgresql

Misskey uses Postgresql as a database instead of Mongodb since version 11.

* Docker: Uncomment about data persistence

This patch will save a lot of databases.

* wip

* wip

* wip

* Update activitypub.ts

* wip

* wip

* wip

* Update logs.ts

* wip

* Update drive-file.ts

* Update register.ts

* wip

* wip

* Update mentions.ts

* wip

* wip

* wip

* Update recommendation.ts

* wip

* Update index.ts

* wip

* Update recommendation.ts

* Doc: Update docker.ja.md and docker.en.md (#1) (#4608)

Update how to set up misskey.

* wip

* ✌️

* wip

* Update note.ts

* Update postgre.ts

* wip

* wip

* wip

* wip

* Update add-file.ts

* wip

* wip

* wip

* Clean up

* Update logs.ts

* wip

* 🍕

* wip

* Ad notes

* wip

* Update api-visibility.ts

* Update note.ts

* Update add-file.ts

* tests

* tests

* Update postgre.ts

* Update utils.ts

* wip

* wip

* Refactor

* wip

* Refactor

* wip

* wip

* Update show-users.ts

* Update update-instance.ts

* wip

* Update feed.ts

* Update outbox.ts

* Update outbox.ts

* Update user.ts

* wip

* Update list.ts

* Update update-hashtag.ts

* wip

* Update update-hashtag.ts

* Refactor

* Update update.ts

* wip

* wip

* ✌️

* clean up

* docs

* Update push.ts

* wip

* Update api.ts

* wip

* ✌️

* Update make-pagination-query.ts

* ✌️

* Delete hashtags.ts

* Update instances.ts

* Update instances.ts

* Update create.ts

* Update search.ts

* Update reversi-game.ts

* Update signup.ts

* Update user.ts

* id

* Update example.yml

* 🎨

* objectid

* fix

* reversi

* reversi

* Fix bug of chart engine

* Add test of chart engine

* Improve test

* Better testing

* Improve chart engine

* Refactor

* Add test of chart engine

* Refactor

* Add chart test

* Fix bug

* コミットし忘れ

* Refactoring

* ✌️

* Add tests

* Add test

* Extarct note tests

* Refactor

* 存在しないユーザーにメンションできなくなっていた問題を修正

* Fix bug

* Update update-meta.ts

* Fix bug

* Update mention.vue

* Fix bug

* Update meta.ts

* Update CONTRIBUTING.md

* Fix bug

* Fix bug

* Fix bug

* Clean up

* Clean up

* Update notification.ts

* Clean up

* Add mute tests

* Add test

* Refactor

* Add test

* Fix test

* Refactor

* Refactor

* Add tests

* Update utils.ts

* Update utils.ts

* Fix test

* Update package.json

* Update update.ts

* Update manifest.ts

* Fix bug

* Fix bug

* Add test

* 🎨

* Update endpoint permissions

* Updaye permisison

* Update person.ts

#4299

* データベースと同期しないように

* Fix bug

* Fix bug

* Update reversi-game.ts

* Use a feature of Node v11.7.0 to extract a public key (#4644)

* wip

* wip

* ✌️

* Refactoring

#1540

* test

* test

* test

* test

* test

* test

* test

* Fix bug

* Fix test

* 🍣

* wip

* #4471

* Add test for #4335

* Refactor

* Fix test

* Add tests

* 🕓

* Fix bug

* Add test

* Add test

* rename

* Fix bug

src/server/api/endpoints/drive/files/attached-notes.ts

@@ -1,9 +1,8 @@
 import $ from 'cafy';
-import ID, { transform } from '../../../../../misc/cafy-id';
-import DriveFile from '../../../../../models/drive-file';
+import { ID } from '../../../../../misc/cafy-id';
 import define from '../../../define';
-import { packMany } from '../../../../../models/note';
 import { ApiError } from '../../../error';
+import { DriveFiles } from '../../../../../models';
 
 export const meta = {
 	stability: 'stable',
@@ -17,12 +16,11 @@ export const meta = {
 
 	requireCredential: true,
 
-	kind: 'drive-read',
+	kind: 'read:drive',
 
 	params: {
 		fileId: {
 			validator: $.type(ID),
-			transform: transform,
 			desc: {
 				'ja-JP': '対象のファイルID',
 				'en-US': 'Target file ID'
@@ -48,18 +46,17 @@ export const meta = {
 
 export default define(meta, async (ps, user) => {
 	// Fetch file
-	const file = await DriveFile
-		.findOne({
-			_id: ps.fileId,
-			'metadata.userId': user._id,
-			'metadata.deletedAt': { $exists: false }
-		});
+	const file = await DriveFiles.findOne({
+		id: ps.fileId,
+		userId: user.id,
+	});
 
-	if (file === null) {
+	if (file == null) {
 		throw new ApiError(meta.errors.noSuchFile);
 	}
 
+	/* v11 TODO
 	return await packMany(file.metadata.attachedNoteIds || [], user, {
 		detail: true
-	});
+	});*/
 });

src/server/api/endpoints/drive/files/check-existence.ts

@@ -1,6 +1,6 @@
 import $ from 'cafy';
-import DriveFile, { pack } from '../../../../../models/drive-file';
 import define from '../../../define';
+import { DriveFiles } from '../../../../../models';
 
 export const meta = {
 	desc: {
@@ -12,7 +12,7 @@ export const meta = {
 
 	requireCredential: true,
 
-	kind: 'drive-read',
+	kind: 'read:drive',
 
 	params: {
 		md5: {
@@ -29,11 +29,12 @@ export const meta = {
 };
 
 export default define(meta, async (ps, user) => {
-	const file = await DriveFile.findOne({
+	const file = await DriveFiles.findOne({
 		md5: ps.md5,
-		'metadata.userId': user._id,
-		'metadata.deletedAt': { $exists: false }
+		userId: user.id,
 	});
 
-	return { file: file ? await pack(file, { self: true }) : null };
+	return {
+		file: file ? await DriveFiles.pack(file, { self: true }) : null
+	};
 });

src/server/api/endpoints/drive/files/create.ts

@@ -1,11 +1,11 @@
 import * as ms from 'ms';
 import $ from 'cafy';
-import ID, { transform } from '../../../../../misc/cafy-id';
-import { validateFileName, pack } from '../../../../../models/drive-file';
+import { ID } from '../../../../../misc/cafy-id';
 import create from '../../../../../services/drive/add-file';
 import define from '../../../define';
 import { apiLogger } from '../../../logger';
 import { ApiError } from '../../../error';
+import { DriveFiles } from '../../../../../models';
 
 export const meta = {
 	desc: {
@@ -24,12 +24,11 @@ export const meta = {
 
 	requireFile: true,
 
-	kind: 'drive-write',
+	kind: 'write:drive',
 
 	params: {
 		folderId: {
 			validator: $.optional.nullable.type(ID),
-			transform: transform,
 			default: null as any,
 			desc: {
 				'ja-JP': 'フォルダID'
@@ -78,7 +77,7 @@ export default define(meta, async (ps, user, app, file, cleanup) => {
 			name = null;
 		} else if (name === 'blob') {
 			name = null;
-		} else if (!validateFileName(name)) {
+		} else if (!DriveFiles.validateFileName(name)) {
 			throw new ApiError(meta.errors.invalidFileName);
 		}
 	} else {
@@ -88,7 +87,7 @@ export default define(meta, async (ps, user, app, file, cleanup) => {
 	try {
 		// Create file
 		const driveFile = await create(user, file.path, name, null, ps.folderId, ps.force, false, null, null, ps.isSensitive);
-		return pack(driveFile, { self: true });
+		return DriveFiles.pack(driveFile, { self: true });
 	} catch (e) {
 		apiLogger.error(e);
 		throw new ApiError();

src/server/api/endpoints/drive/files/delete.ts

@@ -1,10 +1,10 @@
 import $ from 'cafy';
-import ID, { transform } from '../../../../../misc/cafy-id';
-import DriveFile from '../../../../../models/drive-file';
+import { ID } from '../../../../../misc/cafy-id';
 import del from '../../../../../services/drive/delete-file';
 import { publishDriveStream } from '../../../../../services/stream';
 import define from '../../../define';
 import { ApiError } from '../../../error';
+import { DriveFiles } from '../../../../../models';
 
 export const meta = {
 	stability: 'stable',
@@ -18,12 +18,11 @@ export const meta = {
 
 	requireCredential: true,
 
-	kind: 'drive-write',
+	kind: 'write:drive',
 
 	params: {
 		fileId: {
 			validator: $.type(ID),
-			transform: transform,
 			desc: {
 				'ja-JP': '対象のファイルID',
 				'en-US': 'Target file ID'
@@ -47,17 +46,13 @@ export const meta = {
 };
 
 export default define(meta, async (ps, user) => {
-	// Fetch file
-	const file = await DriveFile
-		.findOne({
-			_id: ps.fileId
-		});
+	const file = await DriveFiles.findOne(ps.fileId);
 
-	if (file === null) {
+	if (file == null) {
 		throw new ApiError(meta.errors.noSuchFile);
 	}
 
-	if (!user.isAdmin && !user.isModerator && !file.metadata.userId.equals(user._id)) {
+	if (!user.isAdmin && !user.isModerator && (file.userId !== user.id)) {
 		throw new ApiError(meta.errors.accessDenied);
 	}
 
@@ -65,7 +60,5 @@ export default define(meta, async (ps, user) => {
 	await del(file);
 
 	// Publish fileDeleted event
-	publishDriveStream(user._id, 'fileDeleted', file._id);
-
-	return;
+	publishDriveStream(user.id, 'fileDeleted', file.id);
 });

src/server/api/endpoints/drive/files/find.ts

@@ -1,14 +1,14 @@
 import $ from 'cafy';
-import ID, { transform } from '../../../../../misc/cafy-id';
-import DriveFile, { pack } from '../../../../../models/drive-file';
+import { ID } from '../../../../../misc/cafy-id';
 import define from '../../../define';
+import { DriveFiles } from '../../../../../models';
 
 export const meta = {
 	requireCredential: true,
 
 	tags: ['drive'],
 
-	kind: 'drive-read',
+	kind: 'read:drive',
 
 	params: {
 		name: {
@@ -17,7 +17,6 @@ export const meta = {
 
 		folderId: {
 			validator: $.optional.nullable.type(ID),
-			transform: transform,
 			default: null as any,
 			desc: {
 				'ja-JP': 'フォルダID'
@@ -27,12 +26,11 @@ export const meta = {
 };
 
 export default define(meta, async (ps, user) => {
-	const files = await DriveFile
-		.find({
-			filename: ps.name,
-			'metadata.userId': user._id,
-			'metadata.folderId': ps.folderId
-		});
+	const files = await DriveFiles.find({
+		name: ps.name,
+		userId: user.id,
+		folderId: ps.folderId
+	});
 
-	return await Promise.all(files.map(file => pack(file, { self: true })));
+	return await Promise.all(files.map(file => DriveFiles.pack(file, { self: true })));
 });

src/server/api/endpoints/drive/files/show.ts

@@ -1,10 +1,9 @@
 import $ from 'cafy';
-import * as mongo from 'mongodb';
-import ID, { transform } from '../../../../../misc/cafy-id';
-import DriveFile, { pack, IDriveFile } from '../../../../../models/drive-file';
+import { ID } from '../../../../../misc/cafy-id';
 import define from '../../../define';
-import config from '../../../../../config';
 import { ApiError } from '../../../error';
+import { DriveFile } from '../../../../../models/entities/drive-file';
+import { DriveFiles } from '../../../../../models';
 
 export const meta = {
 	stability: 'stable',
@@ -18,12 +17,11 @@ export const meta = {
 
 	requireCredential: true,
 
-	kind: 'drive-read',
+	kind: 'read:drive',
 
 	params: {
 		fileId: {
 			validator: $.optional.type(ID),
-			transform: transform,
 			desc: {
 				'ja-JP': '対象のファイルID',
 				'en-US': 'Target file ID'
@@ -65,49 +63,33 @@ export const meta = {
 };
 
 export default define(meta, async (ps, user) => {
-	let file: IDriveFile;
+	let file: DriveFile;
 
 	if (ps.fileId) {
-		file = await DriveFile.findOne({
-			_id: ps.fileId,
-			'metadata.deletedAt': { $exists: false }
-		});
+		file = await DriveFiles.findOne(ps.fileId);
 	} else if (ps.url) {
-		const isInternalStorageUrl = ps.url.startsWith(config.driveUrl);
-		if (isInternalStorageUrl) {
-			// Extract file ID from url
-			// e.g.
-			// http://misskey.local/files/foo?original=bar --> foo
-			const fileId = new mongo.ObjectID(ps.url.replace(config.driveUrl, '').replace(/\?(.*)$/, '').replace(/\//g, ''));
-			file = await DriveFile.findOne({
-				_id: fileId,
-				'metadata.deletedAt': { $exists: false }
-			});
-		} else {
-			file = await DriveFile.findOne({
-				$or: [{
-					'metadata.url': ps.url
-				}, {
-					'metadata.webpublicUrl': ps.url
-				}, {
-					'metadata.thumbnailUrl': ps.url
-				}],
-				'metadata.deletedAt': { $exists: false }
-			});
-		}
+		file = await DriveFiles.findOne({
+			where: [{
+				url: ps.url
+			}, {
+				webpublicUrl: ps.url
+			}, {
+				thumbnailUrl: ps.url
+			}],
+		});
 	} else {
 		throw new ApiError(meta.errors.fileIdOrUrlRequired);
 	}
 
-	if (!user.isAdmin && !user.isModerator && !file.metadata.userId.equals(user._id)) {
+	if (!user.isAdmin && !user.isModerator && (file.userId !== user.id)) {
 		throw new ApiError(meta.errors.accessDenied);
 	}
 
-	if (file === null) {
+	if (file == null) {
 		throw new ApiError(meta.errors.noSuchFile);
 	}
 
-	return await pack(file, {
+	return await DriveFiles.pack(file, {
 		detail: true,
 		self: true
 	});

src/server/api/endpoints/drive/files/update.ts

@@ -1,11 +1,9 @@
 import $ from 'cafy';
-import ID, { transform } from '../../../../../misc/cafy-id';
-import DriveFolder from '../../../../../models/drive-folder';
-import DriveFile, { validateFileName, pack } from '../../../../../models/drive-file';
+import { ID } from '../../../../../misc/cafy-id';
 import { publishDriveStream } from '../../../../../services/stream';
 import define from '../../../define';
-import Note from '../../../../../models/note';
 import { ApiError } from '../../../error';
+import { DriveFiles, DriveFolders } from '../../../../../models';
 
 export const meta = {
 	desc: {
@@ -17,12 +15,11 @@ export const meta = {
 
 	requireCredential: true,
 
-	kind: 'drive-write',
+	kind: 'write:drive',
 
 	params: {
 		fileId: {
 			validator: $.type(ID),
-			transform: transform,
 			desc: {
 				'ja-JP': '対象のファイルID'
 			}
@@ -30,7 +27,6 @@ export const meta = {
 
 		folderId: {
 			validator: $.optional.nullable.type(ID),
-			transform: transform,
 			default: undefined as any,
 			desc: {
 				'ja-JP': 'フォルダID'
@@ -38,7 +34,7 @@ export const meta = {
 		},
 
 		name: {
-			validator: $.optional.str.pipe(validateFileName),
+			validator: $.optional.str.pipe(DriveFiles.validateFileName),
 			default: undefined as any,
 			desc: {
 				'ja-JP': 'ファイル名',
@@ -78,69 +74,47 @@ export const meta = {
 };
 
 export default define(meta, async (ps, user) => {
-	// Fetch file
-	const file = await DriveFile
-		.findOne({
-			_id: ps.fileId
-		});
+	const file = await DriveFiles.findOne(ps.fileId);
 
-	if (file === null) {
+	if (file == null) {
 		throw new ApiError(meta.errors.noSuchFile);
 	}
 
-	if (!user.isAdmin && !user.isModerator && !file.metadata.userId.equals(user._id)) {
+	if (!user.isAdmin && !user.isModerator && (file.userId !== user.id)) {
 		throw new ApiError(meta.errors.accessDenied);
 	}
 
-	if (ps.name) file.filename = ps.name;
+	if (ps.name) file.name = ps.name;
 
-	if (ps.isSensitive !== undefined) file.metadata.isSensitive = ps.isSensitive;
+	if (ps.isSensitive !== undefined) file.isSensitive = ps.isSensitive;
 
 	if (ps.folderId !== undefined) {
 		if (ps.folderId === null) {
-			file.metadata.folderId = null;
+			file.folderId = null;
 		} else {
-			// Fetch folder
-			const folder = await DriveFolder
-				.findOne({
-					_id: ps.folderId,
-					userId: user._id
-				});
+			const folder = await DriveFolders.findOne({
+				id: ps.folderId,
+				userId: user.id
+			});
 
-			if (folder === null) {
+			if (folder == null) {
 				throw new ApiError(meta.errors.noSuchFolder);
 			}
 
-			file.metadata.folderId = folder._id;
+			file.folderId = folder.id;
 		}
 	}
 
-	await DriveFile.update(file._id, {
-		$set: {
-			filename: file.filename,
-			'metadata.folderId': file.metadata.folderId,
-			'metadata.isSensitive': file.metadata.isSensitive
-		}
+	await DriveFiles.update(file.id, {
+		name: file.name,
+		folderId: file.folderId,
+		isSensitive: file.isSensitive
 	});
 
-	// ドライブのファイルが非正規化されているドキュメントも更新
-	Note.find({
-		'_files._id': file._id
-	}).then(notes => {
-		for (const note of notes) {
-			note._files[note._files.findIndex(f => f._id.equals(file._id))] = file;
-			Note.update({ _id: note._id }, {
-				$set: {
-					_files: note._files
-				}
-			});
-		}
-	});
-
-	const fileObj = await pack(file, { self: true });
+	const fileObj = await DriveFiles.pack(file, { self: true });
 
 	// Publish fileUpdated event
-	publishDriveStream(user._id, 'fileUpdated', fileObj);
+	publishDriveStream(user.id, 'fileUpdated', fileObj);
 
 	return fileObj;
 });

src/server/api/endpoints/drive/files/upload-from-url.ts

@@ -1,9 +1,9 @@
 import $ from 'cafy';
-import ID, { transform } from '../../../../../misc/cafy-id';
+import { ID } from '../../../../../misc/cafy-id';
 import * as ms from 'ms';
-import { pack } from '../../../../../models/drive-file';
 import uploadFromUrl from '../../../../../services/drive/upload-from-url';
 import define from '../../../define';
+import { DriveFiles } from '../../../../../models';
 
 export const meta = {
 	desc: {
@@ -19,7 +19,7 @@ export const meta = {
 
 	requireCredential: true,
 
-	kind: 'drive-write',
+	kind: 'write:drive',
 
 	params: {
 		url: {
@@ -30,7 +30,6 @@ export const meta = {
 		folderId: {
 			validator: $.optional.nullable.type(ID),
 			default: null as any,
-			transform: transform
 		},
 
 		isSensitive: {
@@ -53,5 +52,5 @@ export const meta = {
 };
 
 export default define(meta, async (ps, user) => {
-	return await pack(await uploadFromUrl(ps.url, user, ps.folderId, null, ps.isSensitive, ps.force), { self: true });
+	return await DriveFiles.pack(await uploadFromUrl(ps.url, user, ps.folderId, null, ps.isSensitive, ps.force), { self: true });
 });