Use PostgreSQL instead of MongoDB (#4572)

* wip

* Update note.ts

* Update timeline.ts

* Update core.ts

* wip

* Update generate-visibility-query.ts

* wip

* wip

* wip

* wip

* wip

* Update global-timeline.ts

* wip

* wip

* wip

* Update vote.ts

* wip

* wip

* Update create.ts

* wip

* wip

* wip

* wip

* wip

* wip

* wip

* wip

* wip

* wip

* wip

* wip

* Update files.ts

* wip

* wip

* Update CONTRIBUTING.md

* wip

* wip

* wip

* wip

* wip

* wip

* wip

* wip

* Update read-notification.ts

* wip

* wip

* wip

* wip

* wip

* wip

* wip

* Update cancel.ts

* wip

* wip

* wip

* Update show.ts

* wip

* wip

* Update gen-id.ts

* Update create.ts

* Update id.ts

* wip

* wip

* wip

* wip

* wip

* wip

* wip

* Docker: Update files about Docker (#4599)

* Docker: Use cache if files used by `yarn install` was not updated

This patch reduces the number of times to installing node_modules.
For example, `yarn install` step will be skipped when only ".config/default.yml" is updated.

* Docker: Migrate MongoDB to Postgresql

Misskey uses Postgresql as a database instead of Mongodb since version 11.

* Docker: Uncomment about data persistence

This patch will save a lot of databases.

* wip

* wip

* wip

* Update activitypub.ts

* wip

* wip

* wip

* Update logs.ts

* wip

* Update drive-file.ts

* Update register.ts

* wip

* wip

* Update mentions.ts

* wip

* wip

* wip

* Update recommendation.ts

* wip

* Update index.ts

* wip

* Update recommendation.ts

* Doc: Update docker.ja.md and docker.en.md (#1) (#4608)

Update how to set up misskey.

* wip

* ✌️

* wip

* Update note.ts

* Update postgre.ts

* wip

* wip

* wip

* wip

* Update add-file.ts

* wip

* wip

* wip

* Clean up

* Update logs.ts

* wip

* 🍕

* wip

* Ad notes

* wip

* Update api-visibility.ts

* Update note.ts

* Update add-file.ts

* tests

* tests

* Update postgre.ts

* Update utils.ts

* wip

* wip

* Refactor

* wip

* Refactor

* wip

* wip

* Update show-users.ts

* Update update-instance.ts

* wip

* Update feed.ts

* Update outbox.ts

* Update outbox.ts

* Update user.ts

* wip

* Update list.ts

* Update update-hashtag.ts

* wip

* Update update-hashtag.ts

* Refactor

* Update update.ts

* wip

* wip

* ✌️

* clean up

* docs

* Update push.ts

* wip

* Update api.ts

* wip

* ✌️

* Update make-pagination-query.ts

* ✌️

* Delete hashtags.ts

* Update instances.ts

* Update instances.ts

* Update create.ts

* Update search.ts

* Update reversi-game.ts

* Update signup.ts

* Update user.ts

* id

* Update example.yml

* 🎨

* objectid

* fix

* reversi

* reversi

* Fix bug of chart engine

* Add test of chart engine

* Improve test

* Better testing

* Improve chart engine

* Refactor

* Add test of chart engine

* Refactor

* Add chart test

* Fix bug

* コミットし忘れ

* Refactoring

* ✌️

* Add tests

* Add test

* Extarct note tests

* Refactor

* 存在しないユーザーにメンションできなくなっていた問題を修正

* Fix bug

* Update update-meta.ts

* Fix bug

* Update mention.vue

* Fix bug

* Update meta.ts

* Update CONTRIBUTING.md

* Fix bug

* Fix bug

* Fix bug

* Clean up

* Clean up

* Update notification.ts

* Clean up

* Add mute tests

* Add test

* Refactor

* Add test

* Fix test

* Refactor

* Refactor

* Add tests

* Update utils.ts

* Update utils.ts

* Fix test

* Update package.json

* Update update.ts

* Update manifest.ts

* Fix bug

* Fix bug

* Add test

* 🎨

* Update endpoint permissions

* Updaye permisison

* Update person.ts

#4299

* データベースと同期しないように

* Fix bug

* Fix bug

* Update reversi-game.ts

* Use a feature of Node v11.7.0 to extract a public key (#4644)

* wip

* wip

* ✌️

* Refactoring

#1540

* test

* test

* test

* test

* test

* test

* test

* Fix bug

* Fix test

* 🍣

* wip

* #4471

* Add test for #4335

* Refactor

* Fix test

* Add tests

* 🕓

* Fix bug

* Add test

* Add test

* rename

* Fix bug

src/server/api/common/generate-mute-query.ts

@@ -0,0 +1,36 @@
+import { User } from '../../../models/entities/user';
+import { Mutings } from '../../../models';
+import { SelectQueryBuilder, Brackets } from 'typeorm';
+
+export function generateMuteQuery(q: SelectQueryBuilder<any>, me: User) {
+	const mutingQuery = Mutings.createQueryBuilder('muting')
+		.select('muting.muteeId')
+		.where('muting.muterId = :muterId', { muterId: me.id });
+
+	// 投稿の作者をミュートしていない かつ
+	// 投稿の返信先の作者をミュートしていない かつ
+	// 投稿の引用元の作者をミュートしていない
+	q
+		.andWhere(`note.userId NOT IN (${ mutingQuery.getQuery() })`)
+		.andWhere(new Brackets(qb => { qb
+			.where(`note.replyUserId IS NULL`)
+			.orWhere(`note.replyUserId NOT IN (${ mutingQuery.getQuery() })`);
+		}))
+		.andWhere(new Brackets(qb => { qb
+			.where(`note.renoteUserId IS NULL`)
+			.orWhere(`note.renoteUserId NOT IN (${ mutingQuery.getQuery() })`);
+		}));
+
+	q.setParameters(mutingQuery.getParameters());
+}
+
+export function generateMuteQueryForUsers(q: SelectQueryBuilder<any>, me: User) {
+	const mutingQuery = Mutings.createQueryBuilder('muting')
+		.select('muting.muteeId')
+		.where('muting.muterId = :muterId', { muterId: me.id });
+
+	q
+		.andWhere(`user.id NOT IN (${ mutingQuery.getQuery() })`);
+
+	q.setParameters(mutingQuery.getParameters());
+}

src/server/api/common/generate-native-user-token.ts

@@ -1,3 +1,3 @@
 import rndstr from 'rndstr';
 
-export default () => `!${rndstr('a-zA-Z0-9', 32)}`;
+export default () => `!${rndstr('a-zA-Z0-9', 31)}`;

src/server/api/common/generate-visibility-query.ts

@@ -0,0 +1,40 @@
+import { User } from '../../../models/entities/user';
+import { Followings } from '../../../models';
+import { Brackets, SelectQueryBuilder } from 'typeorm';
+
+export function generateVisibilityQuery(q: SelectQueryBuilder<any>, me?: User) {
+	if (me == null) {
+		q.andWhere(new Brackets(qb => { qb
+			.where(`note.visibility = 'public'`)
+			.orWhere(`note.visibility = 'home'`);
+		}));
+	} else {
+		const followingQuery = Followings.createQueryBuilder('following')
+			.select('following.followeeId')
+			.where('following.followerId = :followerId', { followerId: me.id });
+
+		q.andWhere(new Brackets(qb => { qb
+			// 公開投稿である
+			.where(new Brackets(qb => { qb
+				.where(`note.visibility = 'public'`)
+				.orWhere(`note.visibility = 'home'`);
+			}))
+			// または 自分自身
+			.orWhere('note.userId = :userId1', { userId1: me.id })
+			// または 自分宛て
+			.orWhere(':userId2 = ANY(note.visibleUserIds)', { userId2: me.id })
+			.orWhere(new Brackets(qb => { qb
+				// または フォロワー宛ての投稿であり、
+				.where('note.visibility = \'followers\'')
+				.andWhere(new Brackets(qb => { qb
+					// 自分がフォロワーである
+					.where(`note.userId IN (${ followingQuery.getQuery() })`)
+					// または 自分の投稿へのリプライ
+					.orWhere('note.replyUserId = :userId3', { userId3: me.id });
+				}));
+			}));
+		}));
+
+		q.setParameters(followingQuery.getParameters());
+	}
+}

src/server/api/common/get-friends.ts

@@ -1,49 +0,0 @@
-import * as mongodb from 'mongodb';
-import Following from '../../../models/following';
-
-export const getFriendIds = async (me: mongodb.ObjectID, includeMe = true) => {
-	// Fetch relation to other users who the I follows
-	// SELECT followee
-	const followings = await Following
-		.find({
-			followerId: me
-		}, {
-			fields: {
-				followeeId: true
-			}
-		});
-
-	// ID list of other users who the I follows
-	const myfollowingIds = followings.map(following => following.followeeId);
-
-	if (includeMe) {
-		myfollowingIds.push(me);
-	}
-
-	return myfollowingIds;
-};
-
-export const getFriends = async (me: mongodb.ObjectID, includeMe = true, remoteOnly = false) => {
-	const q: any = remoteOnly ? {
-		followerId: me,
-		'_followee.host': { $ne: null }
-	} : {
-		followerId: me
-	};
-	// Fetch relation to other users who the I follows
-	const followings = await Following
-		.find(q);
-
-	// ID list of other users who the I follows
-	const myfollowings = followings.map(following => ({
-		id: following.followeeId
-	}));
-
-	if (includeMe) {
-		myfollowings.push({
-			id: me
-		});
-	}
-
-	return myfollowings;
-};

src/server/api/common/get-hide-users.ts

@@ -1,25 +0,0 @@
-import * as mongo from 'mongodb';
-import Mute from '../../../models/mute';
-import User, { IUser } from '../../../models/user';
-import { unique } from '../../../prelude/array';
-
-export async function getHideUserIds(me: IUser) {
-	return await getHideUserIdsById(me ? me._id : null);
-}
-
-export async function getHideUserIdsById(meId?: mongo.ObjectID) {
-	const [suspended, muted] = await Promise.all([
-		User.find({
-			isSuspended: true
-		}, {
-			fields: {
-				_id: true
-			}
-		}),
-		meId ? Mute.find({
-			muterId: meId
-		}) : Promise.resolve([])
-	]);
-
-	return unique(suspended.map(user => user._id).concat(muted.map(mute => mute.muteeId)));
-}

src/server/api/common/getters.ts

@@ -1,18 +1,15 @@
-import * as mongo from 'mongodb';
-import Note from '../../../models/note';
-import User, { isRemoteUser, isLocalUser } from '../../../models/user';
 import { IdentifiableError } from '../../../misc/identifiable-error';
+import { User } from '../../../models/entities/user';
+import { Note } from '../../../models/entities/note';
+import { Notes, Users } from '../../../models';
 
 /**
  * Get note for API processing
  */
-export async function getNote(noteId: mongo.ObjectID) {
-	const note = await Note.findOne({
-		_id: noteId,
-		deletedAt: { $exists: false }
-	});
+export async function getNote(noteId: Note['id']) {
+	const note = await Notes.findOne(noteId);
 
-	if (note === null) {
+	if (note == null) {
 		throw new IdentifiableError('9725d0ce-ba28-4dde-95a7-2cbb2c15de24', 'No such note.');
 	}
 
@@ -22,23 +19,10 @@ export async function getNote(noteId: mongo.ObjectID) {
 /**
  * Get user for API processing
  */
-export async function getUser(userId: mongo.ObjectID) {
-	const user = await User.findOne({
-		_id: userId,
-		$or: [{
-			isDeleted: { $exists: false }
-		}, {
-			isDeleted: false
-		}]
-	}, {
-		fields: {
-			data: false,
-			profile: false,
-			clientSettings: false
-		}
-	});
+export async function getUser(userId: User['id']) {
+	const user = await Users.findOne(userId);
 
-	if (user === null) {
+	if (user == null) {
 		throw new IdentifiableError('15348ddd-432d-49c2-8a5a-8069753becff', 'No such user.');
 	}
 
@@ -48,10 +32,10 @@ export async function getUser(userId: mongo.ObjectID) {
 /**
  * Get remote user for API processing
  */
-export async function getRemoteUser(userId: mongo.ObjectID) {
+export async function getRemoteUser(userId: User['id']) {
 	const user = await getUser(userId);
 
-	if (!isRemoteUser(user)) {
+	if (!Users.isRemoteUser(user)) {
 		throw 'user is not a remote user';
 	}
 
@@ -61,10 +45,10 @@ export async function getRemoteUser(userId: mongo.ObjectID) {
 /**
  * Get local user for API processing
  */
-export async function getLocalUser(userId: mongo.ObjectID) {
+export async function getLocalUser(userId: User['id']) {
 	const user = await getUser(userId);
 
-	if (!isLocalUser(user)) {
+	if (!Users.isLocalUser(user)) {
 		throw 'user is not a local user';
 	}
 

src/server/api/common/make-pagination-query.ts

@@ -0,0 +1,28 @@
+import { SelectQueryBuilder } from 'typeorm';
+
+export function makePaginationQuery<T>(q: SelectQueryBuilder<T>, sinceId: string, untilId: string, sinceDate?: number, untilDate?: number) {
+	if (sinceId && untilId) {
+		q.andWhere(`${q.alias}.id > :sinceId`, { sinceId: sinceId });
+		q.andWhere(`${q.alias}.id < :untilId`, { untilId: untilId });
+		q.orderBy(`${q.alias}.id`, 'DESC');
+	} else if (sinceId) {
+		q.andWhere(`${q.alias}.id > :sinceId`, { sinceId: sinceId });
+		q.orderBy(`${q.alias}.id`, 'ASC');
+	} else if (untilId) {
+		q.andWhere(`${q.alias}.id < :untilId`, { untilId: untilId });
+		q.orderBy(`${q.alias}.id`, 'DESC');
+	} else if (sinceDate && untilDate) {
+		q.andWhere(`${q.alias}.createdAt > :sinceDate`, { sinceDate: new Date(sinceDate) });
+		q.andWhere(`${q.alias}.createdAt < :untilDate`, { untilDate: new Date(untilDate) });
+		q.orderBy(`${q.alias}.createdAt`, 'DESC');
+	} else if (sinceDate) {
+		q.andWhere(`${q.alias}.createdAt > :sinceDate`, { sinceDate: new Date(sinceDate) });
+		q.orderBy(`${q.alias}.createdAt`, 'ASC');
+	} else if (untilDate) {
+		q.andWhere(`${q.alias}.createdAt < :untilDate`, { untilDate: new Date(untilDate) });
+		q.orderBy(`${q.alias}.createdAt`, 'DESC');
+	} else {
+		q.orderBy(`${q.alias}.id`, 'DESC');
+	}
+	return q;
+}

src/server/api/common/read-messaging-message.ts

@@ -1,77 +1,43 @@
-import * as mongo from 'mongodb';
-import isObjectId from '../../../misc/is-objectid';
-import Message from '../../../models/messaging-message';
-import { IMessagingMessage as IMessage } from '../../../models/messaging-message';
 import { publishMainStream } from '../../../services/stream';
 import { publishMessagingStream } from '../../../services/stream';
 import { publishMessagingIndexStream } from '../../../services/stream';
-import User from '../../../models/user';
+import { User } from '../../../models/entities/user';
+import { MessagingMessage } from '../../../models/entities/messaging-message';
+import { MessagingMessages } from '../../../models';
+import { In } from 'typeorm';
 
 /**
  * Mark messages as read
  */
-export default (
-	user: string | mongo.ObjectID,
-	otherparty: string | mongo.ObjectID,
-	message: string | string[] | IMessage | IMessage[] | mongo.ObjectID | mongo.ObjectID[]
-) => new Promise<any>(async (resolve, reject) => {
-
-	const userId = isObjectId(user)
-		? user
-		: new mongo.ObjectID(user);
-
-	const otherpartyId = isObjectId(otherparty)
-		? otherparty
-		: new mongo.ObjectID(otherparty);
-
-	const ids: mongo.ObjectID[] = Array.isArray(message)
-		? isObjectId(message[0])
-			? (message as mongo.ObjectID[])
-			: typeof message[0] === 'string'
-				? (message as string[]).map(m => new mongo.ObjectID(m))
-				: (message as IMessage[]).map(m => m._id)
-		: isObjectId(message)
-			? [(message as mongo.ObjectID)]
-			: typeof message === 'string'
-				? [new mongo.ObjectID(message)]
-				: [(message as IMessage)._id];
+export default async (
+	userId: User['id'],
+	otherpartyId: User['id'],
+	messageIds: MessagingMessage['id'][]
+) => {
+	if (messageIds.length === 0) return;
 
 	// Update documents
-	await Message.update({
-		_id: { $in: ids },
+	await MessagingMessages.update({
+		id: In(messageIds),
 		userId: otherpartyId,
 		recipientId: userId,
 		isRead: false
 	}, {
-			$set: {
-				isRead: true
-			}
-		}, {
-			multi: true
-		});
+		isRead: true
+	});
 
 	// Publish event
-	publishMessagingStream(otherpartyId, userId, 'read', ids.map(id => id.toString()));
-	publishMessagingIndexStream(userId, 'read', ids.map(id => id.toString()));
+	publishMessagingStream(otherpartyId, userId, 'read', messageIds);
+	publishMessagingIndexStream(userId, 'read', messageIds);
 
 	// Calc count of my unread messages
-	const count = await Message
-		.count({
-			recipientId: userId,
-			isRead: false
-		}, {
-				limit: 1
-			});
+	const count = await MessagingMessages.count({
+		recipientId: userId,
+		isRead: false
+	});
 
 	if (count == 0) {
-		// Update flag
-		User.update({ _id: userId }, {
-			$set: {
-				hasUnreadMessagingMessage: false
-			}
-		});
-
 		// 全ての(いままで未読だった)自分宛てのメッセージを(これで)読みましたよというイベントを発行
 		publishMainStream(userId, 'readAllMessagingMessages');
 	}
-});
+};

src/server/api/common/read-notification.ts

@@ -1,72 +1,38 @@
-import * as mongo from 'mongodb';
-import isObjectId from '../../../misc/is-objectid';
-import { default as Notification, INotification } from '../../../models/notification';
 import { publishMainStream } from '../../../services/stream';
-import Mute from '../../../models/mute';
-import User from '../../../models/user';
+import { User } from '../../../models/entities/user';
+import { Notification } from '../../../models/entities/notification';
+import { Mutings, Notifications } from '../../../models';
+import { In, Not } from 'typeorm';
 
 /**
  * Mark notifications as read
  */
-export default (
-	user: string | mongo.ObjectID,
-	message: string | string[] | INotification | INotification[] | mongo.ObjectID | mongo.ObjectID[]
-) => new Promise<any>(async (resolve, reject) => {
-
-	const userId = isObjectId(user)
-		? user
-		: new mongo.ObjectID(user);
-
-	const ids: mongo.ObjectID[] = Array.isArray(message)
-		? isObjectId(message[0])
-			? (message as mongo.ObjectID[])
-			: typeof message[0] === 'string'
-				? (message as string[]).map(m => new mongo.ObjectID(m))
-				: (message as INotification[]).map(m => m._id)
-		: isObjectId(message)
-			? [(message as mongo.ObjectID)]
-			: typeof message === 'string'
-				? [new mongo.ObjectID(message)]
-				: [(message as INotification)._id];
-
-	const mute = await Mute.find({
+export async function readNotification(
+	userId: User['id'],
+	notificationIds: Notification['id'][]
+) {
+	const mute = await Mutings.find({
 		muterId: userId
 	});
 	const mutedUserIds = mute.map(m => m.muteeId);
 
 	// Update documents
-	await Notification.update({
-		_id: { $in: ids },
+	await Notifications.update({
+		id: In(notificationIds),
 		isRead: false
 	}, {
-			$set: {
-				isRead: true
-			}
-		}, {
-			multi: true
-		});
+		isRead: true
+	});
 
 	// Calc count of my unread notifications
-	const count = await Notification
-		.count({
-			notifieeId: userId,
-			notifierId: {
-				$nin: mutedUserIds
-			},
-			isRead: false
-		}, {
-				limit: 1
-			});
-
-	if (count == 0) {
-		// Update flag
-		User.update({ _id: userId }, {
-			$set: {
-				hasUnreadNotification: false
-			}
-		});
+	const count = await Notifications.count({
+		notifieeId: userId,
+		...(mutedUserIds.length > 0 ? { notifierId: Not(In(mutedUserIds)) } : {}),
+		isRead: false
+	});
 
+	if (count === 0) {
 		// 全ての(いままで未読だった)通知を(これで)読みましたよというイベントを発行
 		publishMainStream(userId, 'readAllNotifications');
 	}
-});
+}

src/server/api/common/signin.ts

@@ -1,7 +1,7 @@
 import * as Koa from 'koa';
 
 import config from '../../../config';
-import { ILocalUser } from '../../../models/user';
+import { ILocalUser } from '../../../models/entities/user';
 
 export default function(ctx: Koa.BaseContext, user: ILocalUser, redirect = false) {
 	if (redirect) {