Use PostgreSQL instead of MongoDB (#4572)

* wip

* Update note.ts

* Update timeline.ts

* Update core.ts

* wip

* Update generate-visibility-query.ts

* wip

* wip

* wip

* wip

* wip

* Update global-timeline.ts

* wip

* wip

* wip

* Update vote.ts

* wip

* wip

* Update create.ts

* wip

* wip

* wip

* wip

* wip

* wip

* wip

* wip

* wip

* wip

* wip

* wip

* Update files.ts

* wip

* wip

* Update CONTRIBUTING.md

* wip

* wip

* wip

* wip

* wip

* wip

* wip

* wip

* Update read-notification.ts

* wip

* wip

* wip

* wip

* wip

* wip

* wip

* Update cancel.ts

* wip

* wip

* wip

* Update show.ts

* wip

* wip

* Update gen-id.ts

* Update create.ts

* Update id.ts

* wip

* wip

* wip

* wip

* wip

* wip

* wip

* Docker: Update files about Docker (#4599)

* Docker: Use cache if files used by `yarn install` was not updated

This patch reduces the number of times to installing node_modules.
For example, `yarn install` step will be skipped when only ".config/default.yml" is updated.

* Docker: Migrate MongoDB to Postgresql

Misskey uses Postgresql as a database instead of Mongodb since version 11.

* Docker: Uncomment about data persistence

This patch will save a lot of databases.

* wip

* wip

* wip

* Update activitypub.ts

* wip

* wip

* wip

* Update logs.ts

* wip

* Update drive-file.ts

* Update register.ts

* wip

* wip

* Update mentions.ts

* wip

* wip

* wip

* Update recommendation.ts

* wip

* Update index.ts

* wip

* Update recommendation.ts

* Doc: Update docker.ja.md and docker.en.md (#1) (#4608)

Update how to set up misskey.

* wip

* :v:

* wip

* Update note.ts

* Update postgre.ts

* wip

* wip

* wip

* wip

* Update add-file.ts

* wip

* wip

* wip

* Clean up

* Update logs.ts

* wip

* :pizza:

* wip

* Ad notes

* wip

* Update api-visibility.ts

* Update note.ts

* Update add-file.ts

* tests

* tests

* Update postgre.ts

* Update utils.ts

* wip

* wip

* Refactor

* wip

* Refactor

* wip

* wip

* Update show-users.ts

* Update update-instance.ts

* wip

* Update feed.ts

* Update outbox.ts

* Update outbox.ts

* Update user.ts

* wip

* Update list.ts

* Update update-hashtag.ts

* wip

* Update update-hashtag.ts

* Refactor

* Update update.ts

* wip

* wip

* :v:

* clean up

* docs

* Update push.ts

* wip

* Update api.ts

* wip

* :v:

* Update make-pagination-query.ts

* :v:

* Delete hashtags.ts

* Update instances.ts

* Update instances.ts

* Update create.ts

* Update search.ts

* Update reversi-game.ts

* Update signup.ts

* Update user.ts

* id

* Update example.yml

* :art:

* objectid

* fix

* reversi

* reversi

* Fix bug of chart engine

* Add test of chart engine

* Improve test

* Better testing

* Improve chart engine

* Refactor

* Add test of chart engine

* Refactor

* Add chart test

* Fix bug

* コミットし忘れ

* Refactoring

* :v:

* Add tests

* Add test

* Extarct note tests

* Refactor

* 存在しないユーザーにメンションできなくなっていた問題を修正

* Fix bug

* Update update-meta.ts

* Fix bug

* Update mention.vue

* Fix bug

* Update meta.ts

* Update CONTRIBUTING.md

* Fix bug

* Fix bug

* Fix bug

* Clean up

* Clean up

* Update notification.ts

* Clean up

* Add mute tests

* Add test

* Refactor

* Add test

* Fix test

* Refactor

* Refactor

* Add tests

* Update utils.ts

* Update utils.ts

* Fix test

* Update package.json

* Update update.ts

* Update manifest.ts

* Fix bug

* Fix bug

* Add test

* :art:

* Update endpoint permissions

* Updaye permisison

* Update person.ts

#4299

* データベースと同期しないように

* Fix bug

* Fix bug

* Update reversi-game.ts

* Use a feature of Node v11.7.0 to extract a public key (#4644)

* wip

* wip

* :v:

* Refactoring

#1540

* test

* test

* test

* test

* test

* test

* test

* Fix bug

* Fix test

* :sushi:

* wip

* #4471

* Add test for #4335

* Refactor

* Fix test

* Add tests

* :clock4:

* Fix bug

* Add test

* Add test

* rename

* Fix bug

5 files changed, 71 insertions, 118 deletions

diff --git a/src/server/api/endpoints/messaging/history.ts b/src/server/api/endpoints/messaging/history.ts
index 699dc7c253..c0aec61212 100644
--- a/src/server/api/endpoints/messaging/history.ts
+++ b/src/server/api/endpoints/messaging/history.ts
@@ -1,7 +1,8 @@
 import $ from 'cafy';
-import Mute from '../../../../models/mute';
-import Message, { pack, IMessagingMessage } from '../../../../models/messaging-message';
 import define from '../../define';
+import { MessagingMessage } from '../../../../models/entities/messaging-message';
+import { MessagingMessages, Mutings } from '../../../../models';
+import { Brackets } from 'typeorm';
 
 export const meta = {
 	desc: {
@@ -31,34 +32,33 @@ export const meta = {
 };
 
 export default define(meta, async (ps, user) => {
-	const mute = await Mute.find({
-		muterId: user._id,
-		deletedAt: { $exists: false }
+	const mute = await Mutings.find({
+		muterId: user.id,
 	});
 
-	const history: IMessagingMessage[] = [];
+	const history: MessagingMessage[] = [];
 
 	for (let i = 0; i < ps.limit; i++) {
-		const found = history.map(m => m.userId.equals(user._id) ? m.recipientId : m.userId);
+		const found = history.map(m => (m.userId === user.id) ? m.recipientId : m.userId);
 
-		const message = await Message.findOne({
-			$or: [{
-				userId: user._id
-			}, {
-				recipientId: user._id
-			}],
-			$and: [{
-				userId: { $nin: found },
-				recipientId: { $nin: found }
-			}, {
-				userId: { $nin: mute.map(m => m.muteeId) },
-				recipientId: { $nin: mute.map(m => m.muteeId) }
-			}]
-		}, {
-			sort: {
-				createdAt: -1
-			}
-		});
+		const query = MessagingMessages.createQueryBuilder('message')
+			.where(new Brackets(qb => { qb
+				.where(`message.userId = :userId`, { userId: user.id })
+				.orWhere(`message.recipientId = :userId`, { userId: user.id });
+			}))
+			.orderBy('message.createdAt', 'DESC');
+
+		if (found.length > 0) {
+			query.andWhere(`message.userId NOT IN (:...found)`, { found: found });
+			query.andWhere(`message.recipientId NOT IN (:...found)`, { found: found });
+		}
+
+		if (mute.length > 0) {
+			query.andWhere(`message.userId NOT IN (:...mute)`, { mute: mute.map(m => m.muteeId) });
+			query.andWhere(`message.recipientId NOT IN (:...mute)`, { mute: mute.map(m => m.muteeId) });
+		}
+
+		const message = await query.getOne();
 
 		if (message) {
 			history.push(message);
@@ -67,5 +67,5 @@ export default define(meta, async (ps, user) => {
 		}
 	}
 
-	return await Promise.all(history.map(h => pack(h._id, user)));
+	return await Promise.all(history.map(h => MessagingMessages.pack(h.id, user)));
 });
diff --git a/src/server/api/endpoints/messaging/messages.ts b/src/server/api/endpoints/messaging/messages.ts
index c19db45f1f..02c57b8d03 100644
--- a/src/server/api/endpoints/messaging/messages.ts
+++ b/src/server/api/endpoints/messaging/messages.ts
@@ -1,11 +1,11 @@
 import $ from 'cafy';
-import ID, { transform } from '../../../../misc/cafy-id';
-import Message from '../../../../models/messaging-message';
-import { pack } from '../../../../models/messaging-message';
+import { ID } from '../../../../misc/cafy-id';
 import read from '../../common/read-messaging-message';
 import define from '../../define';
 import { ApiError } from '../../error';
 import { getUser } from '../../common/getters';
+import { MessagingMessages } from '../../../../models';
+import { makePaginationQuery } from '../../common/make-pagination-query';
 
 export const meta = {
 	desc: {
@@ -22,7 +22,6 @@ export const meta = {
 	params: {
 		userId: {
 			validator: $.type(ID),
-			transform: transform,
 			desc: {
 				'ja-JP': '対象のユーザーのID',
 				'en-US': 'Target user ID'
@@ -36,12 +35,10 @@ export const meta = {
 
 		sinceId: {
 			validator: $.optional.type(ID),
-			transform: transform,
 		},
 
 		untilId: {
 			validator: $.optional.type(ID),
-			transform: transform,
 		},
 
 		markAsRead: {
@@ -73,43 +70,17 @@ export default define(meta, async (ps, user) => {
 		throw e;
 	});
 
-	const query = {
-		$or: [{
-			userId: user._id,
-			recipientId: recipient._id
-		}, {
-			userId: recipient._id,
-			recipientId: user._id
-		}]
-	} as any;
+	const query = makePaginationQuery(MessagingMessages.createQueryBuilder('message'), ps.sinceId, ps.untilId)
+		.andWhere(`(message.userId = :meId AND message.recipientId = :recipientId) OR (message.userId = :recipientId AND message.recipientId = :meId)`, { meId: user.id, recipientId: recipient.id });
 
-	const sort = {
-		_id: -1
-	};
-
-	if (ps.sinceId) {
-		sort._id = 1;
-		query._id = {
-			$gt: ps.sinceId
-		};
-	} else if (ps.untilId) {
-		query._id = {
-			$lt: ps.untilId
-		};
-	}
-
-	const messages = await Message
-		.find(query, {
-			limit: ps.limit,
-			sort: sort
-		});
+	const messages = await query.getMany();
 
 	// Mark all as read
 	if (ps.markAsRead) {
-		read(user._id, recipient._id, messages);
+		read(user.id, recipient.id, messages.map(x => x.id));
 	}
 
-	return await Promise.all(messages.map(message => pack(message, user, {
+	return await Promise.all(messages.map(message => MessagingMessages.pack(message, user, {
 		populateRecipient: false
 	})));
 });
diff --git a/src/server/api/endpoints/messaging/messages/create.ts b/src/server/api/endpoints/messaging/messages/create.ts
index fc048e6edd..2c7e5ad2d9 100644
--- a/src/server/api/endpoints/messaging/messages/create.ts
+++ b/src/server/api/endpoints/messaging/messages/create.ts
@@ -1,17 +1,14 @@
 import $ from 'cafy';
-import ID, { transform } from '../../../../../misc/cafy-id';
-import Message from '../../../../../models/messaging-message';
-import { isValidText } from '../../../../../models/messaging-message';
-import User from '../../../../../models/user';
-import Mute from '../../../../../models/mute';
-import DriveFile from '../../../../../models/drive-file';
-import { pack } from '../../../../../models/messaging-message';
+import { ID } from '../../../../../misc/cafy-id';
 import { publishMainStream } from '../../../../../services/stream';
 import { publishMessagingStream, publishMessagingIndexStream } from '../../../../../services/stream';
 import pushSw from '../../../../../services/push-notification';
 import define from '../../../define';
 import { ApiError } from '../../../error';
 import { getUser } from '../../../common/getters';
+import { MessagingMessages, DriveFiles, Mutings } from '../../../../../models';
+import { MessagingMessage } from '../../../../../models/entities/messaging-message';
+import { genId } from '../../../../../misc/gen-id';
 
 export const meta = {
 	desc: {
@@ -28,7 +25,6 @@ export const meta = {
 	params: {
 		userId: {
 			validator: $.type(ID),
-			transform: transform,
 			desc: {
 				'ja-JP': '対象のユーザーのID',
 				'en-US': 'Target user ID'
@@ -36,12 +32,11 @@ export const meta = {
 		},
 
 		text: {
-			validator: $.optional.str.pipe(isValidText)
+			validator: $.optional.str.pipe(MessagingMessages.isValidText)
 		},
 
 		fileId: {
 			validator: $.optional.type(ID),
-			transform: transform,
 		}
 	},
 
@@ -78,7 +73,7 @@ export const meta = {
 
 export default define(meta, async (ps, user) => {
 	// Myself
-	if (ps.userId.equals(user._id)) {
+	if (ps.userId === user.id) {
 		throw new ApiError(meta.errors.recipientIsYourself);
 	}
 
@@ -90,12 +85,12 @@ export default define(meta, async (ps, user) => {
 
 	let file = null;
 	if (ps.fileId != null) {
-		file = await DriveFile.findOne({
-			_id: ps.fileId,
-			'metadata.userId': user._id
+		file = await DriveFiles.findOne({
+			id: ps.fileId,
+			userId: user.id
 		});
 
-		if (file === null) {
+		if (file == null) {
 			throw new ApiError(meta.errors.noSuchFile);
 		}
 	}
@@ -105,16 +100,17 @@ export default define(meta, async (ps, user) => {
 		throw new ApiError(meta.errors.contentRequired);
 	}
 
-	const message = await Message.insert({
+	const message = await MessagingMessages.save({
+		id: genId(),
 		createdAt: new Date(),
-		fileId: file ? file._id : undefined,
-		recipientId: recipient._id,
-		text: ps.text ? ps.text.trim() : undefined,
-		userId: user._id,
+		fileId: file ? file.id : null,
+		recipientId: recipient.id,
+		text: ps.text ? ps.text.trim() : null,
+		userId: user.id,
 		isRead: false
-	});
+	} as MessagingMessage);
 
-	const messageObj = await pack(message);
+	const messageObj = await MessagingMessages.pack(message);
 
 	// 自分のストリーム
 	publishMessagingStream(message.userId, message.recipientId, 'message', messageObj);
@@ -126,25 +122,17 @@ export default define(meta, async (ps, user) => {
 	publishMessagingIndexStream(message.recipientId, 'message', messageObj);
 	publishMainStream(message.recipientId, 'messagingMessage', messageObj);
 
-	// Update flag
-	User.update({ _id: recipient._id }, {
-		$set: {
-			hasUnreadMessagingMessage: true
-		}
-	});
-
 	// 2秒経っても(今回作成した)メッセージが既読にならなかったら「未読のメッセージがありますよ」イベントを発行する
 	setTimeout(async () => {
-		const freshMessage = await Message.findOne({ _id: message._id }, { isRead: true });
+		const freshMessage = await MessagingMessages.findOne({ id: message.id });
 		if (freshMessage == null) return; // メッセージが削除されている場合もある
 		if (!freshMessage.isRead) {
 			//#region ただしミュートされているなら発行しない
-			const mute = await Mute.find({
-				muterId: recipient._id,
-				deletedAt: { $exists: false }
+			const mute = await Mutings.find({
+				muterId: recipient.id,
 			});
 			const mutedUserIds = mute.map(m => m.muteeId.toString());
-			if (mutedUserIds.indexOf(user._id.toString()) != -1) {
+			if (mutedUserIds.indexOf(user.id) != -1) {
 				return;
 			}
 			//#endregion
diff --git a/src/server/api/endpoints/messaging/messages/delete.ts b/src/server/api/endpoints/messaging/messages/delete.ts
index 0ca12846c1..9f55caba62 100644
--- a/src/server/api/endpoints/messaging/messages/delete.ts
+++ b/src/server/api/endpoints/messaging/messages/delete.ts
@@ -1,10 +1,10 @@
 import $ from 'cafy';
-import ID, { transform } from '../../../../../misc/cafy-id';
-import Message from '../../../../../models/messaging-message';
+import { ID } from '../../../../../misc/cafy-id';
 import define from '../../../define';
 import { publishMessagingStream } from '../../../../../services/stream';
 import * as ms from 'ms';
 import { ApiError } from '../../../error';
+import { MessagingMessages } from '../../../../../models';
 
 export const meta = {
 	stability: 'stable',
@@ -29,7 +29,6 @@ export const meta = {
 	params: {
 		messageId: {
 			validator: $.type(ID),
-			transform: transform,
 			desc: {
 				'ja-JP': '対象のメッセージのID',
 				'en-US': 'Target message ID.'
@@ -47,19 +46,17 @@ export const meta = {
 };
 
 export default define(meta, async (ps, user) => {
-	const message = await Message.findOne({
-		_id: ps.messageId,
-		userId: user._id
+	const message = await MessagingMessages.findOne({
+		id: ps.messageId,
+		userId: user.id
 	});
 
-	if (message === null) {
+	if (message == null) {
 		throw new ApiError(meta.errors.noSuchMessage);
 	}
 
-	await Message.remove({ _id: message._id });
+	await MessagingMessages.delete(message.id);
 
-	publishMessagingStream(message.userId, message.recipientId, 'deleted', message._id);
-	publishMessagingStream(message.recipientId, message.userId, 'deleted', message._id);
-
-	return;
+	publishMessagingStream(message.userId, message.recipientId, 'deleted', message.id);
+	publishMessagingStream(message.recipientId, message.userId, 'deleted', message.id);
 });
diff --git a/src/server/api/endpoints/messaging/messages/read.ts b/src/server/api/endpoints/messaging/messages/read.ts
index aa8ecdc4ff..24a28285bf 100644
--- a/src/server/api/endpoints/messaging/messages/read.ts
+++ b/src/server/api/endpoints/messaging/messages/read.ts
@@ -1,9 +1,9 @@
 import $ from 'cafy';
-import ID, { transform } from '../../../../../misc/cafy-id';
-import Message from '../../../../../models/messaging-message';
+import { ID } from '../../../../../misc/cafy-id';
 import read from '../../../common/read-messaging-message';
 import define from '../../../define';
 import { ApiError } from '../../../error';
+import { MessagingMessages } from '../../../../../models';
 
 export const meta = {
 	desc: {
@@ -20,7 +20,6 @@ export const meta = {
 	params: {
 		messageId: {
 			validator: $.type(ID),
-			transform: transform,
 			desc: {
 				'ja-JP': '既読にするメッセージのID',
 				'en-US': 'The ID of a message that you want to mark as read'
@@ -38,16 +37,14 @@ export const meta = {
 };
 
 export default define(meta, async (ps, user) => {
-	const message = await Message.findOne({
-		_id: ps.messageId,
-		recipientId: user._id
+	const message = await MessagingMessages.findOne({
+		id: ps.messageId,
+		recipientId: user.id
 	});
 
 	if (message == null) {
 		throw new ApiError(meta.errors.noSuchMessage);
 	}
 
-	read(user._id, message.userId, message);
-
-	return;
+	read(user.id, message.userId, [message.id]);
 });