Use PostgreSQL instead of MongoDB (#4572)

* wip

* Update note.ts

* Update timeline.ts

* Update core.ts

* wip

* Update generate-visibility-query.ts

* wip

* wip

* wip

* wip

* wip

* Update global-timeline.ts

* wip

* wip

* wip

* Update vote.ts

* wip

* wip

* Update create.ts

* wip

* wip

* wip

* wip

* wip

* wip

* wip

* wip

* wip

* wip

* wip

* wip

* Update files.ts

* wip

* wip

* Update CONTRIBUTING.md

* wip

* wip

* wip

* wip

* wip

* wip

* wip

* wip

* Update read-notification.ts

* wip

* wip

* wip

* wip

* wip

* wip

* wip

* Update cancel.ts

* wip

* wip

* wip

* Update show.ts

* wip

* wip

* Update gen-id.ts

* Update create.ts

* Update id.ts

* wip

* wip

* wip

* wip

* wip

* wip

* wip

* Docker: Update files about Docker (#4599)

* Docker: Use cache if files used by `yarn install` was not updated

This patch reduces the number of times to installing node_modules.
For example, `yarn install` step will be skipped when only ".config/default.yml" is updated.

* Docker: Migrate MongoDB to Postgresql

Misskey uses Postgresql as a database instead of Mongodb since version 11.

* Docker: Uncomment about data persistence

This patch will save a lot of databases.

* wip

* wip

* wip

* Update activitypub.ts

* wip

* wip

* wip

* Update logs.ts

* wip

* Update drive-file.ts

* Update register.ts

* wip

* wip

* Update mentions.ts

* wip

* wip

* wip

* Update recommendation.ts

* wip

* Update index.ts

* wip

* Update recommendation.ts

* Doc: Update docker.ja.md and docker.en.md (#1) (#4608)

Update how to set up misskey.

* wip

* :v:

* wip

* Update note.ts

* Update postgre.ts

* wip

* wip

* wip

* wip

* Update add-file.ts

* wip

* wip

* wip

* Clean up

* Update logs.ts

* wip

* :pizza:

* wip

* Ad notes

* wip

* Update api-visibility.ts

* Update note.ts

* Update add-file.ts

* tests

* tests

* Update postgre.ts

* Update utils.ts

* wip

* wip

* Refactor

* wip

* Refactor

* wip

* wip

* Update show-users.ts

* Update update-instance.ts

* wip

* Update feed.ts

* Update outbox.ts

* Update outbox.ts

* Update user.ts

* wip

* Update list.ts

* Update update-hashtag.ts

* wip

* Update update-hashtag.ts

* Refactor

* Update update.ts

* wip

* wip

* :v:

* clean up

* docs

* Update push.ts

* wip

* Update api.ts

* wip

* :v:

* Update make-pagination-query.ts

* :v:

* Delete hashtags.ts

* Update instances.ts

* Update instances.ts

* Update create.ts

* Update search.ts

* Update reversi-game.ts

* Update signup.ts

* Update user.ts

* id

* Update example.yml

* :art:

* objectid

* fix

* reversi

* reversi

* Fix bug of chart engine

* Add test of chart engine

* Improve test

* Better testing

* Improve chart engine

* Refactor

* Add test of chart engine

* Refactor

* Add chart test

* Fix bug

* コミットし忘れ

* Refactoring

* :v:

* Add tests

* Add test

* Extarct note tests

* Refactor

* 存在しないユーザーにメンションできなくなっていた問題を修正

* Fix bug

* Update update-meta.ts

* Fix bug

* Update mention.vue

* Fix bug

* Update meta.ts

* Update CONTRIBUTING.md

* Fix bug

* Fix bug

* Fix bug

* Clean up

* Clean up

* Update notification.ts

* Clean up

* Add mute tests

* Add test

* Refactor

* Add test

* Fix test

* Refactor

* Refactor

* Add tests

* Update utils.ts

* Update utils.ts

* Fix test

* Update package.json

* Update update.ts

* Update manifest.ts

* Fix bug

* Fix bug

* Add test

* :art:

* Update endpoint permissions

* Updaye permisison

* Update person.ts

#4299

* データベースと同期しないように

* Fix bug

* Fix bug

* Update reversi-game.ts

* Use a feature of Node v11.7.0 to extract a public key (#4644)

* wip

* wip

* :v:

* Refactoring

#1540

* test

* test

* test

* test

* test

* test

* test

* Fix bug

* Fix test

* :sushi:

* wip

* #4471

* Add test for #4335

* Refactor

* Fix test

* Add tests

* :clock4:

* Fix bug

* Add test

* Add test

* rename

* Fix bug

3 files changed, 112 insertions, 87 deletions

diff --git a/src/server/api/service/discord.ts b/src/server/api/service/discord.ts
index 92f5bbf72d..4290e1ff9d 100644
--- a/src/server/api/service/discord.ts
+++ b/src/server/api/service/discord.ts
@@ -2,13 +2,14 @@ import * as Koa from 'koa';
 import * as Router from 'koa-router';
 import * as request from 'request';
 import { OAuth2 } from 'oauth';
-import User, { pack, ILocalUser } from '../../../models/user';
 import config from '../../../config';
 import { publishMainStream } from '../../../services/stream';
 import redis from '../../../db/redis';
 import * as uuid from 'uuid';
 import signin from '../common/signin';
 import fetchMeta from '../../../misc/fetch-meta';
+import { Users, UserServiceLinkings } from '../../../models';
+import { ILocalUser } from '../../../models/entities/user';
 
 function getUserToken(ctx: Koa.BaseContext) {
 	return ((ctx.headers['cookie'] || '').match(/i=(!\w+)/) || [null, null])[1];
@@ -39,19 +40,27 @@ router.get('/disconnect/discord', async ctx => {
 		return;
 	}
 
-	const user = await User.findOneAndUpdate({
+	const user = await Users.findOne({
 		host: null,
-		'token': userToken
+		token: userToken
+	});
+
+	await UserServiceLinkings.update({
+		userId: user.id
 	}, {
-		$set: {
-			'discord': null
-		}
+		discord: false,
+		discordAccessToken: null,
+		discordRefreshToken: null,
+		discordExpiresDate: null,
+		discordId: null,
+		discordUsername: null,
+		discordDiscriminator: null,
 	});
 
 	ctx.body = `Discordの連携を解除しました :v:`;
 
 	// Publish i updated event
-	publishMainStream(user._id, 'meUpdated', await pack(user, user, {
+	publishMainStream(user.id, 'meUpdated', await Users.pack(user, user, {
 		detail: true,
 		includeSecrets: true
 	}));
@@ -193,32 +202,30 @@ router.get('/dc/cb', async ctx => {
 			return;
 		}
 
-		let user = await User.findOne({
-			host: null,
-			'discord.id': id
-		}) as ILocalUser;
+		const link = await UserServiceLinkings.createQueryBuilder()
+			.where('discord @> :discord', {
+				discord: {
+					id: id,
+				},
+			})
+			.andWhere('userHost IS NULL')
+			.getOne();
 
-		if (!user) {
+		if (link == null) {
 			ctx.throw(404, `@${username}#${discriminator}と連携しているMisskeyアカウントはありませんでした...`);
 			return;
 		}
 
-		user = await User.findOneAndUpdate({
-			host: null,
-			'discord.id': id
-		}, {
-			$set: {
-				discord: {
-					accessToken,
-					refreshToken,
-					expiresDate,
-					username,
-					discriminator
-				}
-			}
-		}) as ILocalUser;
+		await UserServiceLinkings.update(link.id, {
+			discord: true,
+			discordAccessToken: accessToken,
+			discordRefreshToken: refreshToken,
+			discordExpiresDate: expiresDate,
+			discordUsername: username,
+			discordDiscriminator: discriminator
+		});
 
-		signin(ctx, user, true);
+		signin(ctx, await Users.findOne(link.userId) as ILocalUser, true);
 	} else {
 		const code = ctx.query.code;
 
@@ -277,26 +284,25 @@ router.get('/dc/cb', async ctx => {
 			return;
 		}
 
-		const user = await User.findOneAndUpdate({
+		const user = await Users.findOne({
 			host: null,
 			token: userToken
-		}, {
-			$set: {
-				discord: {
-					accessToken,
-					refreshToken,
-					expiresDate,
-					id,
-					username,
-					discriminator
-				}
-			}
+		});
+
+		await UserServiceLinkings.update({ userId: user.id }, {
+			discord: true,
+			discordAccessToken: accessToken,
+			discordRefreshToken: refreshToken,
+			discordExpiresDate: expiresDate,
+			discordId: id,
+			discordUsername: username,
+			discordDiscriminator: discriminator
 		});
 
 		ctx.body = `Discord: @${username}#${discriminator} を、Misskey: @${user.username} に接続しました！`;
 
 		// Publish i updated event
-		publishMainStream(user._id, 'meUpdated', await pack(user, user, {
+		publishMainStream(user.id, 'meUpdated', await Users.pack(user, user, {
 			detail: true,
 			includeSecrets: true
 		}));
diff --git a/src/server/api/service/github.ts b/src/server/api/service/github.ts
index cf3589a4b7..e59b149d19 100644
--- a/src/server/api/service/github.ts
+++ b/src/server/api/service/github.ts
@@ -2,13 +2,14 @@ import * as Koa from 'koa';
 import * as Router from 'koa-router';
 import * as request from 'request';
 import { OAuth2 } from 'oauth';
-import User, { pack, ILocalUser } from '../../../models/user';
 import config from '../../../config';
 import { publishMainStream } from '../../../services/stream';
 import redis from '../../../db/redis';
 import * as uuid from 'uuid';
 import signin from '../common/signin';
 import fetchMeta from '../../../misc/fetch-meta';
+import { Users, UserServiceLinkings } from '../../../models';
+import { ILocalUser } from '../../../models/entities/user';
 
 function getUserToken(ctx: Koa.BaseContext) {
 	return ((ctx.headers['cookie'] || '').match(/i=(!\w+)/) || [null, null])[1];
@@ -39,19 +40,24 @@ router.get('/disconnect/github', async ctx => {
 		return;
 	}
 
-	const user = await User.findOneAndUpdate({
+	const user = await Users.findOne({
 		host: null,
-		'token': userToken
+		token: userToken
+	});
+
+	await UserServiceLinkings.update({
+		userId: user.id
 	}, {
-		$set: {
-			'github': null
-		}
+		github: false,
+		githubAccessToken: null,
+		githubId: null,
+		githubLogin: null,
 	});
 
 	ctx.body = `GitHubの連携を解除しました :v:`;
 
 	// Publish i updated event
-	publishMainStream(user._id, 'meUpdated', await pack(user, user, {
+	publishMainStream(user.id, 'meUpdated', await Users.pack(user, user, {
 		detail: true,
 		includeSecrets: true
 	}));
@@ -185,17 +191,21 @@ router.get('/gh/cb', async ctx => {
 			return;
 		}
 
-		const user = await User.findOne({
-			host: null,
-			'github.id': id
-		}) as ILocalUser;
+		const link = await UserServiceLinkings.createQueryBuilder()
+			.where('github @> :github', {
+				github: {
+					id: id,
+				},
+			})
+			.andWhere('userHost IS NULL')
+			.getOne();
 
-		if (!user) {
+		if (link == null) {
 			ctx.throw(404, `@${login}と連携しているMisskeyアカウントはありませんでした...`);
 			return;
 		}
 
-		signin(ctx, user, true);
+		signin(ctx, await Users.findOne(link.userId) as ILocalUser, true);
 	} else {
 		const code = ctx.query.code;
 
@@ -248,23 +258,22 @@ router.get('/gh/cb', async ctx => {
 			return;
 		}
 
-		const user = await User.findOneAndUpdate({
+		const user = await Users.findOne({
 			host: null,
 			token: userToken
-		}, {
-			$set: {
-				github: {
-					accessToken,
-					id,
-					login
-				}
-			}
+		});
+
+		await UserServiceLinkings.update({ userId: user.id }, {
+			github: true,
+			githubAccessToken: accessToken,
+			githubId: id,
+			githubLogin: login,
 		});
 
 		ctx.body = `GitHub: @${login} を、Misskey: @${user.username} に接続しました！`;
 
 		// Publish i updated event
-		publishMainStream(user._id, 'meUpdated', await pack(user, user, {
+		publishMainStream(user.id, 'meUpdated', await Users.pack(user, user, {
 			detail: true,
 			includeSecrets: true
 		}));
diff --git a/src/server/api/service/twitter.ts b/src/server/api/service/twitter.ts
index fc23808e21..77cf71395b 100644
--- a/src/server/api/service/twitter.ts
+++ b/src/server/api/service/twitter.ts
@@ -3,11 +3,12 @@ import * as Router from 'koa-router';
 import * as uuid from 'uuid';
 import autwh from 'autwh';
 import redis from '../../../db/redis';
-import User, { pack, ILocalUser } from '../../../models/user';
 import { publishMainStream } from '../../../services/stream';
 import config from '../../../config';
 import signin from '../common/signin';
 import fetchMeta from '../../../misc/fetch-meta';
+import { Users, UserServiceLinkings } from '../../../models';
+import { ILocalUser } from '../../../models/entities/user';
 
 function getUserToken(ctx: Koa.BaseContext) {
 	return ((ctx.headers['cookie'] || '').match(/i=(!\w+)/) || [null, null])[1];
@@ -38,19 +39,25 @@ router.get('/disconnect/twitter', async ctx => {
 		return;
 	}
 
-	const user = await User.findOneAndUpdate({
+	const user = await Users.findOne({
 		host: null,
-		'token': userToken
+		token: userToken
+	});
+
+	await UserServiceLinkings.update({
+		userId: user.id
 	}, {
-		$set: {
-			'twitter': null
-		}
+		twitter: false,
+		twitterAccessToken: null,
+		twitterAccessTokenSecret: null,
+		twitterUserId: null,
+		twitterScreenName: null,
 	});
 
 	ctx.body = `Twitterの連携を解除しました :v:`;
 
 	// Publish i updated event
-	publishMainStream(user._id, 'meUpdated', await pack(user, user, {
+	publishMainStream(user.id, 'meUpdated', await Users.pack(user, user, {
 		detail: true,
 		includeSecrets: true
 	}));
@@ -132,17 +139,21 @@ router.get('/tw/cb', async ctx => {
 
 		const result = await twAuth.done(JSON.parse(twCtx), ctx.query.oauth_verifier);
 
-		const user = await User.findOne({
-			host: null,
-			'twitter.userId': result.userId
-		}) as ILocalUser;
+		const link = await UserServiceLinkings.createQueryBuilder()
+			.where('twitter @> :twitter', {
+				twitter: {
+					userId: result.userId,
+				},
+			})
+			.andWhere('userHost IS NULL')
+			.getOne();
 
-		if (user == null) {
+		if (link == null) {
 			ctx.throw(404, `@${result.screenName}と連携しているMisskeyアカウントはありませんでした...`);
 			return;
 		}
 
-		signin(ctx, user, true);
+		signin(ctx, await Users.findOne(link.userId) as ILocalUser, true);
 	} else {
 		const verifier = ctx.query.oauth_verifier;
 
@@ -161,24 +172,23 @@ router.get('/tw/cb', async ctx => {
 
 		const result = await twAuth.done(JSON.parse(twCtx), verifier);
 
-		const user = await User.findOneAndUpdate({
+		const user = await Users.findOne({
 			host: null,
 			token: userToken
-		}, {
-			$set: {
-				twitter: {
-					accessToken: result.accessToken,
-					accessTokenSecret: result.accessTokenSecret,
-					userId: result.userId,
-					screenName: result.screenName
-				}
-			}
+		});
+
+		await UserServiceLinkings.update({ userId: user.id }, {
+			twitter: true,
+			twitterAccessToken: result.accessToken,
+			twitterAccessTokenSecret: result.accessTokenSecret,
+			twitterUserId: result.userId,
+			twitterScreenName: result.screenName,
 		});
 
 		ctx.body = `Twitter: @${result.screenName} を、Misskey: @${user.username} に接続しました！`;
 
 		// Publish i updated event
-		publishMainStream(user._id, 'meUpdated', await pack(user, user, {
+		publishMainStream(user.id, 'meUpdated', await Users.pack(user, user, {
 			detail: true,
 			includeSecrets: true
 		}));