Use PostgreSQL instead of MongoDB (#4572)

* wip

* Update note.ts

* Update timeline.ts

* Update core.ts

* wip

* Update generate-visibility-query.ts

* wip

* wip

* wip

* wip

* wip

* Update global-timeline.ts

* wip

* wip

* wip

* Update vote.ts

* wip

* wip

* Update create.ts

* wip

* wip

* wip

* wip

* wip

* wip

* wip

* wip

* wip

* wip

* wip

* wip

* Update files.ts

* wip

* wip

* Update CONTRIBUTING.md

* wip

* wip

* wip

* wip

* wip

* wip

* wip

* wip

* Update read-notification.ts

* wip

* wip

* wip

* wip

* wip

* wip

* wip

* Update cancel.ts

* wip

* wip

* wip

* Update show.ts

* wip

* wip

* Update gen-id.ts

* Update create.ts

* Update id.ts

* wip

* wip

* wip

* wip

* wip

* wip

* wip

* Docker: Update files about Docker (#4599)

* Docker: Use cache if files used by `yarn install` was not updated

This patch reduces the number of times to installing node_modules.
For example, `yarn install` step will be skipped when only ".config/default.yml" is updated.

* Docker: Migrate MongoDB to Postgresql

Misskey uses Postgresql as a database instead of Mongodb since version 11.

* Docker: Uncomment about data persistence

This patch will save a lot of databases.

* wip

* wip

* wip

* Update activitypub.ts

* wip

* wip

* wip

* Update logs.ts

* wip

* Update drive-file.ts

* Update register.ts

* wip

* wip

* Update mentions.ts

* wip

* wip

* wip

* Update recommendation.ts

* wip

* Update index.ts

* wip

* Update recommendation.ts

* Doc: Update docker.ja.md and docker.en.md (#1) (#4608)

Update how to set up misskey.

* wip

* :v:

* wip

* Update note.ts

* Update postgre.ts

* wip

* wip

* wip

* wip

* Update add-file.ts

* wip

* wip

* wip

* Clean up

* Update logs.ts

* wip

* :pizza:

* wip

* Ad notes

* wip

* Update api-visibility.ts

* Update note.ts

* Update add-file.ts

* tests

* tests

* Update postgre.ts

* Update utils.ts

* wip

* wip

* Refactor

* wip

* Refactor

* wip

* wip

* Update show-users.ts

* Update update-instance.ts

* wip

* Update feed.ts

* Update outbox.ts

* Update outbox.ts

* Update user.ts

* wip

* Update list.ts

* Update update-hashtag.ts

* wip

* Update update-hashtag.ts

* Refactor

* Update update.ts

* wip

* wip

* :v:

* clean up

* docs

* Update push.ts

* wip

* Update api.ts

* wip

* :v:

* Update make-pagination-query.ts

* :v:

* Delete hashtags.ts

* Update instances.ts

* Update instances.ts

* Update create.ts

* Update search.ts

* Update reversi-game.ts

* Update signup.ts

* Update user.ts

* id

* Update example.yml

* :art:

* objectid

* fix

* reversi

* reversi

* Fix bug of chart engine

* Add test of chart engine

* Improve test

* Better testing

* Improve chart engine

* Refactor

* Add test of chart engine

* Refactor

* Add chart test

* Fix bug

* コミットし忘れ

* Refactoring

* :v:

* Add tests

* Add test

* Extarct note tests

* Refactor

* 存在しないユーザーにメンションできなくなっていた問題を修正

* Fix bug

* Update update-meta.ts

* Fix bug

* Update mention.vue

* Fix bug

* Update meta.ts

* Update CONTRIBUTING.md

* Fix bug

* Fix bug

* Fix bug

* Clean up

* Clean up

* Update notification.ts

* Clean up

* Add mute tests

* Add test

* Refactor

* Add test

* Fix test

* Refactor

* Refactor

* Add tests

* Update utils.ts

* Update utils.ts

* Fix test

* Update package.json

* Update update.ts

* Update manifest.ts

* Fix bug

* Fix bug

* Add test

* :art:

* Update endpoint permissions

* Updaye permisison

* Update person.ts

#4299

* データベースと同期しないように

* Fix bug

* Fix bug

* Update reversi-game.ts

* Use a feature of Node v11.7.0 to extract a public key (#4644)

* wip

* wip

* :v:

* Refactoring

#1540

* test

* test

* test

* test

* test

* test

* test

* Fix bug

* Fix test

* :sushi:

* wip

* #4471

* Add test for #4335

* Refactor

* Fix test

* Add tests

* :clock4:

* Fix bug

* Add test

* Add test

* rename

* Fix bug

1 files changed, 29 insertions, 23 deletions

diff --git a/src/queue/processors/inbox.ts b/src/queue/processors/inbox.ts
index 436f3335c8..16badabcf7 100644
--- a/src/queue/processors/inbox.ts
+++ b/src/queue/processors/inbox.ts
@@ -1,7 +1,7 @@
 import * as Bull from 'bull';
 import * as httpSignature from 'http-signature';
 import parseAcct from '../../misc/acct/parse';
-import User, { IRemoteUser } from '../../models/user';
+import { IRemoteUser } from '../../models/entities/user';
 import perform from '../../remote/activitypub/perform';
 import { resolvePerson, updatePerson } from '../../remote/activitypub/models/person';
 import { toUnicode } from 'punycode';
@@ -9,8 +9,10 @@ import { URL } from 'url';
 import { publishApLogStream } from '../../services/stream';
 import Logger from '../../services/logger';
 import { registerOrFetchInstanceDoc } from '../../services/register-or-fetch-instance-doc';
-import Instance from '../../models/instance';
-import instanceChart from '../../services/chart/instance';
+import { Instances, Users, UserPublickeys } from '../../models';
+import { instanceChart } from '../../services/chart';
+import { UserPublickey } from '../../models/entities/user-publickey';
+import fetchMeta from '../../misc/fetch-meta';
 
 const logger = new Logger('inbox');
 
@@ -28,6 +30,7 @@ export default async (job: Bull.Job): Promise<void> => {
 
 	const keyIdLower = signature.keyId.toLowerCase();
 	let user: IRemoteUser;
+	let key: UserPublickey;
 
 	if (keyIdLower.startsWith('acct:')) {
 		const { username, host } = parseAcct(keyIdLower.slice('acct:'.length));
@@ -46,13 +49,17 @@ export default async (job: Bull.Job): Promise<void> => {
 
 		// ブロックしてたら中断
 		// TODO: いちいちデータベースにアクセスするのはコスト高そうなのでどっかにキャッシュしておく
-		const instance = await Instance.findOne({ host: host.toLowerCase() });
-		if (instance && instance.isBlocked) {
+		const meta = await fetchMeta();
+		if (meta.blockedHosts.includes(host.toLowerCase())) {
 			logger.info(`Blocked request: ${host}`);
 			return;
 		}
 
-		user = await User.findOne({ usernameLower: username, host: host.toLowerCase() }) as IRemoteUser;
+		user = await Users.findOne({ usernameLower: username, host: host.toLowerCase() }) as IRemoteUser;
+
+		key = await UserPublickeys.findOne({
+			userId: user.id
+		});
 	} else {
 		// アクティビティ内のホストの検証
 		const host = toUnicode(new URL(signature.keyId).hostname.toLowerCase());
@@ -65,16 +72,17 @@ export default async (job: Bull.Job): Promise<void> => {
 
 		// ブロックしてたら中断
 		// TODO: いちいちデータベースにアクセスするのはコスト高そうなのでどっかにキャッシュしておく
-		const instance = await Instance.findOne({ host: host.toLowerCase() });
-		if (instance && instance.isBlocked) {
-			logger.warn(`Blocked request: ${host}`);
+		const meta = await fetchMeta();
+		if (meta.blockedHosts.includes(host.toLowerCase())) {
+			logger.info(`Blocked request: ${host}`);
 			return;
 		}
 
-		user = await User.findOne({
-			host: { $ne: null },
-			'publicKey.id': signature.keyId
-		}) as IRemoteUser;
+		key = await UserPublickeys.findOne({
+			keyId: signature.keyId
+		});
+
+		user = await Users.findOne(key.userId) as IRemoteUser;
 	}
 
 	// Update Person activityの場合は、ここで署名検証/更新処理まで実施して終了
@@ -82,7 +90,7 @@ export default async (job: Bull.Job): Promise<void> => {
 		if (activity.object && activity.object.type === 'Person') {
 			if (user == null) {
 				logger.warn('Update activity received, but user not registed.');
-			} else if (!httpSignature.verifySignature(signature, user.publicKey.publicKeyPem)) {
+			} else if (!httpSignature.verifySignature(signature, key.keyPem)) {
 				logger.warn('Update activity received, but signature verification failed.');
 			} else {
 				updatePerson(activity.actor, null, activity.object);
@@ -92,15 +100,15 @@ export default async (job: Bull.Job): Promise<void> => {
 	}
 
 	// アクティビティを送信してきたユーザーがまだMisskeyサーバーに登録されていなかったら登録する
-	if (user === null) {
+	if (user == null) {
 		user = await resolvePerson(activity.actor) as IRemoteUser;
 	}
 
-	if (user === null) {
+	if (user == null) {
 		throw new Error('failed to resolve user');
 	}
 
-	if (!httpSignature.verifySignature(signature, user.publicKey.publicKeyPem)) {
+	if (!httpSignature.verifySignature(signature, key.keyPem)) {
 		logger.error('signature verification failed');
 		return;
 	}
@@ -116,12 +124,10 @@ export default async (job: Bull.Job): Promise<void> => {
 
 	// Update stats
 	registerOrFetchInstanceDoc(user.host).then(i => {
-		Instance.update({ _id: i._id }, {
-			$set: {
-				latestRequestReceivedAt: new Date(),
-				lastCommunicatedAt: new Date(),
-				isNotResponding: false
-			}
+		Instances.update(i.id, {
+			latestRequestReceivedAt: new Date(),
+			lastCommunicatedAt: new Date(),
+			isNotResponding: false
 		});
 
 		instanceChart.requestReceived(i.host);