Use PostgreSQL instead of MongoDB (#4572)

* wip

* Update note.ts

* Update timeline.ts

* Update core.ts

* wip

* Update generate-visibility-query.ts

* wip

* wip

* wip

* wip

* wip

* Update global-timeline.ts

* wip

* wip

* wip

* Update vote.ts

* wip

* wip

* Update create.ts

* wip

* wip

* wip

* wip

* wip

* wip

* wip

* wip

* wip

* wip

* wip

* wip

* Update files.ts

* wip

* wip

* Update CONTRIBUTING.md

* wip

* wip

* wip

* wip

* wip

* wip

* wip

* wip

* Update read-notification.ts

* wip

* wip

* wip

* wip

* wip

* wip

* wip

* Update cancel.ts

* wip

* wip

* wip

* Update show.ts

* wip

* wip

* Update gen-id.ts

* Update create.ts

* Update id.ts

* wip

* wip

* wip

* wip

* wip

* wip

* wip

* Docker: Update files about Docker (#4599)

* Docker: Use cache if files used by `yarn install` was not updated

This patch reduces the number of times to installing node_modules.
For example, `yarn install` step will be skipped when only ".config/default.yml" is updated.

* Docker: Migrate MongoDB to Postgresql

Misskey uses Postgresql as a database instead of Mongodb since version 11.

* Docker: Uncomment about data persistence

This patch will save a lot of databases.

* wip

* wip

* wip

* Update activitypub.ts

* wip

* wip

* wip

* Update logs.ts

* wip

* Update drive-file.ts

* Update register.ts

* wip

* wip

* Update mentions.ts

* wip

* wip

* wip

* Update recommendation.ts

* wip

* Update index.ts

* wip

* Update recommendation.ts

* Doc: Update docker.ja.md and docker.en.md (#1) (#4608)

Update how to set up misskey.

* wip

* :v:

* wip

* Update note.ts

* Update postgre.ts

* wip

* wip

* wip

* wip

* Update add-file.ts

* wip

* wip

* wip

* Clean up

* Update logs.ts

* wip

* :pizza:

* wip

* Ad notes

* wip

* Update api-visibility.ts

* Update note.ts

* Update add-file.ts

* tests

* tests

* Update postgre.ts

* Update utils.ts

* wip

* wip

* Refactor

* wip

* Refactor

* wip

* wip

* Update show-users.ts

* Update update-instance.ts

* wip

* Update feed.ts

* Update outbox.ts

* Update outbox.ts

* Update user.ts

* wip

* Update list.ts

* Update update-hashtag.ts

* wip

* Update update-hashtag.ts

* Refactor

* Update update.ts

* wip

* wip

* :v:

* clean up

* docs

* Update push.ts

* wip

* Update api.ts

* wip

* :v:

* Update make-pagination-query.ts

* :v:

* Delete hashtags.ts

* Update instances.ts

* Update instances.ts

* Update create.ts

* Update search.ts

* Update reversi-game.ts

* Update signup.ts

* Update user.ts

* id

* Update example.yml

* :art:

* objectid

* fix

* reversi

* reversi

* Fix bug of chart engine

* Add test of chart engine

* Improve test

* Better testing

* Improve chart engine

* Refactor

* Add test of chart engine

* Refactor

* Add chart test

* Fix bug

* コミットし忘れ

* Refactoring

* :v:

* Add tests

* Add test

* Extarct note tests

* Refactor

* 存在しないユーザーにメンションできなくなっていた問題を修正

* Fix bug

* Update update-meta.ts

* Fix bug

* Update mention.vue

* Fix bug

* Update meta.ts

* Update CONTRIBUTING.md

* Fix bug

* Fix bug

* Fix bug

* Clean up

* Clean up

* Update notification.ts

* Clean up

* Add mute tests

* Add test

* Refactor

* Add test

* Fix test

* Refactor

* Refactor

* Add tests

* Update utils.ts

* Update utils.ts

* Fix test

* Update package.json

* Update update.ts

* Update manifest.ts

* Fix bug

* Fix bug

* Add test

* :art:

* Update endpoint permissions

* Updaye permisison

* Update person.ts

#4299

* データベースと同期しないように

* Fix bug

* Fix bug

* Update reversi-game.ts

* Use a feature of Node v11.7.0 to extract a public key (#4644)

* wip

* wip

* :v:

* Refactoring

#1540

* test

* test

* test

* test

* test

* test

* test

* Fix bug

* Fix test

* :sushi:

* wip

* #4471

* Add test for #4335

* Refactor

* Fix test

* Add tests

* :clock4:

* Fix bug

* Add test

* Add test

* rename

* Fix bug

2 files changed, 80 insertions, 100 deletions

diff --git a/src/server/api/endpoints/notes/polls/recommendation.ts b/src/server/api/endpoints/notes/polls/recommendation.ts
index 9adabdf0e9..ff838d4f4f 100644
--- a/src/server/api/endpoints/notes/polls/recommendation.ts
+++ b/src/server/api/endpoints/notes/polls/recommendation.ts
@@ -1,8 +1,7 @@
 import $ from 'cafy';
-import Vote from '../../../../../models/poll-vote';
-import Note, { pack } from '../../../../../models/note';
 import define from '../../../define';
-import { getHideUserIds } from '../../../common/get-hide-users';
+import { Polls, Mutings, Notes, PollVotes } from '../../../../../models';
+import { Brackets, In } from 'typeorm';
 
 export const meta = {
 	desc: {
@@ -28,51 +27,46 @@ export const meta = {
 };
 
 export default define(meta, async (ps, user) => {
-	// Get votes
-	const votes = await Vote.find({
-		userId: user._id
-	}, {
-		fields: {
-			_id: false,
-			noteId: true
-		}
-	});
+	const query = Polls.createQueryBuilder('poll')
+		.where('poll.userHost IS NULL')
+		.andWhere(`poll.userId != :meId`, { meId: user.id })
+		.andWhere(`poll.noteVisibility = 'public'`)
+		.andWhere(new Brackets(qb => { qb
+			.where('poll.expiresAt IS NULL')
+			.orWhere('poll.expiresAt > :now', { now: new Date() });
+		}));
 
-	const nin = votes && votes.length != 0 ? votes.map(v => v.noteId) : [];
+	//#region exclude arleady voted polls
+	const votedQuery = PollVotes.createQueryBuilder('vote')
+		.select('vote.noteId')
+		.where('vote.userId = :meId', { meId: user.id });
 
-	// 隠すユーザーを取得
-	const hideUserIds = await getHideUserIds(user);
+	query
+		.andWhere(`poll.noteId NOT IN (${ votedQuery.getQuery() })`);
 
-	const notes = await Note.find({
-		'_user.host': null,
-		_id: {
-			$nin: nin
-		},
-		userId: {
-			$ne: user._id,
-			$nin: hideUserIds
-		},
-		visibility: 'public',
-		poll: {
-			$exists: true,
-			$ne: null
-		},
-		$or: [{
-			'poll.expiresAt': null
-		}, {
-			'poll.expiresAt': {
-				$gt: new Date()
-			}
-		}],
-	}, {
-		limit: ps.limit,
-		skip: ps.offset,
-		sort: {
-			_id: -1
-		}
+	query.setParameters(votedQuery.getParameters());
+	//#endregion
+
+	//#region mute
+	const mutingQuery = Mutings.createQueryBuilder('muting')
+		.select('muting.muteeId')
+		.where('muting.muterId = :muterId', { muterId: user.id });
+
+	query
+		.andWhere(`poll.userId NOT IN (${ mutingQuery.getQuery() })`);
+
+	query.setParameters(mutingQuery.getParameters());
+	//#endregion
+
+	const polls = await query.take(ps.limit).skip(ps.offset).getMany();
+
+	if (polls.length === 0) return [];
+
+	const notes = await Notes.find({
+		id: In(polls.map(poll => poll.noteId))
 	});
 
-	return await Promise.all(notes.map(note => pack(note, user, {
+	return await Notes.packMany(notes, user, {
 		detail: true
-	})));
+	});
 });
diff --git a/src/server/api/endpoints/notes/polls/vote.ts b/src/server/api/endpoints/notes/polls/vote.ts
index ed20e0221f..7d0ed6e4f9 100644
--- a/src/server/api/endpoints/notes/polls/vote.ts
+++ b/src/server/api/endpoints/notes/polls/vote.ts
@@ -1,19 +1,19 @@
 import $ from 'cafy';
-import ID, { transform } from '../../../../../misc/cafy-id';
-import Vote from '../../../../../models/poll-vote';
-import Note from '../../../../../models/note';
-import Watching from '../../../../../models/note-watching';
+import { ID } from '../../../../../misc/cafy-id';
 import watch from '../../../../../services/note/watch';
 import { publishNoteStream } from '../../../../../services/stream';
-import notify from '../../../../../services/create-notification';
+import { createNotification } from '../../../../../services/create-notification';
 import define from '../../../define';
-import User, { IRemoteUser } from '../../../../../models/user';
 import { ApiError } from '../../../error';
 import { getNote } from '../../../common/getters';
 import { deliver } from '../../../../../queue';
 import { renderActivity } from '../../../../../remote/activitypub/renderer';
 import renderVote from '../../../../../remote/activitypub/renderer/vote';
 import { deliverQuestionUpdate } from '../../../../../services/note/polls/update';
+import { PollVotes, NoteWatchings, Users, Polls } from '../../../../../models';
+import { Not } from 'typeorm';
+import { IRemoteUser } from '../../../../../models/entities/user';
+import { genId } from '../../../../../misc/gen-id';
 
 export const meta = {
 	desc: {
@@ -30,7 +30,6 @@ export const meta = {
 	params: {
 		noteId: {
 			validator: $.type(ID),
-			transform: transform,
 			desc: {
 				'ja-JP': '対象の投稿のID',
 				'en-US': 'Target note ID'
@@ -84,26 +83,28 @@ export default define(meta, async (ps, user) => {
 		throw e;
 	});
 
-	if (note.poll == null) {
+	if (!note.hasPoll) {
 		throw new ApiError(meta.errors.noPoll);
 	}
 
-	if (note.poll.expiresAt && note.poll.expiresAt < createdAt) {
+	const poll = await Polls.findOne({ noteId: note.id });
+
+	if (poll.expiresAt && poll.expiresAt < createdAt) {
 		throw new ApiError(meta.errors.alreadyExpired);
 	}
 
-	if (!note.poll.choices.some(x => x.id == ps.choice)) {
+	if (poll.choices[ps.choice] == null) {
 		throw new ApiError(meta.errors.invalidChoice);
 	}
 
 	// if already voted
-	const exist = await Vote.find({
-		noteId: note._id,
-		userId: user._id
+	const exist = await PollVotes.find({
+		noteId: note.id,
+		userId: user.id
 	});
 
 	if (exist.length) {
-		if (note.poll.multiple) {
+		if (poll.multiple) {
 			if (exist.some(x => x.choice == ps.choice))
 				throw new ApiError(meta.errors.alreadyVoted);
 		} else {
@@ -112,69 +113,54 @@ export default define(meta, async (ps, user) => {
 	}
 
 	// Create vote
-	const vote = await Vote.insert({
+	const vote = await PollVotes.save({
+		id: genId(),
 		createdAt,
-		noteId: note._id,
-		userId: user._id,
+		noteId: note.id,
+		userId: user.id,
 		choice: ps.choice
 	});
 
-	const inc: any = {};
-	inc[`poll.choices.${note.poll.choices.findIndex(c => c.id == ps.choice)}.votes`] = 1;
-
 	// Increment votes count
-	await Note.update({ _id: note._id }, {
-		$inc: inc
-	});
+	const index = ps.choice + 1; // In SQL, array index is 1 based
+	await Polls.query(`UPDATE poll SET votes[${index}] = votes[${index}] + 1 WHERE id = '${poll.id}'`);
 
-	publishNoteStream(note._id, 'pollVoted', {
+	publishNoteStream(note.id, 'pollVoted', {
 		choice: ps.choice,
-		userId: user._id.toHexString()
+		userId: user.id
 	});
 
 	// Notify
-	notify(note.userId, user._id, 'poll_vote', {
-		noteId: note._id,
+	createNotification(note.userId, user.id, 'pollVote', {
+		noteId: note.id,
 		choice: ps.choice
 	});
 
 	// Fetch watchers
-	Watching
-		.find({
-			noteId: note._id,
-			userId: { $ne: user._id },
-			// 削除されたドキュメントは除く
-			deletedAt: { $exists: false }
-		}, {
-			fields: {
-				userId: true
-			}
-		})
-		.then(watchers => {
-			for (const watcher of watchers) {
-				notify(watcher.userId, user._id, 'poll_vote', {
-					noteId: note._id,
-					choice: ps.choice
-				});
-			}
-		});
+	NoteWatchings.find({
+		noteId: note.id,
+		userId: Not(user.id),
+	}).then(watchers => {
+		for (const watcher of watchers) {
+			createNotification(watcher.userId, user.id, 'pollVote', {
+				noteId: note.id,
+				choice: ps.choice
+			});
+		}
+	});
 
 	// この投稿をWatchする
-	if (user.settings.autoWatch !== false) {
-		watch(user._id, note);
+	if (user.autoWatch !== false) {
+		watch(user.id, note);
 	}
 
 	// リモート投票の場合リプライ送信
-	if (note._user.host != null) {
-		const pollOwner: IRemoteUser = await User.findOne({
-			_id: note.userId
-		});
+	if (note.userHost != null) {
+		const pollOwner: IRemoteUser = await Users.findOne(note.userId);
 
-		deliver(user, renderActivity(await renderVote(user, vote, note, pollOwner)), pollOwner.inbox);
+		deliver(user, renderActivity(await renderVote(user, vote, note, poll, pollOwner)), pollOwner.inbox);
 	}
 
 	// リモートフォロワーにUpdate配信
-	deliverQuestionUpdate(note._id);
-
-	return;
+	deliverQuestionUpdate(note.id);
 });