Use PostgreSQL instead of MongoDB (#4572)

* wip

* Update note.ts

* Update timeline.ts

* Update core.ts

* wip

* Update generate-visibility-query.ts

* wip

* wip

* wip

* wip

* wip

* Update global-timeline.ts

* wip

* wip

* wip

* Update vote.ts

* wip

* wip

* Update create.ts

* wip

* wip

* wip

* wip

* wip

* wip

* wip

* wip

* wip

* wip

* wip

* wip

* Update files.ts

* wip

* wip

* Update CONTRIBUTING.md

* wip

* wip

* wip

* wip

* wip

* wip

* wip

* wip

* Update read-notification.ts

* wip

* wip

* wip

* wip

* wip

* wip

* wip

* Update cancel.ts

* wip

* wip

* wip

* Update show.ts

* wip

* wip

* Update gen-id.ts

* Update create.ts

* Update id.ts

* wip

* wip

* wip

* wip

* wip

* wip

* wip

* Docker: Update files about Docker (#4599)

* Docker: Use cache if files used by `yarn install` was not updated

This patch reduces the number of times to installing node_modules.
For example, `yarn install` step will be skipped when only ".config/default.yml" is updated.

* Docker: Migrate MongoDB to Postgresql

Misskey uses Postgresql as a database instead of Mongodb since version 11.

* Docker: Uncomment about data persistence

This patch will save a lot of databases.

* wip

* wip

* wip

* Update activitypub.ts

* wip

* wip

* wip

* Update logs.ts

* wip

* Update drive-file.ts

* Update register.ts

* wip

* wip

* Update mentions.ts

* wip

* wip

* wip

* Update recommendation.ts

* wip

* Update index.ts

* wip

* Update recommendation.ts

* Doc: Update docker.ja.md and docker.en.md (#1) (#4608)

Update how to set up misskey.

* wip

* :v:

* wip

* Update note.ts

* Update postgre.ts

* wip

* wip

* wip

* wip

* Update add-file.ts

* wip

* wip

* wip

* Clean up

* Update logs.ts

* wip

* :pizza:

* wip

* Ad notes

* wip

* Update api-visibility.ts

* Update note.ts

* Update add-file.ts

* tests

* tests

* Update postgre.ts

* Update utils.ts

* wip

* wip

* Refactor

* wip

* Refactor

* wip

* wip

* Update show-users.ts

* Update update-instance.ts

* wip

* Update feed.ts

* Update outbox.ts

* Update outbox.ts

* Update user.ts

* wip

* Update list.ts

* Update update-hashtag.ts

* wip

* Update update-hashtag.ts

* Refactor

* Update update.ts

* wip

* wip

* :v:

* clean up

* docs

* Update push.ts

* wip

* Update api.ts

* wip

* :v:

* Update make-pagination-query.ts

* :v:

* Delete hashtags.ts

* Update instances.ts

* Update instances.ts

* Update create.ts

* Update search.ts

* Update reversi-game.ts

* Update signup.ts

* Update user.ts

* id

* Update example.yml

* :art:

* objectid

* fix

* reversi

* reversi

* Fix bug of chart engine

* Add test of chart engine

* Improve test

* Better testing

* Improve chart engine

* Refactor

* Add test of chart engine

* Refactor

* Add chart test

* Fix bug

* コミットし忘れ

* Refactoring

* :v:

* Add tests

* Add test

* Extarct note tests

* Refactor

* 存在しないユーザーにメンションできなくなっていた問題を修正

* Fix bug

* Update update-meta.ts

* Fix bug

* Update mention.vue

* Fix bug

* Update meta.ts

* Update CONTRIBUTING.md

* Fix bug

* Fix bug

* Fix bug

* Clean up

* Clean up

* Update notification.ts

* Clean up

* Add mute tests

* Add test

* Refactor

* Add test

* Fix test

* Refactor

* Refactor

* Add tests

* Update utils.ts

* Update utils.ts

* Fix test

* Update package.json

* Update update.ts

* Update manifest.ts

* Fix bug

* Fix bug

* Add test

* :art:

* Update endpoint permissions

* Updaye permisison

* Update person.ts

#4299

* データベースと同期しないように

* Fix bug

* Fix bug

* Update reversi-game.ts

* Use a feature of Node v11.7.0 to extract a public key (#4644)

* wip

* wip

* :v:

* Refactoring

#1540

* test

* test

* test

* test

* test

* test

* test

* Fix bug

* Fix test

* :sushi:

* wip

* #4471

* Add test for #4335

* Refactor

* Fix test

* Add tests

* :clock4:

* Fix bug

* Add test

* Add test

* rename

* Fix bug

4 files changed, 62 insertions, 108 deletions

diff --git a/src/server/activitypub/featured.ts b/src/server/activitypub/featured.ts
index fc6150902b..f43312d79a 100644
--- a/src/server/activitypub/featured.ts
+++ b/src/server/activitypub/featured.ts
@@ -1,35 +1,28 @@
-import { ObjectID } from 'mongodb';
 import * as Router from 'koa-router';
 import config from '../../config';
-import User from '../../models/user';
 import { renderActivity } from '../../remote/activitypub/renderer';
 import renderOrderedCollection from '../../remote/activitypub/renderer/ordered-collection';
 import { setResponseType } from '../activitypub';
-import Note from '../../models/note';
 import renderNote from '../../remote/activitypub/renderer/note';
+import { Users, Notes, UserNotePinings } from '../../models';
 
 export default async (ctx: Router.IRouterContext) => {
-	if (!ObjectID.isValid(ctx.params.user)) {
-		ctx.status = 404;
-		return;
-	}
-
-	const userId = new ObjectID(ctx.params.user);
+	const userId = ctx.params.user;
 
 	// Verify user
-	const user = await User.findOne({
-		_id: userId,
+	const user = await Users.findOne({
+		id: userId,
 		host: null
 	});
 
-	if (user === null) {
+	if (user == null) {
 		ctx.status = 404;
 		return;
 	}
 
-	const pinnedNoteIds = user.pinnedNoteIds || [];
+	const pinings = await UserNotePinings.find({ userId: user.id });
 
-	const pinnedNotes = await Promise.all(pinnedNoteIds.filter(ObjectID.isValid).map(id => Note.findOne({ _id: id })));
+	const pinnedNotes = await Promise.all(pinings.map(pining => Notes.findOne(pining.noteId)));
 
 	const renderedNotes = await Promise.all(pinnedNotes.map(note => renderNote(note)));
 
diff --git a/src/server/activitypub/followers.ts b/src/server/activitypub/followers.ts
index 002576b2e7..62c54399ed 100644
--- a/src/server/activitypub/followers.ts
+++ b/src/server/activitypub/followers.ts
@@ -1,24 +1,18 @@
-import { ObjectID } from 'mongodb';
 import * as Router from 'koa-router';
 import config from '../../config';
 import $ from 'cafy';
-import ID, { transform } from '../../misc/cafy-id';
-import User from '../../models/user';
-import Following from '../../models/following';
+import { ID } from '../../misc/cafy-id';
 import * as url from '../../prelude/url';
 import { renderActivity } from '../../remote/activitypub/renderer';
 import renderOrderedCollection from '../../remote/activitypub/renderer/ordered-collection';
 import renderOrderedCollectionPage from '../../remote/activitypub/renderer/ordered-collection-page';
 import renderFollowUser from '../../remote/activitypub/renderer/follow-user';
 import { setResponseType } from '../activitypub';
+import { Users, Followings } from '../../models';
+import { LessThan } from 'typeorm';
 
 export default async (ctx: Router.IRouterContext) => {
-	if (!ObjectID.isValid(ctx.params.user)) {
-		ctx.status = 404;
-		return;
-	}
-
-	const userId = new ObjectID(ctx.params.user);
+	const userId = ctx.params.user;
 
 	// Get 'cursor' parameter
 	const [cursor, cursorErr] = $.optional.type(ID).get(ctx.request.query.cursor);
@@ -34,12 +28,12 @@ export default async (ctx: Router.IRouterContext) => {
 	}
 
 	// Verify user
-	const user = await User.findOne({
-		_id: userId,
+	const user = await Users.findOne({
+		id: userId,
 		host: null
 	});
 
-	if (user === null) {
+	if (user == null) {
 		ctx.status = 404;
 		return;
 	}
@@ -49,22 +43,20 @@ export default async (ctx: Router.IRouterContext) => {
 
 	if (page) {
 		const query = {
-			followeeId: user._id
+			followeeId: user.id
 		} as any;
 
 		// カーソルが指定されている場合
 		if (cursor) {
-			query._id = {
-				$lt: transform(cursor)
-			};
+			query.id = LessThan(cursor);
 		}
 
 		// Get followers
-		const followings = await Following
-			.find(query, {
-				limit: limit + 1,
-				sort: { _id: -1 }
-			});
+		const followings = await Followings.find({
+			where: query,
+			take: limit + 1,
+			order: { id: -1 }
+		});
 
 		// 「次のページ」があるかどうか
 		const inStock = followings.length === limit + 1;
@@ -80,7 +72,7 @@ export default async (ctx: Router.IRouterContext) => {
 			null,
 			inStock ? `${partOf}?${url.query({
 				page: 'true',
-				cursor: followings[followings.length - 1]._id.toHexString()
+				cursor: followings[followings.length - 1].id
 			})}` : null
 		);
 
diff --git a/src/server/activitypub/following.ts b/src/server/activitypub/following.ts
index 0d7486f68a..4894aac1f8 100644
--- a/src/server/activitypub/following.ts
+++ b/src/server/activitypub/following.ts
@@ -1,24 +1,19 @@
-import { ObjectID } from 'mongodb';
 import * as Router from 'koa-router';
 import config from '../../config';
 import $ from 'cafy';
-import ID, { transform } from '../../misc/cafy-id';
-import User from '../../models/user';
-import Following from '../../models/following';
+import { ID } from '../../misc/cafy-id';
 import * as url from '../../prelude/url';
 import { renderActivity } from '../../remote/activitypub/renderer';
 import renderOrderedCollection from '../../remote/activitypub/renderer/ordered-collection';
 import renderOrderedCollectionPage from '../../remote/activitypub/renderer/ordered-collection-page';
 import renderFollowUser from '../../remote/activitypub/renderer/follow-user';
 import { setResponseType } from '../activitypub';
+import { Users, Followings } from '../../models';
+import { LessThan, FindConditions } from 'typeorm';
+import { Following } from '../../models/entities/following';
 
 export default async (ctx: Router.IRouterContext) => {
-	if (!ObjectID.isValid(ctx.params.user)) {
-		ctx.status = 404;
-		return;
-	}
-
-	const userId = new ObjectID(ctx.params.user);
+	const userId = ctx.params.user;
 
 	// Get 'cursor' parameter
 	const [cursor, cursorErr] = $.optional.type(ID).get(ctx.request.query.cursor);
@@ -34,12 +29,12 @@ export default async (ctx: Router.IRouterContext) => {
 	}
 
 	// Verify user
-	const user = await User.findOne({
-		_id: userId,
+	const user = await Users.findOne({
+		id: userId,
 		host: null
 	});
 
-	if (user === null) {
+	if (user == null) {
 		ctx.status = 404;
 		return;
 	}
@@ -49,22 +44,20 @@ export default async (ctx: Router.IRouterContext) => {
 
 	if (page) {
 		const query = {
-			followerId: user._id
-		} as any;
+			followerId: user.id
+		} as FindConditions<Following>;
 
 		// カーソルが指定されている場合
 		if (cursor) {
-			query._id = {
-				$lt: transform(cursor)
-			};
+			query.id = LessThan(cursor);
 		}
 
 		// Get followings
-		const followings = await Following
-			.find(query, {
-				limit: limit + 1,
-				sort: { _id: -1 }
-			});
+		const followings = await Followings.find({
+			where: query,
+			take: limit + 1,
+			order: { id: -1 }
+		});
 
 		// 「次のページ」があるかどうか
 		const inStock = followings.length === limit + 1;
@@ -80,7 +73,7 @@ export default async (ctx: Router.IRouterContext) => {
 			null,
 			inStock ? `${partOf}?${url.query({
 				page: 'true',
-				cursor: followings[followings.length - 1]._id.toHexString()
+				cursor: followings[followings.length - 1].id
 			})}` : null
 		);
 
diff --git a/src/server/activitypub/outbox.ts b/src/server/activitypub/outbox.ts
index ff8f884b19..377f43c986 100644
--- a/src/server/activitypub/outbox.ts
+++ b/src/server/activitypub/outbox.ts
@@ -1,28 +1,23 @@
-import { ObjectID } from 'mongodb';
 import * as Router from 'koa-router';
 import config from '../../config';
 import $ from 'cafy';
-import ID, { transform } from '../../misc/cafy-id';
-import User from '../../models/user';
+import { ID } from '../../misc/cafy-id';
 import { renderActivity } from '../../remote/activitypub/renderer';
 import renderOrderedCollection from '../../remote/activitypub/renderer/ordered-collection';
 import renderOrderedCollectionPage from '../../remote/activitypub/renderer/ordered-collection-page';
 import { setResponseType } from '../activitypub';
-
-import Note, { INote } from '../../models/note';
 import renderNote from '../../remote/activitypub/renderer/note';
 import renderCreate from '../../remote/activitypub/renderer/create';
 import renderAnnounce from '../../remote/activitypub/renderer/announce';
 import { countIf } from '../../prelude/array';
 import * as url from '../../prelude/url';
+import { Users, Notes } from '../../models';
+import { makePaginationQuery } from '../api/common/make-pagination-query';
+import { Brackets } from 'typeorm';
+import { Note } from '../../models/entities/note';
 
 export default async (ctx: Router.IRouterContext) => {
-	if (!ObjectID.isValid(ctx.params.user)) {
-		ctx.status = 404;
-		return;
-	}
-
-	const userId = new ObjectID(ctx.params.user);
+	const userId = ctx.params.user;
 
 	// Get 'sinceId' parameter
 	const [sinceId, sinceIdErr] = $.optional.type(ID).get(ctx.request.query.since_id);
@@ -41,12 +36,12 @@ export default async (ctx: Router.IRouterContext) => {
 	}
 
 	// Verify user
-	const user = await User.findOne({
-		_id: userId,
+	const user = await Users.findOne({
+		id: userId,
 		host: null
 	});
 
-	if (user === null) {
+	if (user == null) {
 		ctx.status = 404;
 		return;
 	}
@@ -55,34 +50,15 @@ export default async (ctx: Router.IRouterContext) => {
 	const partOf = `${config.url}/users/${userId}/outbox`;
 
 	if (page) {
-		//#region Construct query
-		const sort = {
-			_id: -1
-		};
-
-		const query = {
-			userId: user._id,
-			visibility: { $in: ['public', 'home'] },
-			localOnly: { $ne: true }
-		} as any;
-
-		if (sinceId) {
-			sort._id = 1;
-			query._id = {
-				$gt: transform(sinceId)
-			};
-		} else if (untilId) {
-			query._id = {
-				$lt: transform(untilId)
-			};
-		}
-		//#endregion
+		const query = makePaginationQuery(Notes.createQueryBuilder('note'), sinceId, untilId)
+			.andWhere('note.userId = :userId', { userId: user.id })
+			.andWhere(new Brackets(qb => { qb
+				.where(`note.visibility = 'public'`)
+				.orWhere(`note.visibility = 'home'`);
+			}))
+			.andWhere('note.localOnly = FALSE');
 
-		const notes = await Note
-			.find(query, {
-				limit: limit,
-				sort: sort
-			});
+		const notes = await query.take(limit).getMany();
 
 		if (sinceId) notes.reverse();
 
@@ -96,11 +72,11 @@ export default async (ctx: Router.IRouterContext) => {
 			user.notesCount, activities, partOf,
 			notes.length ? `${partOf}?${url.query({
 				page: 'true',
-				since_id: notes[0]._id.toHexString()
+				since_id: notes[0].id
 			})}` : null,
 			notes.length ? `${partOf}?${url.query({
 				page: 'true',
-				until_id: notes[notes.length - 1]._id.toHexString()
+				until_id: notes[notes.length - 1].id
 			})}` : null
 		);
 
@@ -123,10 +99,10 @@ export default async (ctx: Router.IRouterContext) => {
  * Pack Create<Note> or Announce Activity
  * @param note Note
  */
-export async function packActivity(note: INote): Promise<object> {
-	if (note.renoteId && note.text == null && note.poll == null && (note.fileIds == null || note.fileIds.length == 0)) {
-		const renote = await Note.findOne(note.renoteId);
-		return renderAnnounce(renote.uri ? renote.uri : `${config.url}/notes/${renote._id}`, note);
+export async function packActivity(note: Note): Promise<object> {
+	if (note.renoteId && note.text == null && !note.hasPoll && (note.fileIds == null || note.fileIds.length == 0)) {
+		const renote = await Notes.findOne(note.renoteId);
+		return renderAnnounce(renote.uri ? renote.uri : `${config.url}/notes/${renote.id}`, note);
 	}
 
 	return renderCreate(await renderNote(note, false), note);