Use PostgreSQL instead of MongoDB (#4572)

* wip

* Update note.ts

* Update timeline.ts

* Update core.ts

* wip

* Update generate-visibility-query.ts

* wip

* wip

* wip

* wip

* wip

* Update global-timeline.ts

* wip

* wip

* wip

* Update vote.ts

* wip

* wip

* Update create.ts

* wip

* wip

* wip

* wip

* wip

* wip

* wip

* wip

* wip

* wip

* wip

* wip

* Update files.ts

* wip

* wip

* Update CONTRIBUTING.md

* wip

* wip

* wip

* wip

* wip

* wip

* wip

* wip

* Update read-notification.ts

* wip

* wip

* wip

* wip

* wip

* wip

* wip

* Update cancel.ts

* wip

* wip

* wip

* Update show.ts

* wip

* wip

* Update gen-id.ts

* Update create.ts

* Update id.ts

* wip

* wip

* wip

* wip

* wip

* wip

* wip

* Docker: Update files about Docker (#4599)

* Docker: Use cache if files used by `yarn install` was not updated

This patch reduces the number of times to installing node_modules.
For example, `yarn install` step will be skipped when only ".config/default.yml" is updated.

* Docker: Migrate MongoDB to Postgresql

Misskey uses Postgresql as a database instead of Mongodb since version 11.

* Docker: Uncomment about data persistence

This patch will save a lot of databases.

* wip

* wip

* wip

* Update activitypub.ts

* wip

* wip

* wip

* Update logs.ts

* wip

* Update drive-file.ts

* Update register.ts

* wip

* wip

* Update mentions.ts

* wip

* wip

* wip

* Update recommendation.ts

* wip

* Update index.ts

* wip

* Update recommendation.ts

* Doc: Update docker.ja.md and docker.en.md (#1) (#4608)

Update how to set up misskey.

* wip

* :v:

* wip

* Update note.ts

* Update postgre.ts

* wip

* wip

* wip

* wip

* Update add-file.ts

* wip

* wip

* wip

* Clean up

* Update logs.ts

* wip

* :pizza:

* wip

* Ad notes

* wip

* Update api-visibility.ts

* Update note.ts

* Update add-file.ts

* tests

* tests

* Update postgre.ts

* Update utils.ts

* wip

* wip

* Refactor

* wip

* Refactor

* wip

* wip

* Update show-users.ts

* Update update-instance.ts

* wip

* Update feed.ts

* Update outbox.ts

* Update outbox.ts

* Update user.ts

* wip

* Update list.ts

* Update update-hashtag.ts

* wip

* Update update-hashtag.ts

* Refactor

* Update update.ts

* wip

* wip

* :v:

* clean up

* docs

* Update push.ts

* wip

* Update api.ts

* wip

* :v:

* Update make-pagination-query.ts

* :v:

* Delete hashtags.ts

* Update instances.ts

* Update instances.ts

* Update create.ts

* Update search.ts

* Update reversi-game.ts

* Update signup.ts

* Update user.ts

* id

* Update example.yml

* :art:

* objectid

* fix

* reversi

* reversi

* Fix bug of chart engine

* Add test of chart engine

* Improve test

* Better testing

* Improve chart engine

* Refactor

* Add test of chart engine

* Refactor

* Add chart test

* Fix bug

* コミットし忘れ

* Refactoring

* :v:

* Add tests

* Add test

* Extarct note tests

* Refactor

* 存在しないユーザーにメンションできなくなっていた問題を修正

* Fix bug

* Update update-meta.ts

* Fix bug

* Update mention.vue

* Fix bug

* Update meta.ts

* Update CONTRIBUTING.md

* Fix bug

* Fix bug

* Fix bug

* Clean up

* Clean up

* Update notification.ts

* Clean up

* Add mute tests

* Add test

* Refactor

* Add test

* Fix test

* Refactor

* Refactor

* Add tests

* Update utils.ts

* Update utils.ts

* Fix test

* Update package.json

* Update update.ts

* Update manifest.ts

* Fix bug

* Fix bug

* Add test

* :art:

* Update endpoint permissions

* Updaye permisison

* Update person.ts

#4299

* データベースと同期しないように

* Fix bug

* Fix bug

* Update reversi-game.ts

* Use a feature of Node v11.7.0 to extract a public key (#4644)

* wip

* wip

* :v:

* Refactoring

#1540

* test

* test

* test

* test

* test

* test

* test

* Fix bug

* Fix test

* :sushi:

* wip

* #4471

* Add test for #4335

* Refactor

* Fix test

* Add tests

* :clock4:

* Fix bug

* Add test

* Add test

* rename

* Fix bug

1 files changed, 29 insertions, 132 deletions

diff --git a/src/server/api/endpoints/notes/user-list-timeline.ts b/src/server/api/endpoints/notes/user-list-timeline.ts
index 17c24ab119..deda04acb4 100644
--- a/src/server/api/endpoints/notes/user-list-timeline.ts
+++ b/src/server/api/endpoints/notes/user-list-timeline.ts
@@ -1,12 +1,11 @@
 import $ from 'cafy';
-import ID, { transform } from '../../../../misc/cafy-id';
-import Note from '../../../../models/note';
-import { packMany } from '../../../../models/note';
-import UserList from '../../../../models/user-list';
+import { ID } from '../../../../misc/cafy-id';
 import define from '../../define';
-import { getFriends } from '../../common/get-friends';
-import { getHideUserIds } from '../../common/get-hide-users';
 import { ApiError } from '../../error';
+import { UserLists, UserListJoinings, Notes } from '../../../../models';
+import { makePaginationQuery } from '../../common/make-pagination-query';
+import { generateVisibilityQuery } from '../../common/generate-visibility-query';
+import { activeUsersChart } from '../../../../services/chart';
 
 export const meta = {
 	desc: {
@@ -21,7 +20,6 @@ export const meta = {
 	params: {
 		listId: {
 			validator: $.type(ID),
-			transform: transform,
 			desc: {
 				'ja-JP': 'リストのID'
 			}
@@ -37,17 +35,15 @@ export const meta = {
 
 		sinceId: {
 			validator: $.optional.type(ID),
-			transform: transform,
 			desc: {
-				'ja-JP': '指定すると、この投稿を基点としてより新しい投稿を取得します'
+				'ja-JP': '指定すると、その投稿を基点としてより新しい投稿を取得します'
 			}
 		},
 
 		untilId: {
 			validator: $.optional.type(ID),
-			transform: transform,
 			desc: {
-				'ja-JP': '指定すると、この投稿を基点としてより古い投稿を取得します'
+				'ja-JP': '指定すると、その投稿を基点としてより古い投稿を取得します'
 			}
 		},
 
@@ -95,14 +91,6 @@ export const meta = {
 				'ja-JP': 'true にすると、ファイルが添付された投稿だけ取得します'
 			}
 		},
-
-		mediaOnly: {
-			validator: $.optional.bool,
-			deprecated: true,
-			desc: {
-				'ja-JP': 'true にすると、ファイルが添付された投稿だけ取得します (このパラメータは廃止予定です。代わりに withFiles を使ってください。)'
-			}
-		},
 	},
 
 	res: {
@@ -122,94 +110,28 @@ export const meta = {
 };
 
 export default define(meta, async (ps, user) => {
-	const [list, followings, hideUserIds] = await Promise.all([
-		// リストを取得
-		// Fetch the list
-		UserList.findOne({
-			_id: ps.listId,
-			userId: user._id
-		}),
-
-		// フォローを取得
-		// Fetch following
-		getFriends(user._id, true, false),
-
-		// 隠すユーザーを取得
-		getHideUserIds(user)
-	]);
+	const list = await UserLists.findOne({
+		id: ps.listId,
+		userId: user.id
+	});
 
 	if (list == null) {
 		throw new ApiError(meta.errors.noSuchList);
 	}
 
-	if (list.userIds.length == 0) {
-		return [];
-	}
-
 	//#region Construct query
-	const sort = {
-		_id: -1
-	};
-
-	const listQuery = list.userIds.map(u => ({
-		userId: u,
-
-		/*// リプライは含めない(ただし投稿者自身の投稿へのリプライ、自分の投稿へのリプライ、自分のリプライは含める)
-		$or: [{
-			// リプライでない
-			replyId: null
-		}, { // または
-			// リプライだが返信先が投稿者自身の投稿
-			$expr: {
-				$eq: ['$_reply.userId', '$userId']
-			}
-		}, { // または
-			// リプライだが返信先が自分(フォロワー)の投稿
-			'_reply.userId': user._id
-		}, { // または
-			// 自分(フォロワー)が送信したリプライ
-			userId: user._id
-		}]*/
-	}));
-
-	const visibleQuery = [{
-		visibility: { $in: ['public', 'home'] }
-	}, {
-		// myself (for specified/private)
-		userId: user._id
-	}, {
-		// to me (for specified)
-		visibleUserIds: { $in: [user._id] }
-	}, {
-		visibility: 'followers',
-		userId: { $in: followings.map(f => f.id) }
-	}];
-
-	const query = {
-		$and: [{
-			deletedAt: null,
+	const listQuery = UserListJoinings.createQueryBuilder('joining')
+		.select('joining.userId')
+		.where('joining.userListId = :userListId', { userListId: list.id });
 
-			$and: [{
-				// リストに入っている人のタイムラインへの投稿
-				$or: listQuery
-			}, {
-				// visible for me
-				$or: visibleQuery
-			}],
+	const query = makePaginationQuery(Notes.createQueryBuilder('note'), ps.sinceId, ps.untilId)
+		.andWhere(`note.userId IN (${ listQuery.getQuery() })`)
+		.leftJoinAndSelect('note.user', 'user')
+		.setParameters(listQuery.getParameters());
 
-			// mute
-			userId: {
-				$nin: hideUserIds
-			},
-			'_reply.userId': {
-				$nin: hideUserIds
-			},
-			'_renote.userId': {
-				$nin: hideUserIds
-			},
-		}]
-	} as any;
+	generateVisibilityQuery(query, user);
 
+	/* TODO
 	// MongoDBではトップレベルで否定ができないため、De Morganの法則を利用してクエリします。
 	// つまり、「『自分の投稿かつRenote』ではない」を「『自分の投稿ではない』または『Renoteではない』」と表現します。
 	// for details: https://en.wikipedia.org/wiki/De_Morgan%27s_laws
@@ -217,7 +139,7 @@ export default define(meta, async (ps, user) => {
 	if (ps.includeMyRenotes === false) {
 		query.$and.push({
 			$or: [{
-				userId: { $ne: user._id }
+				userId: { $ne: user.id }
 			}, {
 				renoteId: null
 			}, {
@@ -233,7 +155,7 @@ export default define(meta, async (ps, user) => {
 	if (ps.includeRenotedMyNotes === false) {
 		query.$and.push({
 			$or: [{
-				'_renote.userId': { $ne: user._id }
+				'_renote.userId': { $ne: user.id }
 			}, {
 				renoteId: null
 			}, {
@@ -260,41 +182,16 @@ export default define(meta, async (ps, user) => {
 				poll: { $ne: null }
 			}]
 		});
-	}
-
-	const withFiles = ps.withFiles != null ? ps.withFiles : ps.mediaOnly;
-
-	if (withFiles) {
-		query.$and.push({
-			fileIds: { $exists: true, $ne: [] }
-		});
-	}
+	}*/
 
-	if (ps.sinceId) {
-		sort._id = 1;
-		query._id = {
-			$gt: ps.sinceId
-		};
-	} else if (ps.untilId) {
-		query._id = {
-			$lt: ps.untilId
-		};
-	} else if (ps.sinceDate) {
-		sort._id = 1;
-		query.createdAt = {
-			$gt: new Date(ps.sinceDate)
-		};
-	} else if (ps.untilDate) {
-		query.createdAt = {
-			$lt: new Date(ps.untilDate)
-		};
+	if (ps.withFiles) {
+		query.andWhere('note.fileIds != \'{}\'');
 	}
 	//#endregion
 
-	const timeline = await Note.find(query, {
-		limit: ps.limit,
-		sort: sort
-	});
+	const timeline = await query.take(ps.limit).getMany();
+
+	activeUsersChart.update(user);
 
-	return await packMany(timeline, user);
+	return await Notes.packMany(timeline, user);
 });