Use PostgreSQL instead of MongoDB (#4572)

* wip

* Update note.ts

* Update timeline.ts

* Update core.ts

* wip

* Update generate-visibility-query.ts

* wip

* wip

* wip

* wip

* wip

* Update global-timeline.ts

* wip

* wip

* wip

* Update vote.ts

* wip

* wip

* Update create.ts

* wip

* wip

* wip

* wip

* wip

* wip

* wip

* wip

* wip

* wip

* wip

* wip

* Update files.ts

* wip

* wip

* Update CONTRIBUTING.md

* wip

* wip

* wip

* wip

* wip

* wip

* wip

* wip

* Update read-notification.ts

* wip

* wip

* wip

* wip

* wip

* wip

* wip

* Update cancel.ts

* wip

* wip

* wip

* Update show.ts

* wip

* wip

* Update gen-id.ts

* Update create.ts

* Update id.ts

* wip

* wip

* wip

* wip

* wip

* wip

* wip

* Docker: Update files about Docker (#4599)

* Docker: Use cache if files used by `yarn install` was not updated

This patch reduces the number of times to installing node_modules.
For example, `yarn install` step will be skipped when only ".config/default.yml" is updated.

* Docker: Migrate MongoDB to Postgresql

Misskey uses Postgresql as a database instead of Mongodb since version 11.

* Docker: Uncomment about data persistence

This patch will save a lot of databases.

* wip

* wip

* wip

* Update activitypub.ts

* wip

* wip

* wip

* Update logs.ts

* wip

* Update drive-file.ts

* Update register.ts

* wip

* wip

* Update mentions.ts

* wip

* wip

* wip

* Update recommendation.ts

* wip

* Update index.ts

* wip

* Update recommendation.ts

* Doc: Update docker.ja.md and docker.en.md (#1) (#4608)

Update how to set up misskey.

* wip

* :v:

* wip

* Update note.ts

* Update postgre.ts

* wip

* wip

* wip

* wip

* Update add-file.ts

* wip

* wip

* wip

* Clean up

* Update logs.ts

* wip

* :pizza:

* wip

* Ad notes

* wip

* Update api-visibility.ts

* Update note.ts

* Update add-file.ts

* tests

* tests

* Update postgre.ts

* Update utils.ts

* wip

* wip

* Refactor

* wip

* Refactor

* wip

* wip

* Update show-users.ts

* Update update-instance.ts

* wip

* Update feed.ts

* Update outbox.ts

* Update outbox.ts

* Update user.ts

* wip

* Update list.ts

* Update update-hashtag.ts

* wip

* Update update-hashtag.ts

* Refactor

* Update update.ts

* wip

* wip

* :v:

* clean up

* docs

* Update push.ts

* wip

* Update api.ts

* wip

* :v:

* Update make-pagination-query.ts

* :v:

* Delete hashtags.ts

* Update instances.ts

* Update instances.ts

* Update create.ts

* Update search.ts

* Update reversi-game.ts

* Update signup.ts

* Update user.ts

* id

* Update example.yml

* :art:

* objectid

* fix

* reversi

* reversi

* Fix bug of chart engine

* Add test of chart engine

* Improve test

* Better testing

* Improve chart engine

* Refactor

* Add test of chart engine

* Refactor

* Add chart test

* Fix bug

* コミットし忘れ

* Refactoring

* :v:

* Add tests

* Add test

* Extarct note tests

* Refactor

* 存在しないユーザーにメンションできなくなっていた問題を修正

* Fix bug

* Update update-meta.ts

* Fix bug

* Update mention.vue

* Fix bug

* Update meta.ts

* Update CONTRIBUTING.md

* Fix bug

* Fix bug

* Fix bug

* Clean up

* Clean up

* Update notification.ts

* Clean up

* Add mute tests

* Add test

* Refactor

* Add test

* Fix test

* Refactor

* Refactor

* Add tests

* Update utils.ts

* Update utils.ts

* Fix test

* Update package.json

* Update update.ts

* Update manifest.ts

* Fix bug

* Fix bug

* Add test

* :art:

* Update endpoint permissions

* Updaye permisison

* Update person.ts

#4299

* データベースと同期しないように

* Fix bug

* Fix bug

* Update reversi-game.ts

* Use a feature of Node v11.7.0 to extract a public key (#4644)

* wip

* wip

* :v:

* Refactoring

#1540

* test

* test

* test

* test

* test

* test

* test

* Fix bug

* Fix test

* :sushi:

* wip

* #4471

* Add test for #4335

* Refactor

* Fix test

* Add tests

* :clock4:

* Fix bug

* Add test

* Add test

* rename

* Fix bug

2 files changed, 54 insertions, 68 deletions

diff --git a/src/server/api/private/signin.ts b/src/server/api/private/signin.ts
index 40bcd2c5d6..c1fd908d8a 100644
--- a/src/server/api/private/signin.ts
+++ b/src/server/api/private/signin.ts
@@ -1,11 +1,12 @@
 import * as Koa from 'koa';
 import * as bcrypt from 'bcryptjs';
 import * as speakeasy from 'speakeasy';
-import User, { ILocalUser } from '../../../models/user';
-import Signin, { pack } from '../../../models/signin';
 import { publishMainStream } from '../../../services/stream';
 import signin from '../common/signin';
 import config from '../../../config';
+import { Users, Signins } from '../../../models';
+import { ILocalUser } from '../../../models/entities/user';
+import { genId } from '../../../misc/gen-id';
 
 export default async (ctx: Koa.BaseContext) => {
 	ctx.set('Access-Control-Allow-Origin', config.url);
@@ -32,17 +33,12 @@ export default async (ctx: Koa.BaseContext) => {
 	}
 
 	// Fetch user
-	const user = await User.findOne({
+	const user = await Users.findOne({
 		usernameLower: username.toLowerCase(),
 		host: null
-	}, {
-			fields: {
-				data: false,
-				profile: false
-			}
-		}) as ILocalUser;
+	}) as ILocalUser;
 
-	if (user === null) {
+	if (user == null) {
 		ctx.throw(404, {
 			error: 'user not found'
 		});
@@ -77,14 +73,15 @@ export default async (ctx: Koa.BaseContext) => {
 	}
 
 	// Append signin history
-	const record = await Signin.insert({
+	const record = await Signins.save({
+		id: genId(),
 		createdAt: new Date(),
-		userId: user._id,
+		userId: user.id,
 		ip: ctx.ip,
 		headers: ctx.headers,
 		success: same
 	});
 
 	// Publish signin event
-	publishMainStream(user._id, 'signin', await pack(record));
+	publishMainStream(user.id, 'signin', await Signins.pack(record));
 };
diff --git a/src/server/api/private/signup.ts b/src/server/api/private/signup.ts
index 89b7b330d2..1d304b8e11 100644
--- a/src/server/api/private/signup.ts
+++ b/src/server/api/private/signup.ts
@@ -1,14 +1,15 @@
 import * as Koa from 'koa';
 import * as bcrypt from 'bcryptjs';
-import { generate as generateKeypair } from '../../../crypto_key';
-import User, { IUser, validateUsername, validatePassword, pack } from '../../../models/user';
+import { generateKeyPair } from 'crypto';
 import generateUserToken from '../common/generate-native-user-token';
 import config from '../../../config';
-import Meta from '../../../models/meta';
-import RegistrationTicket from '../../../models/registration-tickets';
-import usersChart from '../../../services/chart/users';
 import fetchMeta from '../../../misc/fetch-meta';
 import * as recaptcha from 'recaptcha-promise';
+import { Users, RegistrationTickets, UserServiceLinkings, UserKeypairs } from '../../../models';
+import { genId } from '../../../misc/gen-id';
+import { usersChart } from '../../../services/chart';
+import { UserServiceLinking } from '../../../models/entities/user-service-linking';
+import { User } from '../../../models/entities/user';
 
 export default async (ctx: Koa.BaseContext) => {
 	const body = ctx.request.body as any;
@@ -32,6 +33,7 @@ export default async (ctx: Koa.BaseContext) => {
 
 	const username = body['username'];
 	const password = body['password'];
+	const host = process.env.NODE_ENV === 'test' ? (body['host'] || null) : null;
 	const invitationCode = body['invitationCode'];
 
 	if (instance && instance.disableRegistration) {
@@ -40,7 +42,7 @@ export default async (ctx: Koa.BaseContext) => {
 			return;
 		}
 
-		const ticket = await RegistrationTicket.findOne({
+		const ticket = await RegistrationTickets.findOne({
 			code: invitationCode
 		});
 
@@ -49,39 +51,22 @@ export default async (ctx: Koa.BaseContext) => {
 			return;
 		}
 
-		RegistrationTicket.remove({
-			_id: ticket._id
-		});
+		RegistrationTickets.delete(ticket.id);
 	}
 
 	// Validate username
-	if (!validateUsername(username)) {
+	if (!Users.validateUsername(username)) {
 		ctx.status = 400;
 		return;
 	}
 
 	// Validate password
-	if (!validatePassword(password)) {
+	if (!Users.validatePassword(password)) {
 		ctx.status = 400;
 		return;
 	}
 
-	const usersCount = await User.count({});
-
-	// Fetch exist user that same username
-	const usernameExist = await User
-		.count({
-			usernameLower: username.toLowerCase(),
-			host: null
-		}, {
-			limit: 1
-		});
-
-	// Check username already used
-	if (usernameExist !== 0) {
-		ctx.status = 400;
-		return;
-	}
+	const usersCount = await Users.count({});
 
 	// Generate hash of password
 	const salt = await bcrypt.genSalt(8);
@@ -90,46 +75,50 @@ export default async (ctx: Koa.BaseContext) => {
 	// Generate secret
 	const secret = generateUserToken();
 
-	// Create account
-	const account: IUser = await User.insert({
-		avatarId: null,
-		bannerId: null,
+	if (await Users.findOne({ usernameLower: username.toLowerCase(), host: null })) {
+		ctx.status = 400;
+		return;
+	}
+
+	const account = await Users.save({
+		id: genId(),
 		createdAt: new Date(),
-		description: null,
-		followersCount: 0,
-		followingCount: 0,
-		name: null,
-		notesCount: 0,
 		username: username,
 		usernameLower: username.toLowerCase(),
-		host: null,
-		keypair: generateKeypair(),
+		host: host,
 		token: secret,
 		password: hash,
 		isAdmin: config.autoAdmin && usersCount === 0,
 		autoAcceptFollowed: true,
-		profile: {
-			bio: null,
-			birthday: null,
-			location: null
-		},
-		settings: {
-			autoWatch: false
-		}
+		autoWatch: false
+	} as User);
+
+	await UserKeypairs.save({
+		id: genId(),
+		keyPem: await new Promise<string>((s, j) => generateKeyPair('rsa', {
+			modulusLength: 4096,
+			publicKeyEncoding: {
+				type: 'pkcs1',
+				format: 'pem'
+			},
+			privateKeyEncoding: {
+				type: 'pkcs1',
+				format: 'pem',
+				cipher: undefined,
+				passphrase: undefined
+			}
+		}, (e, _, x) => e ? j(e) : s(x))),
+		userId: account.id
 	});
 
-	//#region Increment users count
-	Meta.update({}, {
-		$inc: {
-			'stats.usersCount': 1,
-			'stats.originalUsersCount': 1
-		}
-	}, { upsert: true });
-	//#endregion
+	await UserServiceLinkings.save({
+		id: genId(),
+		userId: account.id
+	} as UserServiceLinking);
 
 	usersChart.update(account, true);
 
-	const res = await pack(account, account, {
+	const res = await Users.pack(account, account, {
 		detail: true,
 		includeSecrets: true
 	});