Use PostgreSQL instead of MongoDB (#4572)

* wip

* Update note.ts

* Update timeline.ts

* Update core.ts

* wip

* Update generate-visibility-query.ts

* wip

* wip

* wip

* wip

* wip

* Update global-timeline.ts

* wip

* wip

* wip

* Update vote.ts

* wip

* wip

* Update create.ts

* wip

* wip

* wip

* wip

* wip

* wip

* wip

* wip

* wip

* wip

* wip

* wip

* Update files.ts

* wip

* wip

* Update CONTRIBUTING.md

* wip

* wip

* wip

* wip

* wip

* wip

* wip

* wip

* Update read-notification.ts

* wip

* wip

* wip

* wip

* wip

* wip

* wip

* Update cancel.ts

* wip

* wip

* wip

* Update show.ts

* wip

* wip

* Update gen-id.ts

* Update create.ts

* Update id.ts

* wip

* wip

* wip

* wip

* wip

* wip

* wip

* Docker: Update files about Docker (#4599)

* Docker: Use cache if files used by `yarn install` was not updated

This patch reduces the number of times to installing node_modules.
For example, `yarn install` step will be skipped when only ".config/default.yml" is updated.

* Docker: Migrate MongoDB to Postgresql

Misskey uses Postgresql as a database instead of Mongodb since version 11.

* Docker: Uncomment about data persistence

This patch will save a lot of databases.

* wip

* wip

* wip

* Update activitypub.ts

* wip

* wip

* wip

* Update logs.ts

* wip

* Update drive-file.ts

* Update register.ts

* wip

* wip

* Update mentions.ts

* wip

* wip

* wip

* Update recommendation.ts

* wip

* Update index.ts

* wip

* Update recommendation.ts

* Doc: Update docker.ja.md and docker.en.md (#1) (#4608)

Update how to set up misskey.

* wip

* :v:

* wip

* Update note.ts

* Update postgre.ts

* wip

* wip

* wip

* wip

* Update add-file.ts

* wip

* wip

* wip

* Clean up

* Update logs.ts

* wip

* :pizza:

* wip

* Ad notes

* wip

* Update api-visibility.ts

* Update note.ts

* Update add-file.ts

* tests

* tests

* Update postgre.ts

* Update utils.ts

* wip

* wip

* Refactor

* wip

* Refactor

* wip

* wip

* Update show-users.ts

* Update update-instance.ts

* wip

* Update feed.ts

* Update outbox.ts

* Update outbox.ts

* Update user.ts

* wip

* Update list.ts

* Update update-hashtag.ts

* wip

* Update update-hashtag.ts

* Refactor

* Update update.ts

* wip

* wip

* :v:

* clean up

* docs

* Update push.ts

* wip

* Update api.ts

* wip

* :v:

* Update make-pagination-query.ts

* :v:

* Delete hashtags.ts

* Update instances.ts

* Update instances.ts

* Update create.ts

* Update search.ts

* Update reversi-game.ts

* Update signup.ts

* Update user.ts

* id

* Update example.yml

* :art:

* objectid

* fix

* reversi

* reversi

* Fix bug of chart engine

* Add test of chart engine

* Improve test

* Better testing

* Improve chart engine

* Refactor

* Add test of chart engine

* Refactor

* Add chart test

* Fix bug

* コミットし忘れ

* Refactoring

* :v:

* Add tests

* Add test

* Extarct note tests

* Refactor

* 存在しないユーザーにメンションできなくなっていた問題を修正

* Fix bug

* Update update-meta.ts

* Fix bug

* Update mention.vue

* Fix bug

* Update meta.ts

* Update CONTRIBUTING.md

* Fix bug

* Fix bug

* Fix bug

* Clean up

* Clean up

* Update notification.ts

* Clean up

* Add mute tests

* Add test

* Refactor

* Add test

* Fix test

* Refactor

* Refactor

* Add tests

* Update utils.ts

* Update utils.ts

* Fix test

* Update package.json

* Update update.ts

* Update manifest.ts

* Fix bug

* Fix bug

* Add test

* :art:

* Update endpoint permissions

* Updaye permisison

* Update person.ts

#4299

* データベースと同期しないように

* Fix bug

* Fix bug

* Update reversi-game.ts

* Use a feature of Node v11.7.0 to extract a public key (#4644)

* wip

* wip

* :v:

* Refactoring

#1540

* test

* test

* test

* test

* test

* test

* test

* Fix bug

* Fix test

* :sushi:

* wip

* #4471

* Add test for #4335

* Refactor

* Fix test

* Add tests

* :clock4:

* Fix bug

* Add test

* Add test

* rename

* Fix bug

3 files changed, 273 insertions, 0 deletions

diff --git a/src/boot/index.ts b/src/boot/index.ts
new file mode 100644
index 0000000000..2c86d8ed8c
--- /dev/null
+++ b/src/boot/index.ts
@@ -0,0 +1,77 @@
+import * as cluster from 'cluster';
+import chalk from 'chalk';
+import Xev from 'xev';
+
+import Logger from '../services/logger';
+import { program } from '../argv';
+
+// for typeorm
+import 'reflect-metadata';
+import { masterMain } from './master';
+import { workerMain } from './worker';
+
+const logger = new Logger('core', 'cyan');
+const clusterLogger = logger.createSubLogger('cluster', 'orange', false);
+const ev = new Xev();
+
+/**
+ * Init process
+ */
+export default async function() {
+	process.title = `Misskey (${cluster.isMaster ? 'master' : 'worker'})`;
+
+	if (cluster.isMaster || program.disableClustering) {
+		await masterMain();
+
+		if (cluster.isMaster) {
+			ev.mount();
+		}
+	}
+
+	if (cluster.isWorker || program.disableClustering) {
+		await workerMain();
+	}
+
+	// ユニットテスト時にMisskeyが子プロセスで起動された時のため
+	// それ以外のときは process.send は使えないので弾く
+	if (process.send) {
+		process.send('ok');
+	}
+}
+
+//#region Events
+
+// Listen new workers
+cluster.on('fork', worker => {
+	clusterLogger.debug(`Process forked: [${worker.id}]`);
+});
+
+// Listen online workers
+cluster.on('online', worker => {
+	clusterLogger.debug(`Process is now online: [${worker.id}]`);
+});
+
+// Listen for dying workers
+cluster.on('exit', worker => {
+	// Replace the dead worker,
+	// we're not sentimental
+	clusterLogger.error(chalk.red(`[${worker.id}] died :(`));
+	cluster.fork();
+});
+
+// Display detail of unhandled promise rejection
+if (!program.quiet) {
+	process.on('unhandledRejection', console.dir);
+}
+
+// Display detail of uncaught exception
+process.on('uncaughtException', err => {
+	logger.error(err);
+});
+
+// Dying away...
+process.on('exit', code => {
+	logger.info(`The process is going to exit with code ${code}`);
+});
+
+//#endregion
diff --git a/src/boot/master.ts b/src/boot/master.ts
new file mode 100644
index 0000000000..2d4080fdb0
--- /dev/null
+++ b/src/boot/master.ts
@@ -0,0 +1,176 @@
+import * as os from 'os';
+import * as cluster from 'cluster';
+import chalk from 'chalk';
+import * as portscanner from 'portscanner';
+import * as isRoot from 'is-root';
+
+import Logger from '../services/logger';
+import loadConfig from '../config/load';
+import { Config } from '../config/types';
+import { lessThan } from '../prelude/array';
+import * as pkg from '../../package.json';
+import { program } from '../argv';
+import { showMachineInfo } from '../misc/show-machine-info';
+import { initDb } from '../db/postgre';
+
+const logger = new Logger('core', 'cyan');
+const bootLogger = logger.createSubLogger('boot', 'magenta', false);
+
+function greet() {
+	if (!program.quiet) {
+		//#region Misskey logo
+		const v = `v${pkg.version}`;
+		console.log('  _____ _         _           ');
+		console.log(' |     |_|___ ___| |_ ___ _ _ ');
+		console.log(' | | | | |_ -|_ -| \'_| -_| | |');
+		console.log(' |_|_|_|_|___|___|_,_|___|_  |');
+		console.log(' ' + chalk.gray(v) + ('                        |___|\n'.substr(v.length)));
+		//#endregion
+
+		console.log(' Misskey is maintained by @syuilo, @AyaMorisawa, @mei23, and @acid-chicken.');
+		console.log(chalk.keyword('orange')(' If you like Misskey, please donate to support development. https://www.patreon.com/syuilo'));
+
+		console.log('');
+		console.log(chalk`< ${os.hostname()} {gray (PID: ${process.pid.toString()})} >`);
+	}
+
+	bootLogger.info('Welcome to Misskey!');
+	bootLogger.info(`Misskey v${pkg.version}`, null, true);
+}
+
+/**
+ * Init master process
+ */
+export async function masterMain() {
+	greet();
+
+	let config: Config;
+
+	try {
+		// initialize app
+		config = await init();
+
+		if (config.port == null) {
+			bootLogger.error('The port is not configured. Please configure port.', null, true);
+			process.exit(1);
+		}
+
+		if (process.platform === 'linux' && isWellKnownPort(config.port) && !isRoot()) {
+			bootLogger.error('You need root privileges to listen on well-known port on Linux', null, true);
+			process.exit(1);
+		}
+
+		if (!await isPortAvailable(config.port)) {
+			bootLogger.error(`Port ${config.port} is already in use`, null, true);
+			process.exit(1);
+		}
+	} catch (e) {
+		bootLogger.error('Fatal error occurred during initialization', null, true);
+		process.exit(1);
+	}
+
+	bootLogger.succ('Misskey initialized');
+
+	if (!program.disableClustering) {
+		await spawnWorkers(config.clusterLimit);
+	}
+
+	if (!program.noDaemons) {
+		require('../daemons/server-stats').default();
+		require('../daemons/notes-stats').default();
+		require('../daemons/queue-stats').default();
+	}
+
+	bootLogger.succ(`Now listening on port ${config.port} on ${config.url}`, null, true);
+}
+
+const runningNodejsVersion = process.version.slice(1).split('.').map(x => parseInt(x, 10));
+const requiredNodejsVersion = [11, 7, 0];
+const satisfyNodejsVersion = !lessThan(runningNodejsVersion, requiredNodejsVersion);
+
+function isWellKnownPort(port: number): boolean {
+	return port < 1024;
+}
+
+async function isPortAvailable(port: number): Promise<boolean> {
+	return await portscanner.checkPortStatus(port, '127.0.0.1') === 'closed';
+}
+
+function showEnvironment(): void {
+	const env = process.env.NODE_ENV;
+	const logger = bootLogger.createSubLogger('env');
+	logger.info(typeof env == 'undefined' ? 'NODE_ENV is not set' : `NODE_ENV: ${env}`);
+
+	if (env !== 'production') {
+		logger.warn('The environment is not in production mode.');
+		logger.warn('DO NOT USE FOR PRODUCTION PURPOSE!', null, true);
+	}
+
+	logger.info(`You ${isRoot() ? '' : 'do not '}have root privileges`);
+}
+
+/**
+ * Init app
+ */
+async function init(): Promise<Config> {
+	showEnvironment();
+
+	const nodejsLogger = bootLogger.createSubLogger('nodejs');
+
+	nodejsLogger.info(`Version ${runningNodejsVersion.join('.')}`);
+
+	if (!satisfyNodejsVersion) {
+		nodejsLogger.error(`Node.js version is less than ${requiredNodejsVersion.join('.')}. Please upgrade it.`, null, true);
+		process.exit(1);
+	}
+
+	await showMachineInfo(bootLogger);
+
+	const configLogger = bootLogger.createSubLogger('config');
+	let config;
+
+	try {
+		config = loadConfig();
+	} catch (exception) {
+		if (typeof exception === 'string') {
+			configLogger.error(exception);
+			process.exit(1);
+		}
+		if (exception.code === 'ENOENT') {
+			configLogger.error('Configuration file not found', null, true);
+			process.exit(1);
+		}
+		throw exception;
+	}
+
+	configLogger.succ('Loaded');
+
+	// Try to connect to DB
+	try {
+		bootLogger.info('Connecting database...');
+		await initDb();
+	} catch (e) {
+		bootLogger.error('Cannot connect to database', null, true);
+		bootLogger.error(e);
+		process.exit(1);
+	}
+
+	return config;
+}
+
+async function spawnWorkers(limit: number = Infinity) {
+	const workers = Math.min(limit, os.cpus().length);
+	bootLogger.info(`Starting ${workers} worker${workers === 1 ? '' : 's'}...`);
+	await Promise.all([...Array(workers)].map(spawnWorker));
+	bootLogger.succ('All workers started');
+}
+
+function spawnWorker(): Promise<void> {
+	return new Promise(res => {
+		const worker = cluster.fork();
+		worker.on('message', message => {
+			if (message !== 'ready') return;
+			res();
+		});
+	});
+}
diff --git a/src/boot/worker.ts b/src/boot/worker.ts
new file mode 100644
index 0000000000..ca3716972a
--- /dev/null
+++ b/src/boot/worker.ts
@@ -0,0 +1,20 @@
+import * as cluster from 'cluster';
+import { initDb } from '../db/postgre';
+
+/**
+ * Init worker process
+ */
+export async function workerMain() {
+	await initDb();
+
+	// start server
+	await require('../server').default();
+
+	// start job queue
+	require('../queue').default();
+
+	if (cluster.isWorker) {
+		// Send a 'ready' message to parent process
+		process.send('ready');
+	}
+}