From f0a29721c9fb10f97faf386bc9d6b1b2fad97895 Mon Sep 17 00:00:00 2001 From: syuilo Date: Sun, 7 Apr 2019 21:50:36 +0900 Subject: Use PostgreSQL instead of MongoDB (#4572) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * wip * Update note.ts * Update timeline.ts * Update core.ts * wip * Update generate-visibility-query.ts * wip * wip * wip * wip * wip * Update global-timeline.ts * wip * wip * wip * Update vote.ts * wip * wip * Update create.ts * wip * wip * wip * wip * wip * wip * wip * wip * wip * wip * wip * wip * Update files.ts * wip * wip * Update CONTRIBUTING.md * wip * wip * wip * wip * wip * wip * wip * wip * Update read-notification.ts * wip * wip * wip * wip * wip * wip * wip * Update cancel.ts * wip * wip * wip * Update show.ts * wip * wip * Update gen-id.ts * Update create.ts * Update id.ts * wip * wip * wip * wip * wip * wip * wip * Docker: Update files about Docker (#4599) * Docker: Use cache if files used by `yarn install` was not updated This patch reduces the number of times to installing node_modules. For example, `yarn install` step will be skipped when only ".config/default.yml" is updated. * Docker: Migrate MongoDB to Postgresql Misskey uses Postgresql as a database instead of Mongodb since version 11. * Docker: Uncomment about data persistence This patch will save a lot of databases. * wip * wip * wip * Update activitypub.ts * wip * wip * wip * Update logs.ts * wip * Update drive-file.ts * Update register.ts * wip * wip * Update mentions.ts * wip * wip * wip * Update recommendation.ts * wip * Update index.ts * wip * Update recommendation.ts * Doc: Update docker.ja.md and docker.en.md (#1) (#4608) Update how to set up misskey. * wip * :v: * wip * Update note.ts * Update postgre.ts * wip * wip * wip * wip * Update add-file.ts * wip * wip * wip * Clean up * Update logs.ts * wip * :pizza: * wip * Ad notes * wip * Update api-visibility.ts * Update note.ts * Update add-file.ts * tests * tests * Update postgre.ts * Update utils.ts * wip * wip * Refactor * wip * Refactor * wip * wip * Update show-users.ts * Update update-instance.ts * wip * Update feed.ts * Update outbox.ts * Update outbox.ts * Update user.ts * wip * Update list.ts * Update update-hashtag.ts * wip * Update update-hashtag.ts * Refactor * Update update.ts * wip * wip * :v: * clean up * docs * Update push.ts * wip * Update api.ts * wip * :v: * Update make-pagination-query.ts * :v: * Delete hashtags.ts * Update instances.ts * Update instances.ts * Update create.ts * Update search.ts * Update reversi-game.ts * Update signup.ts * Update user.ts * id * Update example.yml * :art: * objectid * fix * reversi * reversi * Fix bug of chart engine * Add test of chart engine * Improve test * Better testing * Improve chart engine * Refactor * Add test of chart engine * Refactor * Add chart test * Fix bug * コミットし忘れ * Refactoring * :v: * Add tests * Add test * Extarct note tests * Refactor * 存在しないユーザーにメンションできなくなっていた問題を修正 * Fix bug * Update update-meta.ts * Fix bug * Update mention.vue * Fix bug * Update meta.ts * Update CONTRIBUTING.md * Fix bug * Fix bug * Fix bug * Clean up * Clean up * Update notification.ts * Clean up * Add mute tests * Add test * Refactor * Add test * Fix test * Refactor * Refactor * Add tests * Update utils.ts * Update utils.ts * Fix test * Update package.json * Update update.ts * Update manifest.ts * Fix bug * Fix bug * Add test * :art: * Update endpoint permissions * Updaye permisison * Update person.ts #4299 * データベースと同期しないように * Fix bug * Fix bug * Update reversi-game.ts * Use a feature of Node v11.7.0 to extract a public key (#4644) * wip * wip * :v: * Refactoring #1540 * test * test * test * test * test * test * test * Fix bug * Fix test * :sushi: * wip * #4471 * Add test for #4335 * Refactor * Fix test * Add tests * :clock4: * Fix bug * Add test * Add test * rename * Fix bug --- src/boot/index.ts | 77 +++++++++++++++++++++++ src/boot/master.ts | 176 +++++++++++++++++++++++++++++++++++++++++++++++++++++ src/boot/worker.ts | 20 ++++++ 3 files changed, 273 insertions(+) create mode 100644 src/boot/index.ts create mode 100644 src/boot/master.ts create mode 100644 src/boot/worker.ts (limited to 'src/boot') diff --git a/src/boot/index.ts b/src/boot/index.ts new file mode 100644 index 0000000000..2c86d8ed8c --- /dev/null +++ b/src/boot/index.ts @@ -0,0 +1,77 @@ +import * as cluster from 'cluster'; +import chalk from 'chalk'; +import Xev from 'xev'; + +import Logger from '../services/logger'; +import { program } from '../argv'; + +// for typeorm +import 'reflect-metadata'; +import { masterMain } from './master'; +import { workerMain } from './worker'; + +const logger = new Logger('core', 'cyan'); +const clusterLogger = logger.createSubLogger('cluster', 'orange', false); +const ev = new Xev(); + +/** + * Init process + */ +export default async function() { + process.title = `Misskey (${cluster.isMaster ? 'master' : 'worker'})`; + + if (cluster.isMaster || program.disableClustering) { + await masterMain(); + + if (cluster.isMaster) { + ev.mount(); + } + } + + if (cluster.isWorker || program.disableClustering) { + await workerMain(); + } + + // ユニットテスト時にMisskeyが子プロセスで起動された時のため + // それ以外のときは process.send は使えないので弾く + if (process.send) { + process.send('ok'); + } +} + +//#region Events + +// Listen new workers +cluster.on('fork', worker => { + clusterLogger.debug(`Process forked: [${worker.id}]`); +}); + +// Listen online workers +cluster.on('online', worker => { + clusterLogger.debug(`Process is now online: [${worker.id}]`); +}); + +// Listen for dying workers +cluster.on('exit', worker => { + // Replace the dead worker, + // we're not sentimental + clusterLogger.error(chalk.red(`[${worker.id}] died :(`)); + cluster.fork(); +}); + +// Display detail of unhandled promise rejection +if (!program.quiet) { + process.on('unhandledRejection', console.dir); +} + +// Display detail of uncaught exception +process.on('uncaughtException', err => { + logger.error(err); +}); + +// Dying away... +process.on('exit', code => { + logger.info(`The process is going to exit with code ${code}`); +}); + +//#endregion diff --git a/src/boot/master.ts b/src/boot/master.ts new file mode 100644 index 0000000000..2d4080fdb0 --- /dev/null +++ b/src/boot/master.ts @@ -0,0 +1,176 @@ +import * as os from 'os'; +import * as cluster from 'cluster'; +import chalk from 'chalk'; +import * as portscanner from 'portscanner'; +import * as isRoot from 'is-root'; + +import Logger from '../services/logger'; +import loadConfig from '../config/load'; +import { Config } from '../config/types'; +import { lessThan } from '../prelude/array'; +import * as pkg from '../../package.json'; +import { program } from '../argv'; +import { showMachineInfo } from '../misc/show-machine-info'; +import { initDb } from '../db/postgre'; + +const logger = new Logger('core', 'cyan'); +const bootLogger = logger.createSubLogger('boot', 'magenta', false); + +function greet() { + if (!program.quiet) { + //#region Misskey logo + const v = `v${pkg.version}`; + console.log(' _____ _ _ '); + console.log(' | |_|___ ___| |_ ___ _ _ '); + console.log(' | | | | |_ -|_ -| \'_| -_| | |'); + console.log(' |_|_|_|_|___|___|_,_|___|_ |'); + console.log(' ' + chalk.gray(v) + (' |___|\n'.substr(v.length))); + //#endregion + + console.log(' Misskey is maintained by @syuilo, @AyaMorisawa, @mei23, and @acid-chicken.'); + console.log(chalk.keyword('orange')(' If you like Misskey, please donate to support development. https://www.patreon.com/syuilo')); + + console.log(''); + console.log(chalk`< ${os.hostname()} {gray (PID: ${process.pid.toString()})} >`); + } + + bootLogger.info('Welcome to Misskey!'); + bootLogger.info(`Misskey v${pkg.version}`, null, true); +} + +/** + * Init master process + */ +export async function masterMain() { + greet(); + + let config: Config; + + try { + // initialize app + config = await init(); + + if (config.port == null) { + bootLogger.error('The port is not configured. Please configure port.', null, true); + process.exit(1); + } + + if (process.platform === 'linux' && isWellKnownPort(config.port) && !isRoot()) { + bootLogger.error('You need root privileges to listen on well-known port on Linux', null, true); + process.exit(1); + } + + if (!await isPortAvailable(config.port)) { + bootLogger.error(`Port ${config.port} is already in use`, null, true); + process.exit(1); + } + } catch (e) { + bootLogger.error('Fatal error occurred during initialization', null, true); + process.exit(1); + } + + bootLogger.succ('Misskey initialized'); + + if (!program.disableClustering) { + await spawnWorkers(config.clusterLimit); + } + + if (!program.noDaemons) { + require('../daemons/server-stats').default(); + require('../daemons/notes-stats').default(); + require('../daemons/queue-stats').default(); + } + + bootLogger.succ(`Now listening on port ${config.port} on ${config.url}`, null, true); +} + +const runningNodejsVersion = process.version.slice(1).split('.').map(x => parseInt(x, 10)); +const requiredNodejsVersion = [11, 7, 0]; +const satisfyNodejsVersion = !lessThan(runningNodejsVersion, requiredNodejsVersion); + +function isWellKnownPort(port: number): boolean { + return port < 1024; +} + +async function isPortAvailable(port: number): Promise { + return await portscanner.checkPortStatus(port, '127.0.0.1') === 'closed'; +} + +function showEnvironment(): void { + const env = process.env.NODE_ENV; + const logger = bootLogger.createSubLogger('env'); + logger.info(typeof env == 'undefined' ? 'NODE_ENV is not set' : `NODE_ENV: ${env}`); + + if (env !== 'production') { + logger.warn('The environment is not in production mode.'); + logger.warn('DO NOT USE FOR PRODUCTION PURPOSE!', null, true); + } + + logger.info(`You ${isRoot() ? '' : 'do not '}have root privileges`); +} + +/** + * Init app + */ +async function init(): Promise { + showEnvironment(); + + const nodejsLogger = bootLogger.createSubLogger('nodejs'); + + nodejsLogger.info(`Version ${runningNodejsVersion.join('.')}`); + + if (!satisfyNodejsVersion) { + nodejsLogger.error(`Node.js version is less than ${requiredNodejsVersion.join('.')}. Please upgrade it.`, null, true); + process.exit(1); + } + + await showMachineInfo(bootLogger); + + const configLogger = bootLogger.createSubLogger('config'); + let config; + + try { + config = loadConfig(); + } catch (exception) { + if (typeof exception === 'string') { + configLogger.error(exception); + process.exit(1); + } + if (exception.code === 'ENOENT') { + configLogger.error('Configuration file not found', null, true); + process.exit(1); + } + throw exception; + } + + configLogger.succ('Loaded'); + + // Try to connect to DB + try { + bootLogger.info('Connecting database...'); + await initDb(); + } catch (e) { + bootLogger.error('Cannot connect to database', null, true); + bootLogger.error(e); + process.exit(1); + } + + return config; +} + +async function spawnWorkers(limit: number = Infinity) { + const workers = Math.min(limit, os.cpus().length); + bootLogger.info(`Starting ${workers} worker${workers === 1 ? '' : 's'}...`); + await Promise.all([...Array(workers)].map(spawnWorker)); + bootLogger.succ('All workers started'); +} + +function spawnWorker(): Promise { + return new Promise(res => { + const worker = cluster.fork(); + worker.on('message', message => { + if (message !== 'ready') return; + res(); + }); + }); +} diff --git a/src/boot/worker.ts b/src/boot/worker.ts new file mode 100644 index 0000000000..ca3716972a --- /dev/null +++ b/src/boot/worker.ts @@ -0,0 +1,20 @@ +import * as cluster from 'cluster'; +import { initDb } from '../db/postgre'; + +/** + * Init worker process + */ +export async function workerMain() { + await initDb(); + + // start server + await require('../server').default(); + + // start job queue + require('../queue').default(); + + if (cluster.isWorker) { + // Send a 'ready' message to parent process + process.send('ready'); + } +} -- cgit v1.2.3-freya