From aa4ef6745ad798bd7d4f05cb397ef1dd85279814 Mon Sep 17 00:00:00 2001 From: syuilo Date: Sat, 7 Jul 2018 19:19:00 +0900 Subject: Refactorng --- src/queue/processors/http/process-inbox.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src/queue') diff --git a/src/queue/processors/http/process-inbox.ts b/src/queue/processors/http/process-inbox.ts index 0da442aefb..7b62f057bc 100644 --- a/src/queue/processors/http/process-inbox.ts +++ b/src/queue/processors/http/process-inbox.ts @@ -2,7 +2,7 @@ import * as kue from 'kue'; import * as debug from 'debug'; const httpSignature = require('http-signature'); -import parseAcct from '../../../acct/parse'; +import parseAcct from '../../../misc/acct/parse'; import User, { IRemoteUser } from '../../../models/user'; import perform from '../../../remote/activitypub/perform'; import { resolvePerson } from '../../../remote/activitypub/models/person'; -- cgit v1.2.3-freya From f778696a76e5ce2fdeef514cca1f7d293149b69b Mon Sep 17 00:00:00 2001 From: syuilo Date: Thu, 26 Jul 2018 05:27:27 +0900 Subject: :v: --- src/queue/index.ts | 7 +++++++ 1 file changed, 7 insertions(+) (limited to 'src/queue') diff --git a/src/queue/index.ts b/src/queue/index.ts index 1efd6e54ef..53853687d7 100644 --- a/src/queue/index.ts +++ b/src/queue/index.ts @@ -12,6 +12,13 @@ const queue = createQueue({ } }); +process.once('SIGTERM', () => { + queue.shutdown(5000, (err: any) => { + console.log('Kue shutdown: ', err || ''); + process.exit(0); + }); +}); + export function createHttp(data: any) { return queue .create('http', data) -- cgit v1.2.3-freya From 77b493c9b065a7af7533e718f8623f810e65341a Mon Sep 17 00:00:00 2001 From: syuilo Date: Thu, 26 Jul 2018 08:11:47 +0900 Subject: Use bee-queue instead of Kue --- docs/manage.en.md | 6 +--- docs/manage.ja.md | 6 +--- package.json | 3 +- src/index.ts | 3 -- src/queue/index.ts | 44 ++++++++++-------------------- src/queue/processors/http/deliver.ts | 4 +-- src/queue/processors/http/process-inbox.ts | 4 +-- src/server/activitypub.ts | 6 ++-- 8 files changed, 24 insertions(+), 52 deletions(-) (limited to 'src/queue') diff --git a/docs/manage.en.md b/docs/manage.en.md index 0538c4ac00..9a4277b399 100644 --- a/docs/manage.en.md +++ b/docs/manage.en.md @@ -1,11 +1,7 @@ # Management guide ## Check the status of the job queue -In the directory of Misskey: -``` shell -node_modules/kue/bin/kue-dashboard -p 3050 -``` -When you access port 3050, you will see the UI. +coming soon ## Mark as 'admin' user ``` shell diff --git a/docs/manage.ja.md b/docs/manage.ja.md index 5f628a8e4b..924c567d14 100644 --- a/docs/manage.ja.md +++ b/docs/manage.ja.md @@ -1,11 +1,7 @@ # 運営ガイド ## ジョブキューの状態を調べる -Misskeyのディレクトリで: -``` shell -node_modules/kue/bin/kue-dashboard -p 3050 -``` -ポート3050にアクセスするとUIが表示されます +coming soon ## 管理者ユーザーを設定する ``` shell diff --git a/package.json b/package.json index 308474b8b2..480f4d0ac7 100644 --- a/package.json +++ b/package.json @@ -55,7 +55,6 @@ "@types/koa-send": "4.1.1", "@types/koa-views": "2.0.3", "@types/koa__cors": "2.2.2", - "@types/kue": "0.11.9", "@types/minio": "6.0.2", "@types/mkdirp": "0.5.2", "@types/mocha": "5.2.3", @@ -86,6 +85,7 @@ "autosize": "4.0.2", "autwh": "0.1.0", "bcryptjs": "2.4.3", + "bee-queue": "1.2.2", "bootstrap-vue": "2.0.0-rc.11", "cafy": "11.3.0", "chalk": "2.4.1", @@ -144,7 +144,6 @@ "koa-send": "5.0.0", "koa-slow": "2.1.0", "koa-views": "6.1.4", - "kue": "0.11.6", "loader-utils": "1.1.0", "mecab-async": "0.1.2", "minio": "6.0.0", diff --git a/src/index.ts b/src/index.ts index 675f862b6f..7545c82c74 100644 --- a/src/index.ts +++ b/src/index.ts @@ -31,9 +31,6 @@ if (process.env.NODE_ENV != 'production') { process.env.DEBUG = 'misskey:*'; } -// https://github.com/Automattic/kue/issues/822 -require('events').EventEmitter.prototype._maxListeners = 512; - // Start app main(); diff --git a/src/queue/index.ts b/src/queue/index.ts index 53853687d7..6f82d0a8b5 100644 --- a/src/queue/index.ts +++ b/src/queue/index.ts @@ -1,52 +1,36 @@ -import { createQueue } from 'kue'; +import * as Queue from 'bee-queue'; import config from '../config'; import http from './processors/http'; import { ILocalUser } from '../models/user'; -const queue = createQueue({ +const queue = new Queue('misskey', { redis: { port: config.redis.port, host: config.redis.host, - auth: config.redis.pass - } -}); + password: config.redis.pass + }, -process.once('SIGTERM', () => { - queue.shutdown(5000, (err: any) => { - console.log('Kue shutdown: ', err || ''); - process.exit(0); - }); + removeOnSuccess: true, + removeOnFailure: true }); -export function createHttp(data: any) { - return queue - .create('http', data) - .removeOnComplete(true) - .events(false) - .attempts(8) - .backoff({ delay: 16384, type: 'exponential' }); +export function createHttpJob(data: any) { + return queue.createJob(data) + .retries(4) + .backoff('exponential', 16384) // 16s + .save(); } export function deliver(user: ILocalUser, content: any, to: any) { - createHttp({ - title: 'deliver', + createHttpJob({ type: 'deliver', user, content, to - }).save(); + }); } export default function() { - /* - 256 is the default concurrency limit of Mozilla Firefox and Google - Chromium. - a8af215e691f3a2205a3758d2d96e9d328e100ff - chromium/src.git - Git at Google - https://chromium.googlesource.com/chromium/src.git/+/a8af215e691f3a2205a3758d2d96e9d328e100ff - Network.http.max-connections - MozillaZine Knowledge Base - http://kb.mozillazine.org/Network.http.max-connections - */ - //queue.process('http', 256, http); - queue.process('http', 128, http); + queue.process(8, http); } diff --git a/src/queue/processors/http/deliver.ts b/src/queue/processors/http/deliver.ts index 946d2f65bd..e06866da4e 100644 --- a/src/queue/processors/http/deliver.ts +++ b/src/queue/processors/http/deliver.ts @@ -1,8 +1,8 @@ -import * as kue from 'kue'; +import * as bq from 'bee-queue'; import request from '../../../remote/activitypub/request'; -export default async (job: kue.Job, done: any): Promise => { +export default async (job: bq.Job, done: any): Promise => { try { await request(job.data.user, job.data.to, job.data.content); done(); diff --git a/src/queue/processors/http/process-inbox.ts b/src/queue/processors/http/process-inbox.ts index 7b62f057bc..0738853dd1 100644 --- a/src/queue/processors/http/process-inbox.ts +++ b/src/queue/processors/http/process-inbox.ts @@ -1,4 +1,4 @@ -import * as kue from 'kue'; +import * as bq from 'bee-queue'; import * as debug from 'debug'; const httpSignature = require('http-signature'); @@ -10,7 +10,7 @@ import { resolvePerson } from '../../../remote/activitypub/models/person'; const log = debug('misskey:queue:inbox'); // ユーザーのinboxにアクティビティが届いた時の処理 -export default async (job: kue.Job, done: any): Promise => { +export default async (job: bq.Job, done: any): Promise => { const signature = job.data.signature; const activity = job.data.activity; diff --git a/src/server/activitypub.ts b/src/server/activitypub.ts index 17cd34ee6f..2d9a4746c6 100644 --- a/src/server/activitypub.ts +++ b/src/server/activitypub.ts @@ -3,7 +3,7 @@ import * as Router from 'koa-router'; const json = require('koa-json-body'); const httpSignature = require('http-signature'); -import { createHttp } from '../queue'; +import { createHttpJob } from '../queue'; import pack from '../remote/activitypub/renderer'; import Note from '../models/note'; import User, { isLocalUser, ILocalUser, IUser } from '../models/user'; @@ -30,11 +30,11 @@ function inbox(ctx: Router.IRouterContext) { return; } - createHttp({ + createHttpJob({ type: 'processInbox', activity: ctx.request.body, signature - }).save(); + }); ctx.status = 202; } -- cgit v1.2.3-freya From e1e1cd057478b7931ca1dfe481d09a49bd262622 Mon Sep 17 00:00:00 2001 From: syuilo Date: Thu, 26 Jul 2018 17:02:34 +0900 Subject: Update job queue settings --- src/queue/index.ts | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) (limited to 'src/queue') diff --git a/src/queue/index.ts b/src/queue/index.ts index 6f82d0a8b5..2589e026e1 100644 --- a/src/queue/index.ts +++ b/src/queue/index.ts @@ -12,7 +12,10 @@ const queue = new Queue('misskey', { }, removeOnSuccess: true, - removeOnFailure: true + removeOnFailure: true, + getEvents: false, + sendEvents: false, + storeJobs: false }); export function createHttpJob(data: any) { @@ -32,5 +35,5 @@ export function deliver(user: ILocalUser, content: any, to: any) { } export default function() { - queue.process(8, http); + queue.process(128, http); } -- cgit v1.2.3-freya From ac07f04ad82a40deb45c763f31ae6ded33a91c79 Mon Sep 17 00:00:00 2001 From: syuilo Date: Thu, 26 Jul 2018 17:15:00 +0900 Subject: Update job queue setting --- src/queue/index.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'src/queue') diff --git a/src/queue/index.ts b/src/queue/index.ts index 2589e026e1..775e5f199d 100644 --- a/src/queue/index.ts +++ b/src/queue/index.ts @@ -20,8 +20,8 @@ const queue = new Queue('misskey', { export function createHttpJob(data: any) { return queue.createJob(data) - .retries(4) - .backoff('exponential', 16384) // 16s + //.retries(4) + //.backoff('exponential', 16384) // 16s .save(); } -- cgit v1.2.3-freya From 8a57f490ceccb6e1e4099cc945fe80fd03906d52 Mon Sep 17 00:00:00 2001 From: syuilo Date: Fri, 27 Jul 2018 18:40:38 +0900 Subject: バギーなのでジョブキュー無効化 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/index.ts | 3 --- src/queue/index.ts | 26 +------------------------- 2 files changed, 1 insertion(+), 28 deletions(-) (limited to 'src/queue') diff --git a/src/index.ts b/src/index.ts index 7545c82c74..9c16c4d223 100644 --- a/src/index.ts +++ b/src/index.ts @@ -79,9 +79,6 @@ async function workerMain() { // start server await require('./server').default(); - // start processor - require('./queue').default(); - // Send a 'ready' message to parent process process.send('ready'); } diff --git a/src/queue/index.ts b/src/queue/index.ts index 775e5f199d..e404f6fc9c 100644 --- a/src/queue/index.ts +++ b/src/queue/index.ts @@ -1,28 +1,8 @@ -import * as Queue from 'bee-queue'; - -import config from '../config'; import http from './processors/http'; import { ILocalUser } from '../models/user'; -const queue = new Queue('misskey', { - redis: { - port: config.redis.port, - host: config.redis.host, - password: config.redis.pass - }, - - removeOnSuccess: true, - removeOnFailure: true, - getEvents: false, - sendEvents: false, - storeJobs: false -}); - export function createHttpJob(data: any) { - return queue.createJob(data) - //.retries(4) - //.backoff('exponential', 16384) // 16s - .save(); + return http(data, () => {}); } export function deliver(user: ILocalUser, content: any, to: any) { @@ -33,7 +13,3 @@ export function deliver(user: ILocalUser, content: any, to: any) { to }); } - -export default function() { - queue.process(128, http); -} -- cgit v1.2.3-freya From 08e1c87fa602f470e7e19d7c162597ddc6980162 Mon Sep 17 00:00:00 2001 From: syuilo Date: Fri, 27 Jul 2018 18:50:15 +0900 Subject: oops --- src/queue/index.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src/queue') diff --git a/src/queue/index.ts b/src/queue/index.ts index e404f6fc9c..5a48dbe648 100644 --- a/src/queue/index.ts +++ b/src/queue/index.ts @@ -2,7 +2,7 @@ import http from './processors/http'; import { ILocalUser } from '../models/user'; export function createHttpJob(data: any) { - return http(data, () => {}); + return http({ data }, () => {}); } export function deliver(user: ILocalUser, content: any, to: any) { -- cgit v1.2.3-freya