From e8b42d7e1668679e6a6ee0a7aea1e2ff7f37005b Mon Sep 17 00:00:00 2001 From: syuilo Date: Wed, 4 Apr 2018 23:12:35 +0900 Subject: wip --- src/queue/index.ts | 37 +++++++++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) create mode 100644 src/queue/index.ts (limited to 'src/queue/index.ts') diff --git a/src/queue/index.ts b/src/queue/index.ts new file mode 100644 index 0000000000..c8c436b18c --- /dev/null +++ b/src/queue/index.ts @@ -0,0 +1,37 @@ +import { createQueue } from 'kue'; +import config from '../config'; +import db from './processors/db'; +import http from './processors/http'; + +const queue = createQueue({ + redis: { + port: config.redis.port, + host: config.redis.host, + auth: config.redis.pass + } +}); + +export function createHttp(data) { + return queue + .create('http', data) + .attempts(16) + .backoff({ delay: 16384, type: 'exponential' }); +} + +export function createDb(data) { + return queue.create('db', data); +} + +export function process() { + queue.process('db', db); + + /* + 256 is the default concurrency limit of Mozilla Firefox and Google + Chromium. + a8af215e691f3a2205a3758d2d96e9d328e100ff - chromium/src.git - Git at Google + https://chromium.googlesource.com/chromium/src.git/+/a8af215e691f3a2205a3758d2d96e9d328e100ff + Network.http.max-connections - MozillaZine Knowledge Base + http://kb.mozillazine.org/Network.http.max-connections + */ + queue.process('http', 256, http); +} -- cgit v1.2.3-freya From 7403f38fb43b0ad747236061a591cbf94e198ba6 Mon Sep 17 00:00:00 2001 From: syuilo Date: Thu, 5 Apr 2018 18:43:06 +0900 Subject: wip --- src/api/post/create.ts | 15 ++++++++------- src/index.ts | 2 +- src/queue/index.ts | 8 +++++++- src/queue/processors/http/deliver.ts | 9 ++++++++- src/queue/processors/http/index.ts | 16 ++++++++++++---- src/queue/processors/http/report-github-failure.ts | 6 +++--- src/remote/activitypub/resolver.ts | 2 +- src/remote/request.ts | 8 ++++++++ src/server/api/endpoints/following/create.ts | 11 ++--------- 9 files changed, 50 insertions(+), 27 deletions(-) (limited to 'src/queue/index.ts') diff --git a/src/api/post/create.ts b/src/api/post/create.ts index 5495117538..dbeb87ae86 100644 --- a/src/api/post/create.ts +++ b/src/api/post/create.ts @@ -18,20 +18,21 @@ import html from '../../text/html'; import { IApp } from '../../models/app'; export default async (user: IUser, content: { - createdAt: Date; - text: string; - reply: IPost; - repost: IPost; - media: IDriveFile[]; - geo: any; + createdAt?: Date; + text?: string; + reply?: IPost; + repost?: IPost; + media?: IDriveFile[]; + geo?: any; poll?: any; - viaMobile: boolean; + viaMobile?: boolean; tags?: string[]; cw?: string; visibility?: string; uri?: string; app?: IApp; }) => new Promise(async (res, rej) => { + if (content.createdAt == null) content.createdAt = new Date(); if (content.visibility == null) content.visibility = 'public'; const tags = content.tags || []; diff --git a/src/index.ts b/src/index.ts index e35c917a44..f45bcaa6ad 100644 --- a/src/index.ts +++ b/src/index.ts @@ -103,7 +103,7 @@ async function workerMain(opt) { if (!opt['only-server']) { // start processor - require('./processor').default(); + require('./queue').default(); } // Send a 'ready' message to parent process diff --git a/src/queue/index.ts b/src/queue/index.ts index c8c436b18c..86600dc265 100644 --- a/src/queue/index.ts +++ b/src/queue/index.ts @@ -1,8 +1,12 @@ import { createQueue } from 'kue'; +import * as debug from 'debug'; + import config from '../config'; import db from './processors/db'; import http from './processors/http'; +const log = debug('misskey:queue'); + const queue = createQueue({ redis: { port: config.redis.port, @@ -12,6 +16,8 @@ const queue = createQueue({ }); export function createHttp(data) { + log(`HTTP job created: ${JSON.stringify(data)}`); + return queue .create('http', data) .attempts(16) @@ -22,7 +28,7 @@ export function createDb(data) { return queue.create('db', data); } -export function process() { +export default function() { queue.process('db', db); /* diff --git a/src/queue/processors/http/deliver.ts b/src/queue/processors/http/deliver.ts index 1700063a5d..da7e8bc368 100644 --- a/src/queue/processors/http/deliver.ts +++ b/src/queue/processors/http/deliver.ts @@ -3,5 +3,12 @@ import * as kue from 'kue'; import request from '../../../remote/request'; export default async (job: kue.Job, done): Promise => { - await request(job.data.user, job.data.to, job.data.content); + try { + await request(job.data.user, job.data.to, job.data.content); + done(); + } catch (e) { + console.warn(`deliver failed: ${e}`); + + done(e); + } }; diff --git a/src/queue/processors/http/index.ts b/src/queue/processors/http/index.ts index 06c6b1d1aa..3d7d941b1a 100644 --- a/src/queue/processors/http/index.ts +++ b/src/queue/processors/http/index.ts @@ -3,9 +3,17 @@ import processInbox from './process-inbox'; import reportGitHubFailure from './report-github-failure'; const handlers = { - deliver, - processInbox, - reportGitHubFailure, + deliver, + processInbox, + reportGitHubFailure }; -export default (job, done) => handlers[job.data.type](job).then(() => done(), done); +export default (job, done) => { + const handler = handlers[job.data.type]; + + if (handler) { + handler(job).then(() => done(), done); + } else { + console.warn(`Unknown job: ${job.data.type}`); + } +}; diff --git a/src/queue/processors/http/report-github-failure.ts b/src/queue/processors/http/report-github-failure.ts index 4f6f5ccee5..e747d062d3 100644 --- a/src/queue/processors/http/report-github-failure.ts +++ b/src/queue/processors/http/report-github-failure.ts @@ -1,6 +1,6 @@ import * as request from 'request-promise-native'; -import User from '../../models/user'; -const createPost = require('../../server/api/endpoints/posts/create'); +import User from '../../../models/user'; +import createPost from '../../../api/post/create'; export default async ({ data }) => { const asyncBot = User.findOne({ _id: data.userId }); @@ -20,5 +20,5 @@ export default async ({ data }) => { `**⚠️BUILD STILL FAILED⚠️**: ?[${data.message}](${data.htmlUrl})` : `**🚨BUILD FAILED🚨**: →→→?[${data.message}](${data.htmlUrl})←←←`; - createPost({ text }, await asyncBot); + createPost(await asyncBot, { text }); }; diff --git a/src/remote/activitypub/resolver.ts b/src/remote/activitypub/resolver.ts index 09a6e70056..38639c6813 100644 --- a/src/remote/activitypub/resolver.ts +++ b/src/remote/activitypub/resolver.ts @@ -59,7 +59,7 @@ export default class Resolver { throw new Error('invalid response'); } - log(`resolved: ${JSON.stringify(object)}`); + log(`resolved: ${JSON.stringify(object, null, 2)}`); return object; } diff --git a/src/remote/request.ts b/src/remote/request.ts index 72262cbf61..a375aebfbb 100644 --- a/src/remote/request.ts +++ b/src/remote/request.ts @@ -1,9 +1,15 @@ import { request } from 'https'; import { sign } from 'http-signature'; import { URL } from 'url'; +import * as debug from 'debug'; + import config from '../config'; +const log = debug('misskey:activitypub:deliver'); + export default ({ account, username }, url, object) => new Promise((resolve, reject) => { + log(`--> ${url}`); + const { protocol, hostname, port, pathname, search } = new URL(url); const req = request({ @@ -14,6 +20,8 @@ export default ({ account, username }, url, object) => new Promise((resolve, rej path: pathname + search, }, res => { res.on('end', () => { + log(`${url} --> ${res.statusCode}`); + if (res.statusCode >= 200 && res.statusCode < 300) { resolve(); } else { diff --git a/src/server/api/endpoints/following/create.ts b/src/server/api/endpoints/following/create.ts index e568595215..fae686ce54 100644 --- a/src/server/api/endpoints/following/create.ts +++ b/src/server/api/endpoints/following/create.ts @@ -4,7 +4,7 @@ import $ from 'cafy'; import User from '../../../../models/user'; import Following from '../../../../models/following'; -import queue from '../../../../queue'; +import create from '../../../../api/following/create'; /** * Follow a user @@ -50,15 +50,8 @@ module.exports = (params, user) => new Promise(async (res, rej) => { } // Create following - const { _id } = await Following.insert({ - createdAt: new Date(), - followerId: follower._id, - followeeId: followee._id - }); - - queue.create('http', { type: 'follow', following: _id }).save(); + create(follower, followee); // Send response res(); - }); -- cgit v1.2.3-freya From 0de40f3a76ef159b099a90b65ded3073dbbd2b78 Mon Sep 17 00:00:00 2001 From: syuilo Date: Thu, 5 Apr 2018 23:24:51 +0900 Subject: wip --- src/api/following/create.ts | 16 +++------------- src/api/following/delete.ts | 9 ++------- src/api/post/create.ts | 9 ++------- src/queue/index.ts | 9 +++++++++ 4 files changed, 16 insertions(+), 27 deletions(-) (limited to 'src/queue/index.ts') diff --git a/src/api/following/create.ts b/src/api/following/create.ts index 353a6c8920..d919f4487f 100644 --- a/src/api/following/create.ts +++ b/src/api/following/create.ts @@ -7,7 +7,7 @@ import notify from '../../publishers/notify'; import context from '../../remote/activitypub/renderer/context'; import renderFollow from '../../remote/activitypub/renderer/follow'; import renderAccept from '../../remote/activitypub/renderer/accept'; -import { createHttp } from '../../queue'; +import { deliver } from '../../queue'; export default async function(follower: IUser, followee: IUser, activity?) { const following = await Following.insert({ @@ -60,23 +60,13 @@ export default async function(follower: IUser, followee: IUser, activity?) { const content = renderFollow(follower, followee); content['@context'] = context; - createHttp({ - type: 'deliver', - user: follower, - content, - to: followee.account.inbox - }).save(); + deliver(follower, content, followee.account.inbox).save(); } if (isRemoteUser(follower) && isLocalUser(followee)) { const content = renderAccept(activity); content['@context'] = context; - createHttp({ - type: 'deliver', - user: followee, - content, - to: follower.account.inbox - }).save(); + deliver(followee, content, follower.account.inbox).save(); } } diff --git a/src/api/following/delete.ts b/src/api/following/delete.ts index 4cdff7ce1b..364a4803b9 100644 --- a/src/api/following/delete.ts +++ b/src/api/following/delete.ts @@ -6,7 +6,7 @@ import event from '../../publishers/stream'; import context from '../../remote/activitypub/renderer/context'; import renderFollow from '../../remote/activitypub/renderer/follow'; import renderUndo from '../../remote/activitypub/renderer/undo'; -import { createHttp } from '../../queue'; +import { deliver } from '../../queue'; export default async function(follower: IUser, followee: IUser, activity?) { const following = await Following.findOne({ @@ -59,11 +59,6 @@ export default async function(follower: IUser, followee: IUser, activity?) { const content = renderUndo(renderFollow(follower, followee)); content['@context'] = context; - createHttp({ - type: 'deliver', - user: follower, - content, - to: followee.account.inbox - }).save(); + deliver(follower, content, followee.account.inbox).save(); } } diff --git a/src/api/post/create.ts b/src/api/post/create.ts index 7b7fceda2a..9723dbe452 100644 --- a/src/api/post/create.ts +++ b/src/api/post/create.ts @@ -2,7 +2,7 @@ import Post, { pack, IPost } from '../../models/post'; import User, { isLocalUser, IUser } from '../../models/user'; import stream from '../../publishers/stream'; import Following from '../../models/following'; -import { createHttp } from '../../queue'; +import { deliver } from '../../queue'; import renderNote from '../../remote/activitypub/renderer/note'; import renderCreate from '../../remote/activitypub/renderer/create'; import context from '../../remote/activitypub/renderer/context'; @@ -132,12 +132,7 @@ export default async (user: IUser, content: { } else { // フォロワーがリモートユーザーかつ投稿者がローカルユーザーなら投稿を配信 if (isLocalUser(user)) { - createHttp({ - type: 'deliver', - user, - content, - to: follower.account.inbox - }).save(); + deliver(user, content, follower.account.inbox).save(); } } })); diff --git a/src/queue/index.ts b/src/queue/index.ts index 86600dc265..689985e0ed 100644 --- a/src/queue/index.ts +++ b/src/queue/index.ts @@ -28,6 +28,15 @@ export function createDb(data) { return queue.create('db', data); } +export function deliver(user, content, to) { + return createHttp({ + type: 'deliver', + user, + content, + to + }); +} + export default function() { queue.process('db', db); -- cgit v1.2.3-freya From eb0809ebb422849e2e362d64ee0efb7cb75dc3f7 Mon Sep 17 00:00:00 2001 From: こぴなたみぽ Date: Fri, 6 Apr 2018 14:57:01 +0900 Subject: Remove needless log --- src/queue/index.ts | 5 ----- 1 file changed, 5 deletions(-) (limited to 'src/queue/index.ts') diff --git a/src/queue/index.ts b/src/queue/index.ts index 689985e0ed..691223de2d 100644 --- a/src/queue/index.ts +++ b/src/queue/index.ts @@ -1,12 +1,9 @@ import { createQueue } from 'kue'; -import * as debug from 'debug'; import config from '../config'; import db from './processors/db'; import http from './processors/http'; -const log = debug('misskey:queue'); - const queue = createQueue({ redis: { port: config.redis.port, @@ -16,8 +13,6 @@ const queue = createQueue({ }); export function createHttp(data) { - log(`HTTP job created: ${JSON.stringify(data)}`); - return queue .create('http', data) .attempts(16) -- cgit v1.2.3-freya From 494597236cea4a40bddd9655b6506464df053bfe Mon Sep 17 00:00:00 2001 From: syuilo Date: Sat, 7 Apr 2018 07:19:30 +0900 Subject: 投稿に関しては論理削除するように MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 処理をシンプルにするため --- src/models/post.ts | 1 + src/queue/index.ts | 7 ------- src/queue/processors/db/delete-post-dependents.ts | 22 ---------------------- src/queue/processors/db/index.ts | 7 ------- src/remote/activitypub/act/delete/note.ts | 16 +++++++++------- 5 files changed, 10 insertions(+), 43 deletions(-) delete mode 100644 src/queue/processors/db/delete-post-dependents.ts delete mode 100644 src/queue/processors/db/index.ts (limited to 'src/queue/index.ts') diff --git a/src/models/post.ts b/src/models/post.ts index 68a638fa2f..ac7890d2e6 100644 --- a/src/models/post.ts +++ b/src/models/post.ts @@ -27,6 +27,7 @@ export type IPost = { _id: mongo.ObjectID; channelId: mongo.ObjectID; createdAt: Date; + deletedAt: Date; mediaIds: mongo.ObjectID[]; replyId: mongo.ObjectID; repostId: mongo.ObjectID; diff --git a/src/queue/index.ts b/src/queue/index.ts index 691223de2d..4aa1dc032d 100644 --- a/src/queue/index.ts +++ b/src/queue/index.ts @@ -1,7 +1,6 @@ import { createQueue } from 'kue'; import config from '../config'; -import db from './processors/db'; import http from './processors/http'; const queue = createQueue({ @@ -19,10 +18,6 @@ export function createHttp(data) { .backoff({ delay: 16384, type: 'exponential' }); } -export function createDb(data) { - return queue.create('db', data); -} - export function deliver(user, content, to) { return createHttp({ type: 'deliver', @@ -33,8 +28,6 @@ export function deliver(user, content, to) { } export default function() { - queue.process('db', db); - /* 256 is the default concurrency limit of Mozilla Firefox and Google Chromium. diff --git a/src/queue/processors/db/delete-post-dependents.ts b/src/queue/processors/db/delete-post-dependents.ts deleted file mode 100644 index 6de21eb053..0000000000 --- a/src/queue/processors/db/delete-post-dependents.ts +++ /dev/null @@ -1,22 +0,0 @@ -import Favorite from '../../../models/favorite'; -import Notification from '../../../models/notification'; -import PollVote from '../../../models/poll-vote'; -import PostReaction from '../../../models/post-reaction'; -import PostWatching from '../../../models/post-watching'; -import Post from '../../../models/post'; - -export default async ({ data }) => Promise.all([ - Favorite.remove({ postId: data._id }), - Notification.remove({ postId: data._id }), - PollVote.remove({ postId: data._id }), - PostReaction.remove({ postId: data._id }), - PostWatching.remove({ postId: data._id }), - Post.find({ repostId: data._id }).then(reposts => Promise.all([ - Notification.remove({ - postId: { - $in: reposts.map(({ _id }) => _id) - } - }), - Post.remove({ repostId: data._id }) - ])) -]); diff --git a/src/queue/processors/db/index.ts b/src/queue/processors/db/index.ts deleted file mode 100644 index 75838c099b..0000000000 --- a/src/queue/processors/db/index.ts +++ /dev/null @@ -1,7 +0,0 @@ -import deletePostDependents from './delete-post-dependents'; - -const handlers = { - deletePostDependents -}; - -export default (job, done) => handlers[job.data.type](job).then(() => done(), done); diff --git a/src/remote/activitypub/act/delete/note.ts b/src/remote/activitypub/act/delete/note.ts index ff9a8ee5f8..8e9447b481 100644 --- a/src/remote/activitypub/act/delete/note.ts +++ b/src/remote/activitypub/act/delete/note.ts @@ -1,7 +1,6 @@ import * as debug from 'debug'; import Post from '../../../../models/post'; -import { createDb } from '../../../../queue'; import { IRemoteUser } from '../../../../models/user'; const log = debug('misskey:activitypub'); @@ -19,10 +18,13 @@ export default async function(actor: IRemoteUser, uri: string): Promise { throw new Error('投稿を削除しようとしているユーザーは投稿の作成者ではありません'); } - Post.remove({ _id: post._id }); - - createDb({ - type: 'deletePostDependents', - id: post._id - }).delay(65536).save(); + Post.update({ _id: post._id }, { + $set: { + deletedAt: new Date(), + text: null, + textHtml: null, + mediaIds: [], + poll: null + } + }); } -- cgit v1.2.3-freya