From e8b42d7e1668679e6a6ee0a7aea1e2ff7f37005b Mon Sep 17 00:00:00 2001 From: syuilo Date: Wed, 4 Apr 2018 23:12:35 +0900 Subject: wip --- src/queue/processors/http/deliver.ts | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) create mode 100644 src/queue/processors/http/deliver.ts (limited to 'src/queue/processors/http/deliver.ts') diff --git a/src/queue/processors/http/deliver.ts b/src/queue/processors/http/deliver.ts new file mode 100644 index 0000000000..8cd9eb624e --- /dev/null +++ b/src/queue/processors/http/deliver.ts @@ -0,0 +1,17 @@ +import * as kue from 'kue'; + +import Channel from '../../models/channel'; +import Following from '../../models/following'; +import ChannelWatching from '../../models/channel-watching'; +import Post, { pack } from '../../models/post'; +import User, { isLocalUser } from '../../models/user'; +import stream, { publishChannelStream } from '../../publishers/stream'; +import context from '../../remote/activitypub/renderer/context'; +import renderCreate from '../../remote/activitypub/renderer/create'; +import renderNote from '../../remote/activitypub/renderer/note'; +import request from '../../remote/request'; + +export default async (job: kue.Job, done): Promise => { + + request(user, following.follower[0].account.inbox, create); +} -- cgit v1.2.3-freya From 5bd1451b610c134c056991e05327990180cbb8d5 Mon Sep 17 00:00:00 2001 From: syuilo Date: Wed, 4 Apr 2018 23:22:48 +0900 Subject: wip --- src/queue/processors/http/deliver.ts | 16 +++------------- 1 file changed, 3 insertions(+), 13 deletions(-) (limited to 'src/queue/processors/http/deliver.ts') diff --git a/src/queue/processors/http/deliver.ts b/src/queue/processors/http/deliver.ts index 8cd9eb624e..1700063a5d 100644 --- a/src/queue/processors/http/deliver.ts +++ b/src/queue/processors/http/deliver.ts @@ -1,17 +1,7 @@ import * as kue from 'kue'; -import Channel from '../../models/channel'; -import Following from '../../models/following'; -import ChannelWatching from '../../models/channel-watching'; -import Post, { pack } from '../../models/post'; -import User, { isLocalUser } from '../../models/user'; -import stream, { publishChannelStream } from '../../publishers/stream'; -import context from '../../remote/activitypub/renderer/context'; -import renderCreate from '../../remote/activitypub/renderer/create'; -import renderNote from '../../remote/activitypub/renderer/note'; -import request from '../../remote/request'; +import request from '../../../remote/request'; export default async (job: kue.Job, done): Promise => { - - request(user, following.follower[0].account.inbox, create); -} + await request(job.data.user, job.data.to, job.data.content); +}; -- cgit v1.2.3-freya From 7403f38fb43b0ad747236061a591cbf94e198ba6 Mon Sep 17 00:00:00 2001 From: syuilo Date: Thu, 5 Apr 2018 18:43:06 +0900 Subject: wip --- src/api/post/create.ts | 15 ++++++++------- src/index.ts | 2 +- src/queue/index.ts | 8 +++++++- src/queue/processors/http/deliver.ts | 9 ++++++++- src/queue/processors/http/index.ts | 16 ++++++++++++---- src/queue/processors/http/report-github-failure.ts | 6 +++--- src/remote/activitypub/resolver.ts | 2 +- src/remote/request.ts | 8 ++++++++ src/server/api/endpoints/following/create.ts | 11 ++--------- 9 files changed, 50 insertions(+), 27 deletions(-) (limited to 'src/queue/processors/http/deliver.ts') diff --git a/src/api/post/create.ts b/src/api/post/create.ts index 5495117538..dbeb87ae86 100644 --- a/src/api/post/create.ts +++ b/src/api/post/create.ts @@ -18,20 +18,21 @@ import html from '../../text/html'; import { IApp } from '../../models/app'; export default async (user: IUser, content: { - createdAt: Date; - text: string; - reply: IPost; - repost: IPost; - media: IDriveFile[]; - geo: any; + createdAt?: Date; + text?: string; + reply?: IPost; + repost?: IPost; + media?: IDriveFile[]; + geo?: any; poll?: any; - viaMobile: boolean; + viaMobile?: boolean; tags?: string[]; cw?: string; visibility?: string; uri?: string; app?: IApp; }) => new Promise(async (res, rej) => { + if (content.createdAt == null) content.createdAt = new Date(); if (content.visibility == null) content.visibility = 'public'; const tags = content.tags || []; diff --git a/src/index.ts b/src/index.ts index e35c917a44..f45bcaa6ad 100644 --- a/src/index.ts +++ b/src/index.ts @@ -103,7 +103,7 @@ async function workerMain(opt) { if (!opt['only-server']) { // start processor - require('./processor').default(); + require('./queue').default(); } // Send a 'ready' message to parent process diff --git a/src/queue/index.ts b/src/queue/index.ts index c8c436b18c..86600dc265 100644 --- a/src/queue/index.ts +++ b/src/queue/index.ts @@ -1,8 +1,12 @@ import { createQueue } from 'kue'; +import * as debug from 'debug'; + import config from '../config'; import db from './processors/db'; import http from './processors/http'; +const log = debug('misskey:queue'); + const queue = createQueue({ redis: { port: config.redis.port, @@ -12,6 +16,8 @@ const queue = createQueue({ }); export function createHttp(data) { + log(`HTTP job created: ${JSON.stringify(data)}`); + return queue .create('http', data) .attempts(16) @@ -22,7 +28,7 @@ export function createDb(data) { return queue.create('db', data); } -export function process() { +export default function() { queue.process('db', db); /* diff --git a/src/queue/processors/http/deliver.ts b/src/queue/processors/http/deliver.ts index 1700063a5d..da7e8bc368 100644 --- a/src/queue/processors/http/deliver.ts +++ b/src/queue/processors/http/deliver.ts @@ -3,5 +3,12 @@ import * as kue from 'kue'; import request from '../../../remote/request'; export default async (job: kue.Job, done): Promise => { - await request(job.data.user, job.data.to, job.data.content); + try { + await request(job.data.user, job.data.to, job.data.content); + done(); + } catch (e) { + console.warn(`deliver failed: ${e}`); + + done(e); + } }; diff --git a/src/queue/processors/http/index.ts b/src/queue/processors/http/index.ts index 06c6b1d1aa..3d7d941b1a 100644 --- a/src/queue/processors/http/index.ts +++ b/src/queue/processors/http/index.ts @@ -3,9 +3,17 @@ import processInbox from './process-inbox'; import reportGitHubFailure from './report-github-failure'; const handlers = { - deliver, - processInbox, - reportGitHubFailure, + deliver, + processInbox, + reportGitHubFailure }; -export default (job, done) => handlers[job.data.type](job).then(() => done(), done); +export default (job, done) => { + const handler = handlers[job.data.type]; + + if (handler) { + handler(job).then(() => done(), done); + } else { + console.warn(`Unknown job: ${job.data.type}`); + } +}; diff --git a/src/queue/processors/http/report-github-failure.ts b/src/queue/processors/http/report-github-failure.ts index 4f6f5ccee5..e747d062d3 100644 --- a/src/queue/processors/http/report-github-failure.ts +++ b/src/queue/processors/http/report-github-failure.ts @@ -1,6 +1,6 @@ import * as request from 'request-promise-native'; -import User from '../../models/user'; -const createPost = require('../../server/api/endpoints/posts/create'); +import User from '../../../models/user'; +import createPost from '../../../api/post/create'; export default async ({ data }) => { const asyncBot = User.findOne({ _id: data.userId }); @@ -20,5 +20,5 @@ export default async ({ data }) => { `**⚠️BUILD STILL FAILED⚠️**: ?[${data.message}](${data.htmlUrl})` : `**🚨BUILD FAILED🚨**: →→→?[${data.message}](${data.htmlUrl})←←←`; - createPost({ text }, await asyncBot); + createPost(await asyncBot, { text }); }; diff --git a/src/remote/activitypub/resolver.ts b/src/remote/activitypub/resolver.ts index 09a6e70056..38639c6813 100644 --- a/src/remote/activitypub/resolver.ts +++ b/src/remote/activitypub/resolver.ts @@ -59,7 +59,7 @@ export default class Resolver { throw new Error('invalid response'); } - log(`resolved: ${JSON.stringify(object)}`); + log(`resolved: ${JSON.stringify(object, null, 2)}`); return object; } diff --git a/src/remote/request.ts b/src/remote/request.ts index 72262cbf61..a375aebfbb 100644 --- a/src/remote/request.ts +++ b/src/remote/request.ts @@ -1,9 +1,15 @@ import { request } from 'https'; import { sign } from 'http-signature'; import { URL } from 'url'; +import * as debug from 'debug'; + import config from '../config'; +const log = debug('misskey:activitypub:deliver'); + export default ({ account, username }, url, object) => new Promise((resolve, reject) => { + log(`--> ${url}`); + const { protocol, hostname, port, pathname, search } = new URL(url); const req = request({ @@ -14,6 +20,8 @@ export default ({ account, username }, url, object) => new Promise((resolve, rej path: pathname + search, }, res => { res.on('end', () => { + log(`${url} --> ${res.statusCode}`); + if (res.statusCode >= 200 && res.statusCode < 300) { resolve(); } else { diff --git a/src/server/api/endpoints/following/create.ts b/src/server/api/endpoints/following/create.ts index e568595215..fae686ce54 100644 --- a/src/server/api/endpoints/following/create.ts +++ b/src/server/api/endpoints/following/create.ts @@ -4,7 +4,7 @@ import $ from 'cafy'; import User from '../../../../models/user'; import Following from '../../../../models/following'; -import queue from '../../../../queue'; +import create from '../../../../api/following/create'; /** * Follow a user @@ -50,15 +50,8 @@ module.exports = (params, user) => new Promise(async (res, rej) => { } // Create following - const { _id } = await Following.insert({ - createdAt: new Date(), - followerId: follower._id, - followeeId: followee._id - }); - - queue.create('http', { type: 'follow', following: _id }).save(); + create(follower, followee); // Send response res(); - }); -- cgit v1.2.3-freya From 66346495e56e07a5b167a8f94ed971f081c94e88 Mon Sep 17 00:00:00 2001 From: syuilo Date: Sat, 7 Apr 2018 06:22:03 +0900 Subject: Fix bug --- src/queue/processors/http/deliver.ts | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) (limited to 'src/queue/processors/http/deliver.ts') diff --git a/src/queue/processors/http/deliver.ts b/src/queue/processors/http/deliver.ts index da7e8bc368..f5d162fd0c 100644 --- a/src/queue/processors/http/deliver.ts +++ b/src/queue/processors/http/deliver.ts @@ -6,9 +6,14 @@ export default async (job: kue.Job, done): Promise => { try { await request(job.data.user, job.data.to, job.data.content); done(); - } catch (e) { - console.warn(`deliver failed: ${e}`); - - done(e); + } catch (res) { + if (res.statusCode >= 300 && res.statusCode < 400) { + // HTTPステータスコード4xxはクライアントエラーであり、それはつまり + // 何回再送しても成功することはないということなのでエラーにはしないでおく + done(); + } else { + console.warn(`deliver failed: ${res.statusMessage}`); + done(new Error(res.statusMessage)); + } } }; -- cgit v1.2.3-freya From 866797a0058a0eac1e5c1d7f6cdeeab845a505ae Mon Sep 17 00:00:00 2001 From: syuilo Date: Sat, 7 Apr 2018 06:23:38 +0900 Subject: Fix bug --- src/queue/processors/http/deliver.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src/queue/processors/http/deliver.ts') diff --git a/src/queue/processors/http/deliver.ts b/src/queue/processors/http/deliver.ts index f5d162fd0c..422e355b5f 100644 --- a/src/queue/processors/http/deliver.ts +++ b/src/queue/processors/http/deliver.ts @@ -7,7 +7,7 @@ export default async (job: kue.Job, done): Promise => { await request(job.data.user, job.data.to, job.data.content); done(); } catch (res) { - if (res.statusCode >= 300 && res.statusCode < 400) { + if (res.statusCode >= 400 && res.statusCode < 500) { // HTTPステータスコード4xxはクライアントエラーであり、それはつまり // 何回再送しても成功することはないということなのでエラーにはしないでおく done(); -- cgit v1.2.3-freya