From e8b42d7e1668679e6a6ee0a7aea1e2ff7f37005b Mon Sep 17 00:00:00 2001 From: syuilo Date: Wed, 4 Apr 2018 23:12:35 +0900 Subject: wip --- src/remote/activitypub/resolver.ts | 75 +++++++++++++++++--------------------- 1 file changed, 34 insertions(+), 41 deletions(-) (limited to 'src/remote/activitypub/resolver.ts') diff --git a/src/remote/activitypub/resolver.ts b/src/remote/activitypub/resolver.ts index 371ccdcc30..de0bba2687 100644 --- a/src/remote/activitypub/resolver.ts +++ b/src/remote/activitypub/resolver.ts @@ -1,20 +1,45 @@ +import { IObject } from "./type"; + const request = require('request-promise-native'); export default class Resolver { - private requesting: Set; + private history: Set; - constructor(iterable?: Iterable) { - this.requesting = new Set(iterable); + constructor() { + this.history = new Set(); } - private async resolveUnrequestedOne(value) { + public async resolveCollection(value) { + const collection = typeof value === 'string' + ? await this.resolve(value) + : value; + + switch (collection.type) { + case 'Collection': + collection.objects = collection.object.items; + break; + + case 'OrderedCollection': + collection.objects = collection.object.orderedItems; + break; + + default: + throw new Error(`unknown collection type: ${collection.type}`); + } + + return collection; + } + + public async resolve(value): Promise { if (typeof value !== 'string') { - return { resolver: this, object: value }; + return value; } - const resolver = new Resolver(this.requesting); + if (this.history.has(value)) { + throw new Error('cannot resolve already resolved one'); + } - resolver.requesting.add(value); + this.history.add(value); const object = await request({ url: value, @@ -29,41 +54,9 @@ export default class Resolver { !object['@context'].includes('https://www.w3.org/ns/activitystreams') : object['@context'] !== 'https://www.w3.org/ns/activitystreams' )) { - throw new Error(); - } - - return { resolver, object }; - } - - public async resolveCollection(value) { - const resolved = typeof value === 'string' ? - await this.resolveUnrequestedOne(value) : - { resolver: this, object: value }; - - switch (resolved.object.type) { - case 'Collection': - resolved.object = resolved.object.items; - break; - - case 'OrderedCollection': - resolved.object = resolved.object.orderedItems; - break; - - default: - if (!Array.isArray(value)) { - resolved.object = [resolved.object]; - } - break; - } - - return resolved; - } - - public resolveOne(value) { - if (this.requesting.has(value)) { - throw new Error(); + throw new Error('invalid response'); } - return this.resolveUnrequestedOne(value); + return object; } } -- cgit v1.2.3-freya From 06347cd71e46ce2b991bc8b872cd0725c8862954 Mon Sep 17 00:00:00 2001 From: syuilo Date: Thu, 5 Apr 2018 18:08:51 +0900 Subject: wip --- src/api/drive/upload-from-url.ts | 8 +++++++- src/index.ts | 4 ++++ src/remote/activitypub/resolve-person.ts | 10 +++++----- src/remote/activitypub/resolver.ts | 8 ++++++-- 4 files changed, 22 insertions(+), 8 deletions(-) (limited to 'src/remote/activitypub/resolver.ts') diff --git a/src/api/drive/upload-from-url.ts b/src/api/drive/upload-from-url.ts index 26c890d157..676586cd15 100644 --- a/src/api/drive/upload-from-url.ts +++ b/src/api/drive/upload-from-url.ts @@ -6,14 +6,18 @@ import * as tmp from 'tmp'; import * as fs from 'fs'; import * as request from 'request'; -const log = debug('misskey:common:drive:upload_from_url'); +const log = debug('misskey:drive:upload-from-url'); export default async (url, user, folderId = null, uri = null): Promise => { + log(`REQUESTED: ${url}`); + let name = URL.parse(url).pathname.split('/').pop(); if (!validateFileName(name)) { name = null; } + log(`name: ${name}`); + // Create temp file const path = await new Promise((res: (string) => void, rej) => { tmp.file((e, path) => { @@ -37,6 +41,8 @@ export default async (url, user, folderId = null, uri = null): Promise { if (e) log(e.stack); diff --git a/src/index.ts b/src/index.ts index 29c4f3431a..e35c917a44 100644 --- a/src/index.ts +++ b/src/index.ts @@ -30,6 +30,10 @@ const ev = new Xev(); process.title = 'Misskey'; +if (process.env.NODE_ENV != 'production') { + process.env.DEBUG = 'misskey:*'; +} + // https://github.com/Automattic/kue/issues/822 require('events').EventEmitter.prototype._maxListeners = 256; diff --git a/src/remote/activitypub/resolve-person.ts b/src/remote/activitypub/resolve-person.ts index 28162497f3..c288a2f009 100644 --- a/src/remote/activitypub/resolve-person.ts +++ b/src/remote/activitypub/resolve-person.ts @@ -31,7 +31,7 @@ export default async (value, verifier?: string) => { const user = await User.insert({ avatarId: null, bannerId: null, - createdAt: Date.parse(object.published), + createdAt: Date.parse(object.published) || null, description: summaryDOM.textContent, followersCount: 0, followingCount: 0, @@ -55,14 +55,14 @@ export default async (value, verifier?: string) => { const [avatarId, bannerId] = await Promise.all([ object.icon, object.image - ].map(async url => { - if (url === undefined) { + ].map(async img => { + if (img === undefined) { return null; } - const img = await uploadFromUrl(url, user); + const file = await uploadFromUrl(img.url, user); - return img._id; + return file._id; })); User.update({ _id: user._id }, { $set: { avatarId, bannerId } }); diff --git a/src/remote/activitypub/resolver.ts b/src/remote/activitypub/resolver.ts index de0bba2687..09a6e70056 100644 --- a/src/remote/activitypub/resolver.ts +++ b/src/remote/activitypub/resolver.ts @@ -1,6 +1,8 @@ -import { IObject } from "./type"; +import * as request from 'request-promise-native'; +import * as debug from 'debug'; +import { IObject } from './type'; -const request = require('request-promise-native'); +const log = debug('misskey:activitypub:resolver'); export default class Resolver { private history: Set; @@ -57,6 +59,8 @@ export default class Resolver { throw new Error('invalid response'); } + log(`resolved: ${JSON.stringify(object)}`); + return object; } } -- cgit v1.2.3-freya From 7403f38fb43b0ad747236061a591cbf94e198ba6 Mon Sep 17 00:00:00 2001 From: syuilo Date: Thu, 5 Apr 2018 18:43:06 +0900 Subject: wip --- src/api/post/create.ts | 15 ++++++++------- src/index.ts | 2 +- src/queue/index.ts | 8 +++++++- src/queue/processors/http/deliver.ts | 9 ++++++++- src/queue/processors/http/index.ts | 16 ++++++++++++---- src/queue/processors/http/report-github-failure.ts | 6 +++--- src/remote/activitypub/resolver.ts | 2 +- src/remote/request.ts | 8 ++++++++ src/server/api/endpoints/following/create.ts | 11 ++--------- 9 files changed, 50 insertions(+), 27 deletions(-) (limited to 'src/remote/activitypub/resolver.ts') diff --git a/src/api/post/create.ts b/src/api/post/create.ts index 5495117538..dbeb87ae86 100644 --- a/src/api/post/create.ts +++ b/src/api/post/create.ts @@ -18,20 +18,21 @@ import html from '../../text/html'; import { IApp } from '../../models/app'; export default async (user: IUser, content: { - createdAt: Date; - text: string; - reply: IPost; - repost: IPost; - media: IDriveFile[]; - geo: any; + createdAt?: Date; + text?: string; + reply?: IPost; + repost?: IPost; + media?: IDriveFile[]; + geo?: any; poll?: any; - viaMobile: boolean; + viaMobile?: boolean; tags?: string[]; cw?: string; visibility?: string; uri?: string; app?: IApp; }) => new Promise(async (res, rej) => { + if (content.createdAt == null) content.createdAt = new Date(); if (content.visibility == null) content.visibility = 'public'; const tags = content.tags || []; diff --git a/src/index.ts b/src/index.ts index e35c917a44..f45bcaa6ad 100644 --- a/src/index.ts +++ b/src/index.ts @@ -103,7 +103,7 @@ async function workerMain(opt) { if (!opt['only-server']) { // start processor - require('./processor').default(); + require('./queue').default(); } // Send a 'ready' message to parent process diff --git a/src/queue/index.ts b/src/queue/index.ts index c8c436b18c..86600dc265 100644 --- a/src/queue/index.ts +++ b/src/queue/index.ts @@ -1,8 +1,12 @@ import { createQueue } from 'kue'; +import * as debug from 'debug'; + import config from '../config'; import db from './processors/db'; import http from './processors/http'; +const log = debug('misskey:queue'); + const queue = createQueue({ redis: { port: config.redis.port, @@ -12,6 +16,8 @@ const queue = createQueue({ }); export function createHttp(data) { + log(`HTTP job created: ${JSON.stringify(data)}`); + return queue .create('http', data) .attempts(16) @@ -22,7 +28,7 @@ export function createDb(data) { return queue.create('db', data); } -export function process() { +export default function() { queue.process('db', db); /* diff --git a/src/queue/processors/http/deliver.ts b/src/queue/processors/http/deliver.ts index 1700063a5d..da7e8bc368 100644 --- a/src/queue/processors/http/deliver.ts +++ b/src/queue/processors/http/deliver.ts @@ -3,5 +3,12 @@ import * as kue from 'kue'; import request from '../../../remote/request'; export default async (job: kue.Job, done): Promise => { - await request(job.data.user, job.data.to, job.data.content); + try { + await request(job.data.user, job.data.to, job.data.content); + done(); + } catch (e) { + console.warn(`deliver failed: ${e}`); + + done(e); + } }; diff --git a/src/queue/processors/http/index.ts b/src/queue/processors/http/index.ts index 06c6b1d1aa..3d7d941b1a 100644 --- a/src/queue/processors/http/index.ts +++ b/src/queue/processors/http/index.ts @@ -3,9 +3,17 @@ import processInbox from './process-inbox'; import reportGitHubFailure from './report-github-failure'; const handlers = { - deliver, - processInbox, - reportGitHubFailure, + deliver, + processInbox, + reportGitHubFailure }; -export default (job, done) => handlers[job.data.type](job).then(() => done(), done); +export default (job, done) => { + const handler = handlers[job.data.type]; + + if (handler) { + handler(job).then(() => done(), done); + } else { + console.warn(`Unknown job: ${job.data.type}`); + } +}; diff --git a/src/queue/processors/http/report-github-failure.ts b/src/queue/processors/http/report-github-failure.ts index 4f6f5ccee5..e747d062d3 100644 --- a/src/queue/processors/http/report-github-failure.ts +++ b/src/queue/processors/http/report-github-failure.ts @@ -1,6 +1,6 @@ import * as request from 'request-promise-native'; -import User from '../../models/user'; -const createPost = require('../../server/api/endpoints/posts/create'); +import User from '../../../models/user'; +import createPost from '../../../api/post/create'; export default async ({ data }) => { const asyncBot = User.findOne({ _id: data.userId }); @@ -20,5 +20,5 @@ export default async ({ data }) => { `**⚠️BUILD STILL FAILED⚠️**: ?[${data.message}](${data.htmlUrl})` : `**🚨BUILD FAILED🚨**: →→→?[${data.message}](${data.htmlUrl})←←←`; - createPost({ text }, await asyncBot); + createPost(await asyncBot, { text }); }; diff --git a/src/remote/activitypub/resolver.ts b/src/remote/activitypub/resolver.ts index 09a6e70056..38639c6813 100644 --- a/src/remote/activitypub/resolver.ts +++ b/src/remote/activitypub/resolver.ts @@ -59,7 +59,7 @@ export default class Resolver { throw new Error('invalid response'); } - log(`resolved: ${JSON.stringify(object)}`); + log(`resolved: ${JSON.stringify(object, null, 2)}`); return object; } diff --git a/src/remote/request.ts b/src/remote/request.ts index 72262cbf61..a375aebfbb 100644 --- a/src/remote/request.ts +++ b/src/remote/request.ts @@ -1,9 +1,15 @@ import { request } from 'https'; import { sign } from 'http-signature'; import { URL } from 'url'; +import * as debug from 'debug'; + import config from '../config'; +const log = debug('misskey:activitypub:deliver'); + export default ({ account, username }, url, object) => new Promise((resolve, reject) => { + log(`--> ${url}`); + const { protocol, hostname, port, pathname, search } = new URL(url); const req = request({ @@ -14,6 +20,8 @@ export default ({ account, username }, url, object) => new Promise((resolve, rej path: pathname + search, }, res => { res.on('end', () => { + log(`${url} --> ${res.statusCode}`); + if (res.statusCode >= 200 && res.statusCode < 300) { resolve(); } else { diff --git a/src/server/api/endpoints/following/create.ts b/src/server/api/endpoints/following/create.ts index e568595215..fae686ce54 100644 --- a/src/server/api/endpoints/following/create.ts +++ b/src/server/api/endpoints/following/create.ts @@ -4,7 +4,7 @@ import $ from 'cafy'; import User from '../../../../models/user'; import Following from '../../../../models/following'; -import queue from '../../../../queue'; +import create from '../../../../api/following/create'; /** * Follow a user @@ -50,15 +50,8 @@ module.exports = (params, user) => new Promise(async (res, rej) => { } // Create following - const { _id } = await Following.insert({ - createdAt: new Date(), - followerId: follower._id, - followeeId: followee._id - }); - - queue.create('http', { type: 'follow', following: _id }).save(); + create(follower, followee); // Send response res(); - }); -- cgit v1.2.3-freya From 5f8ab584464386b67bff6f9b7f60525e3e50cecc Mon Sep 17 00:00:00 2001 From: syuilo Date: Thu, 5 Apr 2018 22:49:41 +0900 Subject: wip --- src/remote/activitypub/act/create.ts | 102 ++++++++++++++++--------------- src/remote/activitypub/act/index.ts | 3 +- src/remote/activitypub/resolve-person.ts | 2 +- src/remote/activitypub/resolver.ts | 4 ++ 4 files changed, 60 insertions(+), 51 deletions(-) (limited to 'src/remote/activitypub/resolver.ts') diff --git a/src/remote/activitypub/act/create.ts b/src/remote/activitypub/act/create.ts index 7d5a9d427a..9669348d58 100644 --- a/src/remote/activitypub/act/create.ts +++ b/src/remote/activitypub/act/create.ts @@ -5,10 +5,12 @@ import Resolver from '../resolver'; import Post from '../../../models/post'; import uploadFromUrl from '../../../api/drive/upload-from-url'; import createPost from '../../../api/post/create'; +import { IRemoteUser, isRemoteUser } from '../../../models/user'; +import resolvePerson from '../resolve-person'; const log = debug('misskey:activitypub'); -export default async (actor, activity): Promise => { +export default async (actor: IRemoteUser, activity): Promise => { if ('actor' in activity && actor.account.uri !== activity.actor) { throw new Error('invalid actor'); } @@ -32,71 +34,73 @@ export default async (actor, activity): Promise => { switch (object.type) { case 'Image': - createImage(object); + createImage(resolver, actor, object); break; case 'Note': - createNote(object); + createNote(resolver, actor, object); break; default: console.warn(`Unknown type: ${object.type}`); break; } +}; - /// +async function createImage(resolver: Resolver, actor: IRemoteUser, image) { + if ('attributedTo' in image && actor.account.uri !== image.attributedTo) { + log(`invalid image: ${JSON.stringify(image, null, 2)}`); + throw new Error('invalid image'); + } - async function createImage(image) { - if ('attributedTo' in image && actor.account.uri !== image.attributedTo) { - log(`invalid image: ${JSON.stringify(image, null, 2)}`); - throw new Error('invalid image'); - } + log(`Creating the Image: ${image.id}`); - log(`Creating the Image: ${uri}`); + return await uploadFromUrl(image.url, actor); +} - return await uploadFromUrl(image.url, actor); +async function createNote(resolver: Resolver, actor: IRemoteUser, note) { + if ( + ('attributedTo' in note && actor.account.uri !== note.attributedTo) || + typeof note.id !== 'string' + ) { + log(`invalid note: ${JSON.stringify(note, null, 2)}`); + throw new Error('invalid note'); } - async function createNote(note) { - if ( - ('attributedTo' in note && actor.account.uri !== note.attributedTo) || - typeof note.id !== 'string' - ) { - log(`invalid note: ${JSON.stringify(note, null, 2)}`); - throw new Error('invalid note'); - } - - log(`Creating the Note: ${uri}`); + log(`Creating the Note: ${note.id}`); - const media = []; - if ('attachment' in note && note.attachment != null) { - note.attachment.forEach(async media => { - const created = await createImage(media); - media.push(created); - }); - } + const media = []; + if ('attachment' in note && note.attachment != null) { + note.attachment.forEach(async media => { + const created = await createImage(resolver, note.actor, media); + media.push(created); + }); + } - let reply = null; - if ('inReplyTo' in note && note.inReplyTo != null) { - const inReplyToPost = await Post.findOne({ uri: note.inReplyTo.id || note.inReplyTo }); - if (inReplyToPost) { - reply = inReplyToPost; - } else { - reply = await createNote(await resolver.resolve(note)); + let reply = null; + if ('inReplyTo' in note && note.inReplyTo != null) { + const inReplyToPost = await Post.findOne({ uri: note.inReplyTo.id || note.inReplyTo }); + if (inReplyToPost) { + reply = inReplyToPost; + } else { + const inReplyTo = await resolver.resolve(note.inReplyTo) as any; + const actor = await resolvePerson(inReplyTo.attributedTo); + if (isRemoteUser(actor)) { + reply = await createNote(resolver, actor, inReplyTo); } } - - const { window } = new JSDOM(note.content); - - return await createPost(actor, { - createdAt: new Date(note.published), - media, - reply, - repost: undefined, - text: window.document.body.textContent, - viaMobile: false, - geo: undefined, - uri: note.id - }); } -}; + + const { window } = new JSDOM(note.content); + + return await createPost(actor, { + createdAt: new Date(note.published), + media, + reply, + repost: undefined, + text: window.document.body.textContent, + viaMobile: false, + geo: undefined, + uri: note.id + }); +} diff --git a/src/remote/activitypub/act/index.ts b/src/remote/activitypub/act/index.ts index 5840227098..f58505b0ad 100644 --- a/src/remote/activitypub/act/index.ts +++ b/src/remote/activitypub/act/index.ts @@ -3,8 +3,9 @@ import performDeleteActivity from './delete'; import follow from './follow'; import undo from './undo'; import { IObject } from '../type'; +import { IUser } from '../../../models/user'; -export default async (actor, activity: IObject): Promise => { +export default async (actor: IUser, activity: IObject): Promise => { switch (activity.type) { case 'Create': await create(actor, activity); diff --git a/src/remote/activitypub/resolve-person.ts b/src/remote/activitypub/resolve-person.ts index b979bb1cd2..2bf7a13540 100644 --- a/src/remote/activitypub/resolve-person.ts +++ b/src/remote/activitypub/resolve-person.ts @@ -11,7 +11,7 @@ export default async (value, verifier?: string) => { const object = await resolver.resolve(value) as any; if ( - object === null || + object == null || object.type !== 'Person' || typeof object.preferredUsername !== 'string' || !validateUsername(object.preferredUsername) || diff --git a/src/remote/activitypub/resolver.ts b/src/remote/activitypub/resolver.ts index 38639c6813..4a97e2ef66 100644 --- a/src/remote/activitypub/resolver.ts +++ b/src/remote/activitypub/resolver.ts @@ -33,6 +33,10 @@ export default class Resolver { } public async resolve(value): Promise { + if (value == null) { + throw new Error('resolvee is null (or undefined)'); + } + if (typeof value !== 'string') { return value; } -- cgit v1.2.3-freya