From 8c4143290735b39bc68bf98439b2735af1910518 Mon Sep 17 00:00:00 2001 From: Akihiko Odaki Date: Sat, 31 Mar 2018 19:45:56 +0900 Subject: Set empty array instead of null to mediaIds property of posts --- src/client/app/desktop/views/components/post-detail.sub.vue | 2 +- src/client/app/desktop/views/components/post-detail.vue | 2 +- src/client/app/desktop/views/components/posts.post.vue | 2 +- src/client/app/desktop/views/components/sub-post-content.vue | 2 +- src/client/app/mobile/views/components/post-detail.vue | 2 +- src/client/app/mobile/views/components/post.vue | 2 +- src/client/app/mobile/views/components/sub-post-content.vue | 2 +- src/client/docs/api/entities/post.yaml | 4 ++-- src/server/api/endpoints/posts/create.ts | 2 +- 9 files changed, 10 insertions(+), 10 deletions(-) (limited to 'src') diff --git a/src/client/app/desktop/views/components/post-detail.sub.vue b/src/client/app/desktop/views/components/post-detail.sub.vue index 35377e7c24..285b5dedee 100644 --- a/src/client/app/desktop/views/components/post-detail.sub.vue +++ b/src/client/app/desktop/views/components/post-detail.sub.vue @@ -17,7 +17,7 @@
-
+
diff --git a/src/client/app/desktop/views/components/post-detail.vue b/src/client/app/desktop/views/components/post-detail.vue index 5c7a7dfdbe..1811e22bad 100644 --- a/src/client/app/desktop/views/components/post-detail.vue +++ b/src/client/app/desktop/views/components/post-detail.vue @@ -39,7 +39,7 @@
-
+
diff --git a/src/client/app/desktop/views/components/posts.post.vue b/src/client/app/desktop/views/components/posts.post.vue index 37c6e63043..aa1f1db41c 100644 --- a/src/client/app/desktop/views/components/posts.post.vue +++ b/src/client/app/desktop/views/components/posts.post.vue @@ -41,7 +41,7 @@ RP:
-
+
diff --git a/src/client/app/desktop/views/components/sub-post-content.vue b/src/client/app/desktop/views/components/sub-post-content.vue index a79e5e0a4e..1f5ce38984 100644 --- a/src/client/app/desktop/views/components/sub-post-content.vue +++ b/src/client/app/desktop/views/components/sub-post-content.vue @@ -5,7 +5,7 @@ RP: ...
-
+
({{ post.media.length }}つのメディア)
diff --git a/src/client/app/mobile/views/components/post-detail.vue b/src/client/app/mobile/views/components/post-detail.vue index f0af1a61aa..6411011b89 100644 --- a/src/client/app/mobile/views/components/post-detail.vue +++ b/src/client/app/mobile/views/components/post-detail.vue @@ -42,7 +42,7 @@
{{ tag }}
-
+
diff --git a/src/client/app/mobile/views/components/post.vue b/src/client/app/mobile/views/components/post.vue index a01eb7669e..52fb095372 100644 --- a/src/client/app/mobile/views/components/post.vue +++ b/src/client/app/mobile/views/components/post.vue @@ -40,7 +40,7 @@ RP:
-
+
diff --git a/src/client/app/mobile/views/components/sub-post-content.vue b/src/client/app/mobile/views/components/sub-post-content.vue index b95883de77..5ff88089aa 100644 --- a/src/client/app/mobile/views/components/sub-post-content.vue +++ b/src/client/app/mobile/views/components/sub-post-content.vue @@ -5,7 +5,7 @@ RP: ...
-
+
({{ post.media.length }}個のメディア)
diff --git a/src/client/docs/api/entities/post.yaml b/src/client/docs/api/entities/post.yaml index 74d7973e38..da79866ba1 100644 --- a/src/client/docs/api/entities/post.yaml +++ b/src/client/docs/api/entities/post.yaml @@ -33,8 +33,8 @@ props: type: "id(DriveFile)[]" optional: true desc: - ja: "添付されているメディアのID" - en: "The IDs of the attached media" + ja: "添付されているメディアのID (なければレスポンスでは空配列)" + en: "The IDs of the attached media (empty array for response if no media is attached)" - name: "media" type: "entity(DriveFile)[]" optional: true diff --git a/src/server/api/endpoints/posts/create.ts b/src/server/api/endpoints/posts/create.ts index 170b667191..aa7e93c28f 100644 --- a/src/server/api/endpoints/posts/create.ts +++ b/src/server/api/endpoints/posts/create.ts @@ -254,7 +254,7 @@ module.exports = (params, user: IUser, app) => new Promise(async (res, rej) => { createdAt: new Date(), channelId: channel ? channel._id : undefined, index: channel ? channel.index + 1 : undefined, - mediaIds: files ? files.map(file => file._id) : undefined, + mediaIds: files ? files.map(file => file._id) : [], replyId: reply ? reply._id : undefined, repostId: repost ? repost._id : undefined, poll: poll, -- cgit v1.2.3-freya From 7da60a0147116130a94274e4a20ae54dd7d59dea Mon Sep 17 00:00:00 2001 From: Akihiko Odaki Date: Sat, 31 Mar 2018 19:53:30 +0900 Subject: Store texts as HTML --- src/client/app/common/views/components/index.ts | 2 +- .../views/components/messaging-room.message.vue | 33 +- .../app/common/views/components/post-html.ts | 141 --------- .../app/common/views/components/post-html.vue | 103 +++++++ src/client/app/common/views/components/url.vue | 66 ---- .../common/views/components/welcome-timeline.vue | 2 +- .../desktop/views/components/post-detail.sub.vue | 2 +- .../app/desktop/views/components/post-detail.vue | 27 +- .../app/desktop/views/components/posts.post.vue | 31 +- .../desktop/views/components/sub-post-content.vue | 2 +- .../app/mobile/views/components/post-detail.vue | 27 +- src/client/app/mobile/views/components/post.vue | 31 +- .../mobile/views/components/sub-post-content.vue | 2 +- src/client/docs/api/entities/post.yaml | 10 +- src/common/text/core/syntax-highlighter.ts | 334 --------------------- src/common/text/elements/bold.ts | 14 - src/common/text/elements/code.ts | 17 -- src/common/text/elements/emoji.ts | 14 - src/common/text/elements/hashtag.ts | 19 -- src/common/text/elements/inline-code.ts | 17 -- src/common/text/elements/link.ts | 19 -- src/common/text/elements/mention.ts | 17 -- src/common/text/elements/quote.ts | 14 - src/common/text/elements/url.ts | 14 - src/common/text/html.ts | 83 +++++ src/common/text/index.ts | 72 ----- src/common/text/parse/core/syntax-highlighter.ts | 334 +++++++++++++++++++++ src/common/text/parse/elements/bold.ts | 14 + src/common/text/parse/elements/code.ts | 17 ++ src/common/text/parse/elements/emoji.ts | 14 + src/common/text/parse/elements/hashtag.ts | 19 ++ src/common/text/parse/elements/inline-code.ts | 17 ++ src/common/text/parse/elements/link.ts | 19 ++ src/common/text/parse/elements/mention.ts | 17 ++ src/common/text/parse/elements/quote.ts | 14 + src/common/text/parse/elements/url.ts | 14 + src/common/text/parse/index.ts | 72 +++++ src/models/messaging-message.ts | 7 +- src/models/post.ts | 7 +- .../api/endpoints/messaging/messages/create.ts | 3 + src/server/api/endpoints/posts/create.ts | 4 +- tools/migration/nighthike/7.js | 16 + 42 files changed, 868 insertions(+), 833 deletions(-) delete mode 100644 src/client/app/common/views/components/post-html.ts create mode 100644 src/client/app/common/views/components/post-html.vue delete mode 100644 src/client/app/common/views/components/url.vue delete mode 100644 src/common/text/core/syntax-highlighter.ts delete mode 100644 src/common/text/elements/bold.ts delete mode 100644 src/common/text/elements/code.ts delete mode 100644 src/common/text/elements/emoji.ts delete mode 100644 src/common/text/elements/hashtag.ts delete mode 100644 src/common/text/elements/inline-code.ts delete mode 100644 src/common/text/elements/link.ts delete mode 100644 src/common/text/elements/mention.ts delete mode 100644 src/common/text/elements/quote.ts delete mode 100644 src/common/text/elements/url.ts create mode 100644 src/common/text/html.ts delete mode 100644 src/common/text/index.ts create mode 100644 src/common/text/parse/core/syntax-highlighter.ts create mode 100644 src/common/text/parse/elements/bold.ts create mode 100644 src/common/text/parse/elements/code.ts create mode 100644 src/common/text/parse/elements/emoji.ts create mode 100644 src/common/text/parse/elements/hashtag.ts create mode 100644 src/common/text/parse/elements/inline-code.ts create mode 100644 src/common/text/parse/elements/link.ts create mode 100644 src/common/text/parse/elements/mention.ts create mode 100644 src/common/text/parse/elements/quote.ts create mode 100644 src/common/text/parse/elements/url.ts create mode 100644 src/common/text/parse/index.ts create mode 100644 tools/migration/nighthike/7.js (limited to 'src') diff --git a/src/client/app/common/views/components/index.ts b/src/client/app/common/views/components/index.ts index b58ba37ecb..8c10bdee28 100644 --- a/src/client/app/common/views/components/index.ts +++ b/src/client/app/common/views/components/index.ts @@ -4,7 +4,7 @@ import signin from './signin.vue'; import signup from './signup.vue'; import forkit from './forkit.vue'; import nav from './nav.vue'; -import postHtml from './post-html'; +import postHtml from './post-html.vue'; import poll from './poll.vue'; import pollEditor from './poll-editor.vue'; import reactionIcon from './reaction-icon.vue'; diff --git a/src/client/app/common/views/components/messaging-room.message.vue b/src/client/app/common/views/components/messaging-room.message.vue index 94f87fd709..25ceab85a1 100644 --- a/src/client/app/common/views/components/messaging-room.message.vue +++ b/src/client/app/common/views/components/messaging-room.message.vue @@ -4,13 +4,13 @@
-
+

%i18n:common.tags.mk-messaging-message.is-read%

- +
@@ -38,21 +38,32 @@ import getAcct from '../../../../../common/user/get-acct'; export default Vue.extend({ props: ['message'], + data() { + return { + urls: [] + }; + }, computed: { acct() { return getAcct(this.message.user); }, isMe(): boolean { return this.message.userId == (this as any).os.i.id; - }, - urls(): string[] { - if (this.message.ast) { - return this.message.ast - .filter(t => (t.type == 'url' || t.type == 'link') && !t.silent) - .map(t => t.url); - } else { - return null; - } + } + }, + watch: { + message: { + handler(newMessage, oldMessage) { + if (!oldMessage || newMessage.textHtml !== oldMessage.textHtml) { + this.$nextTick(() => { + const elements = this.$refs.text.$el.getElementsByTagName('a'); + + this.urls = [].filter.call(elements, ({ origin }) => origin !== location.origin) + .map(({ href }) => href); + }); + } + }, + immediate: true } } }); diff --git a/src/client/app/common/views/components/post-html.ts b/src/client/app/common/views/components/post-html.ts deleted file mode 100644 index 39d783aac5..0000000000 --- a/src/client/app/common/views/components/post-html.ts +++ /dev/null @@ -1,141 +0,0 @@ -import Vue from 'vue'; -import * as emojilib from 'emojilib'; -import getAcct from '../../../../../common/user/get-acct'; -import { url } from '../../../config'; -import MkUrl from './url.vue'; - -const flatten = list => list.reduce( - (a, b) => a.concat(Array.isArray(b) ? flatten(b) : b), [] -); - -export default Vue.component('mk-post-html', { - props: { - ast: { - type: Array, - required: true - }, - shouldBreak: { - type: Boolean, - default: true - }, - i: { - type: Object, - default: null - } - }, - render(createElement) { - const els = flatten((this as any).ast.map(token => { - switch (token.type) { - case 'text': - const text = token.content.replace(/(\r\n|\n|\r)/g, '\n'); - - if ((this as any).shouldBreak) { - const x = text.split('\n') - .map(t => t == '' ? [createElement('br')] : [createElement('span', t), createElement('br')]); - x[x.length - 1].pop(); - return x; - } else { - return createElement('span', text.replace(/\n/g, ' ')); - } - - case 'bold': - return createElement('strong', token.bold); - - case 'url': - return createElement(MkUrl, { - props: { - url: token.content, - target: '_blank' - } - }); - - case 'link': - return createElement('a', { - attrs: { - class: 'link', - href: token.url, - target: '_blank', - title: token.url - } - }, token.title); - - case 'mention': - return (createElement as any)('a', { - attrs: { - href: `${url}/@${getAcct(token)}`, - target: '_blank', - dataIsMe: (this as any).i && getAcct((this as any).i) == getAcct(token) - }, - directives: [{ - name: 'user-preview', - value: token.content - }] - }, token.content); - - case 'hashtag': - return createElement('a', { - attrs: { - href: `${url}/search?q=${token.content}`, - target: '_blank' - } - }, token.content); - - case 'code': - return createElement('pre', [ - createElement('code', { - domProps: { - innerHTML: token.html - } - }) - ]); - - case 'inline-code': - return createElement('code', { - domProps: { - innerHTML: token.html - } - }); - - case 'quote': - const text2 = token.quote.replace(/(\r\n|\n|\r)/g, '\n'); - - if ((this as any).shouldBreak) { - const x = text2.split('\n') - .map(t => [createElement('span', t), createElement('br')]); - x[x.length - 1].pop(); - return createElement('div', { - attrs: { - class: 'quote' - } - }, x); - } else { - return createElement('span', { - attrs: { - class: 'quote' - } - }, text2.replace(/\n/g, ' ')); - } - - case 'emoji': - const emoji = emojilib.lib[token.emoji]; - return createElement('span', emoji ? emoji.char : token.content); - - default: - console.log('unknown ast type:', token.type); - } - })); - - const _els = []; - els.forEach((el, i) => { - if (el.tag == 'br') { - if (els[i - 1].tag != 'div') { - _els.push(el); - } - } else { - _els.push(el); - } - }); - - return createElement('span', _els); - } -}); diff --git a/src/client/app/common/views/components/post-html.vue b/src/client/app/common/views/components/post-html.vue new file mode 100644 index 0000000000..1c949052b9 --- /dev/null +++ b/src/client/app/common/views/components/post-html.vue @@ -0,0 +1,103 @@ + + + + + diff --git a/src/client/app/common/views/components/url.vue b/src/client/app/common/views/components/url.vue deleted file mode 100644 index 14d4fc82f3..0000000000 --- a/src/client/app/common/views/components/url.vue +++ /dev/null @@ -1,66 +0,0 @@ - - - - - diff --git a/src/client/app/common/views/components/welcome-timeline.vue b/src/client/app/common/views/components/welcome-timeline.vue index 8f6199732a..f379029f9f 100644 --- a/src/client/app/common/views/components/welcome-timeline.vue +++ b/src/client/app/common/views/components/welcome-timeline.vue @@ -15,7 +15,7 @@
- +
diff --git a/src/client/app/desktop/views/components/post-detail.sub.vue b/src/client/app/desktop/views/components/post-detail.sub.vue index 285b5dedee..b6148d9b28 100644 --- a/src/client/app/desktop/views/components/post-detail.sub.vue +++ b/src/client/app/desktop/views/components/post-detail.sub.vue @@ -16,7 +16,7 @@
- +
diff --git a/src/client/app/desktop/views/components/post-detail.vue b/src/client/app/desktop/views/components/post-detail.vue index 1811e22bad..e75ebe34b4 100644 --- a/src/client/app/desktop/views/components/post-detail.vue +++ b/src/client/app/desktop/views/components/post-detail.vue @@ -38,7 +38,7 @@
- +
@@ -109,6 +109,7 @@ export default Vue.extend({ context: [], contextFetching: false, replies: [], + urls: [] }; }, computed: { @@ -130,15 +131,6 @@ export default Vue.extend({ }, title(): string { return dateStringify(this.p.createdAt); - }, - urls(): string[] { - if (this.p.ast) { - return this.p.ast - .filter(t => (t.type == 'url' || t.type == 'link') && !t.silent) - .map(t => t.url); - } else { - return null; - } } }, mounted() { @@ -170,6 +162,21 @@ export default Vue.extend({ } } }, + watch: { + post: { + handler(newPost, oldPost) { + if (!oldPost || newPost.text !== oldPost.text) { + this.$nextTick(() => { + const elements = this.$refs.text.$el.getElementsByTagName('a'); + + this.urls = [].filter.call(elements, ({ origin }) => origin !== location.origin) + .map(({ href }) => href); + }); + } + }, + immediate: true + } + }, methods: { fetchContext() { this.contextFetching = true; diff --git a/src/client/app/desktop/views/components/posts.post.vue b/src/client/app/desktop/views/components/posts.post.vue index aa1f1db41c..f3566c81bf 100644 --- a/src/client/app/desktop/views/components/posts.post.vue +++ b/src/client/app/desktop/views/components/posts.post.vue @@ -38,7 +38,7 @@

%fa:reply% - + RP:
@@ -112,7 +112,8 @@ export default Vue.extend({ return { isDetailOpened: false, connection: null, - connectionId: null + connectionId: null, + urls: [] }; }, computed: { @@ -140,15 +141,6 @@ export default Vue.extend({ }, url(): string { return `/@${this.acct}/${this.p.id}`; - }, - urls(): string[] { - if (this.p.ast) { - return this.p.ast - .filter(t => (t.type == 'url' || t.type == 'link') && !t.silent) - .map(t => t.url); - } else { - return null; - } } }, created() { @@ -190,6 +182,21 @@ export default Vue.extend({ (this as any).os.stream.dispose(this.connectionId); } }, + watch: { + post: { + handler(newPost, oldPost) { + if (!oldPost || newPost.textHtml !== oldPost.textHtml) { + this.$nextTick(() => { + const elements = this.$refs.text.$el.getElementsByTagName('a'); + + this.urls = [].filter.call(elements, ({ origin }) => origin !== location.origin) + .map(({ href }) => href); + }); + } + }, + immediate: true + } + }, methods: { capture(withHandler = false) { if ((this as any).os.isSignedIn) { @@ -450,7 +457,7 @@ export default Vue.extend({ font-size 1.1em color #717171 - >>> .quote + >>> blockquote margin 8px padding 6px 12px color #aaa diff --git a/src/client/app/desktop/views/components/sub-post-content.vue b/src/client/app/desktop/views/components/sub-post-content.vue index 1f5ce38984..58c81e7552 100644 --- a/src/client/app/desktop/views/components/sub-post-content.vue +++ b/src/client/app/desktop/views/components/sub-post-content.vue @@ -2,7 +2,7 @@
%fa:reply% - + RP: ...
diff --git a/src/client/app/mobile/views/components/post-detail.vue b/src/client/app/mobile/views/components/post-detail.vue index 6411011b89..77a73426f2 100644 --- a/src/client/app/mobile/views/components/post-detail.vue +++ b/src/client/app/mobile/views/components/post-detail.vue @@ -38,7 +38,7 @@
- +
{{ tag }}
@@ -103,6 +103,7 @@ export default Vue.extend({ context: [], contextFetching: false, replies: [], + urls: [] }; }, computed: { @@ -127,15 +128,6 @@ export default Vue.extend({ .map(key => this.p.reactionCounts[key]) .reduce((a, b) => a + b) : 0; - }, - urls(): string[] { - if (this.p.ast) { - return this.p.ast - .filter(t => (t.type == 'url' || t.type == 'link') && !t.silent) - .map(t => t.url); - } else { - return null; - } } }, mounted() { @@ -167,6 +159,21 @@ export default Vue.extend({ } } }, + watch: { + post: { + handler(newPost, oldPost) { + if (!oldPost || newPost.text !== oldPost.text) { + this.$nextTick(() => { + const elements = this.$refs.text.$el.getElementsByTagName('a'); + + this.urls = [].filter.call(elements, ({ origin }) => origin !== location.origin) + .map(({ href }) => href); + }); + } + }, + immediate: true + } + }, methods: { fetchContext() { this.contextFetching = true; diff --git a/src/client/app/mobile/views/components/post.vue b/src/client/app/mobile/views/components/post.vue index 52fb095372..96ec9632f1 100644 --- a/src/client/app/mobile/views/components/post.vue +++ b/src/client/app/mobile/views/components/post.vue @@ -37,7 +37,7 @@ %fa:reply% - + RP:
@@ -90,7 +90,8 @@ export default Vue.extend({ data() { return { connection: null, - connectionId: null + connectionId: null, + urls: [] }; }, computed: { @@ -118,15 +119,6 @@ export default Vue.extend({ }, url(): string { return `/@${this.pAcct}/${this.p.id}`; - }, - urls(): string[] { - if (this.p.ast) { - return this.p.ast - .filter(t => (t.type == 'url' || t.type == 'link') && !t.silent) - .map(t => t.url); - } else { - return null; - } } }, created() { @@ -168,6 +160,21 @@ export default Vue.extend({ (this as any).os.stream.dispose(this.connectionId); } }, + watch: { + post: { + handler(newPost, oldPost) { + if (!oldPost || newPost.text !== oldPost.text) { + this.$nextTick(() => { + const elements = this.$refs.text.$el.getElementsByTagName('a'); + + this.urls = [].filter.call(elements, ({ origin }) => origin !== location.origin) + .map(({ href }) => href); + }); + } + }, + immediate: true + } + }, methods: { capture(withHandler = false) { if ((this as any).os.isSignedIn) { @@ -389,7 +396,7 @@ export default Vue.extend({ font-size 1.1em color #717171 - >>> .quote + >>> blockquote margin 8px padding 6px 12px color #aaa diff --git a/src/client/app/mobile/views/components/sub-post-content.vue b/src/client/app/mobile/views/components/sub-post-content.vue index 5ff88089aa..955bb406b4 100644 --- a/src/client/app/mobile/views/components/sub-post-content.vue +++ b/src/client/app/mobile/views/components/sub-post-content.vue @@ -2,7 +2,7 @@
%fa:reply% - + RP: ...
diff --git a/src/client/docs/api/entities/post.yaml b/src/client/docs/api/entities/post.yaml index da79866ba1..7077700129 100644 --- a/src/client/docs/api/entities/post.yaml +++ b/src/client/docs/api/entities/post.yaml @@ -27,8 +27,14 @@ props: type: "string" optional: true desc: - ja: "投稿の本文" - en: "The text of this post" + ja: "投稿の本文 (ローカルの場合Markdown風のフォーマット)" + en: "The text of this post (in Markdown like format if local)" + - name: "textHtml" + type: "string" + optional: true + desc: + ja: "投稿の本文 (HTML) (投稿時は無視)" + en: "The text of this post (in HTML. Ignored when posting.)" - name: "mediaIds" type: "id(DriveFile)[]" optional: true diff --git a/src/common/text/core/syntax-highlighter.ts b/src/common/text/core/syntax-highlighter.ts deleted file mode 100644 index c0396b1fc6..0000000000 --- a/src/common/text/core/syntax-highlighter.ts +++ /dev/null @@ -1,334 +0,0 @@ -function escape(text) { - return text - .replace(/>/g, '>') - .replace(/ k[0].toUpperCase() + k.substr(1))) - .concat(_keywords.map(k => k.toUpperCase())) - .sort((a, b) => b.length - a.length); - -const symbols = [ - '=', - '+', - '-', - '*', - '/', - '%', - '~', - '^', - '&', - '|', - '>', - '<', - '!', - '?' -]; - -const elements = [ - // comment - code => { - if (code.substr(0, 2) != '//') return null; - const match = code.match(/^\/\/(.+?)(\n|$)/); - if (!match) return null; - const comment = match[0]; - return { - html: `${escape(comment)}`, - next: comment.length - }; - }, - - // block comment - code => { - const match = code.match(/^\/\*([\s\S]+?)\*\//); - if (!match) return null; - return { - html: `${escape(match[0])}`, - next: match[0].length - }; - }, - - // string - code => { - if (!/^['"`]/.test(code)) return null; - const begin = code[0]; - let str = begin; - let thisIsNotAString = false; - for (let i = 1; i < code.length; i++) { - const char = code[i]; - if (char == '\\') { - str += char; - str += code[i + 1] || ''; - i++; - continue; - } else if (char == begin) { - str += char; - break; - } else if (char == '\n' || i == (code.length - 1)) { - thisIsNotAString = true; - break; - } else { - str += char; - } - } - if (thisIsNotAString) { - return null; - } else { - return { - html: `${escape(str)}`, - next: str.length - }; - } - }, - - // regexp - code => { - if (code[0] != '/') return null; - let regexp = ''; - let thisIsNotARegexp = false; - for (let i = 1; i < code.length; i++) { - const char = code[i]; - if (char == '\\') { - regexp += char; - regexp += code[i + 1] || ''; - i++; - continue; - } else if (char == '/') { - break; - } else if (char == '\n' || i == (code.length - 1)) { - thisIsNotARegexp = true; - break; - } else { - regexp += char; - } - } - - if (thisIsNotARegexp) return null; - if (regexp == '') return null; - if (regexp[0] == ' ' && regexp[regexp.length - 1] == ' ') return null; - - return { - html: `/${escape(regexp)}/`, - next: regexp.length + 2 - }; - }, - - // label - code => { - if (code[0] != '@') return null; - const match = code.match(/^@([a-zA-Z_-]+?)\n/); - if (!match) return null; - const label = match[0]; - return { - html: `${label}`, - next: label.length - }; - }, - - // number - (code, i, source) => { - const prev = source[i - 1]; - if (prev && /[a-zA-Z]/.test(prev)) return null; - if (!/^[\-\+]?[0-9\.]+/.test(code)) return null; - const match = code.match(/^[\-\+]?[0-9\.]+/)[0]; - if (match) { - return { - html: `${match}`, - next: match.length - }; - } else { - return null; - } - }, - - // nan - (code, i, source) => { - const prev = source[i - 1]; - if (prev && /[a-zA-Z]/.test(prev)) return null; - if (code.substr(0, 3) == 'NaN') { - return { - html: `NaN`, - next: 3 - }; - } else { - return null; - } - }, - - // method - code => { - const match = code.match(/^([a-zA-Z_-]+?)\(/); - if (!match) return null; - - if (match[1] == '-') return null; - - return { - html: `${match[1]}`, - next: match[1].length - }; - }, - - // property - (code, i, source) => { - const prev = source[i - 1]; - if (prev != '.') return null; - - const match = code.match(/^[a-zA-Z0-9_-]+/); - if (!match) return null; - - return { - html: `${match[0]}`, - next: match[0].length - }; - }, - - // keyword - (code, i, source) => { - const prev = source[i - 1]; - if (prev && /[a-zA-Z]/.test(prev)) return null; - - const match = keywords.filter(k => code.substr(0, k.length) == k)[0]; - if (match) { - if (/^[a-zA-Z]/.test(code.substr(match.length))) return null; - return { - html: `${match}`, - next: match.length - }; - } else { - return null; - } - }, - - // symbol - code => { - const match = symbols.filter(s => code[0] == s)[0]; - if (match) { - return { - html: `${match}`, - next: 1 - }; - } else { - return null; - } - } -]; - -// specify lang is todo -export default (source: string, lang?: string) => { - let code = source; - let html = ''; - - let i = 0; - - function push(token) { - html += token.html; - code = code.substr(token.next); - i += token.next; - } - - while (code != '') { - const parsed = elements.some(el => { - const e = el(code, i, source); - if (e) { - push(e); - return true; - } else { - return false; - } - }); - - if (!parsed) { - push({ - html: escape(code[0]), - next: 1 - }); - } - } - - return html; -}; diff --git a/src/common/text/elements/bold.ts b/src/common/text/elements/bold.ts deleted file mode 100644 index ce25764457..0000000000 --- a/src/common/text/elements/bold.ts +++ /dev/null @@ -1,14 +0,0 @@ -/** - * Bold - */ - -module.exports = text => { - const match = text.match(/^\*\*(.+?)\*\*/); - if (!match) return null; - const bold = match[0]; - return { - type: 'bold', - content: bold, - bold: bold.substr(2, bold.length - 4) - }; -}; diff --git a/src/common/text/elements/code.ts b/src/common/text/elements/code.ts deleted file mode 100644 index 4821e95fe2..0000000000 --- a/src/common/text/elements/code.ts +++ /dev/null @@ -1,17 +0,0 @@ -/** - * Code (block) - */ - -import genHtml from '../core/syntax-highlighter'; - -module.exports = text => { - const match = text.match(/^```([\s\S]+?)```/); - if (!match) return null; - const code = match[0]; - return { - type: 'code', - content: code, - code: code.substr(3, code.length - 6).trim(), - html: genHtml(code.substr(3, code.length - 6).trim()) - }; -}; diff --git a/src/common/text/elements/emoji.ts b/src/common/text/elements/emoji.ts deleted file mode 100644 index e24231a223..0000000000 --- a/src/common/text/elements/emoji.ts +++ /dev/null @@ -1,14 +0,0 @@ -/** - * Emoji - */ - -module.exports = text => { - const match = text.match(/^:[a-zA-Z0-9+-_]+:/); - if (!match) return null; - const emoji = match[0]; - return { - type: 'emoji', - content: emoji, - emoji: emoji.substr(1, emoji.length - 2) - }; -}; diff --git a/src/common/text/elements/hashtag.ts b/src/common/text/elements/hashtag.ts deleted file mode 100644 index ee57b140b8..0000000000 --- a/src/common/text/elements/hashtag.ts +++ /dev/null @@ -1,19 +0,0 @@ -/** - * Hashtag - */ - -module.exports = (text, i) => { - if (!(/^\s#[^\s]+/.test(text) || (i == 0 && /^#[^\s]+/.test(text)))) return null; - const isHead = text[0] == '#'; - const hashtag = text.match(/^\s?#[^\s]+/)[0]; - const res: any[] = !isHead ? [{ - type: 'text', - content: text[0] - }] : []; - res.push({ - type: 'hashtag', - content: isHead ? hashtag : hashtag.substr(1), - hashtag: isHead ? hashtag.substr(1) : hashtag.substr(2) - }); - return res; -}; diff --git a/src/common/text/elements/inline-code.ts b/src/common/text/elements/inline-code.ts deleted file mode 100644 index 9f9ef51a2b..0000000000 --- a/src/common/text/elements/inline-code.ts +++ /dev/null @@ -1,17 +0,0 @@ -/** - * Code (inline) - */ - -import genHtml from '../core/syntax-highlighter'; - -module.exports = text => { - const match = text.match(/^`(.+?)`/); - if (!match) return null; - const code = match[0]; - return { - type: 'inline-code', - content: code, - code: code.substr(1, code.length - 2).trim(), - html: genHtml(code.substr(1, code.length - 2).trim()) - }; -}; diff --git a/src/common/text/elements/link.ts b/src/common/text/elements/link.ts deleted file mode 100644 index 35563ddc3d..0000000000 --- a/src/common/text/elements/link.ts +++ /dev/null @@ -1,19 +0,0 @@ -/** - * Link - */ - -module.exports = text => { - const match = text.match(/^\??\[([^\[\]]+?)\]\((https?:\/\/[\w\/:%#@\$&\?!\(\)\[\]~\.=\+\-]+?)\)/); - if (!match) return null; - const silent = text[0] == '?'; - const link = match[0]; - const title = match[1]; - const url = match[2]; - return { - type: 'link', - content: link, - title: title, - url: url, - silent: silent - }; -}; diff --git a/src/common/text/elements/mention.ts b/src/common/text/elements/mention.ts deleted file mode 100644 index d05a76649d..0000000000 --- a/src/common/text/elements/mention.ts +++ /dev/null @@ -1,17 +0,0 @@ -/** - * Mention - */ -import parseAcct from '../../../common/user/parse-acct'; - -module.exports = text => { - const match = text.match(/^(?:@[a-zA-Z0-9\-]+){1,2}/); - if (!match) return null; - const mention = match[0]; - const { username, host } = parseAcct(mention.substr(1)); - return { - type: 'mention', - content: mention, - username, - host - }; -}; diff --git a/src/common/text/elements/quote.ts b/src/common/text/elements/quote.ts deleted file mode 100644 index cc8cfffdc4..0000000000 --- a/src/common/text/elements/quote.ts +++ /dev/null @@ -1,14 +0,0 @@ -/** - * Quoted text - */ - -module.exports = text => { - const match = text.match(/^"([\s\S]+?)\n"/); - if (!match) return null; - const quote = match[0]; - return { - type: 'quote', - content: quote, - quote: quote.substr(1, quote.length - 2).trim(), - }; -}; diff --git a/src/common/text/elements/url.ts b/src/common/text/elements/url.ts deleted file mode 100644 index 1003aff9c3..0000000000 --- a/src/common/text/elements/url.ts +++ /dev/null @@ -1,14 +0,0 @@ -/** - * URL - */ - -module.exports = text => { - const match = text.match(/^https?:\/\/[\w\/:%#@\$&\?!\(\)\[\]~\.=\+\-]+/); - if (!match) return null; - const url = match[0]; - return { - type: 'url', - content: url, - url: url - }; -}; diff --git a/src/common/text/html.ts b/src/common/text/html.ts new file mode 100644 index 0000000000..797f3b3f33 --- /dev/null +++ b/src/common/text/html.ts @@ -0,0 +1,83 @@ +import { lib as emojilib } from 'emojilib'; +import { JSDOM } from 'jsdom'; + +const handlers = { + bold({ document }, { bold }) { + const b = document.createElement('b'); + b.textContent = bold; + document.body.appendChild(b); + }, + + code({ document }, { code }) { + const pre = document.createElement('pre'); + const inner = document.createElement('code'); + inner.innerHTML = code; + pre.appendChild(inner); + document.body.appendChild(pre); + }, + + emoji({ document }, { content, emoji }) { + const found = emojilib[emoji]; + const node = document.createTextNode(found ? found.char : content); + document.body.appendChild(node); + }, + + hashtag({ document }, { hashtag }) { + const a = document.createElement('a'); + a.href = '/search?q=#' + hashtag; + a.textContent = hashtag; + }, + + 'inline-code'({ document }, { code }) { + const element = document.createElement('code'); + element.textContent = code; + document.body.appendChild(element); + }, + + link({ document }, { url, title }) { + const a = document.createElement('a'); + a.href = url; + a.textContent = title; + document.body.appendChild(a); + }, + + mention({ document }, { content }) { + const a = document.createElement('a'); + a.href = '/' + content; + a.textContent = content; + document.body.appendChild(a); + }, + + quote({ document }, { quote }) { + const blockquote = document.createElement('blockquote'); + blockquote.textContent = quote; + document.body.appendChild(blockquote); + }, + + text({ document }, { content }) { + for (const text of content.split('\n')) { + const node = document.createTextNode(text); + document.body.appendChild(node); + + const br = document.createElement('br'); + document.body.appendChild(br); + } + }, + + url({ document }, { url }) { + const a = document.createElement('a'); + a.href = url; + a.textContent = url; + document.body.appendChild(a); + } +}; + +export default tokens => { + const { window } = new JSDOM(''); + + for (const token of tokens) { + handlers[token.type](window, token); + } + + return `

${window.document.body.innerHTML}

`; +}; diff --git a/src/common/text/index.ts b/src/common/text/index.ts deleted file mode 100644 index 1e2398dc38..0000000000 --- a/src/common/text/index.ts +++ /dev/null @@ -1,72 +0,0 @@ -/** - * Misskey Text Analyzer - */ - -const elements = [ - require('./elements/bold'), - require('./elements/url'), - require('./elements/link'), - require('./elements/mention'), - require('./elements/hashtag'), - require('./elements/code'), - require('./elements/inline-code'), - require('./elements/quote'), - require('./elements/emoji') -]; - -export default (source: string) => { - - if (source == '') { - return null; - } - - const tokens = []; - - function push(token) { - if (token != null) { - tokens.push(token); - source = source.substr(token.content.length); - } - } - - let i = 0; - - // パース - while (source != '') { - const parsed = elements.some(el => { - let _tokens = el(source, i); - if (_tokens) { - if (!Array.isArray(_tokens)) { - _tokens = [_tokens]; - } - _tokens.forEach(push); - return true; - } else { - return false; - } - }); - - if (!parsed) { - push({ - type: 'text', - content: source[0] - }); - } - - i++; - } - - // テキストを纏める - tokens[0] = [tokens[0]]; - return tokens.reduce((a, b) => { - if (a[a.length - 1].type == 'text' && b.type == 'text') { - const tail = a.pop(); - return a.concat({ - type: 'text', - content: tail.content + b.content - }); - } else { - return a.concat(b); - } - }); -}; diff --git a/src/common/text/parse/core/syntax-highlighter.ts b/src/common/text/parse/core/syntax-highlighter.ts new file mode 100644 index 0000000000..c0396b1fc6 --- /dev/null +++ b/src/common/text/parse/core/syntax-highlighter.ts @@ -0,0 +1,334 @@ +function escape(text) { + return text + .replace(/>/g, '>') + .replace(/ k[0].toUpperCase() + k.substr(1))) + .concat(_keywords.map(k => k.toUpperCase())) + .sort((a, b) => b.length - a.length); + +const symbols = [ + '=', + '+', + '-', + '*', + '/', + '%', + '~', + '^', + '&', + '|', + '>', + '<', + '!', + '?' +]; + +const elements = [ + // comment + code => { + if (code.substr(0, 2) != '//') return null; + const match = code.match(/^\/\/(.+?)(\n|$)/); + if (!match) return null; + const comment = match[0]; + return { + html: `${escape(comment)}`, + next: comment.length + }; + }, + + // block comment + code => { + const match = code.match(/^\/\*([\s\S]+?)\*\//); + if (!match) return null; + return { + html: `${escape(match[0])}`, + next: match[0].length + }; + }, + + // string + code => { + if (!/^['"`]/.test(code)) return null; + const begin = code[0]; + let str = begin; + let thisIsNotAString = false; + for (let i = 1; i < code.length; i++) { + const char = code[i]; + if (char == '\\') { + str += char; + str += code[i + 1] || ''; + i++; + continue; + } else if (char == begin) { + str += char; + break; + } else if (char == '\n' || i == (code.length - 1)) { + thisIsNotAString = true; + break; + } else { + str += char; + } + } + if (thisIsNotAString) { + return null; + } else { + return { + html: `${escape(str)}`, + next: str.length + }; + } + }, + + // regexp + code => { + if (code[0] != '/') return null; + let regexp = ''; + let thisIsNotARegexp = false; + for (let i = 1; i < code.length; i++) { + const char = code[i]; + if (char == '\\') { + regexp += char; + regexp += code[i + 1] || ''; + i++; + continue; + } else if (char == '/') { + break; + } else if (char == '\n' || i == (code.length - 1)) { + thisIsNotARegexp = true; + break; + } else { + regexp += char; + } + } + + if (thisIsNotARegexp) return null; + if (regexp == '') return null; + if (regexp[0] == ' ' && regexp[regexp.length - 1] == ' ') return null; + + return { + html: `/${escape(regexp)}/`, + next: regexp.length + 2 + }; + }, + + // label + code => { + if (code[0] != '@') return null; + const match = code.match(/^@([a-zA-Z_-]+?)\n/); + if (!match) return null; + const label = match[0]; + return { + html: `${label}`, + next: label.length + }; + }, + + // number + (code, i, source) => { + const prev = source[i - 1]; + if (prev && /[a-zA-Z]/.test(prev)) return null; + if (!/^[\-\+]?[0-9\.]+/.test(code)) return null; + const match = code.match(/^[\-\+]?[0-9\.]+/)[0]; + if (match) { + return { + html: `${match}`, + next: match.length + }; + } else { + return null; + } + }, + + // nan + (code, i, source) => { + const prev = source[i - 1]; + if (prev && /[a-zA-Z]/.test(prev)) return null; + if (code.substr(0, 3) == 'NaN') { + return { + html: `NaN`, + next: 3 + }; + } else { + return null; + } + }, + + // method + code => { + const match = code.match(/^([a-zA-Z_-]+?)\(/); + if (!match) return null; + + if (match[1] == '-') return null; + + return { + html: `${match[1]}`, + next: match[1].length + }; + }, + + // property + (code, i, source) => { + const prev = source[i - 1]; + if (prev != '.') return null; + + const match = code.match(/^[a-zA-Z0-9_-]+/); + if (!match) return null; + + return { + html: `${match[0]}`, + next: match[0].length + }; + }, + + // keyword + (code, i, source) => { + const prev = source[i - 1]; + if (prev && /[a-zA-Z]/.test(prev)) return null; + + const match = keywords.filter(k => code.substr(0, k.length) == k)[0]; + if (match) { + if (/^[a-zA-Z]/.test(code.substr(match.length))) return null; + return { + html: `${match}`, + next: match.length + }; + } else { + return null; + } + }, + + // symbol + code => { + const match = symbols.filter(s => code[0] == s)[0]; + if (match) { + return { + html: `${match}`, + next: 1 + }; + } else { + return null; + } + } +]; + +// specify lang is todo +export default (source: string, lang?: string) => { + let code = source; + let html = ''; + + let i = 0; + + function push(token) { + html += token.html; + code = code.substr(token.next); + i += token.next; + } + + while (code != '') { + const parsed = elements.some(el => { + const e = el(code, i, source); + if (e) { + push(e); + return true; + } else { + return false; + } + }); + + if (!parsed) { + push({ + html: escape(code[0]), + next: 1 + }); + } + } + + return html; +}; diff --git a/src/common/text/parse/elements/bold.ts b/src/common/text/parse/elements/bold.ts new file mode 100644 index 0000000000..ce25764457 --- /dev/null +++ b/src/common/text/parse/elements/bold.ts @@ -0,0 +1,14 @@ +/** + * Bold + */ + +module.exports = text => { + const match = text.match(/^\*\*(.+?)\*\*/); + if (!match) return null; + const bold = match[0]; + return { + type: 'bold', + content: bold, + bold: bold.substr(2, bold.length - 4) + }; +}; diff --git a/src/common/text/parse/elements/code.ts b/src/common/text/parse/elements/code.ts new file mode 100644 index 0000000000..4821e95fe2 --- /dev/null +++ b/src/common/text/parse/elements/code.ts @@ -0,0 +1,17 @@ +/** + * Code (block) + */ + +import genHtml from '../core/syntax-highlighter'; + +module.exports = text => { + const match = text.match(/^```([\s\S]+?)```/); + if (!match) return null; + const code = match[0]; + return { + type: 'code', + content: code, + code: code.substr(3, code.length - 6).trim(), + html: genHtml(code.substr(3, code.length - 6).trim()) + }; +}; diff --git a/src/common/text/parse/elements/emoji.ts b/src/common/text/parse/elements/emoji.ts new file mode 100644 index 0000000000..e24231a223 --- /dev/null +++ b/src/common/text/parse/elements/emoji.ts @@ -0,0 +1,14 @@ +/** + * Emoji + */ + +module.exports = text => { + const match = text.match(/^:[a-zA-Z0-9+-_]+:/); + if (!match) return null; + const emoji = match[0]; + return { + type: 'emoji', + content: emoji, + emoji: emoji.substr(1, emoji.length - 2) + }; +}; diff --git a/src/common/text/parse/elements/hashtag.ts b/src/common/text/parse/elements/hashtag.ts new file mode 100644 index 0000000000..ee57b140b8 --- /dev/null +++ b/src/common/text/parse/elements/hashtag.ts @@ -0,0 +1,19 @@ +/** + * Hashtag + */ + +module.exports = (text, i) => { + if (!(/^\s#[^\s]+/.test(text) || (i == 0 && /^#[^\s]+/.test(text)))) return null; + const isHead = text[0] == '#'; + const hashtag = text.match(/^\s?#[^\s]+/)[0]; + const res: any[] = !isHead ? [{ + type: 'text', + content: text[0] + }] : []; + res.push({ + type: 'hashtag', + content: isHead ? hashtag : hashtag.substr(1), + hashtag: isHead ? hashtag.substr(1) : hashtag.substr(2) + }); + return res; +}; diff --git a/src/common/text/parse/elements/inline-code.ts b/src/common/text/parse/elements/inline-code.ts new file mode 100644 index 0000000000..9f9ef51a2b --- /dev/null +++ b/src/common/text/parse/elements/inline-code.ts @@ -0,0 +1,17 @@ +/** + * Code (inline) + */ + +import genHtml from '../core/syntax-highlighter'; + +module.exports = text => { + const match = text.match(/^`(.+?)`/); + if (!match) return null; + const code = match[0]; + return { + type: 'inline-code', + content: code, + code: code.substr(1, code.length - 2).trim(), + html: genHtml(code.substr(1, code.length - 2).trim()) + }; +}; diff --git a/src/common/text/parse/elements/link.ts b/src/common/text/parse/elements/link.ts new file mode 100644 index 0000000000..35563ddc3d --- /dev/null +++ b/src/common/text/parse/elements/link.ts @@ -0,0 +1,19 @@ +/** + * Link + */ + +module.exports = text => { + const match = text.match(/^\??\[([^\[\]]+?)\]\((https?:\/\/[\w\/:%#@\$&\?!\(\)\[\]~\.=\+\-]+?)\)/); + if (!match) return null; + const silent = text[0] == '?'; + const link = match[0]; + const title = match[1]; + const url = match[2]; + return { + type: 'link', + content: link, + title: title, + url: url, + silent: silent + }; +}; diff --git a/src/common/text/parse/elements/mention.ts b/src/common/text/parse/elements/mention.ts new file mode 100644 index 0000000000..2025dfdaad --- /dev/null +++ b/src/common/text/parse/elements/mention.ts @@ -0,0 +1,17 @@ +/** + * Mention + */ +import parseAcct from '../../../../common/user/parse-acct'; + +module.exports = text => { + const match = text.match(/^(?:@[a-zA-Z0-9\-]+){1,2}/); + if (!match) return null; + const mention = match[0]; + const { username, host } = parseAcct(mention.substr(1)); + return { + type: 'mention', + content: mention, + username, + host + }; +}; diff --git a/src/common/text/parse/elements/quote.ts b/src/common/text/parse/elements/quote.ts new file mode 100644 index 0000000000..cc8cfffdc4 --- /dev/null +++ b/src/common/text/parse/elements/quote.ts @@ -0,0 +1,14 @@ +/** + * Quoted text + */ + +module.exports = text => { + const match = text.match(/^"([\s\S]+?)\n"/); + if (!match) return null; + const quote = match[0]; + return { + type: 'quote', + content: quote, + quote: quote.substr(1, quote.length - 2).trim(), + }; +}; diff --git a/src/common/text/parse/elements/url.ts b/src/common/text/parse/elements/url.ts new file mode 100644 index 0000000000..1003aff9c3 --- /dev/null +++ b/src/common/text/parse/elements/url.ts @@ -0,0 +1,14 @@ +/** + * URL + */ + +module.exports = text => { + const match = text.match(/^https?:\/\/[\w\/:%#@\$&\?!\(\)\[\]~\.=\+\-]+/); + if (!match) return null; + const url = match[0]; + return { + type: 'url', + content: url, + url: url + }; +}; diff --git a/src/common/text/parse/index.ts b/src/common/text/parse/index.ts new file mode 100644 index 0000000000..1e2398dc38 --- /dev/null +++ b/src/common/text/parse/index.ts @@ -0,0 +1,72 @@ +/** + * Misskey Text Analyzer + */ + +const elements = [ + require('./elements/bold'), + require('./elements/url'), + require('./elements/link'), + require('./elements/mention'), + require('./elements/hashtag'), + require('./elements/code'), + require('./elements/inline-code'), + require('./elements/quote'), + require('./elements/emoji') +]; + +export default (source: string) => { + + if (source == '') { + return null; + } + + const tokens = []; + + function push(token) { + if (token != null) { + tokens.push(token); + source = source.substr(token.content.length); + } + } + + let i = 0; + + // パース + while (source != '') { + const parsed = elements.some(el => { + let _tokens = el(source, i); + if (_tokens) { + if (!Array.isArray(_tokens)) { + _tokens = [_tokens]; + } + _tokens.forEach(push); + return true; + } else { + return false; + } + }); + + if (!parsed) { + push({ + type: 'text', + content: source[0] + }); + } + + i++; + } + + // テキストを纏める + tokens[0] = [tokens[0]]; + return tokens.reduce((a, b) => { + if (a[a.length - 1].type == 'text' && b.type == 'text') { + const tail = a.pop(); + return a.concat({ + type: 'text', + content: tail.content + b.content + }); + } else { + return a.concat(b); + } + }); +}; diff --git a/src/models/messaging-message.ts b/src/models/messaging-message.ts index 8bee657c34..974ee54ab8 100644 --- a/src/models/messaging-message.ts +++ b/src/models/messaging-message.ts @@ -3,7 +3,6 @@ import deepcopy = require('deepcopy'); import { pack as packUser } from './user'; import { pack as packFile } from './drive-file'; import db from '../db/mongodb'; -import parse from '../common/text'; const MessagingMessage = db.get('messagingMessages'); export default MessagingMessage; @@ -12,6 +11,7 @@ export interface IMessagingMessage { _id: mongo.ObjectID; createdAt: Date; text: string; + textHtml: string; userId: mongo.ObjectID; recipientId: mongo.ObjectID; isRead: boolean; @@ -60,11 +60,6 @@ export const pack = ( _message.id = _message._id; delete _message._id; - // Parse text - if (_message.text) { - _message.ast = parse(_message.text); - } - // Populate user _message.user = await packUser(_message.userId, me); diff --git a/src/models/post.ts b/src/models/post.ts index 9bc0c1d3b9..6c853e4f81 100644 --- a/src/models/post.ts +++ b/src/models/post.ts @@ -8,7 +8,6 @@ import { pack as packChannel } from './channel'; import Vote from './poll-vote'; import Reaction from './post-reaction'; import { pack as packFile } from './drive-file'; -import parse from '../common/text'; const Post = db.get('posts'); @@ -31,6 +30,7 @@ export type IPost = { repostId: mongo.ObjectID; poll: any; // todo text: string; + textHtml: string; cw: string; userId: mongo.ObjectID; appId: mongo.ObjectID; @@ -103,11 +103,6 @@ export const pack = async ( delete _post.mentions; if (_post.geo) delete _post.geo.type; - // Parse text - if (_post.text) { - _post.ast = parse(_post.text); - } - // Populate user _post.user = packUser(_post.userId, meId); diff --git a/src/server/api/endpoints/messaging/messages/create.ts b/src/server/api/endpoints/messaging/messages/create.ts index d8ffa9fdec..3d3b204da5 100644 --- a/src/server/api/endpoints/messaging/messages/create.ts +++ b/src/server/api/endpoints/messaging/messages/create.ts @@ -11,6 +11,8 @@ import DriveFile from '../../../../../models/drive-file'; import { pack } from '../../../../../models/messaging-message'; import publishUserStream from '../../../event'; import { publishMessagingStream, publishMessagingIndexStream, pushSw } from '../../../event'; +import html from '../../../../../common/text/html'; +import parse from '../../../../../common/text/parse'; import config from '../../../../../conf'; /** @@ -74,6 +76,7 @@ module.exports = (params, user) => new Promise(async (res, rej) => { fileId: file ? file._id : undefined, recipientId: recipient._id, text: text ? text : undefined, + textHtml: text ? html(parse(text)) : undefined, userId: user._id, isRead: false }); diff --git a/src/server/api/endpoints/posts/create.ts b/src/server/api/endpoints/posts/create.ts index aa7e93c28f..5342f77728 100644 --- a/src/server/api/endpoints/posts/create.ts +++ b/src/server/api/endpoints/posts/create.ts @@ -3,7 +3,8 @@ */ import $ from 'cafy'; import deepEqual = require('deep-equal'); -import parse from '../../../../common/text'; +import html from '../../../../common/text/html'; +import parse from '../../../../common/text/parse'; import { default as Post, IPost, isValidText, isValidCw } from '../../../../models/post'; import { default as User, ILocalAccount, IUser } from '../../../../models/user'; import { default as Channel, IChannel } from '../../../../models/channel'; @@ -259,6 +260,7 @@ module.exports = (params, user: IUser, app) => new Promise(async (res, rej) => { repostId: repost ? repost._id : undefined, poll: poll, text: text, + textHtml: tokens === null ? null : html(tokens), cw: cw, tags: tags, userId: user._id, diff --git a/tools/migration/nighthike/7.js b/tools/migration/nighthike/7.js new file mode 100644 index 0000000000..c5055da8ba --- /dev/null +++ b/tools/migration/nighthike/7.js @@ -0,0 +1,16 @@ +// for Node.js interpretation + +const Message = require('../../../built/models/messaging-message').default; +const Post = require('../../../built/models/post').default; +const html = require('../../../built/common/text/html').default; +const parse = require('../../../built/common/text/parse').default; + +Promise.all([Message, Post].map(async model => { + const documents = await model.find(); + + return Promise.all(documents.map(({ _id, text }) => model.update(_id, { + $set: { + textHtml: html(parse(text)) + } + }))); +})).catch(console.error).then(process.exit); -- cgit v1.2.3-freya From 68a9aac9573969311dd00a44536c3ee4c05b883d Mon Sep 17 00:00:00 2001 From: Akihiko Odaki Date: Sat, 31 Mar 2018 19:55:00 +0900 Subject: Implement remote status retrieval --- package.json | 1 + src/common/drive/add-file.ts | 307 +++++++++++++++++++++ src/common/drive/upload_from_url.ts | 46 +++ src/common/event.ts | 80 ++++++ src/common/push-sw.ts | 52 ++++ src/common/remote/activitypub/act/create.ts | 9 + src/common/remote/activitypub/act/index.ts | 22 ++ src/common/remote/activitypub/create.ts | 86 ++++++ src/common/remote/activitypub/resolve-person.ts | 104 +++++++ src/common/remote/activitypub/resolver.ts | 97 +++++++ src/common/remote/activitypub/type.ts | 3 + src/common/remote/resolve-user.ts | 26 ++ src/common/remote/webfinger.ts | 25 ++ src/models/remote-user-object.ts | 15 + src/models/user.ts | 3 + src/processor/http/index.ts | 9 + src/processor/http/perform-activitypub.ts | 6 + src/processor/http/report-github-failure.ts | 29 ++ src/processor/index.ts | 13 +- src/processor/report-github-failure.ts | 29 -- src/server/api/common/drive/add-file.ts | 307 --------------------- src/server/api/common/drive/upload_from_url.ts | 46 --- src/server/api/common/notify.ts | 2 +- src/server/api/common/push-sw.ts | 52 ---- src/server/api/common/read-messaging-message.ts | 6 +- src/server/api/common/read-notification.ts | 2 +- src/server/api/endpoints/drive/files/create.ts | 2 +- src/server/api/endpoints/drive/files/update.ts | 2 +- .../api/endpoints/drive/files/upload_from_url.ts | 2 +- src/server/api/endpoints/drive/folders/create.ts | 2 +- src/server/api/endpoints/drive/folders/update.ts | 2 +- src/server/api/endpoints/following/create.ts | 2 +- src/server/api/endpoints/following/delete.ts | 2 +- src/server/api/endpoints/i/regenerate_token.ts | 2 +- src/server/api/endpoints/i/update.ts | 2 +- .../api/endpoints/i/update_client_setting.ts | 2 +- src/server/api/endpoints/i/update_home.ts | 2 +- src/server/api/endpoints/i/update_mobile_home.ts | 2 +- .../api/endpoints/messaging/messages/create.ts | 4 +- .../endpoints/notifications/mark_as_read_all.ts | 2 +- src/server/api/endpoints/othello/match.ts | 2 +- src/server/api/endpoints/posts/create.ts | 2 +- src/server/api/endpoints/posts/polls/vote.ts | 2 +- src/server/api/endpoints/posts/reactions/create.ts | 2 +- src/server/api/endpoints/users/show.ts | 162 +---------- src/server/api/event.ts | 80 ------ src/server/api/private/signin.ts | 2 +- src/server/api/service/github.ts | 3 +- src/server/api/service/twitter.ts | 2 +- src/server/api/stream/othello-game.ts | 2 +- src/server/api/stream/othello.ts | 2 +- 51 files changed, 969 insertions(+), 699 deletions(-) create mode 100644 src/common/drive/add-file.ts create mode 100644 src/common/drive/upload_from_url.ts create mode 100644 src/common/event.ts create mode 100644 src/common/push-sw.ts create mode 100644 src/common/remote/activitypub/act/create.ts create mode 100644 src/common/remote/activitypub/act/index.ts create mode 100644 src/common/remote/activitypub/create.ts create mode 100644 src/common/remote/activitypub/resolve-person.ts create mode 100644 src/common/remote/activitypub/resolver.ts create mode 100644 src/common/remote/activitypub/type.ts create mode 100644 src/common/remote/resolve-user.ts create mode 100644 src/common/remote/webfinger.ts create mode 100644 src/models/remote-user-object.ts create mode 100644 src/processor/http/index.ts create mode 100644 src/processor/http/perform-activitypub.ts create mode 100644 src/processor/http/report-github-failure.ts delete mode 100644 src/processor/report-github-failure.ts delete mode 100644 src/server/api/common/drive/add-file.ts delete mode 100644 src/server/api/common/drive/upload_from_url.ts delete mode 100644 src/server/api/common/push-sw.ts delete mode 100644 src/server/api/event.ts (limited to 'src') diff --git a/package.json b/package.json index d1f544f86e..4275c1c1c3 100644 --- a/package.json +++ b/package.json @@ -103,6 +103,7 @@ "deep-equal": "1.0.1", "deepcopy": "0.6.3", "diskusage": "0.2.4", + "dompurify": "^1.0.3", "elasticsearch": "14.2.2", "element-ui": "2.3.2", "emojilib": "2.2.12", diff --git a/src/common/drive/add-file.ts b/src/common/drive/add-file.ts new file mode 100644 index 0000000000..52a7713dd9 --- /dev/null +++ b/src/common/drive/add-file.ts @@ -0,0 +1,307 @@ +import { Buffer } from 'buffer'; +import * as fs from 'fs'; +import * as tmp from 'tmp'; +import * as stream from 'stream'; + +import * as mongodb from 'mongodb'; +import * as crypto from 'crypto'; +import * as _gm from 'gm'; +import * as debug from 'debug'; +import fileType = require('file-type'); +import prominence = require('prominence'); + +import DriveFile, { getGridFSBucket } from '../../models/drive-file'; +import DriveFolder from '../../models/drive-folder'; +import { pack } from '../../models/drive-file'; +import event, { publishDriveStream } from '../event'; +import getAcct from '../user/get-acct'; +import config from '../../conf'; + +const gm = _gm.subClass({ + imageMagick: true +}); + +const log = debug('misskey:drive:add-file'); + +const tmpFile = (): Promise => new Promise((resolve, reject) => { + tmp.file((e, path) => { + if (e) return reject(e); + resolve(path); + }); +}); + +const addToGridFS = (name: string, readable: stream.Readable, type: string, metadata: any): Promise => + getGridFSBucket() + .then(bucket => new Promise((resolve, reject) => { + const writeStream = bucket.openUploadStream(name, { contentType: type, metadata }); + writeStream.once('finish', (doc) => { resolve(doc); }); + writeStream.on('error', reject); + readable.pipe(writeStream); + })); + +const addFile = async ( + user: any, + path: string, + name: string = null, + comment: string = null, + folderId: mongodb.ObjectID = null, + force: boolean = false +) => { + log(`registering ${name} (user: ${getAcct(user)}, path: ${path})`); + + // Calculate hash, get content type and get file size + const [hash, [mime, ext], size] = await Promise.all([ + // hash + ((): Promise => new Promise((res, rej) => { + const readable = fs.createReadStream(path); + const hash = crypto.createHash('md5'); + const chunks = []; + readable + .on('error', rej) + .pipe(hash) + .on('error', rej) + .on('data', (chunk) => chunks.push(chunk)) + .on('end', () => { + const buffer = Buffer.concat(chunks); + res(buffer.toString('hex')); + }); + }))(), + // mime + ((): Promise<[string, string | null]> => new Promise((res, rej) => { + const readable = fs.createReadStream(path); + readable + .on('error', rej) + .once('data', (buffer: Buffer) => { + readable.destroy(); + const type = fileType(buffer); + if (type) { + return res([type.mime, type.ext]); + } else { + // 種類が同定できなかったら application/octet-stream にする + return res(['application/octet-stream', null]); + } + }); + }))(), + // size + ((): Promise => new Promise((res, rej) => { + fs.stat(path, (err, stats) => { + if (err) return rej(err); + res(stats.size); + }); + }))() + ]); + + log(`hash: ${hash}, mime: ${mime}, ext: ${ext}, size: ${size}`); + + // detect name + const detectedName: string = name || (ext ? `untitled.${ext}` : 'untitled'); + + if (!force) { + // Check if there is a file with the same hash + const much = await DriveFile.findOne({ + md5: hash, + 'metadata.userId': user._id + }); + + if (much !== null) { + log('file with same hash is found'); + return much; + } else { + log('file with same hash is not found'); + } + } + + const [wh, averageColor, folder] = await Promise.all([ + // Width and height (when image) + (async () => { + // 画像かどうか + if (!/^image\/.*$/.test(mime)) { + return null; + } + + const imageType = mime.split('/')[1]; + + // 画像でもPNGかJPEGかGIFでないならスキップ + if (imageType != 'png' && imageType != 'jpeg' && imageType != 'gif') { + return null; + } + + log('calculate image width and height...'); + + // Calculate width and height + const g = gm(fs.createReadStream(path), name); + const size = await prominence(g).size(); + + log(`image width and height is calculated: ${size.width}, ${size.height}`); + + return [size.width, size.height]; + })(), + // average color (when image) + (async () => { + // 画像かどうか + if (!/^image\/.*$/.test(mime)) { + return null; + } + + const imageType = mime.split('/')[1]; + + // 画像でもPNGかJPEGでないならスキップ + if (imageType != 'png' && imageType != 'jpeg') { + return null; + } + + log('calculate average color...'); + + const buffer = await prominence(gm(fs.createReadStream(path), name) + .setFormat('ppm') + .resize(1, 1)) // 1pxのサイズに縮小して平均色を取得するというハック + .toBuffer(); + + const r = buffer.readUInt8(buffer.length - 3); + const g = buffer.readUInt8(buffer.length - 2); + const b = buffer.readUInt8(buffer.length - 1); + + log(`average color is calculated: ${r}, ${g}, ${b}`); + + return [r, g, b]; + })(), + // folder + (async () => { + if (!folderId) { + return null; + } + const driveFolder = await DriveFolder.findOne({ + _id: folderId, + userId: user._id + }); + if (!driveFolder) { + throw 'folder-not-found'; + } + return driveFolder; + })(), + // usage checker + (async () => { + // Calculate drive usage + const usage = await DriveFile + .aggregate([{ + $match: { 'metadata.userId': user._id } + }, { + $project: { + length: true + } + }, { + $group: { + _id: null, + usage: { $sum: '$length' } + } + }]) + .then((aggregates: any[]) => { + if (aggregates.length > 0) { + return aggregates[0].usage; + } + return 0; + }); + + log(`drive usage is ${usage}`); + + // If usage limit exceeded + if (usage + size > user.driveCapacity) { + throw 'no-free-space'; + } + })() + ]); + + const readable = fs.createReadStream(path); + + const properties = {}; + + if (wh) { + properties['width'] = wh[0]; + properties['height'] = wh[1]; + } + + if (averageColor) { + properties['avgColor'] = averageColor; + } + + return addToGridFS(detectedName, readable, mime, { + userId: user._id, + folderId: folder !== null ? folder._id : null, + comment: comment, + properties: properties + }); +}; + +/** + * Add file to drive + * + * @param user User who wish to add file + * @param file File path or readableStream + * @param comment Comment + * @param type File type + * @param folderId Folder ID + * @param force If set to true, forcibly upload the file even if there is a file with the same hash. + * @return Object that represents added file + */ +export default (user: any, file: string | stream.Readable, ...args) => new Promise((resolve, reject) => { + // Get file path + new Promise((res: (v: [string, boolean]) => void, rej) => { + if (typeof file === 'string') { + res([file, false]); + return; + } + if (typeof file === 'object' && typeof file.read === 'function') { + tmpFile() + .then(path => { + const readable: stream.Readable = file; + const writable = fs.createWriteStream(path); + readable + .on('error', rej) + .on('end', () => { + res([path, true]); + }) + .pipe(writable) + .on('error', rej); + }) + .catch(rej); + } + rej(new Error('un-compatible file.')); + }) + .then(([path, shouldCleanup]): Promise => new Promise((res, rej) => { + addFile(user, path, ...args) + .then(file => { + res(file); + if (shouldCleanup) { + fs.unlink(path, (e) => { + if (e) log(e.stack); + }); + } + }) + .catch(rej); + })) + .then(file => { + log(`drive file has been created ${file._id}`); + resolve(file); + + pack(file).then(serializedFile => { + // Publish drive_file_created event + event(user._id, 'drive_file_created', serializedFile); + publishDriveStream(user._id, 'file_created', serializedFile); + + // Register to search database + if (config.elasticsearch.enable) { + const es = require('../../db/elasticsearch'); + es.index({ + index: 'misskey', + type: 'drive_file', + id: file._id.toString(), + body: { + name: file.name, + userId: user._id.toString() + } + }); + } + }); + }) + .catch(reject); +}); diff --git a/src/common/drive/upload_from_url.ts b/src/common/drive/upload_from_url.ts new file mode 100644 index 0000000000..5dd9695936 --- /dev/null +++ b/src/common/drive/upload_from_url.ts @@ -0,0 +1,46 @@ +import * as URL from 'url'; +import { IDriveFile, validateFileName } from '../../models/drive-file'; +import create from './add-file'; +import * as debug from 'debug'; +import * as tmp from 'tmp'; +import * as fs from 'fs'; +import * as request from 'request'; + +const log = debug('misskey:common:drive:upload_from_url'); + +export default async (url, user, folderId = null): Promise => { + let name = URL.parse(url).pathname.split('/').pop(); + if (!validateFileName(name)) { + name = null; + } + + // Create temp file + const path = await new Promise((res: (string) => void, rej) => { + tmp.file((e, path) => { + if (e) return rej(e); + res(path); + }); + }); + + // write content at URL to temp file + await new Promise((res, rej) => { + const writable = fs.createWriteStream(path); + request(url) + .on('error', rej) + .on('end', () => { + writable.close(); + res(path); + }) + .pipe(writable) + .on('error', rej); + }); + + const driveFile = await create(user, path, name, null, folderId); + + // clean-up + fs.unlink(path, (e) => { + if (e) log(e.stack); + }); + + return driveFile; +}; diff --git a/src/common/event.ts b/src/common/event.ts new file mode 100644 index 0000000000..53520f11ce --- /dev/null +++ b/src/common/event.ts @@ -0,0 +1,80 @@ +import * as mongo from 'mongodb'; +import * as redis from 'redis'; +import swPush from './push-sw'; +import config from '../conf'; + +type ID = string | mongo.ObjectID; + +class MisskeyEvent { + private redisClient: redis.RedisClient; + + constructor() { + // Connect to Redis + this.redisClient = redis.createClient( + config.redis.port, config.redis.host); + } + + public publishUserStream(userId: ID, type: string, value?: any): void { + this.publish(`user-stream:${userId}`, type, typeof value === 'undefined' ? null : value); + } + + public publishSw(userId: ID, type: string, value?: any): void { + swPush(userId, type, value); + } + + public publishDriveStream(userId: ID, type: string, value?: any): void { + this.publish(`drive-stream:${userId}`, type, typeof value === 'undefined' ? null : value); + } + + public publishPostStream(postId: ID, type: string, value?: any): void { + this.publish(`post-stream:${postId}`, type, typeof value === 'undefined' ? null : value); + } + + public publishMessagingStream(userId: ID, otherpartyId: ID, type: string, value?: any): void { + this.publish(`messaging-stream:${userId}-${otherpartyId}`, type, typeof value === 'undefined' ? null : value); + } + + public publishMessagingIndexStream(userId: ID, type: string, value?: any): void { + this.publish(`messaging-index-stream:${userId}`, type, typeof value === 'undefined' ? null : value); + } + + public publishOthelloStream(userId: ID, type: string, value?: any): void { + this.publish(`othello-stream:${userId}`, type, typeof value === 'undefined' ? null : value); + } + + public publishOthelloGameStream(gameId: ID, type: string, value?: any): void { + this.publish(`othello-game-stream:${gameId}`, type, typeof value === 'undefined' ? null : value); + } + + public publishChannelStream(channelId: ID, type: string, value?: any): void { + this.publish(`channel-stream:${channelId}`, type, typeof value === 'undefined' ? null : value); + } + + private publish(channel: string, type: string, value?: any): void { + const message = value == null ? + { type: type } : + { type: type, body: value }; + + this.redisClient.publish(`misskey:${channel}`, JSON.stringify(message)); + } +} + +const ev = new MisskeyEvent(); + +export default ev.publishUserStream.bind(ev); + +export const pushSw = ev.publishSw.bind(ev); + +export const publishDriveStream = ev.publishDriveStream.bind(ev); + +export const publishPostStream = ev.publishPostStream.bind(ev); + +export const publishMessagingStream = ev.publishMessagingStream.bind(ev); + +export const publishMessagingIndexStream = ev.publishMessagingIndexStream.bind(ev); + +export const publishOthelloStream = ev.publishOthelloStream.bind(ev); + +export const publishOthelloGameStream = ev.publishOthelloGameStream.bind(ev); + +export const publishChannelStream = ev.publishChannelStream.bind(ev); diff --git a/src/common/push-sw.ts b/src/common/push-sw.ts new file mode 100644 index 0000000000..44c328e833 --- /dev/null +++ b/src/common/push-sw.ts @@ -0,0 +1,52 @@ +const push = require('web-push'); +import * as mongo from 'mongodb'; +import Subscription from '../models/sw-subscription'; +import config from '../conf'; + +if (config.sw) { + // アプリケーションの連絡先と、サーバーサイドの鍵ペアの情報を登録 + push.setVapidDetails( + config.maintainer.url, + config.sw.public_key, + config.sw.private_key); +} + +export default async function(userId: mongo.ObjectID | string, type, body?) { + if (!config.sw) return; + + if (typeof userId === 'string') { + userId = new mongo.ObjectID(userId); + } + + // Fetch + const subscriptions = await Subscription.find({ + userId: userId + }); + + subscriptions.forEach(subscription => { + const pushSubscription = { + endpoint: subscription.endpoint, + keys: { + auth: subscription.auth, + p256dh: subscription.publickey + } + }; + + push.sendNotification(pushSubscription, JSON.stringify({ + type, body + })).catch(err => { + //console.log(err.statusCode); + //console.log(err.headers); + //console.log(err.body); + + if (err.statusCode == 410) { + Subscription.remove({ + userId: userId, + endpoint: subscription.endpoint, + auth: subscription.auth, + publickey: subscription.publickey + }); + } + }); + }); +} diff --git a/src/common/remote/activitypub/act/create.ts b/src/common/remote/activitypub/act/create.ts new file mode 100644 index 0000000000..6c62f7ab9e --- /dev/null +++ b/src/common/remote/activitypub/act/create.ts @@ -0,0 +1,9 @@ +import create from '../create'; + +export default (resolver, actor, activity) => { + if ('actor' in activity && actor.account.uri !== activity.actor) { + throw new Error; + } + + return create(resolver, actor, activity.object); +}; diff --git a/src/common/remote/activitypub/act/index.ts b/src/common/remote/activitypub/act/index.ts new file mode 100644 index 0000000000..0f4084a61e --- /dev/null +++ b/src/common/remote/activitypub/act/index.ts @@ -0,0 +1,22 @@ +import create from './create'; +import createObject from '../create'; +import Resolver from '../resolver'; + +export default (actor, value) => { + return (new Resolver).resolve(value).then(resolved => Promise.all(resolved.map(async asyncResult => { + const { resolver, object } = await asyncResult; + const created = await (await createObject(resolver, actor, [object]))[0]; + + if (created !== null) { + return created; + } + + switch (object.type) { + case 'Create': + return create(resolver, actor, object); + + default: + return null; + } + }))); +} diff --git a/src/common/remote/activitypub/create.ts b/src/common/remote/activitypub/create.ts new file mode 100644 index 0000000000..4aaaeb3060 --- /dev/null +++ b/src/common/remote/activitypub/create.ts @@ -0,0 +1,86 @@ +import { JSDOM } from 'jsdom'; +import config from '../../../conf'; +import Post from '../../../models/post'; +import RemoteUserObject, { IRemoteUserObject } from '../../../models/remote-user-object'; +import uploadFromUrl from '../../drive/upload_from_url'; +const createDOMPurify = require('dompurify'); + +function createRemoteUserObject($ref, $id, { id }) { + const object = { $ref, $id }; + + if (!id) { + return { object }; + } + + return RemoteUserObject.insert({ uri: id, object }); +} + +async function createImage(actor, object) { + if ('attributedTo' in object && actor.account.uri !== object.attributedTo) { + throw new Error; + } + + const { _id } = await uploadFromUrl(object.url, actor); + return createRemoteUserObject('driveFiles.files', _id, object); +} + +async function createNote(resolver, actor, object) { + if ('attributedTo' in object && actor.account.uri !== object.attributedTo) { + throw new Error; + } + + const mediaIds = 'attachment' in object && + (await Promise.all(await create(resolver, actor, object.attachment))) + .filter(media => media !== null && media.object.$ref === 'driveFiles.files') + .map(({ object }) => object.$id); + + const { window } = new JSDOM(object.content); + + const { _id } = await Post.insert({ + channelId: undefined, + index: undefined, + createdAt: new Date(object.published), + mediaIds, + replyId: undefined, + repostId: undefined, + poll: undefined, + text: window.document.body.textContent, + textHtml: object.content && createDOMPurify(window).sanitize(object.content), + userId: actor._id, + appId: null, + viaMobile: false, + geo: undefined + }); + + // Register to search database + if (object.content && config.elasticsearch.enable) { + const es = require('../../db/elasticsearch'); + + es.index({ + index: 'misskey', + type: 'post', + id: _id.toString(), + body: { + text: window.document.body.textContent + } + }); + } + + return createRemoteUserObject('posts', _id, object); +} + +export default async function create(parentResolver, actor, value): Promise[]> { + const results = await parentResolver.resolveRemoteUserObjects(value); + + return results.map(asyncResult => asyncResult.then(({ resolver, object }) => { + switch (object.type) { + case 'Image': + return createImage(actor, object); + + case 'Note': + return createNote(resolver, actor, object); + } + + return null; + })); +}; diff --git a/src/common/remote/activitypub/resolve-person.ts b/src/common/remote/activitypub/resolve-person.ts new file mode 100644 index 0000000000..c7c131b0ea --- /dev/null +++ b/src/common/remote/activitypub/resolve-person.ts @@ -0,0 +1,104 @@ +import { JSDOM } from 'jsdom'; +import { toUnicode } from 'punycode'; +import User, { validateUsername, isValidName, isValidDescription } from '../../../models/user'; +import queue from '../../../queue'; +import webFinger from '../webfinger'; +import create from './create'; +import Resolver from './resolver'; + +async function isCollection(collection) { + return ['Collection', 'OrderedCollection'].includes(collection.type); +} + +export default async (value, usernameLower, hostLower, acctLower) => { + if (!validateUsername(usernameLower)) { + throw new Error; + } + + const { resolver, object } = await (new Resolver).resolveOne(value); + + if ( + object === null || + object.type !== 'Person' || + typeof object.preferredUsername !== 'string' || + object.preferredUsername.toLowerCase() !== usernameLower || + !isValidName(object.name) || + !isValidDescription(object.summary) + ) { + throw new Error; + } + + const [followers, following, outbox, finger] = await Promise.all([ + resolver.resolveOne(object.followers).then( + resolved => isCollection(resolved.object) ? resolved.object : null, + () => null + ), + resolver.resolveOne(object.following).then( + resolved => isCollection(resolved.object) ? resolved.object : null, + () => null + ), + resolver.resolveOne(object.outbox).then( + resolved => isCollection(resolved.object) ? resolved.object : null, + () => null + ), + webFinger(object.id, acctLower), + ]); + + const summaryDOM = JSDOM.fragment(object.summary); + + // Create user + const user = await User.insert({ + avatarId: null, + bannerId: null, + createdAt: Date.parse(object.published), + description: summaryDOM.textContent, + followersCount: followers.totalItem, + followingCount: following.totalItem, + name: object.name, + postsCount: outbox.totalItem, + driveCapacity: 1024 * 1024 * 8, // 8MiB + username: object.preferredUsername, + usernameLower, + host: toUnicode(finger.subject.replace(/^.*?@/, '')), + hostLower, + account: { + uri: object.id, + }, + }); + + queue.create('http', { + type: 'performActivityPub', + actor: user._id, + outbox + }).save(); + + const [avatarId, bannerId] = await Promise.all([ + object.icon, + object.image + ].map(async value => { + if (value === undefined) { + return null; + } + + try { + const created = await create(resolver, user, value); + + await Promise.all(created.map(asyncCreated => asyncCreated.then(created => { + if (created !== null && created.object.$ref === 'driveFiles.files') { + throw created.object.$id; + } + }, () => {}))); + + return null; + } catch (id) { + return id; + } + })); + + User.update({ _id: user._id }, { $set: { avatarId, bannerId } }); + + user.avatarId = avatarId; + user.bannerId = bannerId; + + return user; +}; diff --git a/src/common/remote/activitypub/resolver.ts b/src/common/remote/activitypub/resolver.ts new file mode 100644 index 0000000000..50ac1b0b19 --- /dev/null +++ b/src/common/remote/activitypub/resolver.ts @@ -0,0 +1,97 @@ +import RemoteUserObject from '../../../models/remote-user-object'; +import { IObject } from './type'; +const request = require('request-promise-native'); + +type IResult = { + resolver: Resolver; + object: IObject; +}; + +async function resolveUnrequestedOne(this: Resolver, value) { + if (typeof value !== 'string') { + return { resolver: this, object: value }; + } + + const resolver = new Resolver(this.requesting); + + resolver.requesting.add(value); + + const object = await request({ + url: value, + headers: { + Accept: 'application/activity+json, application/ld+json' + }, + json: true + }); + + if (object === null || ( + Array.isArray(object['@context']) ? + !object['@context'].includes('https://www.w3.org/ns/activitystreams') : + object['@context'] !== 'https://www.w3.org/ns/activitystreams' + )) { + throw new Error; + } + + return { resolver, object }; +} + +async function resolveCollection(this: Resolver, value) { + if (Array.isArray(value)) { + return value; + } + + const resolved = typeof value === 'string' ? + await resolveUnrequestedOne.call(this, value) : + value; + + switch (resolved.type) { + case 'Collection': + return resolved.items; + + case 'OrderedCollection': + return resolved.orderedItems; + + default: + return [resolved]; + } +} + +export default class Resolver { + requesting: Set; + + constructor(iterable?: Iterable) { + this.requesting = new Set(iterable); + } + + async resolve(value): Promise[]> { + const collection = await resolveCollection.call(this, value); + + return collection + .filter(element => !this.requesting.has(element)) + .map(resolveUnrequestedOne.bind(this)); + } + + resolveOne(value) { + if (this.requesting.has(value)) { + throw new Error; + } + + return resolveUnrequestedOne.call(this, value); + } + + async resolveRemoteUserObjects(value) { + const collection = await resolveCollection.call(this, value); + + return collection.filter(element => !this.requesting.has(element)).map(element => { + if (typeof element === 'string') { + const object = RemoteUserObject.findOne({ uri: element }); + + if (object !== null) { + return object; + } + } + + return resolveUnrequestedOne.call(this, element); + }); + } +} diff --git a/src/common/remote/activitypub/type.ts b/src/common/remote/activitypub/type.ts new file mode 100644 index 0000000000..5c4750e140 --- /dev/null +++ b/src/common/remote/activitypub/type.ts @@ -0,0 +1,3 @@ +export type IObject = { + type: string; +} diff --git a/src/common/remote/resolve-user.ts b/src/common/remote/resolve-user.ts new file mode 100644 index 0000000000..13d155830e --- /dev/null +++ b/src/common/remote/resolve-user.ts @@ -0,0 +1,26 @@ +import { toUnicode, toASCII } from 'punycode'; +import User from '../../models/user'; +import resolvePerson from './activitypub/resolve-person'; +import webFinger from './webfinger'; + +export default async (username, host, option) => { + const usernameLower = username.toLowerCase(); + const hostLowerAscii = toASCII(host).toLowerCase(); + const hostLower = toUnicode(hostLowerAscii); + + let user = await User.findOne({ usernameLower, hostLower }, option); + + if (user === null) { + const acctLower = `${usernameLower}@${hostLowerAscii}`; + + const finger = await webFinger(acctLower, acctLower); + const self = finger.links.find(link => link.rel && link.rel.toLowerCase() === 'self'); + if (!self) { + throw new Error; + } + + user = await resolvePerson(self.href, usernameLower, hostLower, acctLower); + } + + return user; +}; diff --git a/src/common/remote/webfinger.ts b/src/common/remote/webfinger.ts new file mode 100644 index 0000000000..23f0aaa55f --- /dev/null +++ b/src/common/remote/webfinger.ts @@ -0,0 +1,25 @@ +const WebFinger = require('webfinger.js'); + +const webFinger = new WebFinger({}); + +type ILink = { + href: string; + rel: string; +} + +type IWebFinger = { + links: Array; + subject: string; +} + +export default (query, verifier): Promise => new Promise((res, rej) => webFinger.lookup(query, (error, result) => { + if (error) { + return rej(error); + } + + if (result.object.subject.toLowerCase().replace(/^acct:/, '') !== verifier) { + return rej('WebFinger verfification failed'); + } + + res(result.object); +})); diff --git a/src/models/remote-user-object.ts b/src/models/remote-user-object.ts new file mode 100644 index 0000000000..fb5b337c90 --- /dev/null +++ b/src/models/remote-user-object.ts @@ -0,0 +1,15 @@ +import * as mongodb from 'mongodb'; +import db from '../db/mongodb'; + +const RemoteUserObject = db.get('remoteUserObjects'); + +export default RemoteUserObject; + +export type IRemoteUserObject = { + _id: mongodb.ObjectID; + uri: string; + object: { + $ref: string; + $id: mongodb.ObjectID; + } +}; diff --git a/src/models/user.ts b/src/models/user.ts index 4fbfdec907..4728682d67 100644 --- a/src/models/user.ts +++ b/src/models/user.ts @@ -97,6 +97,9 @@ export type IUser = { account: ILocalAccount | IRemoteAccount; }; +export type ILocalUser = IUser & { account: ILocalAccount }; +export type IRemoteUser = IUser & { account: IRemoteAccount }; + export function init(user): IUser { user._id = new mongo.ObjectID(user._id); user.avatarId = new mongo.ObjectID(user.avatarId); diff --git a/src/processor/http/index.ts b/src/processor/http/index.ts new file mode 100644 index 0000000000..da942ad2a1 --- /dev/null +++ b/src/processor/http/index.ts @@ -0,0 +1,9 @@ +import performActivityPub from './perform-activitypub'; +import reportGitHubFailure from './report-github-failure'; + +const handlers = { + performActivityPub, + reportGitHubFailure, +}; + +export default (job, done) => handlers[job.data.type](job).then(() => done(), done); diff --git a/src/processor/http/perform-activitypub.ts b/src/processor/http/perform-activitypub.ts new file mode 100644 index 0000000000..5b1a02173d --- /dev/null +++ b/src/processor/http/perform-activitypub.ts @@ -0,0 +1,6 @@ +import User from '../../models/user'; +import act from '../../common/remote/activitypub/act'; + +export default ({ data }, done) => User.findOne({ _id: data.actor }) + .then(actor => act(actor, data.outbox)) + .then(() => done(), done); diff --git a/src/processor/http/report-github-failure.ts b/src/processor/http/report-github-failure.ts new file mode 100644 index 0000000000..53924a0fbd --- /dev/null +++ b/src/processor/http/report-github-failure.ts @@ -0,0 +1,29 @@ +import * as request from 'request'; +import User from '../../models/user'; +const createPost = require('../../server/api/endpoints/posts/create'); + +export default ({ data }, done) => { + const asyncBot = User.findOne({ _id: data.userId }); + + // Fetch parent status + request({ + url: `${data.parentUrl}/statuses`, + headers: { + 'User-Agent': 'misskey' + } + }, async (err, res, body) => { + if (err) { + console.error(err); + return; + } + const parentStatuses = JSON.parse(body); + const parentState = parentStatuses[0].state; + const stillFailed = parentState == 'failure' || parentState == 'error'; + const text = stillFailed ? + `**⚠️BUILD STILL FAILED⚠️**: ?[${data.message}](${data.htmlUrl})` : + `**🚨BUILD FAILED🚨**: →→→?[${data.message}](${data.htmlUrl})←←←`; + + createPost({ text }, await asyncBot); + done(); + }); +}; diff --git a/src/processor/index.ts b/src/processor/index.ts index f06cf24e87..cd271d3720 100644 --- a/src/processor/index.ts +++ b/src/processor/index.ts @@ -1,4 +1,13 @@ import queue from '../queue'; -import reportGitHubFailure from './report-github-failure'; +import http from './http'; -export default () => queue.process('gitHubFailureReport', reportGitHubFailure); +/* + 256 is the default concurrency limit of Mozilla Firefox and Google + Chromium. + + a8af215e691f3a2205a3758d2d96e9d328e100ff - chromium/src.git - Git at Google + https://chromium.googlesource.com/chromium/src.git/+/a8af215e691f3a2205a3758d2d96e9d328e100ff + Network.http.max-connections - MozillaZine Knowledge Base + http://kb.mozillazine.org/Network.http.max-connections +*/ +export default () => queue.process('http', 256, http); diff --git a/src/processor/report-github-failure.ts b/src/processor/report-github-failure.ts deleted file mode 100644 index 610ffe2767..0000000000 --- a/src/processor/report-github-failure.ts +++ /dev/null @@ -1,29 +0,0 @@ -import * as request from 'request'; -import User from '../models/user'; -const createPost = require('../server/api/endpoints/posts/create'); - -export default ({ data }, done) => { - const asyncBot = User.findOne({ _id: data.userId }); - - // Fetch parent status - request({ - url: `${data.parentUrl}/statuses`, - headers: { - 'User-Agent': 'misskey' - } - }, async (err, res, body) => { - if (err) { - console.error(err); - return; - } - const parentStatuses = JSON.parse(body); - const parentState = parentStatuses[0].state; - const stillFailed = parentState == 'failure' || parentState == 'error'; - const text = stillFailed ? - `**⚠️BUILD STILL FAILED⚠️**: ?[${data.message}](${data.htmlUrl})` : - `**🚨BUILD FAILED🚨**: →→→?[${data.message}](${data.htmlUrl})←←←`; - - createPost({ text }, await asyncBot); - done(); - }); -}; diff --git a/src/server/api/common/drive/add-file.ts b/src/server/api/common/drive/add-file.ts deleted file mode 100644 index 4551f55748..0000000000 --- a/src/server/api/common/drive/add-file.ts +++ /dev/null @@ -1,307 +0,0 @@ -import { Buffer } from 'buffer'; -import * as fs from 'fs'; -import * as tmp from 'tmp'; -import * as stream from 'stream'; - -import * as mongodb from 'mongodb'; -import * as crypto from 'crypto'; -import * as _gm from 'gm'; -import * as debug from 'debug'; -import fileType = require('file-type'); -import prominence = require('prominence'); - -import DriveFile, { getGridFSBucket } from '../../../../models/drive-file'; -import DriveFolder from '../../../../models/drive-folder'; -import { pack } from '../../../../models/drive-file'; -import event, { publishDriveStream } from '../../event'; -import getAcct from '../../../../common/user/get-acct'; -import config from '../../../../conf'; - -const gm = _gm.subClass({ - imageMagick: true -}); - -const log = debug('misskey:drive:add-file'); - -const tmpFile = (): Promise => new Promise((resolve, reject) => { - tmp.file((e, path) => { - if (e) return reject(e); - resolve(path); - }); -}); - -const addToGridFS = (name: string, readable: stream.Readable, type: string, metadata: any): Promise => - getGridFSBucket() - .then(bucket => new Promise((resolve, reject) => { - const writeStream = bucket.openUploadStream(name, { contentType: type, metadata }); - writeStream.once('finish', (doc) => { resolve(doc); }); - writeStream.on('error', reject); - readable.pipe(writeStream); - })); - -const addFile = async ( - user: any, - path: string, - name: string = null, - comment: string = null, - folderId: mongodb.ObjectID = null, - force: boolean = false -) => { - log(`registering ${name} (user: ${getAcct(user)}, path: ${path})`); - - // Calculate hash, get content type and get file size - const [hash, [mime, ext], size] = await Promise.all([ - // hash - ((): Promise => new Promise((res, rej) => { - const readable = fs.createReadStream(path); - const hash = crypto.createHash('md5'); - const chunks = []; - readable - .on('error', rej) - .pipe(hash) - .on('error', rej) - .on('data', (chunk) => chunks.push(chunk)) - .on('end', () => { - const buffer = Buffer.concat(chunks); - res(buffer.toString('hex')); - }); - }))(), - // mime - ((): Promise<[string, string | null]> => new Promise((res, rej) => { - const readable = fs.createReadStream(path); - readable - .on('error', rej) - .once('data', (buffer: Buffer) => { - readable.destroy(); - const type = fileType(buffer); - if (type) { - return res([type.mime, type.ext]); - } else { - // 種類が同定できなかったら application/octet-stream にする - return res(['application/octet-stream', null]); - } - }); - }))(), - // size - ((): Promise => new Promise((res, rej) => { - fs.stat(path, (err, stats) => { - if (err) return rej(err); - res(stats.size); - }); - }))() - ]); - - log(`hash: ${hash}, mime: ${mime}, ext: ${ext}, size: ${size}`); - - // detect name - const detectedName: string = name || (ext ? `untitled.${ext}` : 'untitled'); - - if (!force) { - // Check if there is a file with the same hash - const much = await DriveFile.findOne({ - md5: hash, - 'metadata.userId': user._id - }); - - if (much !== null) { - log('file with same hash is found'); - return much; - } else { - log('file with same hash is not found'); - } - } - - const [wh, averageColor, folder] = await Promise.all([ - // Width and height (when image) - (async () => { - // 画像かどうか - if (!/^image\/.*$/.test(mime)) { - return null; - } - - const imageType = mime.split('/')[1]; - - // 画像でもPNGかJPEGかGIFでないならスキップ - if (imageType != 'png' && imageType != 'jpeg' && imageType != 'gif') { - return null; - } - - log('calculate image width and height...'); - - // Calculate width and height - const g = gm(fs.createReadStream(path), name); - const size = await prominence(g).size(); - - log(`image width and height is calculated: ${size.width}, ${size.height}`); - - return [size.width, size.height]; - })(), - // average color (when image) - (async () => { - // 画像かどうか - if (!/^image\/.*$/.test(mime)) { - return null; - } - - const imageType = mime.split('/')[1]; - - // 画像でもPNGかJPEGでないならスキップ - if (imageType != 'png' && imageType != 'jpeg') { - return null; - } - - log('calculate average color...'); - - const buffer = await prominence(gm(fs.createReadStream(path), name) - .setFormat('ppm') - .resize(1, 1)) // 1pxのサイズに縮小して平均色を取得するというハック - .toBuffer(); - - const r = buffer.readUInt8(buffer.length - 3); - const g = buffer.readUInt8(buffer.length - 2); - const b = buffer.readUInt8(buffer.length - 1); - - log(`average color is calculated: ${r}, ${g}, ${b}`); - - return [r, g, b]; - })(), - // folder - (async () => { - if (!folderId) { - return null; - } - const driveFolder = await DriveFolder.findOne({ - _id: folderId, - userId: user._id - }); - if (!driveFolder) { - throw 'folder-not-found'; - } - return driveFolder; - })(), - // usage checker - (async () => { - // Calculate drive usage - const usage = await DriveFile - .aggregate([{ - $match: { 'metadata.userId': user._id } - }, { - $project: { - length: true - } - }, { - $group: { - _id: null, - usage: { $sum: '$length' } - } - }]) - .then((aggregates: any[]) => { - if (aggregates.length > 0) { - return aggregates[0].usage; - } - return 0; - }); - - log(`drive usage is ${usage}`); - - // If usage limit exceeded - if (usage + size > user.driveCapacity) { - throw 'no-free-space'; - } - })() - ]); - - const readable = fs.createReadStream(path); - - const properties = {}; - - if (wh) { - properties['width'] = wh[0]; - properties['height'] = wh[1]; - } - - if (averageColor) { - properties['avgColor'] = averageColor; - } - - return addToGridFS(detectedName, readable, mime, { - userId: user._id, - folderId: folder !== null ? folder._id : null, - comment: comment, - properties: properties - }); -}; - -/** - * Add file to drive - * - * @param user User who wish to add file - * @param file File path or readableStream - * @param comment Comment - * @param type File type - * @param folderId Folder ID - * @param force If set to true, forcibly upload the file even if there is a file with the same hash. - * @return Object that represents added file - */ -export default (user: any, file: string | stream.Readable, ...args) => new Promise((resolve, reject) => { - // Get file path - new Promise((res: (v: [string, boolean]) => void, rej) => { - if (typeof file === 'string') { - res([file, false]); - return; - } - if (typeof file === 'object' && typeof file.read === 'function') { - tmpFile() - .then(path => { - const readable: stream.Readable = file; - const writable = fs.createWriteStream(path); - readable - .on('error', rej) - .on('end', () => { - res([path, true]); - }) - .pipe(writable) - .on('error', rej); - }) - .catch(rej); - } - rej(new Error('un-compatible file.')); - }) - .then(([path, shouldCleanup]): Promise => new Promise((res, rej) => { - addFile(user, path, ...args) - .then(file => { - res(file); - if (shouldCleanup) { - fs.unlink(path, (e) => { - if (e) log(e.stack); - }); - } - }) - .catch(rej); - })) - .then(file => { - log(`drive file has been created ${file._id}`); - resolve(file); - - pack(file).then(serializedFile => { - // Publish drive_file_created event - event(user._id, 'drive_file_created', serializedFile); - publishDriveStream(user._id, 'file_created', serializedFile); - - // Register to search database - if (config.elasticsearch.enable) { - const es = require('../../db/elasticsearch'); - es.index({ - index: 'misskey', - type: 'drive_file', - id: file._id.toString(), - body: { - name: file.name, - userId: user._id.toString() - } - }); - } - }); - }) - .catch(reject); -}); diff --git a/src/server/api/common/drive/upload_from_url.ts b/src/server/api/common/drive/upload_from_url.ts deleted file mode 100644 index b825e4c531..0000000000 --- a/src/server/api/common/drive/upload_from_url.ts +++ /dev/null @@ -1,46 +0,0 @@ -import * as URL from 'url'; -import { IDriveFile, validateFileName } from '../../../../models/drive-file'; -import create from './add-file'; -import * as debug from 'debug'; -import * as tmp from 'tmp'; -import * as fs from 'fs'; -import * as request from 'request'; - -const log = debug('misskey:common:drive:upload_from_url'); - -export default async (url, user, folderId = null): Promise => { - let name = URL.parse(url).pathname.split('/').pop(); - if (!validateFileName(name)) { - name = null; - } - - // Create temp file - const path = await new Promise((res: (string) => void, rej) => { - tmp.file((e, path) => { - if (e) return rej(e); - res(path); - }); - }); - - // write content at URL to temp file - await new Promise((res, rej) => { - const writable = fs.createWriteStream(path); - request(url) - .on('error', rej) - .on('end', () => { - writable.close(); - res(path); - }) - .pipe(writable) - .on('error', rej); - }); - - const driveFile = await create(user, path, name, null, folderId); - - // clean-up - fs.unlink(path, (e) => { - if (e) log(e.stack); - }); - - return driveFile; -}; diff --git a/src/server/api/common/notify.ts b/src/server/api/common/notify.ts index f90506cf3c..69bf8480b0 100644 --- a/src/server/api/common/notify.ts +++ b/src/server/api/common/notify.ts @@ -1,7 +1,7 @@ import * as mongo from 'mongodb'; import Notification from '../../../models/notification'; import Mute from '../../../models/mute'; -import event from '../event'; +import event from '../../../common/event'; import { pack } from '../../../models/notification'; export default ( diff --git a/src/server/api/common/push-sw.ts b/src/server/api/common/push-sw.ts deleted file mode 100644 index 13227af8d5..0000000000 --- a/src/server/api/common/push-sw.ts +++ /dev/null @@ -1,52 +0,0 @@ -const push = require('web-push'); -import * as mongo from 'mongodb'; -import Subscription from '../../../models/sw-subscription'; -import config from '../../../conf'; - -if (config.sw) { - // アプリケーションの連絡先と、サーバーサイドの鍵ペアの情報を登録 - push.setVapidDetails( - config.maintainer.url, - config.sw.public_key, - config.sw.private_key); -} - -export default async function(userId: mongo.ObjectID | string, type, body?) { - if (!config.sw) return; - - if (typeof userId === 'string') { - userId = new mongo.ObjectID(userId); - } - - // Fetch - const subscriptions = await Subscription.find({ - userId: userId - }); - - subscriptions.forEach(subscription => { - const pushSubscription = { - endpoint: subscription.endpoint, - keys: { - auth: subscription.auth, - p256dh: subscription.publickey - } - }; - - push.sendNotification(pushSubscription, JSON.stringify({ - type, body - })).catch(err => { - //console.log(err.statusCode); - //console.log(err.headers); - //console.log(err.body); - - if (err.statusCode == 410) { - Subscription.remove({ - userId: userId, - endpoint: subscription.endpoint, - auth: subscription.auth, - publickey: subscription.publickey - }); - } - }); - }); -} diff --git a/src/server/api/common/read-messaging-message.ts b/src/server/api/common/read-messaging-message.ts index f728130bb3..127ea18651 100644 --- a/src/server/api/common/read-messaging-message.ts +++ b/src/server/api/common/read-messaging-message.ts @@ -1,9 +1,9 @@ import * as mongo from 'mongodb'; import Message from '../../../models/messaging-message'; import { IMessagingMessage as IMessage } from '../../../models/messaging-message'; -import publishUserStream from '../event'; -import { publishMessagingStream } from '../event'; -import { publishMessagingIndexStream } from '../event'; +import publishUserStream from '../../../common/event'; +import { publishMessagingStream } from '../../../common/event'; +import { publishMessagingIndexStream } from '../../../common/event'; /** * Mark as read message(s) diff --git a/src/server/api/common/read-notification.ts b/src/server/api/common/read-notification.ts index 27632c7ecd..9b2012182d 100644 --- a/src/server/api/common/read-notification.ts +++ b/src/server/api/common/read-notification.ts @@ -1,6 +1,6 @@ import * as mongo from 'mongodb'; import { default as Notification, INotification } from '../../../models/notification'; -import publishUserStream from '../event'; +import publishUserStream from '../../../common/event'; /** * Mark as read notification(s) diff --git a/src/server/api/endpoints/drive/files/create.ts b/src/server/api/endpoints/drive/files/create.ts index 53c8c70676..cf4e35cd1e 100644 --- a/src/server/api/endpoints/drive/files/create.ts +++ b/src/server/api/endpoints/drive/files/create.ts @@ -3,7 +3,7 @@ */ import $ from 'cafy'; import { validateFileName, pack } from '../../../../../models/drive-file'; -import create from '../../../common/drive/add-file'; +import create from '../../../../../common/drive/add-file'; /** * Create a file diff --git a/src/server/api/endpoints/drive/files/update.ts b/src/server/api/endpoints/drive/files/update.ts index 836b4cfcd3..5d0b915f95 100644 --- a/src/server/api/endpoints/drive/files/update.ts +++ b/src/server/api/endpoints/drive/files/update.ts @@ -4,7 +4,7 @@ import $ from 'cafy'; import DriveFolder from '../../../../../models/drive-folder'; import DriveFile, { validateFileName, pack } from '../../../../../models/drive-file'; -import { publishDriveStream } from '../../../event'; +import { publishDriveStream } from '../../../../../common/event'; /** * Update a file diff --git a/src/server/api/endpoints/drive/files/upload_from_url.ts b/src/server/api/endpoints/drive/files/upload_from_url.ts index 7262f09bbc..01d8750553 100644 --- a/src/server/api/endpoints/drive/files/upload_from_url.ts +++ b/src/server/api/endpoints/drive/files/upload_from_url.ts @@ -3,7 +3,7 @@ */ import $ from 'cafy'; import { pack } from '../../../../../models/drive-file'; -import uploadFromUrl from '../../../common/drive/upload_from_url'; +import uploadFromUrl from '../../../../../common/drive/upload_from_url'; /** * Create a file from a URL diff --git a/src/server/api/endpoints/drive/folders/create.ts b/src/server/api/endpoints/drive/folders/create.ts index 24e0359307..bd3b0a0b1d 100644 --- a/src/server/api/endpoints/drive/folders/create.ts +++ b/src/server/api/endpoints/drive/folders/create.ts @@ -3,7 +3,7 @@ */ import $ from 'cafy'; import DriveFolder, { isValidFolderName, pack } from '../../../../../models/drive-folder'; -import { publishDriveStream } from '../../../event'; +import { publishDriveStream } from '../../../../../common/event'; /** * Create drive folder diff --git a/src/server/api/endpoints/drive/folders/update.ts b/src/server/api/endpoints/drive/folders/update.ts index 6c5a5c3761..5ac81e5b50 100644 --- a/src/server/api/endpoints/drive/folders/update.ts +++ b/src/server/api/endpoints/drive/folders/update.ts @@ -3,7 +3,7 @@ */ import $ from 'cafy'; import DriveFolder, { isValidFolderName, pack } from '../../../../../models/drive-folder'; -import { publishDriveStream } from '../../../event'; +import { publishDriveStream } from '../../../../../common/event'; /** * Update a folder diff --git a/src/server/api/endpoints/following/create.ts b/src/server/api/endpoints/following/create.ts index 1e24388a7a..a689250e35 100644 --- a/src/server/api/endpoints/following/create.ts +++ b/src/server/api/endpoints/following/create.ts @@ -5,7 +5,7 @@ import $ from 'cafy'; import User, { pack as packUser } from '../../../../models/user'; import Following from '../../../../models/following'; import notify from '../../common/notify'; -import event from '../../event'; +import event from '../../../../common/event'; /** * Follow a user diff --git a/src/server/api/endpoints/following/delete.ts b/src/server/api/endpoints/following/delete.ts index 7fc5f477f7..ecca27d57d 100644 --- a/src/server/api/endpoints/following/delete.ts +++ b/src/server/api/endpoints/following/delete.ts @@ -4,7 +4,7 @@ import $ from 'cafy'; import User, { pack as packUser } from '../../../../models/user'; import Following from '../../../../models/following'; -import event from '../../event'; +import event from '../../../../common/event'; /** * Unfollow a user diff --git a/src/server/api/endpoints/i/regenerate_token.ts b/src/server/api/endpoints/i/regenerate_token.ts index c35778ac0b..0af622fd90 100644 --- a/src/server/api/endpoints/i/regenerate_token.ts +++ b/src/server/api/endpoints/i/regenerate_token.ts @@ -4,7 +4,7 @@ import $ from 'cafy'; import * as bcrypt from 'bcryptjs'; import User from '../../../../models/user'; -import event from '../../event'; +import event from '../../../../common/event'; import generateUserToken from '../../common/generate-native-user-token'; /** diff --git a/src/server/api/endpoints/i/update.ts b/src/server/api/endpoints/i/update.ts index 8e198f3ad0..b465e763e1 100644 --- a/src/server/api/endpoints/i/update.ts +++ b/src/server/api/endpoints/i/update.ts @@ -3,7 +3,7 @@ */ import $ from 'cafy'; import User, { isValidName, isValidDescription, isValidLocation, isValidBirthday, pack } from '../../../../models/user'; -import event from '../../event'; +import event from '../../../../common/event'; import config from '../../../../conf'; /** diff --git a/src/server/api/endpoints/i/update_client_setting.ts b/src/server/api/endpoints/i/update_client_setting.ts index 03867b4017..79789e6640 100644 --- a/src/server/api/endpoints/i/update_client_setting.ts +++ b/src/server/api/endpoints/i/update_client_setting.ts @@ -3,7 +3,7 @@ */ import $ from 'cafy'; import User, { pack } from '../../../../models/user'; -import event from '../../event'; +import event from '../../../../common/event'; /** * Update myself diff --git a/src/server/api/endpoints/i/update_home.ts b/src/server/api/endpoints/i/update_home.ts index 713cf9fcc8..437f51d6fb 100644 --- a/src/server/api/endpoints/i/update_home.ts +++ b/src/server/api/endpoints/i/update_home.ts @@ -3,7 +3,7 @@ */ import $ from 'cafy'; import User from '../../../../models/user'; -import event from '../../event'; +import event from '../../../../common/event'; module.exports = async (params, user) => new Promise(async (res, rej) => { // Get 'home' parameter diff --git a/src/server/api/endpoints/i/update_mobile_home.ts b/src/server/api/endpoints/i/update_mobile_home.ts index 6f28cebf9c..783ca09d1d 100644 --- a/src/server/api/endpoints/i/update_mobile_home.ts +++ b/src/server/api/endpoints/i/update_mobile_home.ts @@ -3,7 +3,7 @@ */ import $ from 'cafy'; import User from '../../../../models/user'; -import event from '../../event'; +import event from '../../../../common/event'; module.exports = async (params, user) => new Promise(async (res, rej) => { // Get 'home' parameter diff --git a/src/server/api/endpoints/messaging/messages/create.ts b/src/server/api/endpoints/messaging/messages/create.ts index 3d3b204da5..b2b6c971db 100644 --- a/src/server/api/endpoints/messaging/messages/create.ts +++ b/src/server/api/endpoints/messaging/messages/create.ts @@ -9,8 +9,8 @@ import User from '../../../../../models/user'; import Mute from '../../../../../models/mute'; import DriveFile from '../../../../../models/drive-file'; import { pack } from '../../../../../models/messaging-message'; -import publishUserStream from '../../../event'; -import { publishMessagingStream, publishMessagingIndexStream, pushSw } from '../../../event'; +import publishUserStream from '../../../../../common/event'; +import { publishMessagingStream, publishMessagingIndexStream, pushSw } from '../../../../../common/event'; import html from '../../../../../common/text/html'; import parse from '../../../../../common/text/parse'; import config from '../../../../../conf'; diff --git a/src/server/api/endpoints/notifications/mark_as_read_all.ts b/src/server/api/endpoints/notifications/mark_as_read_all.ts index 3693ba87bc..f9bc6ebf70 100644 --- a/src/server/api/endpoints/notifications/mark_as_read_all.ts +++ b/src/server/api/endpoints/notifications/mark_as_read_all.ts @@ -2,7 +2,7 @@ * Module dependencies */ import Notification from '../../../../models/notification'; -import event from '../../event'; +import event from '../../../../common/event'; /** * Mark as read all notifications diff --git a/src/server/api/endpoints/othello/match.ts b/src/server/api/endpoints/othello/match.ts index 03168095dc..992b93d418 100644 --- a/src/server/api/endpoints/othello/match.ts +++ b/src/server/api/endpoints/othello/match.ts @@ -2,7 +2,7 @@ import $ from 'cafy'; import Matching, { pack as packMatching } from '../../../../models/othello-matching'; import OthelloGame, { pack as packGame } from '../../../../models/othello-game'; import User from '../../../../models/user'; -import publishUserStream, { publishOthelloStream } from '../../event'; +import publishUserStream, { publishOthelloStream } from '../../../../common/event'; import { eighteight } from '../../../../common/othello/maps'; module.exports = (params, user) => new Promise(async (res, rej) => { diff --git a/src/server/api/endpoints/posts/create.ts b/src/server/api/endpoints/posts/create.ts index 5342f77728..42901ebcbf 100644 --- a/src/server/api/endpoints/posts/create.ts +++ b/src/server/api/endpoints/posts/create.ts @@ -16,7 +16,7 @@ import ChannelWatching from '../../../../models/channel-watching'; import { pack } from '../../../../models/post'; import notify from '../../common/notify'; import watch from '../../common/watch-post'; -import event, { pushSw, publishChannelStream } from '../../event'; +import event, { pushSw, publishChannelStream } from '../../../../common/event'; import getAcct from '../../../../common/user/get-acct'; import parseAcct from '../../../../common/user/parse-acct'; import config from '../../../../conf'; diff --git a/src/server/api/endpoints/posts/polls/vote.ts b/src/server/api/endpoints/posts/polls/vote.ts index b970c05e8d..98df074e5d 100644 --- a/src/server/api/endpoints/posts/polls/vote.ts +++ b/src/server/api/endpoints/posts/polls/vote.ts @@ -7,7 +7,7 @@ import Post from '../../../../../models/post'; import Watching from '../../../../../models/post-watching'; import notify from '../../../common/notify'; import watch from '../../../common/watch-post'; -import { publishPostStream } from '../../../event'; +import { publishPostStream } from '../../../../../common/event'; /** * Vote poll of a post diff --git a/src/server/api/endpoints/posts/reactions/create.ts b/src/server/api/endpoints/posts/reactions/create.ts index 5d2b5a7ed3..8db76d6436 100644 --- a/src/server/api/endpoints/posts/reactions/create.ts +++ b/src/server/api/endpoints/posts/reactions/create.ts @@ -8,7 +8,7 @@ import { pack as packUser } from '../../../../../models/user'; import Watching from '../../../../../models/post-watching'; import notify from '../../../common/notify'; import watch from '../../../common/watch-post'; -import { publishPostStream, pushSw } from '../../../event'; +import { publishPostStream, pushSw } from '../../../../../common/event'; /** * React to a post diff --git a/src/server/api/endpoints/users/show.ts b/src/server/api/endpoints/users/show.ts index fd51d386b8..9cd8716fe5 100644 --- a/src/server/api/endpoints/users/show.ts +++ b/src/server/api/endpoints/users/show.ts @@ -2,49 +2,10 @@ * Module dependencies */ import $ from 'cafy'; -import { JSDOM } from 'jsdom'; -import { toUnicode, toASCII } from 'punycode'; -import uploadFromUrl from '../../common/drive/upload_from_url'; -import User, { pack, validateUsername, isValidName, isValidDescription } from '../../../../models/user'; -const request = require('request-promise-native'); -const WebFinger = require('webfinger.js'); +import User, { pack } from '../../../../models/user'; +import resolveRemoteUser from '../../../../common/remote/resolve-user'; -const webFinger = new WebFinger({}); - -async function getCollectionCount(url) { - if (!url) { - return null; - } - - try { - const collection = await request({ url, json: true }); - return collection ? collection.totalItems : null; - } catch (exception) { - return null; - } -} - -function findUser(q) { - return User.findOne(q, { - fields: { - data: false - } - }); -} - -function webFingerAndVerify(query, verifier) { - return new Promise((res, rej) => webFinger.lookup(query, (error, result) => { - if (error) { - return rej(error); - } - - if (result.object.subject.toLowerCase().replace(/^acct:/, '') !== verifier) { - return rej('WebFinger verfification failed'); - } - - res(result.object); - })); -} +const cursorOption = { fields: { data: false } }; /** * Show a user @@ -74,124 +35,17 @@ module.exports = (params, me) => new Promise(async (res, rej) => { // Lookup user if (typeof host === 'string') { - const usernameLower = username.toLowerCase(); - const hostLowerAscii = toASCII(host).toLowerCase(); - const hostLower = toUnicode(hostLowerAscii); - - user = await findUser({ usernameLower, hostLower }); - - if (user === null) { - const acctLower = `${usernameLower}@${hostLowerAscii}`; - let activityStreams; - let finger; - let followersCount; - let followingCount; - let postsCount; - - if (!validateUsername(username)) { - return rej('username validation failed'); - } - - try { - finger = await webFingerAndVerify(acctLower, acctLower); - } catch (exception) { - return rej('WebFinger lookup failed'); - } - - const self = finger.links.find(link => link.rel && link.rel.toLowerCase() === 'self'); - if (!self) { - return rej('WebFinger has no reference to self representation'); - } - - try { - activityStreams = await request({ - url: self.href, - headers: { - Accept: 'application/activity+json, application/ld+json' - }, - json: true - }); - } catch (exception) { - return rej('failed to retrieve ActivityStreams representation'); - } - - if (!(activityStreams && - (Array.isArray(activityStreams['@context']) ? - activityStreams['@context'].includes('https://www.w3.org/ns/activitystreams') : - activityStreams['@context'] === 'https://www.w3.org/ns/activitystreams') && - activityStreams.type === 'Person' && - typeof activityStreams.preferredUsername === 'string' && - activityStreams.preferredUsername.toLowerCase() === usernameLower && - isValidName(activityStreams.name) && - isValidDescription(activityStreams.summary) - )) { - return rej('failed ActivityStreams validation'); - } - - try { - [followersCount, followingCount, postsCount] = await Promise.all([ - getCollectionCount(activityStreams.followers), - getCollectionCount(activityStreams.following), - getCollectionCount(activityStreams.outbox), - webFingerAndVerify(activityStreams.id, acctLower), - ]); - } catch (exception) { - return rej('failed to fetch assets'); - } - - const summaryDOM = JSDOM.fragment(activityStreams.summary); - - // Create user - user = await User.insert({ - avatarId: null, - bannerId: null, - createdAt: new Date(), - description: summaryDOM.textContent, - followersCount, - followingCount, - name: activityStreams.name, - postsCount, - driveCapacity: 1024 * 1024 * 8, // 8MiB - username, - usernameLower, - host: toUnicode(finger.subject.replace(/^.*?@/, '')), - hostLower, - account: { - uri: activityStreams.id, - }, - }); - - const [icon, image] = await Promise.all([ - activityStreams.icon, - activityStreams.image, - ].map(async image => { - if (!image || image.type !== 'Image') { - return { _id: null }; - } - - try { - return await uploadFromUrl(image.url, user); - } catch (exception) { - return { _id: null }; - } - })); - - User.update({ _id: user._id }, { - $set: { - avatarId: icon._id, - bannerId: image._id, - }, - }); - - user.avatarId = icon._id; - user.bannerId = icon._id; + try { + user = await resolveRemoteUser(username, host, cursorOption); + } catch (exception) { + return rej('failed to resolve remote user'); } } else { const q = userId !== undefined ? { _id: userId } : { usernameLower: username.toLowerCase(), host: null }; - user = await findUser(q); + user = await User.findOne(q, cursorOption); if (user === null) { return rej('user not found'); diff --git a/src/server/api/event.ts b/src/server/api/event.ts deleted file mode 100644 index 98bf161137..0000000000 --- a/src/server/api/event.ts +++ /dev/null @@ -1,80 +0,0 @@ -import * as mongo from 'mongodb'; -import * as redis from 'redis'; -import swPush from './common/push-sw'; -import config from '../../conf'; - -type ID = string | mongo.ObjectID; - -class MisskeyEvent { - private redisClient: redis.RedisClient; - - constructor() { - // Connect to Redis - this.redisClient = redis.createClient( - config.redis.port, config.redis.host); - } - - public publishUserStream(userId: ID, type: string, value?: any): void { - this.publish(`user-stream:${userId}`, type, typeof value === 'undefined' ? null : value); - } - - public publishSw(userId: ID, type: string, value?: any): void { - swPush(userId, type, value); - } - - public publishDriveStream(userId: ID, type: string, value?: any): void { - this.publish(`drive-stream:${userId}`, type, typeof value === 'undefined' ? null : value); - } - - public publishPostStream(postId: ID, type: string, value?: any): void { - this.publish(`post-stream:${postId}`, type, typeof value === 'undefined' ? null : value); - } - - public publishMessagingStream(userId: ID, otherpartyId: ID, type: string, value?: any): void { - this.publish(`messaging-stream:${userId}-${otherpartyId}`, type, typeof value === 'undefined' ? null : value); - } - - public publishMessagingIndexStream(userId: ID, type: string, value?: any): void { - this.publish(`messaging-index-stream:${userId}`, type, typeof value === 'undefined' ? null : value); - } - - public publishOthelloStream(userId: ID, type: string, value?: any): void { - this.publish(`othello-stream:${userId}`, type, typeof value === 'undefined' ? null : value); - } - - public publishOthelloGameStream(gameId: ID, type: string, value?: any): void { - this.publish(`othello-game-stream:${gameId}`, type, typeof value === 'undefined' ? null : value); - } - - public publishChannelStream(channelId: ID, type: string, value?: any): void { - this.publish(`channel-stream:${channelId}`, type, typeof value === 'undefined' ? null : value); - } - - private publish(channel: string, type: string, value?: any): void { - const message = value == null ? - { type: type } : - { type: type, body: value }; - - this.redisClient.publish(`misskey:${channel}`, JSON.stringify(message)); - } -} - -const ev = new MisskeyEvent(); - -export default ev.publishUserStream.bind(ev); - -export const pushSw = ev.publishSw.bind(ev); - -export const publishDriveStream = ev.publishDriveStream.bind(ev); - -export const publishPostStream = ev.publishPostStream.bind(ev); - -export const publishMessagingStream = ev.publishMessagingStream.bind(ev); - -export const publishMessagingIndexStream = ev.publishMessagingIndexStream.bind(ev); - -export const publishOthelloStream = ev.publishOthelloStream.bind(ev); - -export const publishOthelloGameStream = ev.publishOthelloGameStream.bind(ev); - -export const publishChannelStream = ev.publishChannelStream.bind(ev); diff --git a/src/server/api/private/signin.ts b/src/server/api/private/signin.ts index d78fa11b80..4b70644910 100644 --- a/src/server/api/private/signin.ts +++ b/src/server/api/private/signin.ts @@ -3,7 +3,7 @@ import * as bcrypt from 'bcryptjs'; import * as speakeasy from 'speakeasy'; import { default as User, ILocalAccount, IUser } from '../../../models/user'; import Signin, { pack } from '../../../models/signin'; -import event from '../event'; +import event from '../../../common/event'; import signin from '../common/signin'; import config from '../../../conf'; diff --git a/src/server/api/service/github.ts b/src/server/api/service/github.ts index a2359cfb6f..b4068c729e 100644 --- a/src/server/api/service/github.ts +++ b/src/server/api/service/github.ts @@ -42,7 +42,8 @@ module.exports = async (app: express.Application) => { const commit = event.commit; const parent = commit.parents[0]; - queue.create('gitHubFailureReport', { + queue.create('http', { + type: 'gitHubFailureReport', userId: bot._id, parentUrl: parent.url, htmlUrl: commit.html_url, diff --git a/src/server/api/service/twitter.ts b/src/server/api/service/twitter.ts index d77341db2b..73822b0bd2 100644 --- a/src/server/api/service/twitter.ts +++ b/src/server/api/service/twitter.ts @@ -6,7 +6,7 @@ import * as uuid from 'uuid'; import autwh from 'autwh'; import redis from '../../../db/redis'; import User, { pack } from '../../../models/user'; -import event from '../event'; +import event from '../../../common/event'; import config from '../../../conf'; import signin from '../common/signin'; diff --git a/src/server/api/stream/othello-game.ts b/src/server/api/stream/othello-game.ts index b6a251c4c8..b11915f8ff 100644 --- a/src/server/api/stream/othello-game.ts +++ b/src/server/api/stream/othello-game.ts @@ -2,7 +2,7 @@ import * as websocket from 'websocket'; import * as redis from 'redis'; import * as CRC32 from 'crc-32'; import OthelloGame, { pack } from '../../../models/othello-game'; -import { publishOthelloGameStream } from '../event'; +import { publishOthelloGameStream } from '../../../common/event'; import Othello from '../../../common/othello/core'; import * as maps from '../../../common/othello/maps'; import { ParsedUrlQuery } from 'querystring'; diff --git a/src/server/api/stream/othello.ts b/src/server/api/stream/othello.ts index 4205afae7c..1cf9a1494f 100644 --- a/src/server/api/stream/othello.ts +++ b/src/server/api/stream/othello.ts @@ -2,7 +2,7 @@ import * as mongo from 'mongodb'; import * as websocket from 'websocket'; import * as redis from 'redis'; import Matching, { pack } from '../../../models/othello-matching'; -import publishUserStream from '../event'; +import publishUserStream from '../../../common/event'; export default function(request: websocket.request, connection: websocket.connection, subscriber: redis.RedisClient, user: any): void { // Subscribe othello stream -- cgit v1.2.3-freya