From 1f4ae2f63a609d51942daa23772439379496064e Mon Sep 17 00:00:00 2001 From: syuilo Date: Fri, 2 Apr 2021 10:36:11 +0900 Subject: Use mfm-js for MFM parsing (#7415) * wip * Update mfm.ts * wip * update mfmjs * refactor * nanka * Update mfm.ts * Update to-html.ts * Update to-html.ts * wip * fix test * fix test --- src/services/note/create.ts | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) (limited to 'src/services/note/create.ts') diff --git a/src/services/note/create.ts b/src/services/note/create.ts index 64d5513ecc..125285f34a 100644 --- a/src/services/note/create.ts +++ b/src/services/note/create.ts @@ -1,3 +1,4 @@ +import * as mfm from 'mfm-js'; import es from '../../db/elasticsearch'; import { publishMainStream, publishNotesStream } from '../stream'; import DeliverManager from '../../remote/activitypub/deliver-manager'; @@ -5,7 +6,6 @@ import renderNote from '../../remote/activitypub/renderer/note'; import renderCreate from '../../remote/activitypub/renderer/create'; import renderAnnounce from '../../remote/activitypub/renderer/announce'; import { renderActivity } from '../../remote/activitypub/renderer'; -import { parse } from '../../mfm/parse'; import { resolveUser } from '../../remote/resolve-user'; import config from '@/config'; import { updateHashtags } from '../update-hashtag'; @@ -13,7 +13,7 @@ import { concat } from '../../prelude/array'; import insertNoteUnread from './unread'; import { registerOrFetchInstanceDoc } from '../register-or-fetch-instance-doc'; import extractMentions from '@/misc/extract-mentions'; -import extractEmojis from '@/misc/extract-emojis'; +import { extractCustomEmojisFromMfm } from '@/misc/extract-custom-emojis-from-mfm'; import extractHashtags from '@/misc/extract-hashtags'; import { Note, IMentionedRemoteUsers } from '../../models/entities/note'; import { Mutings, Users, NoteWatchings, Notes, Instances, UserProfiles, Antennas, Followings, MutedNotes, Channels, ChannelFollowings } from '../../models'; @@ -182,17 +182,17 @@ export default async (user: { id: User['id']; username: User['username']; host: // Parse MFM if needed if (!tags || !emojis || !mentionedUsers) { - const tokens = data.text ? parse(data.text)! : []; - const cwTokens = data.cw ? parse(data.cw)! : []; + const tokens = data.text ? mfm.parse(data.text)! : []; + const cwTokens = data.cw ? mfm.parse(data.cw)! : []; const choiceTokens = data.poll && data.poll.choices - ? concat(data.poll.choices.map(choice => parse(choice)!)) + ? concat(data.poll.choices.map(choice => mfm.parse(choice)!)) : []; const combinedTokens = tokens.concat(cwTokens).concat(choiceTokens); tags = data.apHashtags || extractHashtags(combinedTokens); - emojis = data.apEmojis || extractEmojis(combinedTokens); + emojis = data.apEmojis || extractCustomEmojisFromMfm(combinedTokens); mentionedUsers = data.apMentions || await extractMentionedUsers(user, combinedTokens); } @@ -604,7 +604,7 @@ function incNotesCountOfUser(user: { id: User['id']; }) { .execute(); } -async function extractMentionedUsers(user: { host: User['host']; }, tokens: ReturnType): Promise { +async function extractMentionedUsers(user: { host: User['host']; }, tokens: mfm.MfmNode[]): Promise { if (tokens == null) return []; const mentions = extractMentions(tokens); -- cgit v1.2.3-freya From 3a6331693a2c165fa6bb1a377ad7bf471f2b5550 Mon Sep 17 00:00:00 2001 From: marihachi Date: Sat, 10 Apr 2021 17:50:18 +0900 Subject: refactor mfm extract (#7434) * refactor extractCustomEmojisFromMfm() * refactor extract-hashtags * refactor extract-mentions * refactor extract-hashtags * refactor extract-url-from-mfm * refactor extract-mentions --- src/client/components/post-form.vue | 2 +- src/client/ui/chat/post-form.vue | 2 +- src/misc/extract-custom-emojis-from-mfm.ts | 14 +++++--------- src/misc/extract-hashtags.ts | 16 ++++++---------- src/misc/extract-mentions.ts | 17 +++++++---------- src/misc/extract-url-from-mfm.ts | 28 ++++++++++------------------ src/server/api/endpoints/i/update.ts | 2 +- src/services/note/create.ts | 4 ++-- test/extract-mentions.ts | 2 +- 9 files changed, 34 insertions(+), 53 deletions(-) (limited to 'src/services/note/create.ts') diff --git a/src/client/components/post-form.vue b/src/client/components/post-form.vue index a67f3ed10a..ce79f34d62 100644 --- a/src/client/components/post-form.vue +++ b/src/client/components/post-form.vue @@ -61,7 +61,7 @@ import XNotePreview from './note-preview.vue'; import * as mfm from 'mfm-js'; import { host, url } from '@client/config'; import { erase, unique } from '../../prelude/array'; -import extractMentions from '@/misc/extract-mentions'; +import { extractMentions } from '@/misc/extract-mentions'; import getAcct from '@/misc/acct/render'; import { formatTimeString } from '@/misc/format-time-string'; import { Autocomplete } from '@client/scripts/autocomplete'; diff --git a/src/client/ui/chat/post-form.vue b/src/client/ui/chat/post-form.vue index bdf18cf290..a9413ea8bc 100644 --- a/src/client/ui/chat/post-form.vue +++ b/src/client/ui/chat/post-form.vue @@ -56,7 +56,7 @@ import { toASCII } from 'punycode/'; import * as mfm from 'mfm-js'; import { host, url } from '@client/config'; import { erase, unique } from '../../../prelude/array'; -import extractMentions from '@/misc/extract-mentions'; +import { extractMentions } from '@/misc/extract-mentions'; import getAcct from '@/misc/acct/render'; import { formatTimeString } from '@/misc/format-time-string'; import { Autocomplete } from '@client/scripts/autocomplete'; diff --git a/src/misc/extract-custom-emojis-from-mfm.ts b/src/misc/extract-custom-emojis-from-mfm.ts index f1477a79f0..ef2cb1fd6c 100644 --- a/src/misc/extract-custom-emojis-from-mfm.ts +++ b/src/misc/extract-custom-emojis-from-mfm.ts @@ -2,17 +2,13 @@ import * as mfm from 'mfm-js'; import { unique } from '@/prelude/array'; export function extractCustomEmojisFromMfm(nodes: mfm.MfmNode[]): string[] { - const emojiNodes = [] as mfm.MfmEmojiCode[]; + const emojis = [] as string[]; - function scan(nodes: mfm.MfmNode[]) { - for (const node of nodes) { - if (node.type === 'emojiCode') emojiNodes.push(node); - else if (node.children) scan(node.children); + mfm.inspect(nodes, (node) => { + if (node.type === 'emojiCode' && node.props.name.length <= 100) { + emojis.push(node.props.name); } - } + }); - scan(nodes); - - const emojis = emojiNodes.filter(x => x.props.name.length <= 100).map(x => x.props.name!); return unique(emojis); } diff --git a/src/misc/extract-hashtags.ts b/src/misc/extract-hashtags.ts index 9961755ccd..f89b699a94 100644 --- a/src/misc/extract-hashtags.ts +++ b/src/misc/extract-hashtags.ts @@ -1,18 +1,14 @@ import * as mfm from 'mfm-js'; import { unique } from '@/prelude/array'; -export default function(nodes: mfm.MfmNode[]): string[] { - const hashtagNodes = [] as mfm.MfmHashtag[]; +export function extractHashtags(nodes: mfm.MfmNode[]): string[] { + const hashtags = [] as string[]; - function scan(nodes: mfm.MfmNode[]) { - for (const node of nodes) { - if (node.type === 'hashtag') hashtagNodes.push(node); - else if (node.children) scan(node.children); + mfm.inspect(nodes, (node) => { + if (node.type === 'hashtag') { + hashtags.push(node.props.hashtag); } - } + }); - scan(nodes); - - const hashtags = hashtagNodes.map(x => x.props.hashtag); return unique(hashtags); } diff --git a/src/misc/extract-mentions.ts b/src/misc/extract-mentions.ts index a9d4b378f3..a0e2fb6d57 100644 --- a/src/misc/extract-mentions.ts +++ b/src/misc/extract-mentions.ts @@ -2,18 +2,15 @@ import * as mfm from 'mfm-js'; -export default function(nodes: mfm.MfmNode[]): mfm.MfmMention['props'][] { +export function extractMentions(nodes: mfm.MfmNode[]): mfm.MfmMention['props'][] { // TODO: 重複を削除 - const mentionNodes = [] as mfm.MfmMention[]; + const mentions = [] as mfm.MfmMention['props'][]; - function scan(nodes: mfm.MfmNode[]) { - for (const node of nodes) { - if (node.type === 'mention') mentionNodes.push(node); - else if (node.children) scan(node.children); + mfm.inspect(nodes, (node) => { + if (node.type === 'mention') { + mentions.push(node.props); } - } + }); - scan(nodes); - - return mentionNodes.map(x => x.props); + return mentions; } diff --git a/src/misc/extract-url-from-mfm.ts b/src/misc/extract-url-from-mfm.ts index aa7f5f2540..ec3b019ea5 100644 --- a/src/misc/extract-url-from-mfm.ts +++ b/src/misc/extract-url-from-mfm.ts @@ -6,29 +6,21 @@ import { unique } from '@/prelude/array'; const removeHash = (x: string) => x.replace(/#[^#]*$/, ''); export function extractUrlFromMfm(nodes: mfm.MfmNode[], respectSilentFlag = true): string[] { - const urlNodes = [] as (mfm.MfmUrl | mfm.MfmLink)[]; + let urls = [] as string[]; - function scan(nodes: mfm.MfmNode[]) { - for (const node of nodes) { - if (node.type === 'url') { - urlNodes.push(node); - } else if (node.type === 'link') { - if (!respectSilentFlag || !node.props.silent) { - urlNodes.push(node); - } - } else if (node.children) { - scan(node.children); - } + mfm.inspect(nodes, (node) => { + if (node.type === 'url') { + urls.push(node.props.url); + } else if (node.type === 'link' && (!respectSilentFlag || !node.props.silent)) { + urls.push(node.props.url); } - } + }); - scan(nodes); - - const urls = unique(urlNodes.map(x => x.props.url)); + urls = unique(urls); return urls.reduce((array, url) => { - const removed = removeHash(url); - if (!array.map(x => removeHash(x)).includes(removed)) array.push(url); + const urlWithoutHash = removeHash(url); + if (!array.map(x => removeHash(x)).includes(urlWithoutHash)) array.push(url); return array; }, [] as string[]); } diff --git a/src/server/api/endpoints/i/update.ts b/src/server/api/endpoints/i/update.ts index 2c20da41c5..c0ffd75e23 100644 --- a/src/server/api/endpoints/i/update.ts +++ b/src/server/api/endpoints/i/update.ts @@ -6,7 +6,7 @@ import acceptAllFollowRequests from '../../../../services/following/requests/acc import { publishToFollowers } from '../../../../services/i/update'; import define from '../../define'; import { extractCustomEmojisFromMfm } from '@/misc/extract-custom-emojis-from-mfm'; -import extractHashtags from '@/misc/extract-hashtags'; +import { extractHashtags } from '@/misc/extract-hashtags'; import * as langmap from 'langmap'; import { updateUsertags } from '../../../../services/update-hashtag'; import { ApiError } from '../../error'; diff --git a/src/services/note/create.ts b/src/services/note/create.ts index 125285f34a..b9b39d25e4 100644 --- a/src/services/note/create.ts +++ b/src/services/note/create.ts @@ -12,9 +12,9 @@ import { updateHashtags } from '../update-hashtag'; import { concat } from '../../prelude/array'; import insertNoteUnread from './unread'; import { registerOrFetchInstanceDoc } from '../register-or-fetch-instance-doc'; -import extractMentions from '@/misc/extract-mentions'; +import { extractMentions } from '@/misc/extract-mentions'; import { extractCustomEmojisFromMfm } from '@/misc/extract-custom-emojis-from-mfm'; -import extractHashtags from '@/misc/extract-hashtags'; +import { extractHashtags } from '@/misc/extract-hashtags'; import { Note, IMentionedRemoteUsers } from '../../models/entities/note'; import { Mutings, Users, NoteWatchings, Notes, Instances, UserProfiles, Antennas, Followings, MutedNotes, Channels, ChannelFollowings } from '../../models'; import { DriveFile } from '../../models/entities/drive-file'; diff --git a/test/extract-mentions.ts b/test/extract-mentions.ts index 68b59f7f55..9e6d041906 100644 --- a/test/extract-mentions.ts +++ b/test/extract-mentions.ts @@ -1,6 +1,6 @@ import * as assert from 'assert'; -import extractMentions from '../src/misc/extract-mentions'; +import { extractMentions } from '../src/misc/extract-mentions'; import { parse } from 'mfm-js'; describe('Extract mentions', () => { -- cgit v1.2.3-freya