From 1a964cb6c037efe8e71c6e91e5c2ce032d22e107 Mon Sep 17 00:00:00 2001 From: Hazelnoot Date: Wed, 21 May 2025 19:17:51 -0400 Subject: pcleanup dependencies: * Consolidate multiple different HTML/XML/RSS libraries to use the Cheerio stack * Remove unused deps * Move dev dependencies to correct section * Pin versions where missing --- .../backend/src/server/api/endpoints/fetch-rss.ts | 213 ++++++++------------- 1 file changed, 76 insertions(+), 137 deletions(-) (limited to 'packages/backend/src/server/api') diff --git a/packages/backend/src/server/api/endpoints/fetch-rss.ts b/packages/backend/src/server/api/endpoints/fetch-rss.ts index 03f35f16a5..11244b30f6 100644 --- a/packages/backend/src/server/api/endpoints/fetch-rss.ts +++ b/packages/backend/src/server/api/endpoints/fetch-rss.ts @@ -3,12 +3,12 @@ * SPDX-License-Identifier: AGPL-3.0-only */ -import Parser from 'rss-parser'; import { Injectable } from '@nestjs/common'; +import { parseFeed } from 'htmlparser2'; import { Endpoint } from '@/server/api/endpoint-base.js'; import { HttpRequestService } from '@/core/HttpRequestService.js'; - -const rssParser = new Parser(); +import { ApiError } from '../error.js'; +import type { FeedItem } from 'domutils'; export const meta = { tags: ['meta'], @@ -17,52 +17,32 @@ export const meta = { allowGet: true, cacheSec: 60 * 3, + errors: { + fetchFailed: { + id: '88f4356f-719d-4715-b4fc-703a10a812d2', + code: 'FETCH_FAILED', + message: 'Failed to fetch RSS feed', + }, + }, + res: { type: 'object', properties: { - image: { - type: 'object', + type: { + type: 'string', + optional: false, + }, + id: { + type: 'string', optional: true, - properties: { - link: { - type: 'string', - optional: true, - }, - url: { - type: 'string', - optional: false, - }, - title: { - type: 'string', - optional: true, - }, - }, }, - paginationLinks: { - type: 'object', + updated: { + type: 'string', + optional: true, + }, + author: { + type: 'string', optional: true, - properties: { - self: { - type: 'string', - optional: true, - }, - first: { - type: 'string', - optional: true, - }, - next: { - type: 'string', - optional: true, - }, - last: { - type: 'string', - optional: true, - }, - prev: { - type: 'string', - optional: true, - }, - }, }, link: { type: 'string', @@ -94,113 +74,42 @@ export const meta = { type: 'string', optional: true, }, - creator: { - type: 'string', - optional: true, - }, - summary: { - type: 'string', - optional: true, - }, - content: { - type: 'string', - optional: true, - }, - isoDate: { + description: { type: 'string', optional: true, }, - categories: { + media: { type: 'array', - optional: true, + optional: false, items: { - type: 'string', - }, - }, - contentSnippet: { - type: 'string', - optional: true, - }, - enclosure: { - type: 'object', - optional: true, - properties: { - url: { - type: 'string', - optional: false, - }, - length: { - type: 'number', - optional: true, - }, - type: { - type: 'string', - optional: true, + type: 'object', + properties: { + medium: { + type: 'string', + optional: true, + }, + url: { + type: 'string', + optional: true, + }, + type: { + type: 'string', + optional: true, + }, + lang: { + type: 'string', + optional: true, + }, }, }, }, }, }, }, - feedUrl: { - type: 'string', - optional: true, - }, description: { type: 'string', optional: true, }, - itunes: { - type: 'object', - optional: true, - additionalProperties: true, - properties: { - image: { - type: 'string', - optional: true, - }, - owner: { - type: 'object', - optional: true, - properties: { - name: { - type: 'string', - optional: true, - }, - email: { - type: 'string', - optional: true, - }, - }, - }, - author: { - type: 'string', - optional: true, - }, - summary: { - type: 'string', - optional: true, - }, - explicit: { - type: 'string', - optional: true, - }, - categories: { - type: 'array', - optional: true, - items: { - type: 'string', - }, - }, - keywords: { - type: 'array', - optional: true, - items: { - type: 'string', - }, - }, - }, - }, }, }, @@ -224,7 +133,7 @@ export default class extends Endpoint { // eslint- constructor( private httpRequestService: HttpRequestService, ) { - super(meta, paramDef, async (ps, me) => { + super(meta, paramDef, async (ps) => { const res = await this.httpRequestService.send(ps.url, { method: 'GET', headers: { @@ -234,8 +143,38 @@ export default class extends Endpoint { // eslint- }); const text = await res.text(); + const feed = parseFeed(text, { + xmlMode: true, + }); + + if (!feed) { + throw new ApiError(meta.errors.fetchFailed); + } - return rssParser.parseString(text); + return { + type: feed.type, + id: feed.id, + title: feed.title, + link: feed.link, + description: feed.description, + updated: feed.updated?.toISOString(), + author: feed.author, + items: feed.items + .filter((item): item is FeedItem & { link: string, title: string } => !!item.link && !!item.title) + .map(item => ({ + guid: item.id, + title: item.title, + link: item.link, + description: item.description, + pubDate: item.pubDate?.toISOString(), + media: item.media.map(media => ({ + medium: media.medium, + url: media.url, + type: media.type, + lang: media.lang, + })), + })), + }; }); } } -- cgit v1.2.3-freya From 7eca06d672604fe03ce10591263bd04cf58e54c6 Mon Sep 17 00:00:00 2001 From: Hazelnoot Date: Fri, 23 May 2025 19:48:55 -0400 Subject: use pnpm package alias for @transfem-org/sfm-js --- packages/backend/package.json | 2 +- packages/backend/src/core/MfmService.ts | 2 +- packages/backend/src/core/NoteCreateService.ts | 2 +- packages/backend/src/core/NoteEditService.ts | 2 +- packages/backend/src/core/activitypub/ApMfmService.ts | 2 +- packages/backend/src/core/activitypub/ApRendererService.ts | 2 +- packages/backend/src/misc/extract-custom-emojis-from-mfm.ts | 2 +- packages/backend/src/misc/extract-hashtags.ts | 2 +- packages/backend/src/misc/extract-mentions.ts | 2 +- packages/backend/src/server/api/endpoints/i/update.ts | 2 +- packages/backend/src/server/api/mastodon/MastodonConverters.ts | 2 +- packages/backend/src/server/web/FeedService.ts | 2 +- packages/backend/test/unit/MfmService.ts | 2 +- packages/backend/test/unit/extract-mentions.ts | 2 +- packages/frontend-embed/package.json | 4 ++-- packages/frontend-embed/src/components/EmMfm.ts | 2 +- packages/frontend-embed/src/components/EmNote.vue | 2 +- packages/frontend-embed/src/components/EmNoteDetailed.vue | 2 +- packages/frontend/package.json | 2 +- packages/frontend/src/components/MkNote.vue | 2 +- packages/frontend/src/components/MkNoteDetailed.vue | 2 +- packages/frontend/src/components/MkPostForm.vue | 2 +- packages/frontend/src/components/MkSubNoteContent.vue | 2 +- packages/frontend/src/components/SkNote.vue | 2 +- packages/frontend/src/components/SkNoteDetailed.vue | 2 +- packages/frontend/src/components/SkOldNoteWindow.vue | 2 +- packages/frontend/src/components/global/MkMfm.ts | 2 +- packages/frontend/src/pages/chat/XMessage.vue | 2 +- packages/frontend/src/utility/check-animated-mfm.ts | 2 +- packages/frontend/src/utility/extract-mentions.ts | 2 +- packages/frontend/src/utility/extract-preview-urls.ts | 2 +- packages/frontend/src/utility/extract-url-from-mfm.ts | 2 +- 32 files changed, 33 insertions(+), 33 deletions(-) (limited to 'packages/backend/src/server/api') diff --git a/packages/backend/package.json b/packages/backend/package.json index 69d38f3bfb..70432d1bd1 100644 --- a/packages/backend/package.json +++ b/packages/backend/package.json @@ -92,7 +92,7 @@ "@smithy/node-http-handler": "2.5.0", "@swc/cli": "0.7.3", "@swc/core": "1.11.24", - "@transfem-org/sfm-js": "0.24.6", + "mfm-js": "npm:@transfem-org/sfm-js@0.24.6", "@twemoji/parser": "15.1.1", "accepts": "1.3.8", "ajv": "8.17.1", diff --git a/packages/backend/src/core/MfmService.ts b/packages/backend/src/core/MfmService.ts index d85ac7c807..551b25394a 100644 --- a/packages/backend/src/core/MfmService.ts +++ b/packages/backend/src/core/MfmService.ts @@ -15,7 +15,7 @@ import { intersperse } from '@/misc/prelude/array.js'; import { normalizeForSearch } from '@/misc/normalize-for-search.js'; import type { IMentionedRemoteUsers } from '@/models/Note.js'; import { bindThis } from '@/decorators.js'; -import type * as mfm from '@transfem-org/sfm-js'; +import type * as mfm from 'mfm-js'; const urlRegex = /^https?:\/\/[\w\/:%#@$&?!()\[\]~.,=+\-]+/; const urlRegexFull = /^https?:\/\/[\w\/:%#@$&?!()\[\]~.,=+\-]+$/; diff --git a/packages/backend/src/core/NoteCreateService.ts b/packages/backend/src/core/NoteCreateService.ts index a9f4083446..f8584a4a48 100644 --- a/packages/backend/src/core/NoteCreateService.ts +++ b/packages/backend/src/core/NoteCreateService.ts @@ -4,7 +4,7 @@ */ import { setImmediate } from 'node:timers/promises'; -import * as mfm from '@transfem-org/sfm-js'; +import * as mfm from 'mfm-js'; import { In, DataSource, IsNull, LessThan } from 'typeorm'; import * as Redis from 'ioredis'; import { Inject, Injectable, OnApplicationShutdown } from '@nestjs/common'; diff --git a/packages/backend/src/core/NoteEditService.ts b/packages/backend/src/core/NoteEditService.ts index a359381573..d963bf1945 100644 --- a/packages/backend/src/core/NoteEditService.ts +++ b/packages/backend/src/core/NoteEditService.ts @@ -4,7 +4,7 @@ */ import { setImmediate } from 'node:timers/promises'; -import * as mfm from '@transfem-org/sfm-js'; +import * as mfm from 'mfm-js'; import { DataSource, In, IsNull, LessThan } from 'typeorm'; import * as Redis from 'ioredis'; import { Inject, Injectable, OnApplicationShutdown } from '@nestjs/common'; diff --git a/packages/backend/src/core/activitypub/ApMfmService.ts b/packages/backend/src/core/activitypub/ApMfmService.ts index c4a948429a..ddb6461746 100644 --- a/packages/backend/src/core/activitypub/ApMfmService.ts +++ b/packages/backend/src/core/activitypub/ApMfmService.ts @@ -4,7 +4,7 @@ */ import { Injectable } from '@nestjs/common'; -import * as mfm from '@transfem-org/sfm-js'; +import * as mfm from 'mfm-js'; import { MfmService, Appender } from '@/core/MfmService.js'; import type { MiNote } from '@/models/Note.js'; import { bindThis } from '@/decorators.js'; diff --git a/packages/backend/src/core/activitypub/ApRendererService.ts b/packages/backend/src/core/activitypub/ApRendererService.ts index 789611fd97..08a8f30049 100644 --- a/packages/backend/src/core/activitypub/ApRendererService.ts +++ b/packages/backend/src/core/activitypub/ApRendererService.ts @@ -6,7 +6,7 @@ import { createPublicKey, randomUUID } from 'node:crypto'; import { Inject, Injectable } from '@nestjs/common'; import { In } from 'typeorm'; -import * as mfm from '@transfem-org/sfm-js'; +import * as mfm from 'mfm-js'; import { UnrecoverableError } from 'bullmq'; import { Element, Text } from 'domhandler'; import { DI } from '@/di-symbols.js'; diff --git a/packages/backend/src/misc/extract-custom-emojis-from-mfm.ts b/packages/backend/src/misc/extract-custom-emojis-from-mfm.ts index 36a9b8e1f4..73ae9abb54 100644 --- a/packages/backend/src/misc/extract-custom-emojis-from-mfm.ts +++ b/packages/backend/src/misc/extract-custom-emojis-from-mfm.ts @@ -3,7 +3,7 @@ * SPDX-License-Identifier: AGPL-3.0-only */ -import * as mfm from '@transfem-org/sfm-js'; +import * as mfm from 'mfm-js'; import { unique } from '@/misc/prelude/array.js'; export function extractCustomEmojisFromMfm(nodes: mfm.MfmNode[]): string[] { diff --git a/packages/backend/src/misc/extract-hashtags.ts b/packages/backend/src/misc/extract-hashtags.ts index ed7606d995..d3d245d414 100644 --- a/packages/backend/src/misc/extract-hashtags.ts +++ b/packages/backend/src/misc/extract-hashtags.ts @@ -3,7 +3,7 @@ * SPDX-License-Identifier: AGPL-3.0-only */ -import * as mfm from '@transfem-org/sfm-js'; +import * as mfm from 'mfm-js'; import { unique } from '@/misc/prelude/array.js'; export function extractHashtags(nodes: mfm.MfmNode[]): string[] { diff --git a/packages/backend/src/misc/extract-mentions.ts b/packages/backend/src/misc/extract-mentions.ts index bb21c32ffb..2ec9349718 100644 --- a/packages/backend/src/misc/extract-mentions.ts +++ b/packages/backend/src/misc/extract-mentions.ts @@ -5,7 +5,7 @@ // test is located in test/extract-mentions -import * as mfm from '@transfem-org/sfm-js'; +import * as mfm from 'mfm-js'; export function extractMentions(nodes: mfm.MfmNode[]): mfm.MfmMention['props'][] { // TODO: 重複を削除 diff --git a/packages/backend/src/server/api/endpoints/i/update.ts b/packages/backend/src/server/api/endpoints/i/update.ts index e632915f62..5767880531 100644 --- a/packages/backend/src/server/api/endpoints/i/update.ts +++ b/packages/backend/src/server/api/endpoints/i/update.ts @@ -3,7 +3,7 @@ * SPDX-License-Identifier: AGPL-3.0-only */ -import * as mfm from '@transfem-org/sfm-js'; +import * as mfm from 'mfm-js'; import { Inject, Injectable } from '@nestjs/common'; import ms from 'ms'; import { extractCustomEmojisFromMfm } from '@/misc/extract-custom-emojis-from-mfm.js'; diff --git a/packages/backend/src/server/api/mastodon/MastodonConverters.ts b/packages/backend/src/server/api/mastodon/MastodonConverters.ts index 02ce31c4f8..df8d68042a 100644 --- a/packages/backend/src/server/api/mastodon/MastodonConverters.ts +++ b/packages/backend/src/server/api/mastodon/MastodonConverters.ts @@ -5,7 +5,7 @@ import { Inject, Injectable } from '@nestjs/common'; import { Entity, MastodonEntity, MisskeyEntity } from 'megalodon'; -import mfm from '@transfem-org/sfm-js'; +import mfm from 'mfm-js'; import { MastodonNotificationType } from 'megalodon/lib/src/mastodon/notification.js'; import { NotificationType } from 'megalodon/lib/src/notification.js'; import { DI } from '@/di-symbols.js'; diff --git a/packages/backend/src/server/web/FeedService.ts b/packages/backend/src/server/web/FeedService.ts index dcd4d80303..a622ae7e34 100644 --- a/packages/backend/src/server/web/FeedService.ts +++ b/packages/backend/src/server/web/FeedService.ts @@ -15,7 +15,7 @@ import { DriveFileEntityService } from '@/core/entities/DriveFileEntityService.j import { bindThis } from '@/decorators.js'; import { IdService } from '@/core/IdService.js'; import { MfmService } from "@/core/MfmService.js"; -import { parse as mfmParse } from '@transfem-org/sfm-js'; +import { parse as mfmParse } from 'mfm-js'; @Injectable() export class FeedService { diff --git a/packages/backend/test/unit/MfmService.ts b/packages/backend/test/unit/MfmService.ts index f96f3977d0..af1fc4e132 100644 --- a/packages/backend/test/unit/MfmService.ts +++ b/packages/backend/test/unit/MfmService.ts @@ -4,7 +4,7 @@ */ import * as assert from 'assert'; -import * as mfm from '@transfem-org/sfm-js'; +import * as mfm from 'mfm-js'; import { Test } from '@nestjs/testing'; import { CoreModule } from '@/core/CoreModule.js'; diff --git a/packages/backend/test/unit/extract-mentions.ts b/packages/backend/test/unit/extract-mentions.ts index 2aad89d65b..3403387e30 100644 --- a/packages/backend/test/unit/extract-mentions.ts +++ b/packages/backend/test/unit/extract-mentions.ts @@ -5,7 +5,7 @@ import * as assert from 'assert'; -import { parse } from '@transfem-org/sfm-js'; +import { parse } from 'mfm-js'; import { extractMentions } from '@/misc/extract-mentions.js'; describe('Extract mentions', () => { diff --git a/packages/frontend-embed/package.json b/packages/frontend-embed/package.json index 5191fe6852..5b4774c7af 100644 --- a/packages/frontend-embed/package.json +++ b/packages/frontend-embed/package.json @@ -11,8 +11,8 @@ }, "dependencies": { "@discordapp/twemoji": "15.1.0", - "@phosphor-icons/web": "^2.0.3", - "@transfem-org/sfm-js": "0.24.5", + "@phosphor-icons/web": "2.1.2", + "mfm-js": "npm:@transfem-org/sfm-js@0.24.6", "buraha": "0.0.1", "frontend-shared": "workspace:*", "json5": "2.2.3", diff --git a/packages/frontend-embed/src/components/EmMfm.ts b/packages/frontend-embed/src/components/EmMfm.ts index d377d492e0..74ae3373ef 100644 --- a/packages/frontend-embed/src/components/EmMfm.ts +++ b/packages/frontend-embed/src/components/EmMfm.ts @@ -5,7 +5,7 @@ import { h, provide } from 'vue'; import type { VNode, SetupContext } from 'vue'; -import * as mfm from '@transfem-org/sfm-js'; +import * as mfm from 'mfm-js'; import * as Misskey from 'misskey-js'; import { host } from '@@/js/config.js'; import EmUrl from '@/components/EmUrl.vue'; diff --git a/packages/frontend-embed/src/components/EmNote.vue b/packages/frontend-embed/src/components/EmNote.vue index 666cbde72d..0dc77d09a7 100644 --- a/packages/frontend-embed/src/components/EmNote.vue +++ b/packages/frontend-embed/src/components/EmNote.vue @@ -105,7 +105,7 @@ SPDX-License-Identifier: AGPL-3.0-only