From 354cb2a6754b55fd3ad01388a4a17d3a76d4a09b Mon Sep 17 00:00:00 2001 From: dakkar Date: Sat, 9 Mar 2024 12:17:48 +0000 Subject: handle non-ASCII emoji names MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * use the more inclusive regexp for validating emoji names * always normalize emoji names, aliases, categories the latter point is necessary to allow matching, for example, `ä` against `a`+combining diaeresis this will also need to bump the version of `sfm-js` once we merge https://activitypub.software/TransFem-org/sfm-js/-/merge_requests/2 --- .../queue/processors/ExportCustomEmojisProcessorService.ts | 2 +- .../queue/processors/ImportCustomEmojisProcessorService.ts | 13 +++++++------ 2 files changed, 8 insertions(+), 7 deletions(-) (limited to 'packages/backend/src/queue') diff --git a/packages/backend/src/queue/processors/ExportCustomEmojisProcessorService.ts b/packages/backend/src/queue/processors/ExportCustomEmojisProcessorService.ts index e4eb4791bd..45087927a5 100644 --- a/packages/backend/src/queue/processors/ExportCustomEmojisProcessorService.ts +++ b/packages/backend/src/queue/processors/ExportCustomEmojisProcessorService.ts @@ -85,7 +85,7 @@ export class ExportCustomEmojisProcessorService { }); for (const emoji of customEmojis) { - if (!/^[a-zA-Z0-9_]+$/.test(emoji.name)) { + if (!/^[\p{Letter}\p{Number}\p{Mark}_+-]+$/u.test(emoji.name)) { this.logger.error(`invalid emoji name: ${emoji.name}`); continue; } diff --git a/packages/backend/src/queue/processors/ImportCustomEmojisProcessorService.ts b/packages/backend/src/queue/processors/ImportCustomEmojisProcessorService.ts index 171809d25c..04ad74ee01 100644 --- a/packages/backend/src/queue/processors/ImportCustomEmojisProcessorService.ts +++ b/packages/backend/src/queue/processors/ImportCustomEmojisProcessorService.ts @@ -79,13 +79,14 @@ export class ImportCustomEmojisProcessorService { continue; } const emojiInfo = record.emoji; - if (!/^[a-zA-Z0-9_]+$/.test(emojiInfo.name)) { - this.logger.error(`invalid emojiname: ${emojiInfo.name}`); + const nameNfc = emojiInfo.name.normalize('NFC'); + if (!/^[\p{Letter}\p{Number}\p{Mark}_+-]+$/u.test(nameNfc)) { + this.logger.error(`invalid emojiname: ${nameNfc}`); continue; } const emojiPath = outputPath + '/' + record.fileName; await this.emojisRepository.delete({ - name: emojiInfo.name, + name: nameNfc, }); const driveFile = await this.driveService.addFile({ user: null, @@ -94,10 +95,10 @@ export class ImportCustomEmojisProcessorService { force: true, }); await this.customEmojiService.add({ - name: emojiInfo.name, - category: emojiInfo.category, + name: nameNfc, + category: emojiInfo.category?.normalize('NFC'), host: null, - aliases: emojiInfo.aliases, + aliases: emojiInfo.aliases?.map((a: string) => a.normalize('NFC')), driveFile, license: emojiInfo.license, isSensitive: emojiInfo.isSensitive, -- cgit v1.2.3-freya From dd3d562a1e4ae547ddb0b54d77678abf801f2e26 Mon Sep 17 00:00:00 2001 From: Latte macchiato Date: Fri, 19 Apr 2024 21:58:37 +0000 Subject: Rework cache clearing to be fault tolerant --- .../processors/CleanRemoteFilesProcessorService.ts | 27 ++++++++++++++-------- 1 file changed, 18 insertions(+), 9 deletions(-) (limited to 'packages/backend/src/queue') diff --git a/packages/backend/src/queue/processors/CleanRemoteFilesProcessorService.ts b/packages/backend/src/queue/processors/CleanRemoteFilesProcessorService.ts index 917de8b72c..ec75f3ba01 100644 --- a/packages/backend/src/queue/processors/CleanRemoteFilesProcessorService.ts +++ b/packages/backend/src/queue/processors/CleanRemoteFilesProcessorService.ts @@ -33,6 +33,12 @@ export class CleanRemoteFilesProcessorService { let deletedCount = 0; let cursor: MiDriveFile['id'] | null = null; + let errorCount = 0; + + const total = await this.driveFilesRepository.countBy({ + userHost: Not(IsNull()), + isLink: false, + }); while (true) { const files = await this.driveFilesRepository.find({ @@ -41,7 +47,7 @@ export class CleanRemoteFilesProcessorService { isLink: false, ...(cursor ? { id: MoreThan(cursor) } : {}), }, - take: 8, + take: 256, // Adjust the batch size as needed order: { id: 1, }, @@ -54,18 +60,21 @@ export class CleanRemoteFilesProcessorService { cursor = files.at(-1)?.id ?? null; - await Promise.all(files.map(file => this.driveService.deleteFileSync(file, true))); - - deletedCount += 8; + // Handle deletion in a batch + const results = await Promise.allSettled(files.map(file => this.driveService.deleteFileSync(file, true))); - const total = await this.driveFilesRepository.countBy({ - userHost: Not(IsNull()), - isLink: false, + results.forEach((result, index) => { + if (result.status === 'fulfilled') { + deletedCount++; + } else { + this.logger.error(`Failed to delete file ID ${files[index].id}: ${result.reason}`); + errorCount++; + } }); - job.updateProgress(deletedCount / total); + await job.updateProgress((deletedCount / total) * 100); } - this.logger.succ('All cached remote files has been deleted.'); + this.logger.succ(`All cached remote files processed. Total deleted: ${deletedCount}, Failed: ${errorCount}.`); } } -- cgit v1.2.3-freya From 493775ad7b1ae20d40a3a6b6dd7eb505ced6648a Mon Sep 17 00:00:00 2001 From: PrivateGER Date: Wed, 24 Apr 2024 16:05:30 +0200 Subject: reformat expression --- .../backend/src/queue/processors/CleanRemoteFilesProcessorService.ts | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) (limited to 'packages/backend/src/queue') diff --git a/packages/backend/src/queue/processors/CleanRemoteFilesProcessorService.ts b/packages/backend/src/queue/processors/CleanRemoteFilesProcessorService.ts index ec75f3ba01..4fa414b0b5 100644 --- a/packages/backend/src/queue/processors/CleanRemoteFilesProcessorService.ts +++ b/packages/backend/src/queue/processors/CleanRemoteFilesProcessorService.ts @@ -47,7 +47,7 @@ export class CleanRemoteFilesProcessorService { isLink: false, ...(cursor ? { id: MoreThan(cursor) } : {}), }, - take: 256, // Adjust the batch size as needed + take: 256, order: { id: 1, }, @@ -72,7 +72,8 @@ export class CleanRemoteFilesProcessorService { } }); - await job.updateProgress((deletedCount / total) * 100); + await job.updateProgress(100 / total * deletedCount); + } this.logger.succ(`All cached remote files processed. Total deleted: ${deletedCount}, Failed: ${errorCount}.`); -- cgit v1.2.3-freya From 6ae01e28aa717d54743f1ab44fd099853a969d3d Mon Sep 17 00:00:00 2001 From: dakkar Date: Tue, 30 Apr 2024 10:12:54 +0100 Subject: Compact LD-signed activities against well-known context This should defend against some spoofing attacks, see also https://nvd.nist.gov/vuln/detail/CVE-2022-24307 for Mastodon, https://iceshrimp.dev/iceshrimp/iceshrimp/commit/febb499fcb5fe3d56ca79025e4b5851464660c38 from Iceshrimp and https://firefish.dev/firefish/firefish/-/commit/e790d6be90dfd5dc6471b650a54520761bb9d745 for Firefish Thanks to @tesaguri@fedibird.com for reporting and providing the patch. --- .../src/core/activitypub/ApRendererService.ts | 44 +-------------------- .../src/core/activitypub/LdSignatureService.ts | 10 +++++ .../backend/src/core/activitypub/misc/contexts.ts | 46 +++++++++++++++++++++- .../src/queue/processors/InboxProcessorService.ts | 14 ++++++- 4 files changed, 70 insertions(+), 44 deletions(-) (limited to 'packages/backend/src/queue') diff --git a/packages/backend/src/core/activitypub/ApRendererService.ts b/packages/backend/src/core/activitypub/ApRendererService.ts index a84feffac6..d06bb2d8df 100644 --- a/packages/backend/src/core/activitypub/ApRendererService.ts +++ b/packages/backend/src/core/activitypub/ApRendererService.ts @@ -31,6 +31,7 @@ import { IdService } from '@/core/IdService.js'; import { MetaService } from '../MetaService.js'; import { LdSignatureService } from './LdSignatureService.js'; import { ApMfmService } from './ApMfmService.js'; +import { CONTEXT } from './misc/contexts.js'; import type { IAccept, IActivity, IAdd, IAnnounce, IApDocument, IApEmoji, IApHashtag, IApImage, IApMention, IBlock, ICreate, IDelete, IFlag, IFollow, IKey, ILike, IMove, IObject, IPost, IQuestion, IReject, IRemove, ITombstone, IUndo, IUpdate } from './type.js'; @Injectable() @@ -785,48 +786,7 @@ export class ApRendererService { x.id = `${this.config.url}/${randomUUID()}`; } - return Object.assign({ - '@context': [ - 'https://www.w3.org/ns/activitystreams', - 'https://w3id.org/security/v1', - { - Key: 'sec:Key', - // as non-standards - manuallyApprovesFollowers: 'as:manuallyApprovesFollowers', - sensitive: 'as:sensitive', - Hashtag: 'as:Hashtag', - quoteUrl: 'as:quoteUrl', - fedibird: 'http://fedibird.com/ns#', - quoteUri: 'fedibird:quoteUri', - // Mastodon - toot: 'http://joinmastodon.org/ns#', - Emoji: 'toot:Emoji', - featured: 'toot:featured', - discoverable: 'toot:discoverable', - // schema - schema: 'http://schema.org#', - PropertyValue: 'schema:PropertyValue', - value: 'schema:value', - // Misskey - misskey: 'https://misskey-hub.net/ns#', - '_misskey_content': 'misskey:_misskey_content', - '_misskey_quote': 'misskey:_misskey_quote', - '_misskey_reaction': 'misskey:_misskey_reaction', - '_misskey_votes': 'misskey:_misskey_votes', - '_misskey_summary': 'misskey:_misskey_summary', - 'isCat': 'misskey:isCat', - // Firefish - firefish: 'https://joinfirefish.org/ns#', - speakAsCat: 'firefish:speakAsCat', - // Sharkey - sharkey: 'https://joinsharkey.org/ns#', - backgroundUrl: 'sharkey:backgroundUrl', - listenbrainz: 'sharkey:listenbrainz', - // vcard - vcard: 'http://www.w3.org/2006/vcard/ns#', - }, - ], - }, x as T & { id: string }); + return Object.assign({ '@context': CONTEXT }, x as T & { id: string }); } @bindThis diff --git a/packages/backend/src/core/activitypub/LdSignatureService.ts b/packages/backend/src/core/activitypub/LdSignatureService.ts index 9de184336f..a4add22551 100644 --- a/packages/backend/src/core/activitypub/LdSignatureService.ts +++ b/packages/backend/src/core/activitypub/LdSignatureService.ts @@ -88,6 +88,16 @@ class LdSignature { return verifyData; } + @bindThis + public async compact(data: any, context: any = CONTEXT): Promise { + const customLoader = this.getLoader(); + // XXX: Importing jsonld dynamically since Jest frequently fails to import it statically + // https://github.com/misskey-dev/misskey/pull/9894#discussion_r1103753595 + return (await import('jsonld')).default.compact(data, context, { + documentLoader: customLoader, + }); + } + @bindThis public async normalize(data: JsonLdDocument): Promise { const customLoader = this.getLoader(); diff --git a/packages/backend/src/core/activitypub/misc/contexts.ts b/packages/backend/src/core/activitypub/misc/contexts.ts index 88afdefcd3..4ff114bbf5 100644 --- a/packages/backend/src/core/activitypub/misc/contexts.ts +++ b/packages/backend/src/core/activitypub/misc/contexts.ts @@ -3,7 +3,7 @@ * SPDX-License-Identifier: AGPL-3.0-only */ -import type { JsonLd } from 'jsonld/jsonld-spec.js'; +import type { Context, JsonLd } from 'jsonld/jsonld-spec.js'; /* eslint:disable:quotemark indent */ const id_v1 = { @@ -526,6 +526,50 @@ const activitystreams = { }, } satisfies JsonLd; +const context_iris = [ + 'https://www.w3.org/ns/activitystreams', + 'https://w3id.org/security/v1', +]; + +const extension_context_definition = { + Key: 'sec:Key', + // as non-standards + manuallyApprovesFollowers: 'as:manuallyApprovesFollowers', + sensitive: 'as:sensitive', + Hashtag: 'as:Hashtag', + quoteUrl: 'as:quoteUrl', + fedibird: 'http://fedibird.com/ns#', + quoteUri: 'fedibird:quoteUri', + // Mastodon + toot: 'http://joinmastodon.org/ns#', + Emoji: 'toot:Emoji', + featured: 'toot:featured', + discoverable: 'toot:discoverable', + // schema + schema: 'http://schema.org#', + PropertyValue: 'schema:PropertyValue', + value: 'schema:value', + // Misskey + misskey: 'https://misskey-hub.net/ns#', + '_misskey_content': 'misskey:_misskey_content', + '_misskey_quote': 'misskey:_misskey_quote', + '_misskey_reaction': 'misskey:_misskey_reaction', + '_misskey_votes': 'misskey:_misskey_votes', + '_misskey_summary': 'misskey:_misskey_summary', + 'isCat': 'misskey:isCat', + // Firefish + firefish: 'https://joinfirefish.org/ns#', + speakAsCat: 'firefish:speakAsCat', + // Sharkey + sharkey: 'https://joinsharkey.org/ns#', + backgroundUrl: 'sharkey:backgroundUrl', + listenbrainz: 'sharkey:listenbrainz', + // vcard + vcard: 'http://www.w3.org/2006/vcard/ns#', +} satisfies Context; + +export const CONTEXT: (string | Context)[] = [...context_iris, extension_context_definition]; + export const CONTEXTS: Record = { 'https://w3id.org/identity/v1': id_v1, 'https://w3id.org/security/v1': security_v1, diff --git a/packages/backend/src/queue/processors/InboxProcessorService.ts b/packages/backend/src/queue/processors/InboxProcessorService.ts index ad1d9799a7..2b5b7c5619 100644 --- a/packages/backend/src/queue/processors/InboxProcessorService.ts +++ b/packages/backend/src/queue/processors/InboxProcessorService.ts @@ -15,6 +15,7 @@ import InstanceChart from '@/core/chart/charts/instance.js'; import ApRequestChart from '@/core/chart/charts/ap-request.js'; import FederationChart from '@/core/chart/charts/federation.js'; import { getApId } from '@/core/activitypub/type.js'; +import type { IActivity } from '@/core/activitypub/type.js'; import type { MiRemoteUser } from '@/models/User.js'; import type { MiUserPublickey } from '@/models/UserPublickey.js'; import { ApDbResolverService } from '@/core/activitypub/ApDbResolverService.js'; @@ -52,7 +53,7 @@ export class InboxProcessorService { @bindThis public async process(job: Bull.Job): Promise { const signature = job.data.signature; // HTTP-signature - const activity = job.data.activity; + let activity = job.data.activity; //#region Log const info = Object.assign({}, activity); @@ -150,6 +151,17 @@ export class InboxProcessorService { throw new Bull.UnrecoverableError('skip: LD-Signatureの検証に失敗しました'); } + // アクティビティを正規化 + delete activity.signature; + try { + activity = await ldSignature.compact(activity) as IActivity; + } catch (e) { + throw new Bull.UnrecoverableError(`skip: failed to compact activity: ${e}`); + } + // TODO: 元のアクティビティと非互換な形に正規化される場合は転送をスキップする + // https://github.com/mastodon/mastodon/blob/664b0ca/app/services/activitypub/process_collection_service.rb#L24-L29 + activity.signature = ldSignature; + // もう一度actorチェック if (authUser.user.uri !== activity.actor) { throw new Bull.UnrecoverableError(`skip: LD-Signature user(${authUser.user.uri}) !== activity.actor(${activity.actor})`); -- cgit v1.2.3-freya