From 354cb2a6754b55fd3ad01388a4a17d3a76d4a09b Mon Sep 17 00:00:00 2001 From: dakkar Date: Sat, 9 Mar 2024 12:17:48 +0000 Subject: handle non-ASCII emoji names MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * use the more inclusive regexp for validating emoji names * always normalize emoji names, aliases, categories the latter point is necessary to allow matching, for example, `ä` against `a`+combining diaeresis this will also need to bump the version of `sfm-js` once we merge https://activitypub.software/TransFem-org/sfm-js/-/merge_requests/2 --- .../processors/ExportCustomEmojisProcessorService.ts | 2 +- .../processors/ImportCustomEmojisProcessorService.ts | 13 +++++++------ .../api/endpoints/admin/emoji/add-aliases-bulk.ts | 2 +- .../backend/src/server/api/endpoints/admin/emoji/add.ts | 11 ++++++----- .../src/server/api/endpoints/admin/emoji/copy.ts | 9 +++++---- .../src/server/api/endpoints/admin/emoji/list-remote.ts | 2 +- .../src/server/api/endpoints/admin/emoji/list.ts | 11 ++++++----- .../api/endpoints/admin/emoji/remove-aliases-bulk.ts | 2 +- .../api/endpoints/admin/emoji/set-aliases-bulk.ts | 2 +- .../api/endpoints/admin/emoji/set-category-bulk.ts | 2 +- .../src/server/api/endpoints/admin/emoji/update.ts | 17 +++++++++-------- 11 files changed, 39 insertions(+), 34 deletions(-) (limited to 'packages/backend/src') diff --git a/packages/backend/src/queue/processors/ExportCustomEmojisProcessorService.ts b/packages/backend/src/queue/processors/ExportCustomEmojisProcessorService.ts index e4eb4791bd..45087927a5 100644 --- a/packages/backend/src/queue/processors/ExportCustomEmojisProcessorService.ts +++ b/packages/backend/src/queue/processors/ExportCustomEmojisProcessorService.ts @@ -85,7 +85,7 @@ export class ExportCustomEmojisProcessorService { }); for (const emoji of customEmojis) { - if (!/^[a-zA-Z0-9_]+$/.test(emoji.name)) { + if (!/^[\p{Letter}\p{Number}\p{Mark}_+-]+$/u.test(emoji.name)) { this.logger.error(`invalid emoji name: ${emoji.name}`); continue; } diff --git a/packages/backend/src/queue/processors/ImportCustomEmojisProcessorService.ts b/packages/backend/src/queue/processors/ImportCustomEmojisProcessorService.ts index 171809d25c..04ad74ee01 100644 --- a/packages/backend/src/queue/processors/ImportCustomEmojisProcessorService.ts +++ b/packages/backend/src/queue/processors/ImportCustomEmojisProcessorService.ts @@ -79,13 +79,14 @@ export class ImportCustomEmojisProcessorService { continue; } const emojiInfo = record.emoji; - if (!/^[a-zA-Z0-9_]+$/.test(emojiInfo.name)) { - this.logger.error(`invalid emojiname: ${emojiInfo.name}`); + const nameNfc = emojiInfo.name.normalize('NFC'); + if (!/^[\p{Letter}\p{Number}\p{Mark}_+-]+$/u.test(nameNfc)) { + this.logger.error(`invalid emojiname: ${nameNfc}`); continue; } const emojiPath = outputPath + '/' + record.fileName; await this.emojisRepository.delete({ - name: emojiInfo.name, + name: nameNfc, }); const driveFile = await this.driveService.addFile({ user: null, @@ -94,10 +95,10 @@ export class ImportCustomEmojisProcessorService { force: true, }); await this.customEmojiService.add({ - name: emojiInfo.name, - category: emojiInfo.category, + name: nameNfc, + category: emojiInfo.category?.normalize('NFC'), host: null, - aliases: emojiInfo.aliases, + aliases: emojiInfo.aliases?.map((a: string) => a.normalize('NFC')), driveFile, license: emojiInfo.license, isSensitive: emojiInfo.isSensitive, diff --git a/packages/backend/src/server/api/endpoints/admin/emoji/add-aliases-bulk.ts b/packages/backend/src/server/api/endpoints/admin/emoji/add-aliases-bulk.ts index a30a080e59..f4fc79bdb3 100644 --- a/packages/backend/src/server/api/endpoints/admin/emoji/add-aliases-bulk.ts +++ b/packages/backend/src/server/api/endpoints/admin/emoji/add-aliases-bulk.ts @@ -34,7 +34,7 @@ export default class extends Endpoint { // eslint- private customEmojiService: CustomEmojiService, ) { super(meta, paramDef, async (ps, me) => { - await this.customEmojiService.addAliasesBulk(ps.ids, ps.aliases); + await this.customEmojiService.addAliasesBulk(ps.ids, ps.aliases.map(a => a.normalize('NFC'))); }); } } diff --git a/packages/backend/src/server/api/endpoints/admin/emoji/add.ts b/packages/backend/src/server/api/endpoints/admin/emoji/add.ts index 767e517b80..b45a3c7156 100644 --- a/packages/backend/src/server/api/endpoints/admin/emoji/add.ts +++ b/packages/backend/src/server/api/endpoints/admin/emoji/add.ts @@ -40,7 +40,7 @@ export const meta = { export const paramDef = { type: 'object', properties: { - name: { type: 'string', pattern: '^[a-zA-Z0-9_]+$' }, + name: { type: 'string', pattern: '^[\\p{Letter}\\p{Number}\\p{Mark}_+-]+$' }, fileId: { type: 'string', format: 'misskey:id' }, category: { type: 'string', @@ -73,18 +73,19 @@ export default class extends Endpoint { // eslint- private emojiEntityService: EmojiEntityService, ) { super(meta, paramDef, async (ps, me) => { + const nameNfc = ps.name.normalize('NFC'); const driveFile = await this.driveFilesRepository.findOneBy({ id: ps.fileId }); if (driveFile == null) throw new ApiError(meta.errors.noSuchFile); - const isDuplicate = await this.customEmojiService.checkDuplicate(ps.name); + const isDuplicate = await this.customEmojiService.checkDuplicate(nameNfc); if (isDuplicate) throw new ApiError(meta.errors.duplicateName); if (driveFile.user !== null) await this.driveFilesRepository.update(driveFile.id, { user: null }); const emoji = await this.customEmojiService.add({ driveFile, - name: ps.name, - category: ps.category ?? null, - aliases: ps.aliases ?? [], + name: nameNfc, + category: ps.category?.normalize('NFC') ?? null, + aliases: ps.aliases?.map(a => a.normalize('NFC')) ?? [], host: null, license: ps.license ?? null, isSensitive: ps.isSensitive ?? false, diff --git a/packages/backend/src/server/api/endpoints/admin/emoji/copy.ts b/packages/backend/src/server/api/endpoints/admin/emoji/copy.ts index 29af7598ed..f968813197 100644 --- a/packages/backend/src/server/api/endpoints/admin/emoji/copy.ts +++ b/packages/backend/src/server/api/endpoints/admin/emoji/copy.ts @@ -82,15 +82,16 @@ export default class extends Endpoint { // eslint- throw new ApiError(); } + const nameNfc = emoji.name.normalize('NFC'); // Duplication Check - const isDuplicate = await this.customEmojiService.checkDuplicate(emoji.name); + const isDuplicate = await this.customEmojiService.checkDuplicate(nameNfc); if (isDuplicate) throw new ApiError(meta.errors.duplicateName); const addedEmoji = await this.customEmojiService.add({ driveFile, - name: emoji.name, - category: emoji.category, - aliases: emoji.aliases, + name: nameNfc, + category: emoji.category?.normalize('NFC'), + aliases: emoji.aliases?.map(a => a.normalize('NFC')), host: null, license: emoji.license, isSensitive: emoji.isSensitive, diff --git a/packages/backend/src/server/api/endpoints/admin/emoji/list-remote.ts b/packages/backend/src/server/api/endpoints/admin/emoji/list-remote.ts index e423f440d0..1182918ea2 100644 --- a/packages/backend/src/server/api/endpoints/admin/emoji/list-remote.ts +++ b/packages/backend/src/server/api/endpoints/admin/emoji/list-remote.ts @@ -98,7 +98,7 @@ export default class extends Endpoint { // eslint- } if (ps.query) { - q.andWhere('emoji.name like :query', { query: '%' + sqlLikeEscape(ps.query) + '%' }) + q.andWhere('emoji.name like :query', { query: '%' + sqlLikeEscape(ps.query.normalize('NFC')) + '%' }) .orderBy('length(emoji.name)', 'ASC'); } diff --git a/packages/backend/src/server/api/endpoints/admin/emoji/list.ts b/packages/backend/src/server/api/endpoints/admin/emoji/list.ts index 53810d1d16..5e21111f9f 100644 --- a/packages/backend/src/server/api/endpoints/admin/emoji/list.ts +++ b/packages/backend/src/server/api/endpoints/admin/emoji/list.ts @@ -92,17 +92,18 @@ export default class extends Endpoint { // eslint- //const emojis = await q.limit(ps.limit).getMany(); emojis = await q.orderBy('length(emoji.name)', 'ASC').getMany(); - const queryarry = ps.query.match(/\:([a-z0-9_]*)\:/g); + const queryarry = ps.query.match(/:([\p{Letter}\p{Number}\p{Mark}_+-]*):/ug); if (queryarry) { emojis = emojis.filter(emoji => - queryarry.includes(`:${emoji.name}:`), + queryarry.includes(`:${emoji.name.normalize('NFC')}:`), ); } else { + const queryNfc = ps.query!.normalize('NFC'); emojis = emojis.filter(emoji => - emoji.name.includes(ps.query!) || - emoji.aliases.some(a => a.includes(ps.query!)) || - emoji.category?.includes(ps.query!)); + emoji.name.includes(queryNfc) || + emoji.aliases.some(a => a.includes(queryNfc)) || + emoji.category?.includes(queryNfc)); } emojis.splice(ps.limit + 1); } else { diff --git a/packages/backend/src/server/api/endpoints/admin/emoji/remove-aliases-bulk.ts b/packages/backend/src/server/api/endpoints/admin/emoji/remove-aliases-bulk.ts index 0fa119eabe..e78620eac1 100644 --- a/packages/backend/src/server/api/endpoints/admin/emoji/remove-aliases-bulk.ts +++ b/packages/backend/src/server/api/endpoints/admin/emoji/remove-aliases-bulk.ts @@ -34,7 +34,7 @@ export default class extends Endpoint { // eslint- private customEmojiService: CustomEmojiService, ) { super(meta, paramDef, async (ps, me) => { - await this.customEmojiService.removeAliasesBulk(ps.ids, ps.aliases); + await this.customEmojiService.removeAliasesBulk(ps.ids, ps.aliases.map(a => a.normalize('NFC'))); }); } } diff --git a/packages/backend/src/server/api/endpoints/admin/emoji/set-aliases-bulk.ts b/packages/backend/src/server/api/endpoints/admin/emoji/set-aliases-bulk.ts index d9ee18699c..69fc8e0cb5 100644 --- a/packages/backend/src/server/api/endpoints/admin/emoji/set-aliases-bulk.ts +++ b/packages/backend/src/server/api/endpoints/admin/emoji/set-aliases-bulk.ts @@ -34,7 +34,7 @@ export default class extends Endpoint { // eslint- private customEmojiService: CustomEmojiService, ) { super(meta, paramDef, async (ps, me) => { - await this.customEmojiService.setAliasesBulk(ps.ids, ps.aliases); + await this.customEmojiService.setAliasesBulk(ps.ids, ps.aliases.map(a => a.normalize('NFC'))); }); } } diff --git a/packages/backend/src/server/api/endpoints/admin/emoji/set-category-bulk.ts b/packages/backend/src/server/api/endpoints/admin/emoji/set-category-bulk.ts index dc25df2767..679a9f9c42 100644 --- a/packages/backend/src/server/api/endpoints/admin/emoji/set-category-bulk.ts +++ b/packages/backend/src/server/api/endpoints/admin/emoji/set-category-bulk.ts @@ -36,7 +36,7 @@ export default class extends Endpoint { // eslint- private customEmojiService: CustomEmojiService, ) { super(meta, paramDef, async (ps, me) => { - await this.customEmojiService.setCategoryBulk(ps.ids, ps.category ?? null); + await this.customEmojiService.setCategoryBulk(ps.ids, ps.category?.normalize('NFC') ?? null); }); } } diff --git a/packages/backend/src/server/api/endpoints/admin/emoji/update.ts b/packages/backend/src/server/api/endpoints/admin/emoji/update.ts index 22609a16a3..3caa0f84a3 100644 --- a/packages/backend/src/server/api/endpoints/admin/emoji/update.ts +++ b/packages/backend/src/server/api/endpoints/admin/emoji/update.ts @@ -40,7 +40,7 @@ export const paramDef = { type: 'object', properties: { id: { type: 'string', format: 'misskey:id' }, - name: { type: 'string', pattern: '^[a-zA-Z0-9_]+$' }, + name: { type: 'string', pattern: '^[\\p{Letter}\\p{Number}\\p{Mark}_+-]+$' }, fileId: { type: 'string', format: 'misskey:id' }, category: { type: 'string', @@ -72,6 +72,7 @@ export default class extends Endpoint { // eslint- private customEmojiService: CustomEmojiService, ) { super(meta, paramDef, async (ps, me) => { + const nameNfc = ps.name?.normalize('NFC'); let driveFile; if (ps.fileId) { driveFile = await this.driveFilesRepository.findOneBy({ id: ps.fileId }); @@ -83,22 +84,22 @@ export default class extends Endpoint { // eslint- emojiId = ps.id; const emoji = await this.customEmojiService.getEmojiById(ps.id); if (!emoji) throw new ApiError(meta.errors.noSuchEmoji); - if (ps.name && (ps.name !== emoji.name)) { - const isDuplicate = await this.customEmojiService.checkDuplicate(ps.name); + if (nameNfc && (nameNfc !== emoji.name)) { + const isDuplicate = await this.customEmojiService.checkDuplicate(nameNfc); if (isDuplicate) throw new ApiError(meta.errors.sameNameEmojiExists); } } else { - if (!ps.name) throw new Error('Invalid Params unexpectedly passed. This is a BUG. Please report it to the development team.'); - const emoji = await this.customEmojiService.getEmojiByName(ps.name); + if (!nameNfc) throw new Error('Invalid Params unexpectedly passed. This is a BUG. Please report it to the development team.'); + const emoji = await this.customEmojiService.getEmojiByName(nameNfc); if (!emoji) throw new ApiError(meta.errors.noSuchEmoji); emojiId = emoji.id; } await this.customEmojiService.update(emojiId, { driveFile, - name: ps.name, - category: ps.category, - aliases: ps.aliases, + name: nameNfc, + category: ps.category?.normalize('NFC'), + aliases: ps.aliases?.map(a => a.normalize('NFC')), license: ps.license, isSensitive: ps.isSensitive, localOnly: ps.localOnly, -- cgit v1.2.3-freya From 56b19ab6bbec024abe23ce73e078d5c3ca534252 Mon Sep 17 00:00:00 2001 From: Marie Date: Sun, 24 Mar 2024 23:41:23 +0000 Subject: fix: incorrect type for quote property --- packages/backend/src/server/api/mastodon/converters.ts | 2 +- packages/megalodon/src/misskey/api_client.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) (limited to 'packages/backend/src') diff --git a/packages/backend/src/server/api/mastodon/converters.ts b/packages/backend/src/server/api/mastodon/converters.ts index 20fccec21d..ca6f233b7f 100644 --- a/packages/backend/src/server/api/mastodon/converters.ts +++ b/packages/backend/src/server/api/mastodon/converters.ts @@ -278,7 +278,7 @@ export class MastoConverters { reactions: status.emoji_reactions, emoji_reactions: status.emoji_reactions, bookmarked: false, - quote: isQuote ? await this.convertReblog(status.reblog) : false, + quote: isQuote ? await this.convertReblog(status.reblog) : null, edited_at: note.updatedAt?.toISOString(), }); } diff --git a/packages/megalodon/src/misskey/api_client.ts b/packages/megalodon/src/misskey/api_client.ts index 520928c9fe..02828dcf9a 100644 --- a/packages/megalodon/src/misskey/api_client.ts +++ b/packages/megalodon/src/misskey/api_client.ts @@ -303,7 +303,7 @@ namespace MisskeyAPI { pinned: null, emoji_reactions: typeof n.reactions === 'object' ? mapReactions(n.reactions, n.myReaction) : [], bookmarked: false, - quote: n.renote && n.text ? note(n.renote, n.user.host ? n.user.host : host ? host : null) : false + quote: n.renote && n.text ? note(n.renote, n.user.host ? n.user.host : host ? host : null) : null } } -- cgit v1.2.3-freya From dd3d562a1e4ae547ddb0b54d77678abf801f2e26 Mon Sep 17 00:00:00 2001 From: Latte macchiato Date: Fri, 19 Apr 2024 21:58:37 +0000 Subject: Rework cache clearing to be fault tolerant --- .../processors/CleanRemoteFilesProcessorService.ts | 27 ++++++++++++++-------- 1 file changed, 18 insertions(+), 9 deletions(-) (limited to 'packages/backend/src') diff --git a/packages/backend/src/queue/processors/CleanRemoteFilesProcessorService.ts b/packages/backend/src/queue/processors/CleanRemoteFilesProcessorService.ts index 917de8b72c..ec75f3ba01 100644 --- a/packages/backend/src/queue/processors/CleanRemoteFilesProcessorService.ts +++ b/packages/backend/src/queue/processors/CleanRemoteFilesProcessorService.ts @@ -33,6 +33,12 @@ export class CleanRemoteFilesProcessorService { let deletedCount = 0; let cursor: MiDriveFile['id'] | null = null; + let errorCount = 0; + + const total = await this.driveFilesRepository.countBy({ + userHost: Not(IsNull()), + isLink: false, + }); while (true) { const files = await this.driveFilesRepository.find({ @@ -41,7 +47,7 @@ export class CleanRemoteFilesProcessorService { isLink: false, ...(cursor ? { id: MoreThan(cursor) } : {}), }, - take: 8, + take: 256, // Adjust the batch size as needed order: { id: 1, }, @@ -54,18 +60,21 @@ export class CleanRemoteFilesProcessorService { cursor = files.at(-1)?.id ?? null; - await Promise.all(files.map(file => this.driveService.deleteFileSync(file, true))); - - deletedCount += 8; + // Handle deletion in a batch + const results = await Promise.allSettled(files.map(file => this.driveService.deleteFileSync(file, true))); - const total = await this.driveFilesRepository.countBy({ - userHost: Not(IsNull()), - isLink: false, + results.forEach((result, index) => { + if (result.status === 'fulfilled') { + deletedCount++; + } else { + this.logger.error(`Failed to delete file ID ${files[index].id}: ${result.reason}`); + errorCount++; + } }); - job.updateProgress(deletedCount / total); + await job.updateProgress((deletedCount / total) * 100); } - this.logger.succ('All cached remote files has been deleted.'); + this.logger.succ(`All cached remote files processed. Total deleted: ${deletedCount}, Failed: ${errorCount}.`); } } -- cgit v1.2.3-freya From 0f3764ff716802bbed41cc7c608b669a27030ded Mon Sep 17 00:00:00 2001 From: dakkar Date: Tue, 23 Apr 2024 14:42:02 +0100 Subject: teach ReactionService about non-ASCII emoji names --- packages/backend/src/core/ReactionService.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'packages/backend/src') diff --git a/packages/backend/src/core/ReactionService.ts b/packages/backend/src/core/ReactionService.ts index 90586b500e..e70d427e98 100644 --- a/packages/backend/src/core/ReactionService.ts +++ b/packages/backend/src/core/ReactionService.ts @@ -64,8 +64,8 @@ type DecodedReaction = { host?: string | null; }; -const isCustomEmojiRegexp = /^:([\w+-]+)(?:@\.)?:$/; -const decodeCustomEmojiRegexp = /^:([\w+-]+)(?:@([\w.-]+))?:$/; +const isCustomEmojiRegexp = /^:([\p{Letter}\p{Number}\p{Mark}_+-]+)(?:@\.)?:$/; +const decodeCustomEmojiRegexp = /^:([\p{Letter}\p{Number}\p{Mark}_+-]+)(?:@([\w.-]+))?:$/; @Injectable() export class ReactionService { -- cgit v1.2.3-freya From 493775ad7b1ae20d40a3a6b6dd7eb505ced6648a Mon Sep 17 00:00:00 2001 From: PrivateGER Date: Wed, 24 Apr 2024 16:05:30 +0200 Subject: reformat expression --- .../backend/src/queue/processors/CleanRemoteFilesProcessorService.ts | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) (limited to 'packages/backend/src') diff --git a/packages/backend/src/queue/processors/CleanRemoteFilesProcessorService.ts b/packages/backend/src/queue/processors/CleanRemoteFilesProcessorService.ts index ec75f3ba01..4fa414b0b5 100644 --- a/packages/backend/src/queue/processors/CleanRemoteFilesProcessorService.ts +++ b/packages/backend/src/queue/processors/CleanRemoteFilesProcessorService.ts @@ -47,7 +47,7 @@ export class CleanRemoteFilesProcessorService { isLink: false, ...(cursor ? { id: MoreThan(cursor) } : {}), }, - take: 256, // Adjust the batch size as needed + take: 256, order: { id: 1, }, @@ -72,7 +72,8 @@ export class CleanRemoteFilesProcessorService { } }); - await job.updateProgress((deletedCount / total) * 100); + await job.updateProgress(100 / total * deletedCount); + } this.logger.succ(`All cached remote files processed. Total deleted: ${deletedCount}, Failed: ${errorCount}.`); -- cgit v1.2.3-freya From 6ae01e28aa717d54743f1ab44fd099853a969d3d Mon Sep 17 00:00:00 2001 From: dakkar Date: Tue, 30 Apr 2024 10:12:54 +0100 Subject: Compact LD-signed activities against well-known context This should defend against some spoofing attacks, see also https://nvd.nist.gov/vuln/detail/CVE-2022-24307 for Mastodon, https://iceshrimp.dev/iceshrimp/iceshrimp/commit/febb499fcb5fe3d56ca79025e4b5851464660c38 from Iceshrimp and https://firefish.dev/firefish/firefish/-/commit/e790d6be90dfd5dc6471b650a54520761bb9d745 for Firefish Thanks to @tesaguri@fedibird.com for reporting and providing the patch. --- .../src/core/activitypub/ApRendererService.ts | 44 +-------------------- .../src/core/activitypub/LdSignatureService.ts | 10 +++++ .../backend/src/core/activitypub/misc/contexts.ts | 46 +++++++++++++++++++++- .../src/queue/processors/InboxProcessorService.ts | 14 ++++++- 4 files changed, 70 insertions(+), 44 deletions(-) (limited to 'packages/backend/src') diff --git a/packages/backend/src/core/activitypub/ApRendererService.ts b/packages/backend/src/core/activitypub/ApRendererService.ts index a84feffac6..d06bb2d8df 100644 --- a/packages/backend/src/core/activitypub/ApRendererService.ts +++ b/packages/backend/src/core/activitypub/ApRendererService.ts @@ -31,6 +31,7 @@ import { IdService } from '@/core/IdService.js'; import { MetaService } from '../MetaService.js'; import { LdSignatureService } from './LdSignatureService.js'; import { ApMfmService } from './ApMfmService.js'; +import { CONTEXT } from './misc/contexts.js'; import type { IAccept, IActivity, IAdd, IAnnounce, IApDocument, IApEmoji, IApHashtag, IApImage, IApMention, IBlock, ICreate, IDelete, IFlag, IFollow, IKey, ILike, IMove, IObject, IPost, IQuestion, IReject, IRemove, ITombstone, IUndo, IUpdate } from './type.js'; @Injectable() @@ -785,48 +786,7 @@ export class ApRendererService { x.id = `${this.config.url}/${randomUUID()}`; } - return Object.assign({ - '@context': [ - 'https://www.w3.org/ns/activitystreams', - 'https://w3id.org/security/v1', - { - Key: 'sec:Key', - // as non-standards - manuallyApprovesFollowers: 'as:manuallyApprovesFollowers', - sensitive: 'as:sensitive', - Hashtag: 'as:Hashtag', - quoteUrl: 'as:quoteUrl', - fedibird: 'http://fedibird.com/ns#', - quoteUri: 'fedibird:quoteUri', - // Mastodon - toot: 'http://joinmastodon.org/ns#', - Emoji: 'toot:Emoji', - featured: 'toot:featured', - discoverable: 'toot:discoverable', - // schema - schema: 'http://schema.org#', - PropertyValue: 'schema:PropertyValue', - value: 'schema:value', - // Misskey - misskey: 'https://misskey-hub.net/ns#', - '_misskey_content': 'misskey:_misskey_content', - '_misskey_quote': 'misskey:_misskey_quote', - '_misskey_reaction': 'misskey:_misskey_reaction', - '_misskey_votes': 'misskey:_misskey_votes', - '_misskey_summary': 'misskey:_misskey_summary', - 'isCat': 'misskey:isCat', - // Firefish - firefish: 'https://joinfirefish.org/ns#', - speakAsCat: 'firefish:speakAsCat', - // Sharkey - sharkey: 'https://joinsharkey.org/ns#', - backgroundUrl: 'sharkey:backgroundUrl', - listenbrainz: 'sharkey:listenbrainz', - // vcard - vcard: 'http://www.w3.org/2006/vcard/ns#', - }, - ], - }, x as T & { id: string }); + return Object.assign({ '@context': CONTEXT }, x as T & { id: string }); } @bindThis diff --git a/packages/backend/src/core/activitypub/LdSignatureService.ts b/packages/backend/src/core/activitypub/LdSignatureService.ts index 9de184336f..a4add22551 100644 --- a/packages/backend/src/core/activitypub/LdSignatureService.ts +++ b/packages/backend/src/core/activitypub/LdSignatureService.ts @@ -88,6 +88,16 @@ class LdSignature { return verifyData; } + @bindThis + public async compact(data: any, context: any = CONTEXT): Promise { + const customLoader = this.getLoader(); + // XXX: Importing jsonld dynamically since Jest frequently fails to import it statically + // https://github.com/misskey-dev/misskey/pull/9894#discussion_r1103753595 + return (await import('jsonld')).default.compact(data, context, { + documentLoader: customLoader, + }); + } + @bindThis public async normalize(data: JsonLdDocument): Promise { const customLoader = this.getLoader(); diff --git a/packages/backend/src/core/activitypub/misc/contexts.ts b/packages/backend/src/core/activitypub/misc/contexts.ts index 88afdefcd3..4ff114bbf5 100644 --- a/packages/backend/src/core/activitypub/misc/contexts.ts +++ b/packages/backend/src/core/activitypub/misc/contexts.ts @@ -3,7 +3,7 @@ * SPDX-License-Identifier: AGPL-3.0-only */ -import type { JsonLd } from 'jsonld/jsonld-spec.js'; +import type { Context, JsonLd } from 'jsonld/jsonld-spec.js'; /* eslint:disable:quotemark indent */ const id_v1 = { @@ -526,6 +526,50 @@ const activitystreams = { }, } satisfies JsonLd; +const context_iris = [ + 'https://www.w3.org/ns/activitystreams', + 'https://w3id.org/security/v1', +]; + +const extension_context_definition = { + Key: 'sec:Key', + // as non-standards + manuallyApprovesFollowers: 'as:manuallyApprovesFollowers', + sensitive: 'as:sensitive', + Hashtag: 'as:Hashtag', + quoteUrl: 'as:quoteUrl', + fedibird: 'http://fedibird.com/ns#', + quoteUri: 'fedibird:quoteUri', + // Mastodon + toot: 'http://joinmastodon.org/ns#', + Emoji: 'toot:Emoji', + featured: 'toot:featured', + discoverable: 'toot:discoverable', + // schema + schema: 'http://schema.org#', + PropertyValue: 'schema:PropertyValue', + value: 'schema:value', + // Misskey + misskey: 'https://misskey-hub.net/ns#', + '_misskey_content': 'misskey:_misskey_content', + '_misskey_quote': 'misskey:_misskey_quote', + '_misskey_reaction': 'misskey:_misskey_reaction', + '_misskey_votes': 'misskey:_misskey_votes', + '_misskey_summary': 'misskey:_misskey_summary', + 'isCat': 'misskey:isCat', + // Firefish + firefish: 'https://joinfirefish.org/ns#', + speakAsCat: 'firefish:speakAsCat', + // Sharkey + sharkey: 'https://joinsharkey.org/ns#', + backgroundUrl: 'sharkey:backgroundUrl', + listenbrainz: 'sharkey:listenbrainz', + // vcard + vcard: 'http://www.w3.org/2006/vcard/ns#', +} satisfies Context; + +export const CONTEXT: (string | Context)[] = [...context_iris, extension_context_definition]; + export const CONTEXTS: Record = { 'https://w3id.org/identity/v1': id_v1, 'https://w3id.org/security/v1': security_v1, diff --git a/packages/backend/src/queue/processors/InboxProcessorService.ts b/packages/backend/src/queue/processors/InboxProcessorService.ts index ad1d9799a7..2b5b7c5619 100644 --- a/packages/backend/src/queue/processors/InboxProcessorService.ts +++ b/packages/backend/src/queue/processors/InboxProcessorService.ts @@ -15,6 +15,7 @@ import InstanceChart from '@/core/chart/charts/instance.js'; import ApRequestChart from '@/core/chart/charts/ap-request.js'; import FederationChart from '@/core/chart/charts/federation.js'; import { getApId } from '@/core/activitypub/type.js'; +import type { IActivity } from '@/core/activitypub/type.js'; import type { MiRemoteUser } from '@/models/User.js'; import type { MiUserPublickey } from '@/models/UserPublickey.js'; import { ApDbResolverService } from '@/core/activitypub/ApDbResolverService.js'; @@ -52,7 +53,7 @@ export class InboxProcessorService { @bindThis public async process(job: Bull.Job): Promise { const signature = job.data.signature; // HTTP-signature - const activity = job.data.activity; + let activity = job.data.activity; //#region Log const info = Object.assign({}, activity); @@ -150,6 +151,17 @@ export class InboxProcessorService { throw new Bull.UnrecoverableError('skip: LD-Signatureの検証に失敗しました'); } + // アクティビティを正規化 + delete activity.signature; + try { + activity = await ldSignature.compact(activity) as IActivity; + } catch (e) { + throw new Bull.UnrecoverableError(`skip: failed to compact activity: ${e}`); + } + // TODO: 元のアクティビティと非互換な形に正規化される場合は転送をスキップする + // https://github.com/mastodon/mastodon/blob/664b0ca/app/services/activitypub/process_collection_service.rb#L24-L29 + activity.signature = ldSignature; + // もう一度actorチェック if (authUser.user.uri !== activity.actor) { throw new Bull.UnrecoverableError(`skip: LD-Signature user(${authUser.user.uri}) !== activity.actor(${activity.actor})`); -- cgit v1.2.3-freya From 45182c17e2e8a103a0cf39fde4c30fec9c5f6cc6 Mon Sep 17 00:00:00 2001 From: dakkar Date: Wed, 1 May 2024 17:41:33 +0100 Subject: fix imports --- packages/backend/src/core/activitypub/LdSignatureService.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'packages/backend/src') diff --git a/packages/backend/src/core/activitypub/LdSignatureService.ts b/packages/backend/src/core/activitypub/LdSignatureService.ts index a4add22551..2ec6dc4585 100644 --- a/packages/backend/src/core/activitypub/LdSignatureService.ts +++ b/packages/backend/src/core/activitypub/LdSignatureService.ts @@ -7,7 +7,7 @@ import * as crypto from 'node:crypto'; import { Injectable } from '@nestjs/common'; import { HttpRequestService } from '@/core/HttpRequestService.js'; import { bindThis } from '@/decorators.js'; -import { CONTEXTS } from './misc/contexts.js'; +import { CONTEXT, CONTEXTS } from './misc/contexts.js'; import { validateContentTypeSetAsJsonLD } from './misc/validator.js'; import type { JsonLdDocument } from 'jsonld'; import type { JsonLd, RemoteDocument } from 'jsonld/jsonld-spec.js'; -- cgit v1.2.3-freya From e333283905d2fd25f2fcfbe449d4fa05bd85d09c Mon Sep 17 00:00:00 2001 From: ShittyKopper Date: Thu, 2 May 2024 02:37:59 +0300 Subject: Send default reactions as Like activities to Iceshrimp.NET instances --- packages/backend/src/core/activitypub/ApRendererService.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'packages/backend/src') diff --git a/packages/backend/src/core/activitypub/ApRendererService.ts b/packages/backend/src/core/activitypub/ApRendererService.ts index a84feffac6..bdf92edb2e 100644 --- a/packages/backend/src/core/activitypub/ApRendererService.ts +++ b/packages/backend/src/core/activitypub/ApRendererService.ts @@ -283,9 +283,10 @@ export class ApRendererService { if (instance && instance.softwareName === 'mastodon') isMastodon = true; if (instance && instance.softwareName === 'akkoma') isMastodon = true; if (instance && instance.softwareName === 'pleroma') isMastodon = true; + if (instance && instance.softwareName === 'iceshrimp.net') isMastodon = true; } } - + const object: ILike = { type: 'Like', id: `${this.config.url}/likes/${noteReaction.id}`, -- cgit v1.2.3-freya From f843bf6c17bec84ec4e5a114be67d98b0e2e404c Mon Sep 17 00:00:00 2001 From: Essem Date: Fri, 3 May 2024 11:46:22 -0500 Subject: fix: Add unicode flag to custom emoji regexes --- packages/backend/src/core/ReactionService.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'packages/backend/src') diff --git a/packages/backend/src/core/ReactionService.ts b/packages/backend/src/core/ReactionService.ts index e70d427e98..c0b59e635d 100644 --- a/packages/backend/src/core/ReactionService.ts +++ b/packages/backend/src/core/ReactionService.ts @@ -64,8 +64,8 @@ type DecodedReaction = { host?: string | null; }; -const isCustomEmojiRegexp = /^:([\p{Letter}\p{Number}\p{Mark}_+-]+)(?:@\.)?:$/; -const decodeCustomEmojiRegexp = /^:([\p{Letter}\p{Number}\p{Mark}_+-]+)(?:@([\w.-]+))?:$/; +const isCustomEmojiRegexp = /^:([\p{Letter}\p{Number}\p{Mark}_+-]+)(?:@\.)?:$/u; +const decodeCustomEmojiRegexp = /^:([\p{Letter}\p{Number}\p{Mark}_+-]+)(?:@([\w.-]+))?:$/u; @Injectable() export class ReactionService { -- cgit v1.2.3-freya From eab690a5e380432198634de770ddbd9aeabc0563 Mon Sep 17 00:00:00 2001 From: dakkar Date: Tue, 7 May 2024 20:16:38 +0000 Subject: really edit notes in more cases - fixes #424 --- packages/backend/src/core/NoteEditService.ts | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) (limited to 'packages/backend/src') diff --git a/packages/backend/src/core/NoteEditService.ts b/packages/backend/src/core/NoteEditService.ts index 72fc01ae3b..a01dfec664 100644 --- a/packages/backend/src/core/NoteEditService.ts +++ b/packages/backend/src/core/NoteEditService.ts @@ -430,11 +430,16 @@ export class NoteEditService implements OnApplicationShutdown { update.hasPoll = !!data.poll; } + // technically we should check if the two sets of files are + // different, or if their descriptions have changed. In practice + // this is good enough. + const filesChanged = oldnote.fileIds?.length || data.files?.length; + const poll = await this.pollsRepository.findOneBy({ noteId: oldnote.id }); const oldPoll = poll ? { choices: poll.choices, multiple: poll.multiple, expiresAt: poll.expiresAt } : null; - if (Object.keys(update).length > 0) { + if (Object.keys(update).length > 0 || filesChanged) { const exists = await this.noteEditRepository.findOneBy({ noteId: oldnote.id }); await this.noteEditRepository.insert({ -- cgit v1.2.3-freya From 89f4f0e5f4df75552d2aaf111a56f8cc0f796e2c Mon Sep 17 00:00:00 2001 From: dakkar Date: Tue, 7 May 2024 20:17:53 +0000 Subject: don't count "system" local accounts in user chart - fixes #451 --- packages/backend/src/core/chart/charts/users.ts | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) (limited to 'packages/backend/src') diff --git a/packages/backend/src/core/chart/charts/users.ts b/packages/backend/src/core/chart/charts/users.ts index d148fc629b..840522ae9b 100644 --- a/packages/backend/src/core/chart/charts/users.ts +++ b/packages/backend/src/core/chart/charts/users.ts @@ -4,7 +4,7 @@ */ import { Injectable, Inject } from '@nestjs/common'; -import { Not, IsNull, DataSource } from 'typeorm'; +import { Not, IsNull, Like, DataSource } from 'typeorm'; import type { MiUser } from '@/models/User.js'; import { AppLockService } from '@/core/AppLockService.js'; import { DI } from '@/di-symbols.js'; @@ -37,7 +37,10 @@ export default class UsersChart extends Chart { // eslint-disable protected async tickMajor(): Promise>> { const [localCount, remoteCount] = await Promise.all([ - this.usersRepository.countBy({ host: IsNull() }), + // that Not(Like()) is ugly, but it matches the logic in + // packages/backend/src/models/User.ts to not count "system" + // accounts + this.usersRepository.countBy({ host: IsNull(), username: Not(Like('%.%')) }), this.usersRepository.countBy({ host: Not(IsNull()) }), ]); -- cgit v1.2.3-freya