From 354cb2a6754b55fd3ad01388a4a17d3a76d4a09b Mon Sep 17 00:00:00 2001 From: dakkar Date: Sat, 9 Mar 2024 12:17:48 +0000 Subject: handle non-ASCII emoji names MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * use the more inclusive regexp for validating emoji names * always normalize emoji names, aliases, categories the latter point is necessary to allow matching, for example, `ä` against `a`+combining diaeresis this will also need to bump the version of `sfm-js` once we merge https://activitypub.software/TransFem-org/sfm-js/-/merge_requests/2 --- .../api/endpoints/admin/emoji/add-aliases-bulk.ts | 2 +- .../backend/src/server/api/endpoints/admin/emoji/add.ts | 11 ++++++----- .../src/server/api/endpoints/admin/emoji/copy.ts | 9 +++++---- .../src/server/api/endpoints/admin/emoji/list-remote.ts | 2 +- .../src/server/api/endpoints/admin/emoji/list.ts | 11 ++++++----- .../api/endpoints/admin/emoji/remove-aliases-bulk.ts | 2 +- .../api/endpoints/admin/emoji/set-aliases-bulk.ts | 2 +- .../api/endpoints/admin/emoji/set-category-bulk.ts | 2 +- .../src/server/api/endpoints/admin/emoji/update.ts | 17 +++++++++-------- 9 files changed, 31 insertions(+), 27 deletions(-) (limited to 'packages/backend/src/server/api') diff --git a/packages/backend/src/server/api/endpoints/admin/emoji/add-aliases-bulk.ts b/packages/backend/src/server/api/endpoints/admin/emoji/add-aliases-bulk.ts index a30a080e59..f4fc79bdb3 100644 --- a/packages/backend/src/server/api/endpoints/admin/emoji/add-aliases-bulk.ts +++ b/packages/backend/src/server/api/endpoints/admin/emoji/add-aliases-bulk.ts @@ -34,7 +34,7 @@ export default class extends Endpoint { // eslint- private customEmojiService: CustomEmojiService, ) { super(meta, paramDef, async (ps, me) => { - await this.customEmojiService.addAliasesBulk(ps.ids, ps.aliases); + await this.customEmojiService.addAliasesBulk(ps.ids, ps.aliases.map(a => a.normalize('NFC'))); }); } } diff --git a/packages/backend/src/server/api/endpoints/admin/emoji/add.ts b/packages/backend/src/server/api/endpoints/admin/emoji/add.ts index 767e517b80..b45a3c7156 100644 --- a/packages/backend/src/server/api/endpoints/admin/emoji/add.ts +++ b/packages/backend/src/server/api/endpoints/admin/emoji/add.ts @@ -40,7 +40,7 @@ export const meta = { export const paramDef = { type: 'object', properties: { - name: { type: 'string', pattern: '^[a-zA-Z0-9_]+$' }, + name: { type: 'string', pattern: '^[\\p{Letter}\\p{Number}\\p{Mark}_+-]+$' }, fileId: { type: 'string', format: 'misskey:id' }, category: { type: 'string', @@ -73,18 +73,19 @@ export default class extends Endpoint { // eslint- private emojiEntityService: EmojiEntityService, ) { super(meta, paramDef, async (ps, me) => { + const nameNfc = ps.name.normalize('NFC'); const driveFile = await this.driveFilesRepository.findOneBy({ id: ps.fileId }); if (driveFile == null) throw new ApiError(meta.errors.noSuchFile); - const isDuplicate = await this.customEmojiService.checkDuplicate(ps.name); + const isDuplicate = await this.customEmojiService.checkDuplicate(nameNfc); if (isDuplicate) throw new ApiError(meta.errors.duplicateName); if (driveFile.user !== null) await this.driveFilesRepository.update(driveFile.id, { user: null }); const emoji = await this.customEmojiService.add({ driveFile, - name: ps.name, - category: ps.category ?? null, - aliases: ps.aliases ?? [], + name: nameNfc, + category: ps.category?.normalize('NFC') ?? null, + aliases: ps.aliases?.map(a => a.normalize('NFC')) ?? [], host: null, license: ps.license ?? null, isSensitive: ps.isSensitive ?? false, diff --git a/packages/backend/src/server/api/endpoints/admin/emoji/copy.ts b/packages/backend/src/server/api/endpoints/admin/emoji/copy.ts index 29af7598ed..f968813197 100644 --- a/packages/backend/src/server/api/endpoints/admin/emoji/copy.ts +++ b/packages/backend/src/server/api/endpoints/admin/emoji/copy.ts @@ -82,15 +82,16 @@ export default class extends Endpoint { // eslint- throw new ApiError(); } + const nameNfc = emoji.name.normalize('NFC'); // Duplication Check - const isDuplicate = await this.customEmojiService.checkDuplicate(emoji.name); + const isDuplicate = await this.customEmojiService.checkDuplicate(nameNfc); if (isDuplicate) throw new ApiError(meta.errors.duplicateName); const addedEmoji = await this.customEmojiService.add({ driveFile, - name: emoji.name, - category: emoji.category, - aliases: emoji.aliases, + name: nameNfc, + category: emoji.category?.normalize('NFC'), + aliases: emoji.aliases?.map(a => a.normalize('NFC')), host: null, license: emoji.license, isSensitive: emoji.isSensitive, diff --git a/packages/backend/src/server/api/endpoints/admin/emoji/list-remote.ts b/packages/backend/src/server/api/endpoints/admin/emoji/list-remote.ts index e423f440d0..1182918ea2 100644 --- a/packages/backend/src/server/api/endpoints/admin/emoji/list-remote.ts +++ b/packages/backend/src/server/api/endpoints/admin/emoji/list-remote.ts @@ -98,7 +98,7 @@ export default class extends Endpoint { // eslint- } if (ps.query) { - q.andWhere('emoji.name like :query', { query: '%' + sqlLikeEscape(ps.query) + '%' }) + q.andWhere('emoji.name like :query', { query: '%' + sqlLikeEscape(ps.query.normalize('NFC')) + '%' }) .orderBy('length(emoji.name)', 'ASC'); } diff --git a/packages/backend/src/server/api/endpoints/admin/emoji/list.ts b/packages/backend/src/server/api/endpoints/admin/emoji/list.ts index 53810d1d16..5e21111f9f 100644 --- a/packages/backend/src/server/api/endpoints/admin/emoji/list.ts +++ b/packages/backend/src/server/api/endpoints/admin/emoji/list.ts @@ -92,17 +92,18 @@ export default class extends Endpoint { // eslint- //const emojis = await q.limit(ps.limit).getMany(); emojis = await q.orderBy('length(emoji.name)', 'ASC').getMany(); - const queryarry = ps.query.match(/\:([a-z0-9_]*)\:/g); + const queryarry = ps.query.match(/:([\p{Letter}\p{Number}\p{Mark}_+-]*):/ug); if (queryarry) { emojis = emojis.filter(emoji => - queryarry.includes(`:${emoji.name}:`), + queryarry.includes(`:${emoji.name.normalize('NFC')}:`), ); } else { + const queryNfc = ps.query!.normalize('NFC'); emojis = emojis.filter(emoji => - emoji.name.includes(ps.query!) || - emoji.aliases.some(a => a.includes(ps.query!)) || - emoji.category?.includes(ps.query!)); + emoji.name.includes(queryNfc) || + emoji.aliases.some(a => a.includes(queryNfc)) || + emoji.category?.includes(queryNfc)); } emojis.splice(ps.limit + 1); } else { diff --git a/packages/backend/src/server/api/endpoints/admin/emoji/remove-aliases-bulk.ts b/packages/backend/src/server/api/endpoints/admin/emoji/remove-aliases-bulk.ts index 0fa119eabe..e78620eac1 100644 --- a/packages/backend/src/server/api/endpoints/admin/emoji/remove-aliases-bulk.ts +++ b/packages/backend/src/server/api/endpoints/admin/emoji/remove-aliases-bulk.ts @@ -34,7 +34,7 @@ export default class extends Endpoint { // eslint- private customEmojiService: CustomEmojiService, ) { super(meta, paramDef, async (ps, me) => { - await this.customEmojiService.removeAliasesBulk(ps.ids, ps.aliases); + await this.customEmojiService.removeAliasesBulk(ps.ids, ps.aliases.map(a => a.normalize('NFC'))); }); } } diff --git a/packages/backend/src/server/api/endpoints/admin/emoji/set-aliases-bulk.ts b/packages/backend/src/server/api/endpoints/admin/emoji/set-aliases-bulk.ts index d9ee18699c..69fc8e0cb5 100644 --- a/packages/backend/src/server/api/endpoints/admin/emoji/set-aliases-bulk.ts +++ b/packages/backend/src/server/api/endpoints/admin/emoji/set-aliases-bulk.ts @@ -34,7 +34,7 @@ export default class extends Endpoint { // eslint- private customEmojiService: CustomEmojiService, ) { super(meta, paramDef, async (ps, me) => { - await this.customEmojiService.setAliasesBulk(ps.ids, ps.aliases); + await this.customEmojiService.setAliasesBulk(ps.ids, ps.aliases.map(a => a.normalize('NFC'))); }); } } diff --git a/packages/backend/src/server/api/endpoints/admin/emoji/set-category-bulk.ts b/packages/backend/src/server/api/endpoints/admin/emoji/set-category-bulk.ts index dc25df2767..679a9f9c42 100644 --- a/packages/backend/src/server/api/endpoints/admin/emoji/set-category-bulk.ts +++ b/packages/backend/src/server/api/endpoints/admin/emoji/set-category-bulk.ts @@ -36,7 +36,7 @@ export default class extends Endpoint { // eslint- private customEmojiService: CustomEmojiService, ) { super(meta, paramDef, async (ps, me) => { - await this.customEmojiService.setCategoryBulk(ps.ids, ps.category ?? null); + await this.customEmojiService.setCategoryBulk(ps.ids, ps.category?.normalize('NFC') ?? null); }); } } diff --git a/packages/backend/src/server/api/endpoints/admin/emoji/update.ts b/packages/backend/src/server/api/endpoints/admin/emoji/update.ts index 22609a16a3..3caa0f84a3 100644 --- a/packages/backend/src/server/api/endpoints/admin/emoji/update.ts +++ b/packages/backend/src/server/api/endpoints/admin/emoji/update.ts @@ -40,7 +40,7 @@ export const paramDef = { type: 'object', properties: { id: { type: 'string', format: 'misskey:id' }, - name: { type: 'string', pattern: '^[a-zA-Z0-9_]+$' }, + name: { type: 'string', pattern: '^[\\p{Letter}\\p{Number}\\p{Mark}_+-]+$' }, fileId: { type: 'string', format: 'misskey:id' }, category: { type: 'string', @@ -72,6 +72,7 @@ export default class extends Endpoint { // eslint- private customEmojiService: CustomEmojiService, ) { super(meta, paramDef, async (ps, me) => { + const nameNfc = ps.name?.normalize('NFC'); let driveFile; if (ps.fileId) { driveFile = await this.driveFilesRepository.findOneBy({ id: ps.fileId }); @@ -83,22 +84,22 @@ export default class extends Endpoint { // eslint- emojiId = ps.id; const emoji = await this.customEmojiService.getEmojiById(ps.id); if (!emoji) throw new ApiError(meta.errors.noSuchEmoji); - if (ps.name && (ps.name !== emoji.name)) { - const isDuplicate = await this.customEmojiService.checkDuplicate(ps.name); + if (nameNfc && (nameNfc !== emoji.name)) { + const isDuplicate = await this.customEmojiService.checkDuplicate(nameNfc); if (isDuplicate) throw new ApiError(meta.errors.sameNameEmojiExists); } } else { - if (!ps.name) throw new Error('Invalid Params unexpectedly passed. This is a BUG. Please report it to the development team.'); - const emoji = await this.customEmojiService.getEmojiByName(ps.name); + if (!nameNfc) throw new Error('Invalid Params unexpectedly passed. This is a BUG. Please report it to the development team.'); + const emoji = await this.customEmojiService.getEmojiByName(nameNfc); if (!emoji) throw new ApiError(meta.errors.noSuchEmoji); emojiId = emoji.id; } await this.customEmojiService.update(emojiId, { driveFile, - name: ps.name, - category: ps.category, - aliases: ps.aliases, + name: nameNfc, + category: ps.category?.normalize('NFC'), + aliases: ps.aliases?.map(a => a.normalize('NFC')), license: ps.license, isSensitive: ps.isSensitive, localOnly: ps.localOnly, -- cgit v1.2.3-freya From 56b19ab6bbec024abe23ce73e078d5c3ca534252 Mon Sep 17 00:00:00 2001 From: Marie Date: Sun, 24 Mar 2024 23:41:23 +0000 Subject: fix: incorrect type for quote property --- packages/backend/src/server/api/mastodon/converters.ts | 2 +- packages/megalodon/src/misskey/api_client.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) (limited to 'packages/backend/src/server/api') diff --git a/packages/backend/src/server/api/mastodon/converters.ts b/packages/backend/src/server/api/mastodon/converters.ts index 20fccec21d..ca6f233b7f 100644 --- a/packages/backend/src/server/api/mastodon/converters.ts +++ b/packages/backend/src/server/api/mastodon/converters.ts @@ -278,7 +278,7 @@ export class MastoConverters { reactions: status.emoji_reactions, emoji_reactions: status.emoji_reactions, bookmarked: false, - quote: isQuote ? await this.convertReblog(status.reblog) : false, + quote: isQuote ? await this.convertReblog(status.reblog) : null, edited_at: note.updatedAt?.toISOString(), }); } diff --git a/packages/megalodon/src/misskey/api_client.ts b/packages/megalodon/src/misskey/api_client.ts index 520928c9fe..02828dcf9a 100644 --- a/packages/megalodon/src/misskey/api_client.ts +++ b/packages/megalodon/src/misskey/api_client.ts @@ -303,7 +303,7 @@ namespace MisskeyAPI { pinned: null, emoji_reactions: typeof n.reactions === 'object' ? mapReactions(n.reactions, n.myReaction) : [], bookmarked: false, - quote: n.renote && n.text ? note(n.renote, n.user.host ? n.user.host : host ? host : null) : false + quote: n.renote && n.text ? note(n.renote, n.user.host ? n.user.host : host ? host : null) : null } } -- cgit v1.2.3-freya From e779c1e6674230c13db1784dcc3350aa21f9f5ca Mon Sep 17 00:00:00 2001 From: Sugar🍬🍭🏳️‍⚧ Date: Mon, 1 Apr 2024 20:17:53 +0200 Subject: fix: send null for empty edited_at in mastodon api --- packages/backend/src/server/api/mastodon/converters.ts | 5 +++-- packages/megalodon/src/entities/status.ts | 1 + packages/megalodon/src/friendica/api_client.ts | 1 + packages/megalodon/src/friendica/entities/status.ts | 1 + packages/megalodon/src/mastodon/api_client.ts | 1 + packages/megalodon/src/mastodon/entities/status.ts | 1 + packages/megalodon/src/misskey/api_client.ts | 1 + packages/megalodon/src/misskey/entities/note.ts | 1 + packages/megalodon/src/pleroma/api_client.ts | 1 + packages/megalodon/src/pleroma/entities/status.ts | 1 + packages/megalodon/test/integration/mastodon/api_client.spec.ts | 1 + packages/megalodon/test/unit/parser.spec.ts | 1 + packages/megalodon/test/unit/webo_socket.spec.ts | 1 + 13 files changed, 15 insertions(+), 2 deletions(-) (limited to 'packages/backend/src/server/api') diff --git a/packages/backend/src/server/api/mastodon/converters.ts b/packages/backend/src/server/api/mastodon/converters.ts index 20fccec21d..60e25c0dfd 100644 --- a/packages/backend/src/server/api/mastodon/converters.ts +++ b/packages/backend/src/server/api/mastodon/converters.ts @@ -83,7 +83,7 @@ export class MastoConverters { } return 'unknown'; } - + public encodeFile(f: any): Entity.Attachment { return { id: f.id, @@ -279,7 +279,8 @@ export class MastoConverters { emoji_reactions: status.emoji_reactions, bookmarked: false, quote: isQuote ? await this.convertReblog(status.reblog) : false, - edited_at: note.updatedAt?.toISOString(), + // optional chaining cannot be used, as it evaluates to undefined, not null + edited_at: note.updatedAt ? note.updatedAt.toISOString() : null, }); } } diff --git a/packages/megalodon/src/entities/status.ts b/packages/megalodon/src/entities/status.ts index da36a04717..a38e1ea9f7 100644 --- a/packages/megalodon/src/entities/status.ts +++ b/packages/megalodon/src/entities/status.ts @@ -19,6 +19,7 @@ namespace Entity { content: string plain_content?: string | null created_at: string + edited_at: string | null emojis: Emoji[] replies_count: number reblogs_count: number diff --git a/packages/megalodon/src/friendica/api_client.ts b/packages/megalodon/src/friendica/api_client.ts index 1f27fd6d20..b0d3399784 100644 --- a/packages/megalodon/src/friendica/api_client.ts +++ b/packages/megalodon/src/friendica/api_client.ts @@ -725,6 +725,7 @@ namespace FriendicaAPI { content: s.content, plain_content: null, created_at: s.created_at, + edited_at: s.edited_at || null, emojis: Array.isArray(s.emojis) ? s.emojis.map(e => emoji(e)) : [], replies_count: s.replies_count, reblogs_count: s.reblogs_count, diff --git a/packages/megalodon/src/friendica/entities/status.ts b/packages/megalodon/src/friendica/entities/status.ts index 9d2e8b9666..014da84ee1 100644 --- a/packages/megalodon/src/friendica/entities/status.ts +++ b/packages/megalodon/src/friendica/entities/status.ts @@ -17,6 +17,7 @@ namespace FriendicaEntity { reblog: Status | null content: string created_at: string + edited_at?: string | null emojis: Emoji[] replies_count: number reblogs_count: number diff --git a/packages/megalodon/src/mastodon/api_client.ts b/packages/megalodon/src/mastodon/api_client.ts index 58f56d1013..ba4bd36ead 100644 --- a/packages/megalodon/src/mastodon/api_client.ts +++ b/packages/megalodon/src/mastodon/api_client.ts @@ -628,6 +628,7 @@ namespace MastodonAPI { content: s.content, plain_content: null, created_at: s.created_at, + edited_at: s.edited_at || null, emojis: Array.isArray(s.emojis) ? s.emojis.map(e => emoji(e)) : [], replies_count: s.replies_count, reblogs_count: s.reblogs_count, diff --git a/packages/megalodon/src/mastodon/entities/status.ts b/packages/megalodon/src/mastodon/entities/status.ts index 9624e9c72a..54b5d3bfe3 100644 --- a/packages/megalodon/src/mastodon/entities/status.ts +++ b/packages/megalodon/src/mastodon/entities/status.ts @@ -18,6 +18,7 @@ namespace MastodonEntity { reblog: Status | null content: string created_at: string + edited_at?: string | null emojis: Emoji[] replies_count: number reblogs_count: number diff --git a/packages/megalodon/src/misskey/api_client.ts b/packages/megalodon/src/misskey/api_client.ts index 520928c9fe..0591691110 100644 --- a/packages/megalodon/src/misskey/api_client.ts +++ b/packages/megalodon/src/misskey/api_client.ts @@ -283,6 +283,7 @@ namespace MisskeyAPI { : '', plain_content: n.text ? n.text : null, created_at: n.createdAt, + edited_at: n.updatedAt || null, emojis: mapEmojis(n.emojis).concat(mapReactionEmojis(n.reactionEmojis)), replies_count: n.repliesCount, reblogs_count: n.renoteCount, diff --git a/packages/megalodon/src/misskey/entities/note.ts b/packages/megalodon/src/misskey/entities/note.ts index 08c5f10aea..a7b208b153 100644 --- a/packages/megalodon/src/misskey/entities/note.ts +++ b/packages/megalodon/src/misskey/entities/note.ts @@ -7,6 +7,7 @@ namespace MisskeyEntity { export type Note = { id: string createdAt: string + updatedAt?: string | null userId: string user: User text: string | null diff --git a/packages/megalodon/src/pleroma/api_client.ts b/packages/megalodon/src/pleroma/api_client.ts index 99d964353e..c20350b67c 100644 --- a/packages/megalodon/src/pleroma/api_client.ts +++ b/packages/megalodon/src/pleroma/api_client.ts @@ -357,6 +357,7 @@ namespace PleromaAPI { content: s.content, plain_content: s.pleroma.content?.['text/plain'] ? s.pleroma.content['text/plain'] : null, created_at: s.created_at, + edited_at: s.edited_at || null, emojis: Array.isArray(s.emojis) ? s.emojis.map(e => emoji(e)) : [], replies_count: s.replies_count, reblogs_count: s.reblogs_count, diff --git a/packages/megalodon/src/pleroma/entities/status.ts b/packages/megalodon/src/pleroma/entities/status.ts index 1949ec954c..7c2b887e53 100644 --- a/packages/megalodon/src/pleroma/entities/status.ts +++ b/packages/megalodon/src/pleroma/entities/status.ts @@ -18,6 +18,7 @@ namespace PleromaEntity { reblog: Status | null content: string created_at: string + edited_at?: string | null emojis: Emoji[] replies_count: number reblogs_count: number diff --git a/packages/megalodon/test/integration/mastodon/api_client.spec.ts b/packages/megalodon/test/integration/mastodon/api_client.spec.ts index 950105152c..51caf4e227 100644 --- a/packages/megalodon/test/integration/mastodon/api_client.spec.ts +++ b/packages/megalodon/test/integration/mastodon/api_client.spec.ts @@ -49,6 +49,7 @@ const status: Entity.Status = { content: 'hoge', plain_content: null, created_at: '2019-03-26T21:40:32', + edited_at: null, emojis: [], replies_count: 0, reblogs_count: 0, diff --git a/packages/megalodon/test/unit/parser.spec.ts b/packages/megalodon/test/unit/parser.spec.ts index 94c1d98029..74264552d6 100644 --- a/packages/megalodon/test/unit/parser.spec.ts +++ b/packages/megalodon/test/unit/parser.spec.ts @@ -38,6 +38,7 @@ const status: Entity.Status = { content: 'hoge', plain_content: 'hoge', created_at: '2019-03-26T21:40:32', + edited_at: null, emojis: [], replies_count: 0, reblogs_count: 0, diff --git a/packages/megalodon/test/unit/webo_socket.spec.ts b/packages/megalodon/test/unit/webo_socket.spec.ts index bb9f997a57..b3b684efb4 100644 --- a/packages/megalodon/test/unit/webo_socket.spec.ts +++ b/packages/megalodon/test/unit/webo_socket.spec.ts @@ -37,6 +37,7 @@ const status: Entity.Status = { content: 'hoge', plain_content: 'hoge', created_at: '2019-03-26T21:40:32', + edited_at: null, emojis: [], replies_count: 0, reblogs_count: 0, -- cgit v1.2.3-freya