From 2c6f25b710b4f8095458fe88ddd56e6c6a41d006 Mon Sep 17 00:00:00 2001 From: tamaina Date: Sun, 25 Feb 2024 12:36:10 +0900 Subject: fix: 古いキャッシュを使うのを修正 (#13453) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- packages/backend/src/queue/processors/RelationshipProcessorService.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'packages/backend/src/queue/processors') diff --git a/packages/backend/src/queue/processors/RelationshipProcessorService.ts b/packages/backend/src/queue/processors/RelationshipProcessorService.ts index 53dbb42169..408b02fb38 100644 --- a/packages/backend/src/queue/processors/RelationshipProcessorService.ts +++ b/packages/backend/src/queue/processors/RelationshipProcessorService.ts @@ -35,7 +35,7 @@ export class RelationshipProcessorService { @bindThis public async processFollow(job: Bull.Job): Promise { this.logger.info(`${job.data.from.id} is trying to follow ${job.data.to.id} ${job.data.withReplies ? "with replies" : "without replies"}`); - await this.userFollowingService.followByThinUser(job.data.from, job.data.to, { + await this.userFollowingService.follow(job.data.from, job.data.to, { requestId: job.data.requestId, silent: job.data.silent, withReplies: job.data.withReplies, -- cgit v1.2.3-freya From b7d9d1620161a728e34ab20d8ea99160b3eb4196 Mon Sep 17 00:00:00 2001 From: okayurisotto <47853651+okayurisotto@users.noreply.github.com> Date: Wed, 28 Feb 2024 15:34:58 +0900 Subject: refactor(backend): ノートのエクスポート処理でStreams APIを使うように (#13465) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * refactor(backend): ノートのエクスポート処理でStreams APIを使うように * fixup! refactor(backend): ノートのエクスポート処理でStreams APIを使うように `await`忘れにより、ジョブがすぐに完了したことになり削除されてしまっていた。 それによって、`NoteStream`内での`updateProgress`メソッドの呼び出しで、`Missing key for job`のエラーが発生することがあった。 --------- Co-authored-by: syuilo <4439005+syuilo@users.noreply.github.com> --- packages/backend/src/misc/FileWriterStream.ts | 31 ++++ packages/backend/src/misc/JsonArrayStream.ts | 30 ++++ .../processors/ExportNotesProcessorService.ts | 164 +++++++++++---------- 3 files changed, 146 insertions(+), 79 deletions(-) create mode 100644 packages/backend/src/misc/FileWriterStream.ts create mode 100644 packages/backend/src/misc/JsonArrayStream.ts (limited to 'packages/backend/src/queue/processors') diff --git a/packages/backend/src/misc/FileWriterStream.ts b/packages/backend/src/misc/FileWriterStream.ts new file mode 100644 index 0000000000..828851df0e --- /dev/null +++ b/packages/backend/src/misc/FileWriterStream.ts @@ -0,0 +1,31 @@ +import * as fs from 'node:fs/promises'; +import type { PathLike } from 'node:fs'; + +/** + * `fs.createWriteStream()`相当のことを行う`WritableStream` (Web標準) + */ +export class FileWriterStream extends WritableStream { + constructor(path: PathLike) { + let file: fs.FileHandle | null = null; + + super({ + start: async () => { + file = await fs.open(path, 'a'); + }, + write: async (chunk, controller) => { + if (file === null) { + controller.error(); + throw new Error(); + } + + await file.write(chunk); + }, + close: async () => { + await file?.close(); + }, + abort: async () => { + await file?.close(); + }, + }); + } +} diff --git a/packages/backend/src/misc/JsonArrayStream.ts b/packages/backend/src/misc/JsonArrayStream.ts new file mode 100644 index 0000000000..ad35bb3a79 --- /dev/null +++ b/packages/backend/src/misc/JsonArrayStream.ts @@ -0,0 +1,30 @@ +import { TransformStream } from 'node:stream/web'; + +/** + * ストリームに流れてきた各データについて`JSON.stringify()`した上で、それらを一つの配列にまとめる + */ +export class JsonArrayStream extends TransformStream { + constructor() { + /** 最初の要素かどうかを変数に記録 */ + let isFirst = true; + + super({ + start(controller) { + controller.enqueue('['); + }, + flush(controller) { + controller.enqueue(']'); + }, + transform(chunk, controller) { + if (isFirst) { + isFirst = false; + } else { + // 妥当なJSON配列にするためには最初以外の要素の前に`,`を挿入しなければならない + controller.enqueue(',\n'); + } + + controller.enqueue(JSON.stringify(chunk)); + }, + }); + } +} diff --git a/packages/backend/src/queue/processors/ExportNotesProcessorService.ts b/packages/backend/src/queue/processors/ExportNotesProcessorService.ts index f2ae0ce4b4..c7611012d7 100644 --- a/packages/backend/src/queue/processors/ExportNotesProcessorService.ts +++ b/packages/backend/src/queue/processors/ExportNotesProcessorService.ts @@ -3,7 +3,7 @@ * SPDX-License-Identifier: AGPL-3.0-only */ -import * as fs from 'node:fs'; +import { ReadableStream, TextEncoderStream } from 'node:stream/web'; import { Inject, Injectable } from '@nestjs/common'; import { MoreThan } from 'typeorm'; import { format as dateFormat } from 'date-fns'; @@ -18,10 +18,82 @@ import { bindThis } from '@/decorators.js'; import { DriveFileEntityService } from '@/core/entities/DriveFileEntityService.js'; import { Packed } from '@/misc/json-schema.js'; import { IdService } from '@/core/IdService.js'; +import { JsonArrayStream } from '@/misc/JsonArrayStream.js'; +import { FileWriterStream } from '@/misc/FileWriterStream.js'; import { QueueLoggerService } from '../QueueLoggerService.js'; import type * as Bull from 'bullmq'; import type { DbJobDataWithUser } from '../types.js'; +class NoteStream extends ReadableStream> { + constructor( + job: Bull.Job, + notesRepository: NotesRepository, + pollsRepository: PollsRepository, + driveFileEntityService: DriveFileEntityService, + idService: IdService, + userId: string, + ) { + let exportedNotesCount = 0; + let cursor: MiNote['id'] | null = null; + + const serialize = ( + note: MiNote, + poll: MiPoll | null, + files: Packed<'DriveFile'>[], + ): Record => { + return { + id: note.id, + text: note.text, + createdAt: idService.parse(note.id).date.toISOString(), + fileIds: note.fileIds, + files: files, + replyId: note.replyId, + renoteId: note.renoteId, + poll: poll, + cw: note.cw, + visibility: note.visibility, + visibleUserIds: note.visibleUserIds, + localOnly: note.localOnly, + reactionAcceptance: note.reactionAcceptance, + }; + }; + + super({ + async pull(controller): Promise { + const notes = await notesRepository.find({ + where: { + userId, + ...(cursor !== null ? { id: MoreThan(cursor) } : {}), + }, + take: 100, // 100件ずつ取得 + order: { id: 1 }, + }); + + if (notes.length === 0) { + job.updateProgress(100); + controller.close(); + } + + cursor = notes.at(-1)?.id ?? null; + + for (const note of notes) { + const poll = note.hasPoll + ? await pollsRepository.findOneByOrFail({ noteId: note.id }) // N+1 + : null; + const files = await driveFileEntityService.packManyByIds(note.fileIds); // N+1 + const content = serialize(note, poll, files); + + controller.enqueue(content); + exportedNotesCount++; + } + + const total = await notesRepository.countBy({ userId }); + job.updateProgress(exportedNotesCount / total); + }, + }); + } +} + @Injectable() export class ExportNotesProcessorService { private logger: Logger; @@ -59,67 +131,19 @@ export class ExportNotesProcessorService { this.logger.info(`Temp file is ${path}`); try { - const stream = fs.createWriteStream(path, { flags: 'a' }); - - const write = (text: string): Promise => { - return new Promise((res, rej) => { - stream.write(text, err => { - if (err) { - this.logger.error(err); - rej(err); - } else { - res(); - } - }); - }); - }; - - await write('['); - - let exportedNotesCount = 0; - let cursor: MiNote['id'] | null = null; - - while (true) { - const notes = await this.notesRepository.find({ - where: { - userId: user.id, - ...(cursor ? { id: MoreThan(cursor) } : {}), - }, - take: 100, - order: { - id: 1, - }, - }) as MiNote[]; - - if (notes.length === 0) { - job.updateProgress(100); - break; - } - - cursor = notes.at(-1)?.id ?? null; + // メモリが足りなくならないようにストリームで処理する + await new NoteStream( + job, + this.notesRepository, + this.pollsRepository, + this.driveFileEntityService, + this.idService, + user.id, + ) + .pipeThrough(new JsonArrayStream()) + .pipeThrough(new TextEncoderStream()) + .pipeTo(new FileWriterStream(path)); - for (const note of notes) { - let poll: MiPoll | undefined; - if (note.hasPoll) { - poll = await this.pollsRepository.findOneByOrFail({ noteId: note.id }); - } - const files = await this.driveFileEntityService.packManyByIds(note.fileIds); - const content = JSON.stringify(this.serialize(note, poll, files)); - const isFirst = exportedNotesCount === 0; - await write(isFirst ? content : ',\n' + content); - exportedNotesCount++; - } - - const total = await this.notesRepository.countBy({ - userId: user.id, - }); - - job.updateProgress(exportedNotesCount / total); - } - - await write(']'); - - stream.end(); this.logger.succ(`Exported to: ${path}`); const fileName = 'notes-' + dateFormat(new Date(), 'yyyy-MM-dd-HH-mm-ss') + '.json'; @@ -130,22 +154,4 @@ export class ExportNotesProcessorService { cleanup(); } } - - private serialize(note: MiNote, poll: MiPoll | null = null, files: Packed<'DriveFile'>[]): Record { - return { - id: note.id, - text: note.text, - createdAt: this.idService.parse(note.id).date.toISOString(), - fileIds: note.fileIds, - files: files, - replyId: note.replyId, - renoteId: note.renoteId, - poll: poll, - cw: note.cw, - visibility: note.visibility, - visibleUserIds: note.visibleUserIds, - localOnly: note.localOnly, - reactionAcceptance: note.reactionAcceptance, - }; - } } -- cgit v1.2.3-freya From eb60460d28be24513b567d378cec6ecba5c158c7 Mon Sep 17 00:00:00 2001 From: tamaina Date: Fri, 1 Mar 2024 11:57:26 +0900 Subject: enhance: 禁止ワードチェック強化 (#27) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * enhance: 禁止ワードチェック強化 * リモートの禁止ワードチェックを添付ファイルとユーザーを登録する前に行うなど Resolve https://github.com/misskey-dev/misskey/issues/13374 * 禁止ワートの対象の見直し * performActivityで特定のエラーが出た際にDelayedに追加しないように * use IdentifiableError * NoteCreateService.checkProhibitedWords * https://github.com/misskey-dev/misskey-private/pull/27/files#r1507416135 * remove comment --- packages/backend/src/core/NoteCreateService.ts | 25 ++++++++- packages/backend/src/core/UtilityService.ts | 14 +++++ .../backend/src/core/activitypub/ApInboxService.ts | 1 - .../src/core/activitypub/models/ApNoteService.ts | 62 ++++++++++++++-------- .../src/queue/processors/InboxProcessorService.ts | 5 +- 5 files changed, 83 insertions(+), 24 deletions(-) (limited to 'packages/backend/src/queue/processors') diff --git a/packages/backend/src/core/NoteCreateService.ts b/packages/backend/src/core/NoteCreateService.ts index 727787f868..81ae2908d3 100644 --- a/packages/backend/src/core/NoteCreateService.ts +++ b/packages/backend/src/core/NoteCreateService.ts @@ -263,7 +263,13 @@ export class NoteCreateService implements OnApplicationShutdown { } } - if (this.utilityService.isKeyWordIncluded(data.cw ?? data.text ?? '', meta.prohibitedWords)) { + const hasProhibitedWords = await this.checkProhibitedWordsContain({ + cw: data.cw, + text: data.text, + pollChoices: data.poll?.choices, + }, meta.prohibitedWords); + + if (hasProhibitedWords) { throw new IdentifiableError('689ee33f-f97c-479a-ac49-1b9f8140af99', 'Note contains prohibited words'); } @@ -995,6 +1001,23 @@ export class NoteCreateService implements OnApplicationShutdown { } } + public async checkProhibitedWordsContain(content: Parameters[0], prohibitedWords?: string[]) { + if (prohibitedWords == null) { + prohibitedWords = (await this.metaService.fetch()).prohibitedWords; + } + + if ( + this.utilityService.isKeyWordIncluded( + this.utilityService.concatNoteContentsForKeyWordCheck(content), + prohibitedWords, + ) + ) { + return true; + } + + return false; + } + @bindThis public dispose(): void { this.#shutdownController.abort(); diff --git a/packages/backend/src/core/UtilityService.ts b/packages/backend/src/core/UtilityService.ts index 638a0c019e..652e8f7449 100644 --- a/packages/backend/src/core/UtilityService.ts +++ b/packages/backend/src/core/UtilityService.ts @@ -42,6 +42,20 @@ export class UtilityService { return silencedHosts.some(x => `.${host.toLowerCase()}`.endsWith(`.${x}`)); } + @bindThis + public concatNoteContentsForKeyWordCheck(content: { + cw?: string | null; + text?: string | null; + pollChoices?: string[] | null; + others?: string[] | null; + }): string { + /** + * ノートの内容を結合してキーワードチェック用の文字列を生成する + * cwとtextは内容が繋がっているかもしれないので間に何も入れずにチェックする + */ + return `${content.cw ?? ''}${content.text ?? ''}\n${(content.pollChoices ?? []).join('\n')}\n${(content.others ?? []).join('\n')}`; + } + @bindThis public isKeyWordIncluded(text: string, keyWords: string[]): boolean { if (keyWords.length === 0) return false; diff --git a/packages/backend/src/core/activitypub/ApInboxService.ts b/packages/backend/src/core/activitypub/ApInboxService.ts index b0f56a5d82..1621c41bcc 100644 --- a/packages/backend/src/core/activitypub/ApInboxService.ts +++ b/packages/backend/src/core/activitypub/ApInboxService.ts @@ -36,7 +36,6 @@ import { ApResolverService } from './ApResolverService.js'; import { ApAudienceService } from './ApAudienceService.js'; import { ApPersonService } from './models/ApPersonService.js'; import { ApQuestionService } from './models/ApQuestionService.js'; -import { CacheService } from '@/core/CacheService.js'; import { GlobalEventService } from '@/core/GlobalEventService.js'; import type { Resolver } from './ApResolverService.js'; import type { IAccept, IAdd, IAnnounce, IBlock, ICreate, IDelete, IFlag, IFollow, ILike, IObject, IReject, IRemove, IUndo, IUpdate, IMove } from './type.js'; diff --git a/packages/backend/src/core/activitypub/models/ApNoteService.ts b/packages/backend/src/core/activitypub/models/ApNoteService.ts index e201b88173..b2fd435f93 100644 --- a/packages/backend/src/core/activitypub/models/ApNoteService.ts +++ b/packages/backend/src/core/activitypub/models/ApNoteService.ts @@ -24,6 +24,8 @@ import { StatusError } from '@/misc/status-error.js'; import { UtilityService } from '@/core/UtilityService.js'; import { bindThis } from '@/decorators.js'; import { checkHttps } from '@/misc/check-https.js'; +import { IdentifiableError } from '@/misc/identifiable-error.js'; +import { isNotNull } from '@/misc/is-not-null.js'; import { getOneApId, getApId, getOneApHrefNullable, validPost, isEmoji, getApType } from '../type.js'; import { ApLoggerService } from '../ApLoggerService.js'; import { ApMfmService } from '../ApMfmService.js'; @@ -37,7 +39,6 @@ import { ApQuestionService } from './ApQuestionService.js'; import { ApImageService } from './ApImageService.js'; import type { Resolver } from '../ApResolverService.js'; import type { IObject, IPost } from '../type.js'; -import { isNotNull } from '@/misc/is-not-null.js'; @Injectable() export class ApNoteService { @@ -152,11 +153,47 @@ export class ApNoteService { throw new Error('invalid note.attributedTo: ' + note.attributedTo); } - const actor = await this.apPersonService.resolvePerson(getOneApId(note.attributedTo), resolver) as MiRemoteUser; + const uri = getOneApId(note.attributedTo); + + // ローカルで投稿者を検索し、もし凍結されていたらスキップ + const cachedActor = await this.apPersonService.fetchPerson(uri) as MiRemoteUser; + if (cachedActor && cachedActor.isSuspended) { + throw new IdentifiableError('85ab9bd7-3a41-4530-959d-f07073900109', 'actor has been suspended'); + } + + const apMentions = await this.apMentionService.extractApMentions(note.tag, resolver); + const apHashtags = extractApHashtags(note.tag); + + const cw = note.summary === '' ? null : note.summary; + + // テキストのパース + let text: string | null = null; + if (note.source?.mediaType === 'text/x.misskeymarkdown' && typeof note.source.content === 'string') { + text = note.source.content; + } else if (typeof note._misskey_content !== 'undefined') { + text = note._misskey_content; + } else if (typeof note.content === 'string') { + text = this.apMfmService.htmlToMfm(note.content, note.tag); + } + + const poll = await this.apQuestionService.extractPollFromQuestion(note, resolver).catch(() => undefined); - // 投稿者が凍結されていたらスキップ + //#region Contents Check + // 添付ファイルとユーザーをこのサーバーで登録する前に内容をチェックする + /** + * 禁止ワードチェック + */ + const hasProhibitedWords = await this.noteCreateService.checkProhibitedWordsContain({ cw, text, pollChoices: poll?.choices }); + if (hasProhibitedWords) { + throw new IdentifiableError('689ee33f-f97c-479a-ac49-1b9f8140af99', 'Note contains prohibited words'); + } + //#endregion + + const actor = cachedActor ?? await this.apPersonService.resolvePerson(uri, resolver) as MiRemoteUser; + + // 解決した投稿者が凍結されていたらスキップ if (actor.isSuspended) { - throw new Error('actor has been suspended'); + throw new IdentifiableError('85ab9bd7-3a41-4530-959d-f07073900109', 'actor has been suspended'); } const noteAudience = await this.apAudienceService.parseAudience(actor, note.to, note.cc, resolver); @@ -171,9 +208,6 @@ export class ApNoteService { } } - const apMentions = await this.apMentionService.extractApMentions(note.tag, resolver); - const apHashtags = extractApHashtags(note.tag); - // 添付ファイル // TODO: attachmentは必ずしもImageではない // TODO: attachmentは必ずしも配列ではない @@ -233,18 +267,6 @@ export class ApNoteService { } } - const cw = note.summary === '' ? null : note.summary; - - // テキストのパース - let text: string | null = null; - if (note.source?.mediaType === 'text/x.misskeymarkdown' && typeof note.source.content === 'string') { - text = note.source.content; - } else if (typeof note._misskey_content !== 'undefined') { - text = note._misskey_content; - } else if (typeof note.content === 'string') { - text = this.apMfmService.htmlToMfm(note.content, note.tag); - } - // vote if (reply && reply.hasPoll) { const poll = await this.pollsRepository.findOneByOrFail({ noteId: reply.id }); @@ -274,8 +296,6 @@ export class ApNoteService { const apEmojis = emojis.map(emoji => emoji.name); - const poll = await this.apQuestionService.extractPollFromQuestion(note, resolver).catch(() => undefined); - try { return await this.noteCreateService.create(actor, { createdAt: note.published ? new Date(note.published) : null, diff --git a/packages/backend/src/queue/processors/InboxProcessorService.ts b/packages/backend/src/queue/processors/InboxProcessorService.ts index 0a713149e5..3addead058 100644 --- a/packages/backend/src/queue/processors/InboxProcessorService.ts +++ b/packages/backend/src/queue/processors/InboxProcessorService.ts @@ -185,7 +185,10 @@ export class InboxProcessorService { await this.apInboxService.performActivity(authUser.user, activity); } catch (e) { if (e instanceof IdentifiableError) { - if (e.id === '689ee33f-f97c-479a-ac49-1b9f8140af99') return 'blocked notes with prohibited words'; + if (e.id === '689ee33f-f97c-479a-ac49-1b9f8140af99') { + return 'blocked notes with prohibited words'; + } + if (e.id === '85ab9bd7-3a41-4530-959d-f07073900109') return 'actor has been suspended'; } throw e; } -- cgit v1.2.3-freya From f4838e50b4043f917020dd1cfa7b75da087ff8f2 Mon Sep 17 00:00:00 2001 From: かっこかり <67428053+kakkokari-gtyih@users.noreply.github.com> Date: Thu, 21 Mar 2024 07:51:01 +0900 Subject: enhance(antenna): Botの投稿を除外できるように (#13603) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * enhance(antenna): Botの投稿を除外できるように (MisskeyIO#545) (cherry picked from commit a95ce067c6cf0a93647e358aabc984bdbe99e952) * Update Changelog * remove translations * spdx --------- Co-authored-by: まっちゃとーにゅ <17376330+u1-liquid@users.noreply.github.com> --- CHANGELOG.md | 2 ++ locales/index.d.ts | 4 ++++ locales/ja-JP.yml | 1 + .../migration/1710919614510-antenna-exclude-bots.js | 16 ++++++++++++++++ packages/backend/src/core/AntennaService.ts | 6 ++++-- .../backend/src/core/entities/AntennaEntityService.ts | 1 + packages/backend/src/models/Antenna.ts | 5 +++++ packages/backend/src/models/json-schema/antenna.ts | 5 +++++ .../queue/processors/ExportAntennasProcessorService.ts | 1 + .../queue/processors/ImportAntennasProcessorService.ts | 2 ++ .../backend/src/server/api/endpoints/antennas/create.ts | 2 ++ .../backend/src/server/api/endpoints/antennas/update.ts | 2 ++ packages/backend/test/e2e/antennas.ts | 2 ++ packages/frontend/src/pages/my-antennas/create.vue | 1 + packages/frontend/src/pages/my-antennas/editor.vue | 3 +++ packages/misskey-js/src/autogen/types.ts | 4 ++++ 16 files changed, 55 insertions(+), 2 deletions(-) create mode 100644 packages/backend/migration/1710919614510-antenna-exclude-bots.js (limited to 'packages/backend/src/queue/processors') diff --git a/CHANGELOG.md b/CHANGELOG.md index 18dd07f1c3..0dce1a0496 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,8 @@ ## Unreleased ### General +- Enhance: アンテナでBotによるノートを除外できるように + (Cherry-picked from https://github.com/MisskeyIO/misskey/pull/545) - Fix: Play作成時に設定した公開範囲が機能していない問題を修正 ### Client diff --git a/locales/index.d.ts b/locales/index.d.ts index 7f4ec7ecb0..afb4adac6c 100644 --- a/locales/index.d.ts +++ b/locales/index.d.ts @@ -1616,6 +1616,10 @@ export interface Locale extends ILocale { * 除外キーワード */ "antennaExcludeKeywords": string; + /** + * Botアカウントを除外 + */ + "antennaExcludeBots": string; /** * スペースで区切るとAND指定になり、改行で区切るとOR指定になります */ diff --git a/locales/ja-JP.yml b/locales/ja-JP.yml index 8b44ac2121..a64c83b10f 100644 --- a/locales/ja-JP.yml +++ b/locales/ja-JP.yml @@ -400,6 +400,7 @@ name: "名前" antennaSource: "受信ソース" antennaKeywords: "受信キーワード" antennaExcludeKeywords: "除外キーワード" +antennaExcludeBots: "Botアカウントを除外" antennaKeywordsDescription: "スペースで区切るとAND指定になり、改行で区切るとOR指定になります" notifyAntenna: "新しいノートを通知する" withFileAntenna: "ファイルが添付されたノートのみ" diff --git a/packages/backend/migration/1710919614510-antenna-exclude-bots.js b/packages/backend/migration/1710919614510-antenna-exclude-bots.js new file mode 100644 index 0000000000..fac84317cc --- /dev/null +++ b/packages/backend/migration/1710919614510-antenna-exclude-bots.js @@ -0,0 +1,16 @@ +/* + * SPDX-FileCopyrightText: syuilo and misskey-project + * SPDX-License-Identifier: AGPL-3.0-only + */ + +export class AntennaExcludeBots1710919614510 { + name = 'AntennaExcludeBots1710919614510' + + async up(queryRunner) { + await queryRunner.query(`ALTER TABLE "antenna" ADD "excludeBots" boolean NOT NULL DEFAULT false`); + } + + async down(queryRunner) { + await queryRunner.query(`ALTER TABLE "antenna" DROP COLUMN "excludeBots"`); + } +} diff --git a/packages/backend/src/core/AntennaService.ts b/packages/backend/src/core/AntennaService.ts index 4f956a43ed..793d8974b3 100644 --- a/packages/backend/src/core/AntennaService.ts +++ b/packages/backend/src/core/AntennaService.ts @@ -92,7 +92,7 @@ export class AntennaService implements OnApplicationShutdown { } @bindThis - public async addNoteToAntennas(note: MiNote, noteUser: { id: MiUser['id']; username: string; host: string | null; }): Promise { + public async addNoteToAntennas(note: MiNote, noteUser: { id: MiUser['id']; username: string; host: string | null; isBot: boolean; }): Promise { const antennas = await this.getAntennas(); const antennasWithMatchResult = await Promise.all(antennas.map(antenna => this.checkHitAntenna(antenna, note, noteUser).then(hit => [antenna, hit] as const))); const matchedAntennas = antennasWithMatchResult.filter(([, hit]) => hit).map(([antenna]) => antenna); @@ -110,10 +110,12 @@ export class AntennaService implements OnApplicationShutdown { // NOTE: フォローしているユーザーのノート、リストのユーザーのノート、グループのユーザーのノート指定はパフォーマンス上の理由で無効になっている @bindThis - public async checkHitAntenna(antenna: MiAntenna, note: (MiNote | Packed<'Note'>), noteUser: { id: MiUser['id']; username: string; host: string | null; }): Promise { + public async checkHitAntenna(antenna: MiAntenna, note: (MiNote | Packed<'Note'>), noteUser: { id: MiUser['id']; username: string; host: string | null; isBot: boolean; }): Promise { if (note.visibility === 'specified') return false; if (note.visibility === 'followers') return false; + if (antenna.excludeBots && noteUser.isBot) return false; + if (antenna.localOnly && noteUser.host != null) return false; if (!antenna.withReplies && note.replyId != null) return false; diff --git a/packages/backend/src/core/entities/AntennaEntityService.ts b/packages/backend/src/core/entities/AntennaEntityService.ts index 64d6a3c978..3ec8efa6bf 100644 --- a/packages/backend/src/core/entities/AntennaEntityService.ts +++ b/packages/backend/src/core/entities/AntennaEntityService.ts @@ -39,6 +39,7 @@ export class AntennaEntityService { caseSensitive: antenna.caseSensitive, localOnly: antenna.localOnly, notify: antenna.notify, + excludeBots: antenna.excludeBots, withReplies: antenna.withReplies, withFile: antenna.withFile, isActive: antenna.isActive, diff --git a/packages/backend/src/models/Antenna.ts b/packages/backend/src/models/Antenna.ts index 332a899768..f5e819059e 100644 --- a/packages/backend/src/models/Antenna.ts +++ b/packages/backend/src/models/Antenna.ts @@ -72,6 +72,11 @@ export class MiAntenna { }) public caseSensitive: boolean; + @Column('boolean', { + default: false, + }) + public excludeBots: boolean; + @Column('boolean', { default: false, }) diff --git a/packages/backend/src/models/json-schema/antenna.ts b/packages/backend/src/models/json-schema/antenna.ts index 74622b6193..78cf6d3ba2 100644 --- a/packages/backend/src/models/json-schema/antenna.ts +++ b/packages/backend/src/models/json-schema/antenna.ts @@ -76,6 +76,11 @@ export const packedAntennaSchema = { type: 'boolean', optional: false, nullable: false, }, + excludeBots: { + type: 'boolean', + optional: false, nullable: false, + default: false, + }, withReplies: { type: 'boolean', optional: false, nullable: false, diff --git a/packages/backend/src/queue/processors/ExportAntennasProcessorService.ts b/packages/backend/src/queue/processors/ExportAntennasProcessorService.ts index af48bad417..1d8e90f367 100644 --- a/packages/backend/src/queue/processors/ExportAntennasProcessorService.ts +++ b/packages/backend/src/queue/processors/ExportAntennasProcessorService.ts @@ -81,6 +81,7 @@ export class ExportAntennasProcessorService { }) : null, caseSensitive: antenna.caseSensitive, localOnly: antenna.localOnly, + excludeBots: antenna.excludeBots, withReplies: antenna.withReplies, withFile: antenna.withFile, notify: antenna.notify, diff --git a/packages/backend/src/queue/processors/ImportAntennasProcessorService.ts b/packages/backend/src/queue/processors/ImportAntennasProcessorService.ts index 951b560597..ff1c04de06 100644 --- a/packages/backend/src/queue/processors/ImportAntennasProcessorService.ts +++ b/packages/backend/src/queue/processors/ImportAntennasProcessorService.ts @@ -44,6 +44,7 @@ const validate = new Ajv().compile({ } }, caseSensitive: { type: 'boolean' }, localOnly: { type: 'boolean' }, + excludeBots: { type: 'boolean' }, withReplies: { type: 'boolean' }, withFile: { type: 'boolean' }, notify: { type: 'boolean' }, @@ -88,6 +89,7 @@ export class ImportAntennasProcessorService { users: (antenna.src === 'list' && antenna.userListAccts !== null ? antenna.userListAccts : antenna.users).filter(Boolean), caseSensitive: antenna.caseSensitive, localOnly: antenna.localOnly, + excludeBots: antenna.excludeBots, withReplies: antenna.withReplies, withFile: antenna.withFile, notify: antenna.notify, diff --git a/packages/backend/src/server/api/endpoints/antennas/create.ts b/packages/backend/src/server/api/endpoints/antennas/create.ts index 191de8f833..57c8eb4958 100644 --- a/packages/backend/src/server/api/endpoints/antennas/create.ts +++ b/packages/backend/src/server/api/endpoints/antennas/create.ts @@ -64,6 +64,7 @@ export const paramDef = { } }, caseSensitive: { type: 'boolean' }, localOnly: { type: 'boolean' }, + excludeBots: { type: 'boolean' }, withReplies: { type: 'boolean' }, withFile: { type: 'boolean' }, notify: { type: 'boolean' }, @@ -124,6 +125,7 @@ export default class extends Endpoint { // eslint- users: ps.users, caseSensitive: ps.caseSensitive, localOnly: ps.localOnly, + excludeBots: ps.excludeBots, withReplies: ps.withReplies, withFile: ps.withFile, notify: ps.notify, diff --git a/packages/backend/src/server/api/endpoints/antennas/update.ts b/packages/backend/src/server/api/endpoints/antennas/update.ts index 76a34924a0..e6720aacf8 100644 --- a/packages/backend/src/server/api/endpoints/antennas/update.ts +++ b/packages/backend/src/server/api/endpoints/antennas/update.ts @@ -63,6 +63,7 @@ export const paramDef = { } }, caseSensitive: { type: 'boolean' }, localOnly: { type: 'boolean' }, + excludeBots: { type: 'boolean' }, withReplies: { type: 'boolean' }, withFile: { type: 'boolean' }, notify: { type: 'boolean' }, @@ -120,6 +121,7 @@ export default class extends Endpoint { // eslint- users: ps.users, caseSensitive: ps.caseSensitive, localOnly: ps.localOnly, + excludeBots: ps.excludeBots, withReplies: ps.withReplies, withFile: ps.withFile, notify: ps.notify, diff --git a/packages/backend/test/e2e/antennas.ts b/packages/backend/test/e2e/antennas.ts index 7370b1963c..cf5c7dd130 100644 --- a/packages/backend/test/e2e/antennas.ts +++ b/packages/backend/test/e2e/antennas.ts @@ -44,6 +44,7 @@ describe('アンテナ', () => { users: [''], withFile: false, withReplies: false, + excludeBots: false, }; let root: User; @@ -156,6 +157,7 @@ describe('アンテナ', () => { users: [''], withFile: false, withReplies: false, + excludeBots: false, localOnly: false, }; assert.deepStrictEqual(response, expected); diff --git a/packages/frontend/src/pages/my-antennas/create.vue b/packages/frontend/src/pages/my-antennas/create.vue index 8b3b3cfbfd..2d026d2fa9 100644 --- a/packages/frontend/src/pages/my-antennas/create.vue +++ b/packages/frontend/src/pages/my-antennas/create.vue @@ -26,6 +26,7 @@ const draft = ref({ users: [], keywords: [], excludeKeywords: [], + excludeBots: false, withReplies: false, caseSensitive: false, localOnly: false, diff --git a/packages/frontend/src/pages/my-antennas/editor.vue b/packages/frontend/src/pages/my-antennas/editor.vue index c6dcbadd9b..97edbc44ce 100644 --- a/packages/frontend/src/pages/my-antennas/editor.vue +++ b/packages/frontend/src/pages/my-antennas/editor.vue @@ -26,6 +26,7 @@ SPDX-License-Identifier: AGPL-3.0-only + {{ i18n.ts.antennaExcludeBots }} {{ i18n.ts.withReplies }} @@ -78,6 +79,7 @@ const keywords = ref(props.antenna.keywords.map(x => x.join(' ')).join(' const excludeKeywords = ref(props.antenna.excludeKeywords.map(x => x.join(' ')).join('\n')); const caseSensitive = ref(props.antenna.caseSensitive); const localOnly = ref(props.antenna.localOnly); +const excludeBots = ref(props.antenna.excludeBots); const withReplies = ref(props.antenna.withReplies); const withFile = ref(props.antenna.withFile); const notify = ref(props.antenna.notify); @@ -94,6 +96,7 @@ async function saveAntenna() { name: name.value, src: src.value, userListId: userListId.value, + excludeBots: excludeBots.value, withReplies: withReplies.value, withFile: withFile.value, notify: notify.value, diff --git a/packages/misskey-js/src/autogen/types.ts b/packages/misskey-js/src/autogen/types.ts index 3c862f690e..636bc62aaa 100644 --- a/packages/misskey-js/src/autogen/types.ts +++ b/packages/misskey-js/src/autogen/types.ts @@ -4434,6 +4434,8 @@ export type components = { localOnly: boolean; notify: boolean; /** @default false */ + excludeBots: boolean; + /** @default false */ withReplies: boolean; withFile: boolean; isActive: boolean; @@ -9654,6 +9656,7 @@ export type operations = { users: string[]; caseSensitive: boolean; localOnly?: boolean; + excludeBots?: boolean; withReplies: boolean; withFile: boolean; notify: boolean; @@ -9935,6 +9938,7 @@ export type operations = { users?: string[]; caseSensitive?: boolean; localOnly?: boolean; + excludeBots?: boolean; withReplies?: boolean; withFile?: boolean; notify?: boolean; -- cgit v1.2.3-freya From f3500ffda96913e41708a6ca04ef9bbf07af74e4 Mon Sep 17 00:00:00 2001 From: Nila <43315617+nilathedragon@users.noreply.github.com> Date: Sat, 30 Mar 2024 02:28:47 +0100 Subject: fix: report progress out of 100% in CleanRemoteFilesProcessorService (#13633) * Report progress out of 100% in CleanRemoteFilesProcessorService * Add changelog entry --- CHANGELOG.md | 1 + .../backend/src/queue/processors/CleanRemoteFilesProcessorService.ts | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) (limited to 'packages/backend/src/queue/processors') diff --git a/CHANGELOG.md b/CHANGELOG.md index f41ff2171f..3cfbb5f9c8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -35,6 +35,7 @@ - Enhance: misskey-dev/summaly@5.1.0の取り込み（プレビュー生成処理の効率化） - Fix: フォローリクエストを作成する際に既存のものは削除するように (Cherry-picked from https://activitypub.software/TransFem-org/Sharkey/-/merge_requests/440) +- Fix: CleanRemoteFilesProcessorService report progress from 100% (#13632) ## 2024.3.1 diff --git a/packages/backend/src/queue/processors/CleanRemoteFilesProcessorService.ts b/packages/backend/src/queue/processors/CleanRemoteFilesProcessorService.ts index 917de8b72c..728fc9e72b 100644 --- a/packages/backend/src/queue/processors/CleanRemoteFilesProcessorService.ts +++ b/packages/backend/src/queue/processors/CleanRemoteFilesProcessorService.ts @@ -63,7 +63,7 @@ export class CleanRemoteFilesProcessorService { isLink: false, }); - job.updateProgress(deletedCount / total); + job.updateProgress(100 / total * deletedCount); } this.logger.succ('All cached remote files has been deleted.'); -- cgit v1.2.3-freya From d2a5bb39e344fcb84a24ae60faafe4694b227b88 Mon Sep 17 00:00:00 2001 From: Daiki Mizukami Date: Wed, 1 May 2024 07:33:58 +0000 Subject: Merge pull request from GHSA-2vxv-pv3m-3wvj * fix: normalize incoming signed activities * Tweak style * Update CHANGELOG.md * Log compacted activity as well --------- Co-authored-by: syuilo <4439005+syuilo@users.noreply.github.com> --- CHANGELOG.md | 2 + packages/backend/src/core/CoreModule.ts | 12 +- .../src/core/activitypub/ApRendererService.ts | 45 +----- .../backend/src/core/activitypub/JsonLdService.ts | 179 +++++++++++++++++++++ .../src/core/activitypub/LdSignatureService.ts | 171 -------------------- .../backend/src/core/activitypub/misc/contexts.ts | 39 ++++- .../src/queue/processors/InboxProcessorService.ts | 44 +++-- packages/backend/test/unit/activitypub.ts | 42 +++++ 8 files changed, 305 insertions(+), 229 deletions(-) create mode 100644 packages/backend/src/core/activitypub/JsonLdService.ts delete mode 100644 packages/backend/src/core/activitypub/LdSignatureService.ts (limited to 'packages/backend/src/queue/processors') diff --git a/CHANGELOG.md b/CHANGELOG.md index 4b65550daf..4394ab0c55 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,7 @@ ### Note - コントロールパネル内にあるサマリープロキシの設定個所がセキュリティから全般へ変更となります。 +- 悪意のある第三者がリモートユーザーになりすましたアクティビティを受け取れてしまう問題を修正しました。詳しくは[GitHub security advisory](https://github.com/misskey-dev/misskey/security/advisories/GHSA-2vxv-pv3m-3wvj)をご覧ください。 ### General - Enhance: URLプレビューの有効化・無効化を設定できるように #13569 @@ -61,6 +62,7 @@ ### Server - Enhance: エンドポイント`antennas/update`の必須項目を`antennaId`のみに - Enhance: misskey-dev/summaly@5.1.0の取り込み（プレビュー生成処理の効率化） +- Fix: リモートから配送されたアクティビティにJSON-LD compactionをかける - Fix: フォローリクエストを作成する際に既存のものは削除するように (Cherry-picked from https://activitypub.software/TransFem-org/Sharkey/-/merge_requests/440) - Fix: エンドポイント`notes/translate`のエラーを改善 diff --git a/packages/backend/src/core/CoreModule.ts b/packages/backend/src/core/CoreModule.ts index 2c27d33c06..5953155872 100644 --- a/packages/backend/src/core/CoreModule.ts +++ b/packages/backend/src/core/CoreModule.ts @@ -127,7 +127,7 @@ import { ApMfmService } from './activitypub/ApMfmService.js'; import { ApRendererService } from './activitypub/ApRendererService.js'; import { ApRequestService } from './activitypub/ApRequestService.js'; import { ApResolverService } from './activitypub/ApResolverService.js'; -import { LdSignatureService } from './activitypub/LdSignatureService.js'; +import { JsonLdService } from './activitypub/JsonLdService.js'; import { RemoteLoggerService } from './RemoteLoggerService.js'; import { RemoteUserResolveService } from './RemoteUserResolveService.js'; import { WebfingerService } from './WebfingerService.js'; @@ -266,7 +266,7 @@ const $ApMfmService: Provider = { provide: 'ApMfmService', useExisting: ApMfmSer const $ApRendererService: Provider = { provide: 'ApRendererService', useExisting: ApRendererService }; const $ApRequestService: Provider = { provide: 'ApRequestService', useExisting: ApRequestService }; const $ApResolverService: Provider = { provide: 'ApResolverService', useExisting: ApResolverService }; -const $LdSignatureService: Provider = { provide: 'LdSignatureService', useExisting: LdSignatureService }; +const $JsonLdService: Provider = { provide: 'JsonLdService', useExisting: JsonLdService }; const $RemoteLoggerService: Provider = { provide: 'RemoteLoggerService', useExisting: RemoteLoggerService }; const $RemoteUserResolveService: Provider = { provide: 'RemoteUserResolveService', useExisting: RemoteUserResolveService }; const $WebfingerService: Provider = { provide: 'WebfingerService', useExisting: WebfingerService }; @@ -406,7 +406,7 @@ const $ApQuestionService: Provider = { provide: 'ApQuestionService', useExisting ApRendererService, ApRequestService, ApResolverService, - LdSignatureService, + JsonLdService, RemoteLoggerService, RemoteUserResolveService, WebfingerService, @@ -542,7 +542,7 @@ const $ApQuestionService: Provider = { provide: 'ApQuestionService', useExisting $ApRendererService, $ApRequestService, $ApResolverService, - $LdSignatureService, + $JsonLdService, $RemoteLoggerService, $RemoteUserResolveService, $WebfingerService, @@ -678,7 +678,7 @@ const $ApQuestionService: Provider = { provide: 'ApQuestionService', useExisting ApRendererService, ApRequestService, ApResolverService, - LdSignatureService, + JsonLdService, RemoteLoggerService, RemoteUserResolveService, WebfingerService, @@ -813,7 +813,7 @@ const $ApQuestionService: Provider = { provide: 'ApQuestionService', useExisting $ApRendererService, $ApRequestService, $ApResolverService, - $LdSignatureService, + $JsonLdService, $RemoteLoggerService, $RemoteUserResolveService, $WebfingerService, diff --git a/packages/backend/src/core/activitypub/ApRendererService.ts b/packages/backend/src/core/activitypub/ApRendererService.ts index d7fb977a99..d3553b6f73 100644 --- a/packages/backend/src/core/activitypub/ApRendererService.ts +++ b/packages/backend/src/core/activitypub/ApRendererService.ts @@ -28,8 +28,9 @@ import { bindThis } from '@/decorators.js'; import { CustomEmojiService } from '@/core/CustomEmojiService.js'; import { isNotNull } from '@/misc/is-not-null.js'; import { IdService } from '@/core/IdService.js'; -import { LdSignatureService } from './LdSignatureService.js'; +import { JsonLdService } from './JsonLdService.js'; import { ApMfmService } from './ApMfmService.js'; +import { CONTEXT } from './misc/contexts.js'; import type { IAccept, IActivity, IAdd, IAnnounce, IApDocument, IApEmoji, IApHashtag, IApImage, IApMention, IBlock, ICreate, IDelete, IFlag, IFollow, IKey, ILike, IMove, IObject, IPost, IQuestion, IReject, IRemove, ITombstone, IUndo, IUpdate } from './type.js'; @Injectable() @@ -56,7 +57,7 @@ export class ApRendererService { private customEmojiService: CustomEmojiService, private userEntityService: UserEntityService, private driveFileEntityService: DriveFileEntityService, - private ldSignatureService: LdSignatureService, + private jsonLdService: JsonLdService, private userKeypairService: UserKeypairService, private apMfmService: ApMfmService, private mfmService: MfmService, @@ -617,48 +618,16 @@ export class ApRendererService { x.id = `${this.config.url}/${randomUUID()}`; } - return Object.assign({ - '@context': [ - 'https://www.w3.org/ns/activitystreams', - 'https://w3id.org/security/v1', - { - Key: 'sec:Key', - // as non-standards - manuallyApprovesFollowers: 'as:manuallyApprovesFollowers', - sensitive: 'as:sensitive', - Hashtag: 'as:Hashtag', - quoteUrl: 'as:quoteUrl', - // Mastodon - toot: 'http://joinmastodon.org/ns#', - Emoji: 'toot:Emoji', - featured: 'toot:featured', - discoverable: 'toot:discoverable', - // schema - schema: 'http://schema.org#', - PropertyValue: 'schema:PropertyValue', - value: 'schema:value', - // Misskey - misskey: 'https://misskey-hub.net/ns#', - '_misskey_content': 'misskey:_misskey_content', - '_misskey_quote': 'misskey:_misskey_quote', - '_misskey_reaction': 'misskey:_misskey_reaction', - '_misskey_votes': 'misskey:_misskey_votes', - '_misskey_summary': 'misskey:_misskey_summary', - 'isCat': 'misskey:isCat', - // vcard - vcard: 'http://www.w3.org/2006/vcard/ns#', - }, - ], - }, x as T & { id: string }); + return Object.assign({ '@context': CONTEXT }, x as T & { id: string }); } @bindThis public async attachLdSignature(activity: any, user: { id: MiUser['id']; host: null; }): Promise { const keypair = await this.userKeypairService.getUserKeypair(user.id); - const ldSignature = this.ldSignatureService.use(); - ldSignature.debug = false; - activity = await ldSignature.signRsaSignature2017(activity, keypair.privateKey, `${this.config.url}/users/${user.id}#main-key`); + const jsonLd = this.jsonLdService.use(); + jsonLd.debug = false; + activity = await jsonLd.signRsaSignature2017(activity, keypair.privateKey, `${this.config.url}/users/${user.id}#main-key`); return activity; } diff --git a/packages/backend/src/core/activitypub/JsonLdService.ts b/packages/backend/src/core/activitypub/JsonLdService.ts new file mode 100644 index 0000000000..100d4fa19f --- /dev/null +++ b/packages/backend/src/core/activitypub/JsonLdService.ts @@ -0,0 +1,179 @@ +/* + * SPDX-FileCopyrightText: syuilo and misskey-project + * SPDX-License-Identifier: AGPL-3.0-only + */ + +import * as crypto from 'node:crypto'; +import { Injectable } from '@nestjs/common'; +import { HttpRequestService } from '@/core/HttpRequestService.js'; +import { bindThis } from '@/decorators.js'; +import { CONTEXT, PRELOADED_CONTEXTS } from './misc/contexts.js'; +import { validateContentTypeSetAsJsonLD } from './misc/validator.js'; +import type { JsonLdDocument } from 'jsonld'; +import type { JsonLd as JsonLdObject, RemoteDocument } from 'jsonld/jsonld-spec.js'; + +// RsaSignature2017 implementation is based on https://github.com/transmute-industries/RsaSignature2017 + +class JsonLd { + public debug = false; + public preLoad = true; + public loderTimeout = 5000; + + constructor( + private httpRequestService: HttpRequestService, + ) { + } + + @bindThis + public async signRsaSignature2017(data: any, privateKey: string, creator: string, domain?: string, created?: Date): Promise { + const options: { + type: string; + creator: string; + domain?: string; + nonce: string; + created: string; + } = { + type: 'RsaSignature2017', + creator, + nonce: crypto.randomBytes(16).toString('hex'), + created: (created ?? new Date()).toISOString(), + }; + + if (domain) { + options.domain = domain; + } + + const toBeSigned = await this.createVerifyData(data, options); + + const signer = crypto.createSign('sha256'); + signer.update(toBeSigned); + signer.end(); + + const signature = signer.sign(privateKey); + + return { + ...data, + signature: { + ...options, + signatureValue: signature.toString('base64'), + }, + }; + } + + @bindThis + public async verifyRsaSignature2017(data: any, publicKey: string): Promise { + const toBeSigned = await this.createVerifyData(data, data.signature); + const verifier = crypto.createVerify('sha256'); + verifier.update(toBeSigned); + return verifier.verify(publicKey, data.signature.signatureValue, 'base64'); + } + + @bindThis + public async createVerifyData(data: any, options: any): Promise { + const transformedOptions = { + ...options, + '@context': 'https://w3id.org/identity/v1', + }; + delete transformedOptions['type']; + delete transformedOptions['id']; + delete transformedOptions['signatureValue']; + const canonizedOptions = await this.normalize(transformedOptions); + const optionsHash = this.sha256(canonizedOptions.toString()); + const transformedData = { ...data }; + delete transformedData['signature']; + const cannonidedData = await this.normalize(transformedData); + if (this.debug) console.debug(`cannonidedData: ${cannonidedData}`); + const documentHash = this.sha256(cannonidedData.toString()); + const verifyData = `${optionsHash}${documentHash}`; + return verifyData; + } + + @bindThis + public async compact(data: any, context: any = CONTEXT): Promise { + const customLoader = this.getLoader(); + // XXX: Importing jsonld dynamically since Jest frequently fails to import it statically + // https://github.com/misskey-dev/misskey/pull/9894#discussion_r1103753595 + return (await import('jsonld')).default.compact(data, context, { + documentLoader: customLoader, + }); + } + + @bindThis + public async normalize(data: JsonLdDocument): Promise { + const customLoader = this.getLoader(); + return (await import('jsonld')).default.normalize(data, { + documentLoader: customLoader, + }); + } + + @bindThis + private getLoader() { + return async (url: string): Promise => { + if (!/^https?:\/\//.test(url)) throw new Error(`Invalid URL ${url}`); + + if (this.preLoad) { + if (url in PRELOADED_CONTEXTS) { + if (this.debug) console.debug(`HIT: ${url}`); + return { + contextUrl: undefined, + document: PRELOADED_CONTEXTS[url], + documentUrl: url, + }; + } + } + + if (this.debug) console.debug(`MISS: ${url}`); + const document = await this.fetchDocument(url); + return { + contextUrl: undefined, + document: document, + documentUrl: url, + }; + }; + } + + @bindThis + private async fetchDocument(url: string): Promise { + const json = await this.httpRequestService.send( + url, + { + headers: { + Accept: 'application/ld+json, application/json', + }, + timeout: this.loderTimeout, + }, + { + throwErrorWhenResponseNotOk: false, + validators: [validateContentTypeSetAsJsonLD], + }, + ).then(res => { + if (!res.ok) { + throw new Error(`${res.status} ${res.statusText}`); + } else { + return res.json(); + } + }); + + return json as JsonLdObject; + } + + @bindThis + public sha256(data: string): string { + const hash = crypto.createHash('sha256'); + hash.update(data); + return hash.digest('hex'); + } +} + +@Injectable() +export class JsonLdService { + constructor( + private httpRequestService: HttpRequestService, + ) { + } + + @bindThis + public use(): JsonLd { + return new JsonLd(this.httpRequestService); + } +} diff --git a/packages/backend/src/core/activitypub/LdSignatureService.ts b/packages/backend/src/core/activitypub/LdSignatureService.ts deleted file mode 100644 index 9de184336f..0000000000 --- a/packages/backend/src/core/activitypub/LdSignatureService.ts +++ /dev/null @@ -1,171 +0,0 @@ -/* - * SPDX-FileCopyrightText: syuilo and misskey-project - * SPDX-License-Identifier: AGPL-3.0-only - */ - -import * as crypto from 'node:crypto'; -import { Injectable } from '@nestjs/common'; -import { HttpRequestService } from '@/core/HttpRequestService.js'; -import { bindThis } from '@/decorators.js'; -import { CONTEXTS } from './misc/contexts.js'; -import { validateContentTypeSetAsJsonLD } from './misc/validator.js'; -import type { JsonLdDocument } from 'jsonld'; -import type { JsonLd, RemoteDocument } from 'jsonld/jsonld-spec.js'; - -// RsaSignature2017 based from https://github.com/transmute-industries/RsaSignature2017 - -class LdSignature { - public debug = false; - public preLoad = true; - public loderTimeout = 5000; - - constructor( - private httpRequestService: HttpRequestService, - ) { - } - - @bindThis - public async signRsaSignature2017(data: any, privateKey: string, creator: string, domain?: string, created?: Date): Promise { - const options: { - type: string; - creator: string; - domain?: string; - nonce: string; - created: string; - } = { - type: 'RsaSignature2017', - creator, - nonce: crypto.randomBytes(16).toString('hex'), - created: (created ?? new Date()).toISOString(), - }; - - if (domain) { - options.domain = domain; - } - - const toBeSigned = await this.createVerifyData(data, options); - - const signer = crypto.createSign('sha256'); - signer.update(toBeSigned); - signer.end(); - - const signature = signer.sign(privateKey); - - return { - ...data, - signature: { - ...options, - signatureValue: signature.toString('base64'), - }, - }; - } - - @bindThis - public async verifyRsaSignature2017(data: any, publicKey: string): Promise { - const toBeSigned = await this.createVerifyData(data, data.signature); - const verifier = crypto.createVerify('sha256'); - verifier.update(toBeSigned); - return verifier.verify(publicKey, data.signature.signatureValue, 'base64'); - } - - @bindThis - public async createVerifyData(data: any, options: any): Promise { - const transformedOptions = { - ...options, - '@context': 'https://w3id.org/identity/v1', - }; - delete transformedOptions['type']; - delete transformedOptions['id']; - delete transformedOptions['signatureValue']; - const canonizedOptions = await this.normalize(transformedOptions); - const optionsHash = this.sha256(canonizedOptions.toString()); - const transformedData = { ...data }; - delete transformedData['signature']; - const cannonidedData = await this.normalize(transformedData); - if (this.debug) console.debug(`cannonidedData: ${cannonidedData}`); - const documentHash = this.sha256(cannonidedData.toString()); - const verifyData = `${optionsHash}${documentHash}`; - return verifyData; - } - - @bindThis - public async normalize(data: JsonLdDocument): Promise { - const customLoader = this.getLoader(); - // XXX: Importing jsonld dynamically since Jest frequently fails to import it statically - // https://github.com/misskey-dev/misskey/pull/9894#discussion_r1103753595 - return (await import('jsonld')).default.normalize(data, { - documentLoader: customLoader, - }); - } - - @bindThis - private getLoader() { - return async (url: string): Promise => { - if (!/^https?:\/\//.test(url)) throw new Error(`Invalid URL ${url}`); - - if (this.preLoad) { - if (url in CONTEXTS) { - if (this.debug) console.debug(`HIT: ${url}`); - return { - contextUrl: undefined, - document: CONTEXTS[url], - documentUrl: url, - }; - } - } - - if (this.debug) console.debug(`MISS: ${url}`); - const document = await this.fetchDocument(url); - return { - contextUrl: undefined, - document: document, - documentUrl: url, - }; - }; - } - - @bindThis - private async fetchDocument(url: string): Promise { - const json = await this.httpRequestService.send( - url, - { - headers: { - Accept: 'application/ld+json, application/json', - }, - timeout: this.loderTimeout, - }, - { - throwErrorWhenResponseNotOk: false, - validators: [validateContentTypeSetAsJsonLD], - }, - ).then(res => { - if (!res.ok) { - throw new Error(`${res.status} ${res.statusText}`); - } else { - return res.json(); - } - }); - - return json as JsonLd; - } - - @bindThis - public sha256(data: string): string { - const hash = crypto.createHash('sha256'); - hash.update(data); - return hash.digest('hex'); - } -} - -@Injectable() -export class LdSignatureService { - constructor( - private httpRequestService: HttpRequestService, - ) { - } - - @bindThis - public use(): LdSignature { - return new LdSignature(this.httpRequestService); - } -} diff --git a/packages/backend/src/core/activitypub/misc/contexts.ts b/packages/backend/src/core/activitypub/misc/contexts.ts index 88afdefcd3..feb8c42c56 100644 --- a/packages/backend/src/core/activitypub/misc/contexts.ts +++ b/packages/backend/src/core/activitypub/misc/contexts.ts @@ -3,7 +3,7 @@ * SPDX-License-Identifier: AGPL-3.0-only */ -import type { JsonLd } from 'jsonld/jsonld-spec.js'; +import type { Context, JsonLd } from 'jsonld/jsonld-spec.js'; /* eslint:disable:quotemark indent */ const id_v1 = { @@ -526,7 +526,42 @@ const activitystreams = { }, } satisfies JsonLd; -export const CONTEXTS: Record = { +const context_iris = [ + 'https://www.w3.org/ns/activitystreams', + 'https://w3id.org/security/v1', +]; + +const extension_context_definition = { + Key: 'sec:Key', + // as non-standards + manuallyApprovesFollowers: 'as:manuallyApprovesFollowers', + sensitive: 'as:sensitive', + Hashtag: 'as:Hashtag', + quoteUrl: 'as:quoteUrl', + // Mastodon + toot: 'http://joinmastodon.org/ns#', + Emoji: 'toot:Emoji', + featured: 'toot:featured', + discoverable: 'toot:discoverable', + // schema + schema: 'http://schema.org#', + PropertyValue: 'schema:PropertyValue', + value: 'schema:value', + // Misskey + misskey: 'https://misskey-hub.net/ns#', + '_misskey_content': 'misskey:_misskey_content', + '_misskey_quote': 'misskey:_misskey_quote', + '_misskey_reaction': 'misskey:_misskey_reaction', + '_misskey_votes': 'misskey:_misskey_votes', + '_misskey_summary': 'misskey:_misskey_summary', + 'isCat': 'misskey:isCat', + // vcard + vcard: 'http://www.w3.org/2006/vcard/ns#', +} satisfies Context; + +export const CONTEXT: (string | Context)[] = [...context_iris, extension_context_definition]; + +export const PRELOADED_CONTEXTS: Record = { 'https://w3id.org/identity/v1': id_v1, 'https://w3id.org/security/v1': security_v1, 'https://www.w3.org/ns/activitystreams': activitystreams, diff --git a/packages/backend/src/queue/processors/InboxProcessorService.ts b/packages/backend/src/queue/processors/InboxProcessorService.ts index 3addead058..1d05f4ade1 100644 --- a/packages/backend/src/queue/processors/InboxProcessorService.ts +++ b/packages/backend/src/queue/processors/InboxProcessorService.ts @@ -15,13 +15,14 @@ import InstanceChart from '@/core/chart/charts/instance.js'; import ApRequestChart from '@/core/chart/charts/ap-request.js'; import FederationChart from '@/core/chart/charts/federation.js'; import { getApId } from '@/core/activitypub/type.js'; +import type { IActivity } from '@/core/activitypub/type.js'; import type { MiRemoteUser } from '@/models/User.js'; import type { MiUserPublickey } from '@/models/UserPublickey.js'; import { ApDbResolverService } from '@/core/activitypub/ApDbResolverService.js'; import { StatusError } from '@/misc/status-error.js'; import { UtilityService } from '@/core/UtilityService.js'; import { ApPersonService } from '@/core/activitypub/models/ApPersonService.js'; -import { LdSignatureService } from '@/core/activitypub/LdSignatureService.js'; +import { JsonLdService } from '@/core/activitypub/JsonLdService.js'; import { ApInboxService } from '@/core/activitypub/ApInboxService.js'; import { bindThis } from '@/decorators.js'; import { IdentifiableError } from '@/misc/identifiable-error.js'; @@ -38,7 +39,7 @@ export class InboxProcessorService { private apInboxService: ApInboxService, private federatedInstanceService: FederatedInstanceService, private fetchInstanceMetadataService: FetchInstanceMetadataService, - private ldSignatureService: LdSignatureService, + private jsonLdService: JsonLdService, private apPersonService: ApPersonService, private apDbResolverService: ApDbResolverService, private instanceChart: InstanceChart, @@ -52,7 +53,7 @@ export class InboxProcessorService { @bindThis public async process(job: Bull.Job): Promise { const signature = job.data.signature; // HTTP-signature - const activity = job.data.activity; + let activity = job.data.activity; //#region Log const info = Object.assign({}, activity); @@ -110,20 +111,21 @@ export class InboxProcessorService { // また、signatureのsignerは、activity.actorと一致する必要がある if (!httpSignatureValidated || authUser.user.uri !== activity.actor) { // 一致しなくても、でもLD-Signatureがありそうならそっちも見る - if (activity.signature) { - if (activity.signature.type !== 'RsaSignature2017') { - throw new Bull.UnrecoverableError(`skip: unsupported LD-signature type ${activity.signature.type}`); + const ldSignature = activity.signature; + if (ldSignature) { + if (ldSignature.type !== 'RsaSignature2017') { + throw new Bull.UnrecoverableError(`skip: unsupported LD-signature type ${ldSignature.type}`); } - // activity.signature.creator: https://example.oom/users/user#main-key + // ldSignature.creator: https://example.oom/users/user#main-key // みたいになっててUserを引っ張れば公開キーも入ることを期待する - if (activity.signature.creator) { - const candicate = activity.signature.creator.replace(/#.*/, ''); + if (ldSignature.creator) { + const candicate = ldSignature.creator.replace(/#.*/, ''); await this.apPersonService.resolvePerson(candicate).catch(() => null); } // keyIdからLD-Signatureのユーザーを取得 - authUser = await this.apDbResolverService.getAuthUserFromKeyId(activity.signature.creator); + authUser = await this.apDbResolverService.getAuthUserFromKeyId(ldSignature.creator); if (authUser == null) { throw new Bull.UnrecoverableError('skip: LD-Signatureのユーザーが取得できませんでした'); } @@ -132,13 +134,31 @@ export class InboxProcessorService { throw new Bull.UnrecoverableError('skip: LD-SignatureのユーザーはpublicKeyを持っていませんでした'); } + const jsonLd = this.jsonLdService.use(); + // LD-Signature検証 - const ldSignature = this.ldSignatureService.use(); - const verified = await ldSignature.verifyRsaSignature2017(activity, authUser.key.keyPem).catch(() => false); + const verified = await jsonLd.verifyRsaSignature2017(activity, authUser.key.keyPem).catch(() => false); if (!verified) { throw new Bull.UnrecoverableError('skip: LD-Signatureの検証に失敗しました'); } + // アクティビティを正規化 + delete activity.signature; + try { + activity = await jsonLd.compact(activity) as IActivity; + } catch (e) { + throw new Bull.UnrecoverableError(`skip: failed to compact activity: ${e}`); + } + // TODO: 元のアクティビティと非互換な形に正規化される場合は転送をスキップする + // https://github.com/mastodon/mastodon/blob/664b0ca/app/services/activitypub/process_collection_service.rb#L24-L29 + activity.signature = ldSignature; + + //#region Log + const compactedInfo = Object.assign({}, activity); + delete compactedInfo['@context']; + this.logger.debug(`compacted: ${JSON.stringify(compactedInfo, null, 2)}`); + //#endregion + // もう一度actorチェック if (authUser.user.uri !== activity.actor) { throw new Bull.UnrecoverableError(`skip: LD-Signature user(${authUser.user.uri}) !== activity.actor(${activity.actor})`); diff --git a/packages/backend/test/unit/activitypub.ts b/packages/backend/test/unit/activitypub.ts index aa3f3a4ff1..6962608106 100644 --- a/packages/backend/test/unit/activitypub.ts +++ b/packages/backend/test/unit/activitypub.ts @@ -13,6 +13,8 @@ import { ApImageService } from '@/core/activitypub/models/ApImageService.js'; import { ApNoteService } from '@/core/activitypub/models/ApNoteService.js'; import { ApPersonService } from '@/core/activitypub/models/ApPersonService.js'; import { ApRendererService } from '@/core/activitypub/ApRendererService.js'; +import { JsonLdService } from '@/core/activitypub/JsonLdService.js'; +import { CONTEXT } from '@/core/activitypub/misc/contexts.js'; import { GlobalModule } from '@/GlobalModule.js'; import { CoreModule } from '@/core/CoreModule.js'; import { FederatedInstanceService } from '@/core/FederatedInstanceService.js'; @@ -88,6 +90,7 @@ describe('ActivityPub', () => { let noteService: ApNoteService; let personService: ApPersonService; let rendererService: ApRendererService; + let jsonLdService: JsonLdService; let resolver: MockResolver; const metaInitial = { @@ -128,6 +131,7 @@ describe('ActivityPub', () => { personService = app.get(ApPersonService); rendererService = app.get(ApRendererService); imageService = app.get(ApImageService); + jsonLdService = app.get(JsonLdService); resolver = new MockResolver(await app.resolve(LoggerService)); // Prevent ApPersonService from fetching instance, as it causes Jest import-after-test error @@ -381,4 +385,42 @@ describe('ActivityPub', () => { assert.strictEqual(driveFile, null); }); }); + + describe('JSON-LD', () =>{ + test('Compaction', async () => { + const jsonLd = jsonLdService.use(); + + const object = { + '@context': [ + 'https://www.w3.org/ns/activitystreams', + { + _misskey_quote: 'https://misskey-hub.net/ns#_misskey_quote', + unknown: 'https://example.org/ns#unknown', + undefined: null, + }, + ], + id: 'https://example.com/notes/42', + type: 'Note', + attributedTo: 'https://example.com/users/1', + to: ['https://www.w3.org/ns/activitystreams#Public'], + content: 'test test foo', + _misskey_quote: 'https://example.com/notes/1', + unknown: 'test test bar', + undefined: 'test test baz', + }; + const compacted = await jsonLd.compact(object); + + assert.deepStrictEqual(compacted, { + '@context': CONTEXT, + id: 'https://example.com/notes/42', + type: 'Note', + attributedTo: 'https://example.com/users/1', + to: 'as:Public', + content: 'test test foo', + _misskey_quote: 'https://example.com/notes/1', + 'https://example.org/ns#unknown': 'test test bar', + // undefined: 'test test baz', + }); + }); + }); }); -- cgit v1.2.3-freya From 87326683720fdc8998370d06b9459b81ee36e805 Mon Sep 17 00:00:00 2001 From: dakkar Date: Sat, 11 May 2024 14:31:56 +0100 Subject: lint --- .../backend/src/queue/processors/CleanRemoteFilesProcessorService.ts | 1 - 1 file changed, 1 deletion(-) (limited to 'packages/backend/src/queue/processors') diff --git a/packages/backend/src/queue/processors/CleanRemoteFilesProcessorService.ts b/packages/backend/src/queue/processors/CleanRemoteFilesProcessorService.ts index 4fa414b0b5..81842b221f 100644 --- a/packages/backend/src/queue/processors/CleanRemoteFilesProcessorService.ts +++ b/packages/backend/src/queue/processors/CleanRemoteFilesProcessorService.ts @@ -73,7 +73,6 @@ export class CleanRemoteFilesProcessorService { }); await job.updateProgress(100 / total * deletedCount); - } this.logger.succ(`All cached remote files processed. Total deleted: ${deletedCount}, Failed: ${errorCount}.`); -- cgit v1.2.3-freya From 83a9aa4533912c685a74a107be3894c4a85a338c Mon Sep 17 00:00:00 2001 From: anatawa12 Date: Thu, 23 May 2024 15:55:47 +0900 Subject: feat: suspend instance improvements (#13861) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat(backend): dead instance detection * feat(backend): suspend type detection * feat(frontend): show suspend reason on frontend * feat(backend): resume federation automatically if the server is automatically suspended * docs(changelog): 配信停止まわりの改善 * lint: fix lint errors * Update packages/frontend/src/pages/instance-info.vue * lint: fix lint error * chore: suspendedState => suspensionState --------- Co-authored-by: syuilo <4439005+syuilo@users.noreply.github.com> --- CHANGELOG.md | 3 ++ locales/index.d.ts | 32 ++++++++++++++ locales/ja-JP.yml | 10 +++++ .../migration/1716345015347-NotRespondingSince.js | 16 +++++++ ...447138870-SuspensionStateInsteadOfIsSspended.js | 50 ++++++++++++++++++++++ .../src/core/entities/InstanceEntityService.ts | 3 +- packages/backend/src/models/Instance.ts | 17 ++++++-- .../src/models/json-schema/federation-instance.ts | 5 +++ .../queue/processors/DeliverProcessorService.ts | 14 +++++- .../src/queue/processors/InboxProcessorService.ts | 2 + .../backend/src/server/api/ApiServerService.ts | 2 +- .../endpoints/admin/federation/update-instance.ts | 11 ++++- packages/frontend/src/pages/admin/federation.vue | 14 +++++- packages/frontend/src/pages/instance-info.vue | 29 ++++++++++--- packages/misskey-js/src/autogen/types.ts | 2 + 15 files changed, 193 insertions(+), 17 deletions(-) create mode 100644 packages/backend/migration/1716345015347-NotRespondingSince.js create mode 100644 packages/backend/migration/1716447138870-SuspensionStateInsteadOfIsSspended.js (limited to 'packages/backend/src/queue/processors') diff --git a/CHANGELOG.md b/CHANGELOG.md index ce66d779a3..21bb3b2e8e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,9 @@ - サスペンド済みユーザーか - 鍵アカウントユーザーか - 「アカウントを見つけやすくする」が有効なユーザーか +- Enhance: Goneを出さずに終了したサーバーへの配信停止を自動的に行うように + - もしそのようなサーバーからから配信が届いた場合には自動的に配信を再開します +- Enhance: 配信停止の理由を表示するように - Fix: Play作成時に設定した公開範囲が機能していない問題を修正 - Fix: 正規化されていない状態のhashtagが連合されてきたhtmlに含まれているとhashtagが正しくhashtagに復元されない問題を修正 - Fix: みつけるのアンケート欄にてチャンネルのアンケートが含まれてしまう問題を修正 diff --git a/locales/index.d.ts b/locales/index.d.ts index d5d6ef0f34..991ec1ac1d 100644 --- a/locales/index.d.ts +++ b/locales/index.d.ts @@ -4972,6 +4972,38 @@ export interface Locale extends ILocale { * お問い合わせ */ "inquiry": string; + "_delivery": { + /** + * 配信状態 + */ + "status": string; + /** + * 配信停止 + */ + "stop": string; + /** + * 配信再開 + */ + "resume": string; + "_type": { + /** + * 配信中 + */ + "none": string; + /** + * 手動停止中 + */ + "manuallySuspended": string; + /** + * サーバー削除のため停止中 + */ + "goneSuspended": string; + /** + * サーバー応答なしのため停止中 + */ + "autoSuspendedForNotResponding": string; + }; + }; "_bubbleGame": { /** * 遊び方 diff --git a/locales/ja-JP.yml b/locales/ja-JP.yml index 9aa1e6e6a0..d7635acc2e 100644 --- a/locales/ja-JP.yml +++ b/locales/ja-JP.yml @@ -1240,6 +1240,16 @@ noDescription: "説明文はありません" alwaysConfirmFollow: "フォローの際常に確認する" inquiry: "お問い合わせ" +_delivery: + status: "配信状態" + stop: "配信停止" + resume: "配信再開" + _type: + none: "配信中" + manuallySuspended: "手動停止中" + goneSuspended: "サーバー削除のため停止中" + autoSuspendedForNotResponding: "サーバー応答なしのため停止中" + _bubbleGame: howToPlay: "遊び方" hold: "ホールド" diff --git a/packages/backend/migration/1716345015347-NotRespondingSince.js b/packages/backend/migration/1716345015347-NotRespondingSince.js new file mode 100644 index 0000000000..fc4ee6639a --- /dev/null +++ b/packages/backend/migration/1716345015347-NotRespondingSince.js @@ -0,0 +1,16 @@ +/* + * SPDX-FileCopyrightText: syuilo and misskey-project + * SPDX-License-Identifier: AGPL-3.0-only + */ + +export class NotRespondingSince1716345015347 { + name = 'NotRespondingSince1716345015347' + + async up(queryRunner) { + await queryRunner.query(`ALTER TABLE "instance" ADD "notRespondingSince" TIMESTAMP WITH TIME ZONE`); + } + + async down(queryRunner) { + await queryRunner.query(`ALTER TABLE "instance" DROP COLUMN "notRespondingSince"`); + } +} diff --git a/packages/backend/migration/1716447138870-SuspensionStateInsteadOfIsSspended.js b/packages/backend/migration/1716447138870-SuspensionStateInsteadOfIsSspended.js new file mode 100644 index 0000000000..4808a9a3db --- /dev/null +++ b/packages/backend/migration/1716447138870-SuspensionStateInsteadOfIsSspended.js @@ -0,0 +1,50 @@ +/* + * SPDX-FileCopyrightText: syuilo and misskey-project + * SPDX-License-Identifier: AGPL-3.0-only + */ + +export class SuspensionStateInsteadOfIsSspended1716345771510 { + name = 'SuspensionStateInsteadOfIsSspended1716345771510' + + async up(queryRunner) { + await queryRunner.query(`CREATE TYPE "public"."instance_suspensionstate_enum" AS ENUM('none', 'manuallySuspended', 'goneSuspended', 'autoSuspendedForNotResponding')`); + + await queryRunner.query(`DROP INDEX "public"."IDX_34500da2e38ac393f7bb6b299c"`); + + await queryRunner.query(`ALTER TABLE "instance" RENAME COLUMN "isSuspended" TO "suspensionState"`); + + await queryRunner.query(`ALTER TABLE "instance" ALTER COLUMN "suspensionState" DROP DEFAULT`); + + await queryRunner.query(`ALTER TABLE "instance" ALTER COLUMN "suspensionState" TYPE "public"."instance_suspensionstate_enum" USING ( + CASE "suspensionState" + WHEN TRUE THEN 'manuallySuspended'::instance_suspensionstate_enum + ELSE 'none'::instance_suspensionstate_enum + END + )`); + + await queryRunner.query(`ALTER TABLE "instance" ALTER COLUMN "suspensionState" SET DEFAULT 'none'`); + + await queryRunner.query(`CREATE INDEX "IDX_3ede46f507c87ad698051d56a8" ON "instance" ("suspensionState") `); + } + + async down(queryRunner) { + await queryRunner.query(`DROP INDEX "public"."IDX_3ede46f507c87ad698051d56a8"`); + + await queryRunner.query(`ALTER TABLE "instance" ALTER COLUMN "suspensionState" DROP DEFAULT`); + + await queryRunner.query(`ALTER TABLE "instance" ALTER COLUMN "suspensionState" TYPE boolean USING ( + CASE "suspensionState" + WHEN 'none'::instance_suspensionstate_enum THEN FALSE + ELSE TRUE + END + )`); + + await queryRunner.query(`ALTER TABLE "instance" ALTER COLUMN "suspensionState" SET DEFAULT false`); + + await queryRunner.query(`ALTER TABLE "instance" RENAME COLUMN "suspensionState" TO "isSuspended"`); + + await queryRunner.query(`CREATE INDEX "IDX_34500da2e38ac393f7bb6b299c" ON "instance" ("isSuspended") `); + + await queryRunner.query(`DROP TYPE "public"."instance_suspensionstate_enum"`); + } +} diff --git a/packages/backend/src/core/entities/InstanceEntityService.ts b/packages/backend/src/core/entities/InstanceEntityService.ts index e46bd8b963..9117b13914 100644 --- a/packages/backend/src/core/entities/InstanceEntityService.ts +++ b/packages/backend/src/core/entities/InstanceEntityService.ts @@ -39,7 +39,8 @@ export class InstanceEntityService { followingCount: instance.followingCount, followersCount: instance.followersCount, isNotResponding: instance.isNotResponding, - isSuspended: instance.isSuspended, + isSuspended: instance.suspensionState !== 'none', + suspensionState: instance.suspensionState, isBlocked: this.utilityService.isBlockedHost(meta.blockedHosts, instance.host), softwareName: instance.softwareName, softwareVersion: instance.softwareVersion, diff --git a/packages/backend/src/models/Instance.ts b/packages/backend/src/models/Instance.ts index 9863c9d75d..17cd5c6665 100644 --- a/packages/backend/src/models/Instance.ts +++ b/packages/backend/src/models/Instance.ts @@ -81,13 +81,22 @@ export class MiInstance { public isNotResponding: boolean; /** - * このインスタンスへの配信を停止するか + * このインスタンスと不通になった日時 + */ + @Column('timestamp with time zone', { + nullable: true, + }) + public notRespondingSince: Date | null; + + /** + * このインスタンスへの配信状態 */ @Index() - @Column('boolean', { - default: false, + @Column('enum', { + default: 'none', + enum: ['none', 'manuallySuspended', 'goneSuspended', 'autoSuspendedForNotResponding'], }) - public isSuspended: boolean; + public suspensionState: 'none' | 'manuallySuspended' | 'goneSuspended' | 'autoSuspendedForNotResponding'; @Column('varchar', { length: 64, nullable: true, diff --git a/packages/backend/src/models/json-schema/federation-instance.ts b/packages/backend/src/models/json-schema/federation-instance.ts index 42d98fe523..ed40d405c6 100644 --- a/packages/backend/src/models/json-schema/federation-instance.ts +++ b/packages/backend/src/models/json-schema/federation-instance.ts @@ -45,6 +45,11 @@ export const packedFederationInstanceSchema = { type: 'boolean', optional: false, nullable: false, }, + suspensionState: { + type: 'string', + nullable: false, optional: false, + enum: ['none', 'manuallySuspended', 'goneSuspended', 'autoSuspendedForNotResponding'], + }, isBlocked: { type: 'boolean', optional: false, nullable: false, diff --git a/packages/backend/src/queue/processors/DeliverProcessorService.ts b/packages/backend/src/queue/processors/DeliverProcessorService.ts index 5fed070929..b73195afc3 100644 --- a/packages/backend/src/queue/processors/DeliverProcessorService.ts +++ b/packages/backend/src/queue/processors/DeliverProcessorService.ts @@ -5,6 +5,7 @@ import { Inject, Injectable } from '@nestjs/common'; import * as Bull from 'bullmq'; +import { Not } from 'typeorm'; import { DI } from '@/di-symbols.js'; import type { InstancesRepository } from '@/models/_.js'; import type Logger from '@/logger.js'; @@ -62,7 +63,7 @@ export class DeliverProcessorService { if (suspendedHosts == null) { suspendedHosts = await this.instancesRepository.find({ where: { - isSuspended: true, + suspensionState: Not('none'), }, }); this.suspendedHostsCache.set(suspendedHosts); @@ -79,6 +80,7 @@ export class DeliverProcessorService { if (i.isNotResponding) { this.federatedInstanceService.update(i.id, { isNotResponding: false, + notRespondingSince: null, }); } @@ -98,7 +100,15 @@ export class DeliverProcessorService { if (!i.isNotResponding) { this.federatedInstanceService.update(i.id, { isNotResponding: true, + notRespondingSince: new Date(), }); + } else if (i.notRespondingSince) { + // 1週間以上不通ならサスペンド + if (i.suspensionState === 'none' && i.notRespondingSince.getTime() <= Date.now() - 1000 * 60 * 60 * 24 * 7) { + this.federatedInstanceService.update(i.id, { + suspensionState: 'autoSuspendedForNotResponding', + }); + } } this.apRequestChart.deliverFail(); @@ -116,7 +126,7 @@ export class DeliverProcessorService { if (job.data.isSharedInbox && res.statusCode === 410) { this.federatedInstanceService.fetch(host).then(i => { this.federatedInstanceService.update(i.id, { - isSuspended: true, + suspensionState: 'goneSuspended', }); }); throw new Bull.UnrecoverableError(`${host} is gone`); diff --git a/packages/backend/src/queue/processors/InboxProcessorService.ts b/packages/backend/src/queue/processors/InboxProcessorService.ts index 1d05f4ade1..f465339075 100644 --- a/packages/backend/src/queue/processors/InboxProcessorService.ts +++ b/packages/backend/src/queue/processors/InboxProcessorService.ts @@ -188,6 +188,8 @@ export class InboxProcessorService { this.federatedInstanceService.update(i.id, { latestRequestReceivedAt: new Date(), isNotResponding: false, + // もしサーバーが死んでるために配信が止まっていた場合には自動的に復活させてあげる + suspensionState: i.suspensionState === 'autoSuspendedForNotResponding' ? 'none' : undefined, }); this.fetchInstanceMetadataService.fetchInstanceMetadata(i); diff --git a/packages/backend/src/server/api/ApiServerService.ts b/packages/backend/src/server/api/ApiServerService.ts index e99244cdd0..4a5935f930 100644 --- a/packages/backend/src/server/api/ApiServerService.ts +++ b/packages/backend/src/server/api/ApiServerService.ts @@ -137,7 +137,7 @@ export class ApiServerService { const instances = await this.instancesRepository.find({ select: ['host'], where: { - isSuspended: false, + suspensionState: 'none', }, }); diff --git a/packages/backend/src/server/api/endpoints/admin/federation/update-instance.ts b/packages/backend/src/server/api/endpoints/admin/federation/update-instance.ts index 0bcdc2a4b8..fed7bfbbde 100644 --- a/packages/backend/src/server/api/endpoints/admin/federation/update-instance.ts +++ b/packages/backend/src/server/api/endpoints/admin/federation/update-instance.ts @@ -46,12 +46,19 @@ export default class extends Endpoint { // eslint- throw new Error('instance not found'); } + const isSuspendedBefore = instance.suspensionState !== 'none'; + let suspensionState: undefined | 'manuallySuspended' | 'none'; + + if (ps.isSuspended != null && isSuspendedBefore !== ps.isSuspended) { + suspensionState = ps.isSuspended ? 'manuallySuspended' : 'none'; + } + await this.federatedInstanceService.update(instance.id, { - isSuspended: ps.isSuspended, + suspensionState, moderationNote: ps.moderationNote, }); - if (ps.isSuspended != null && instance.isSuspended !== ps.isSuspended) { + if (ps.isSuspended != null && isSuspendedBefore !== ps.isSuspended) { if (ps.isSuspended) { this.moderationLogService.log(me, 'suspendRemoteInstance', { id: instance.id, diff --git a/packages/frontend/src/pages/admin/federation.vue b/packages/frontend/src/pages/admin/federation.vue index de27e1f67a..0aaa398584 100644 --- a/packages/frontend/src/pages/admin/federation.vue +++ b/packages/frontend/src/pages/admin/federation.vue @@ -58,6 +58,7 @@ SPDX-License-Identifier: AGPL-3.0-only + + diff --git a/packages/frontend/src/components/MkFormDialog.vue b/packages/frontend/src/components/MkFormDialog.vue index deedc5badb..124f114111 100644 --- a/packages/frontend/src/components/MkFormDialog.vue +++ b/packages/frontend/src/components/MkFormDialog.vue @@ -21,8 +21,9 @@ SPDX-License-Identifier: AGPL-3.0-only
- - + diff --git a/packages/frontend/src/ui/deck/deck-store.ts b/packages/frontend/src/ui/deck/deck-store.ts index 70b55e8172..bb3c04cd5c 100644 --- a/packages/frontend/src/ui/deck/deck-store.ts +++ b/packages/frontend/src/ui/deck/deck-store.ts @@ -9,6 +9,7 @@ import { notificationTypes } from 'misskey-js'; import { Storage } from '@/pizzax.js'; import { misskeyApi } from '@/scripts/misskey-api.js'; import { deepClone } from '@/scripts/clone.js'; +import { SoundStore } from '@/store.js'; type ColumnWidget = { name: string; @@ -33,6 +34,7 @@ export type Column = { withRenotes?: boolean; withReplies?: boolean; onlyFiles?: boolean; + soundSetting: SoundStore; }; export const deckStore = markRaw(new Storage('deck', { diff --git a/packages/frontend/src/ui/deck/list-column.vue b/packages/frontend/src/ui/deck/list-column.vue index 70ea54326f..5369112494 100644 --- a/packages/frontend/src/ui/deck/list-column.vue +++ b/packages/frontend/src/ui/deck/list-column.vue @@ -9,7 +9,7 @@ SPDX-License-Identifier: AGPL-3.0-only {{ column.name }} - + @@ -21,6 +21,10 @@ import MkTimeline from '@/components/MkTimeline.vue'; import * as os from '@/os.js'; import { misskeyApi } from '@/scripts/misskey-api.js'; import { i18n } from '@/i18n.js'; +import { MenuItem } from '@/types/menu.js'; +import { SoundStore } from '@/store.js'; +import { soundSettingsButton } from '@/ui/deck/tl-note-notification.js'; +import * as sound from '@/scripts/sound.js'; const props = defineProps<{ column: Column; @@ -29,6 +33,7 @@ const props = defineProps<{ const timeline = shallowRef>(); const withRenotes = ref(props.column.withRenotes ?? true); +const soundSetting = ref(props.column.soundSetting ?? { type: null, volume: 1 }); if (props.column.listId == null) { setList(); @@ -40,6 +45,10 @@ watch(withRenotes, v => { }); }); +watch(soundSetting, v => { + updateColumn(props.column.id, { soundSetting: v }); +}); + async function setList() { const lists = await misskeyApi('users/lists/list'); const { canceled, result: list } = await os.select({ @@ -59,7 +68,11 @@ function editList() { os.pageWindow('my/lists/' + props.column.listId); } -const menu = [ +function onNote() { + sound.playMisskeySfxFile(soundSetting.value); +} + +const menu: MenuItem[] = [ { icon: 'ti ti-pencil', text: i18n.ts.selectList, @@ -75,5 +88,10 @@ const menu = [ text: i18n.ts.showRenotes, ref: withRenotes, }, + { + icon: 'ti ti-bell', + text: i18n.ts._deck.newNoteNotificationSettings, + action: () => soundSettingsButton(soundSetting), + }, ]; diff --git a/packages/frontend/src/ui/deck/role-timeline-column.vue b/packages/frontend/src/ui/deck/role-timeline-column.vue index eae2ee13f3..32ab7527b4 100644 --- a/packages/frontend/src/ui/deck/role-timeline-column.vue +++ b/packages/frontend/src/ui/deck/role-timeline-column.vue @@ -9,18 +9,22 @@ SPDX-License-Identifier: AGPL-3.0-only {{ column.name }} - +