From b36e6b1a777848ec8553b297e956ada240dcacc9 Mon Sep 17 00:00:00 2001 From: anatawa12 Date: Thu, 22 Feb 2024 00:59:59 +0900 Subject: fix: 禁止キーワードを含むノートがDelayed Queueに追加されて再処理される問題 (#13428) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * refactor: use IdentifiableError instead of NoteCreateService.ContainsProhibitedWordsError * fix: notes with prohibited words are reprocessed with delay * docs(changelog): 禁止キーワードを含むノートがDelayed Queueに追加されて再処理される問題 * lint: fix lint errors * fix: rethrowするべきなのにrethrowし忘れていたのを修正 --- packages/backend/src/queue/processors/InboxProcessorService.ts | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) (limited to 'packages/backend/src/queue/processors') diff --git a/packages/backend/src/queue/processors/InboxProcessorService.ts b/packages/backend/src/queue/processors/InboxProcessorService.ts index 7adadd799b..0a713149e5 100644 --- a/packages/backend/src/queue/processors/InboxProcessorService.ts +++ b/packages/backend/src/queue/processors/InboxProcessorService.ts @@ -24,6 +24,7 @@ import { ApPersonService } from '@/core/activitypub/models/ApPersonService.js'; import { LdSignatureService } from '@/core/activitypub/LdSignatureService.js'; import { ApInboxService } from '@/core/activitypub/ApInboxService.js'; import { bindThis } from '@/decorators.js'; +import { IdentifiableError } from '@/misc/identifiable-error.js'; import { QueueLoggerService } from '../QueueLoggerService.js'; import type { InboxJobData } from '../types.js'; @@ -180,7 +181,14 @@ export class InboxProcessorService { }); // アクティビティを処理 - await this.apInboxService.performActivity(authUser.user, activity); + try { + await this.apInboxService.performActivity(authUser.user, activity); + } catch (e) { + if (e instanceof IdentifiableError) { + if (e.id === '689ee33f-f97c-479a-ac49-1b9f8140af99') return 'blocked notes with prohibited words'; + } + throw e; + } return 'ok'; } } -- cgit v1.2.3-freya From 600d91beda206fa22cfa1c1a3f94ca9e5a0cac68 Mon Sep 17 00:00:00 2001 From: tamaina Date: Fri, 23 Feb 2024 18:04:30 +0900 Subject: enhance: リモートのフォロワーから再度Followが来た場合、acceptを返してあげる (#13388) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * enhance: リモートのフォロワーから再度Followが来た場合、acceptを返してあげる * nanka meccha kaeta * ブロックチェックの後にフォロー関係の存在チェックをする --- packages/backend/src/core/UserFollowingService.ts | 60 ++++++++++++++++++---- .../processors/RelationshipProcessorService.ts | 2 +- .../src/server/api/endpoints/following/create.ts | 13 +---- 3 files changed, 52 insertions(+), 23 deletions(-) (limited to 'packages/backend/src/queue/processors') diff --git a/packages/backend/src/core/UserFollowingService.ts b/packages/backend/src/core/UserFollowingService.ts index 8ad85391c6..d87cbacdcb 100644 --- a/packages/backend/src/core/UserFollowingService.ts +++ b/packages/backend/src/core/UserFollowingService.ts @@ -30,6 +30,7 @@ import type { Config } from '@/config.js'; import { AccountMoveService } from '@/core/AccountMoveService.js'; import { UtilityService } from '@/core/UtilityService.js'; import { FanoutTimelineService } from '@/core/FanoutTimelineService.js'; +import type { ThinUser } from '@/queue/types.js'; import Logger from '../logger.js'; const logger = new Logger('following/create'); @@ -94,20 +95,43 @@ export class UserFollowingService implements OnModuleInit { this.userBlockingService = this.moduleRef.get('UserBlockingService'); } + @bindThis + public async deliverAccept(follower: MiRemoteUser, followee: MiPartialLocalUser, requestId?: string) { + const content = this.apRendererService.addContext(this.apRendererService.renderAccept(this.apRendererService.renderFollow(follower, followee, requestId), followee)); + this.queueService.deliver(followee, content, follower.inbox, false); + } + + /** + * ThinUserでなくともユーザーの情報が最新でない場合はこちらを使うべき + */ + @bindThis + public async followByThinUser( + _follower: ThinUser, + _followee: ThinUser, + options: Parameters[2] = {}, + ) { + const [follower, followee] = await Promise.all([ + this.usersRepository.findOneByOrFail({ id: _follower.id }), + this.usersRepository.findOneByOrFail({ id: _followee.id }), + ]) as [MiLocalUser | MiRemoteUser, MiLocalUser | MiRemoteUser]; + + await this.follow(follower, followee, options); + } + @bindThis public async follow( - _follower: { id: MiUser['id'] }, - _followee: { id: MiUser['id'] }, + follower: MiLocalUser | MiRemoteUser, + followee: MiLocalUser | MiRemoteUser, { requestId, silent = false, withReplies }: { requestId?: string, silent?: boolean, withReplies?: boolean, } = {}, ): Promise { - const [follower, followee] = await Promise.all([ - this.usersRepository.findOneByOrFail({ id: _follower.id }), - this.usersRepository.findOneByOrFail({ id: _followee.id }), - ]) as [MiLocalUser | MiRemoteUser, MiLocalUser | MiRemoteUser]; + if (this.userEntityService.isRemoteUser(follower) && this.userEntityService.isRemoteUser(followee)) { + // What? + throw new Error('Remote user cannot follow remote user.'); + } // check blocking const [blocking, blocked] = await Promise.all([ @@ -129,6 +153,24 @@ export class UserFollowingService implements OnModuleInit { if (blocked) throw new IdentifiableError('3338392a-f764-498d-8855-db939dcf8c48', 'blocked'); } + if (await this.followingsRepository.exists({ + where: { + followerId: follower.id, + followeeId: followee.id, + }, + })) { + // すでにフォロー関係が存在している場合 + if (this.userEntityService.isRemoteUser(follower) && this.userEntityService.isLocalUser(followee)) { + // リモート → ローカル: acceptを送り返しておしまい + this.deliverAccept(follower, followee, requestId); + return; + } + if (this.userEntityService.isLocalUser(follower)) { + // ローカル → リモート/ローカル: 例外 + throw new IdentifiableError('ec3f65c0-a9d1-47d9-8791-b2e7b9dcdced', 'already following'); + } + } + const followeeProfile = await this.userProfilesRepository.findOneByOrFail({ userId: followee.id }); // フォロー対象が鍵アカウントである or // フォロワーがBotであり、フォロー対象がBotからのフォローに慎重である or @@ -189,8 +231,7 @@ export class UserFollowingService implements OnModuleInit { await this.insertFollowingDoc(followee, follower, silent, withReplies); if (this.userEntityService.isRemoteUser(follower) && this.userEntityService.isLocalUser(followee)) { - const content = this.apRendererService.addContext(this.apRendererService.renderAccept(this.apRendererService.renderFollow(follower, followee, requestId), followee)); - this.queueService.deliver(followee, content, follower.inbox, false); + this.deliverAccept(follower, followee, requestId); } } @@ -571,8 +612,7 @@ export class UserFollowingService implements OnModuleInit { await this.insertFollowingDoc(followee, follower, false, request.withReplies); if (this.userEntityService.isRemoteUser(follower) && this.userEntityService.isLocalUser(followee)) { - const content = this.apRendererService.addContext(this.apRendererService.renderAccept(this.apRendererService.renderFollow(follower, followee as MiPartialLocalUser, request.requestId!), followee)); - this.queueService.deliver(followee, content, follower.inbox, false); + this.deliverAccept(follower, followee as MiPartialLocalUser, request.requestId ?? undefined); } this.userEntityService.pack(followee.id, followee, { diff --git a/packages/backend/src/queue/processors/RelationshipProcessorService.ts b/packages/backend/src/queue/processors/RelationshipProcessorService.ts index 408b02fb38..53dbb42169 100644 --- a/packages/backend/src/queue/processors/RelationshipProcessorService.ts +++ b/packages/backend/src/queue/processors/RelationshipProcessorService.ts @@ -35,7 +35,7 @@ export class RelationshipProcessorService { @bindThis public async processFollow(job: Bull.Job): Promise { this.logger.info(`${job.data.from.id} is trying to follow ${job.data.to.id} ${job.data.withReplies ? "with replies" : "without replies"}`); - await this.userFollowingService.follow(job.data.from, job.data.to, { + await this.userFollowingService.followByThinUser(job.data.from, job.data.to, { requestId: job.data.requestId, silent: job.data.silent, withReplies: job.data.withReplies, diff --git a/packages/backend/src/server/api/endpoints/following/create.ts b/packages/backend/src/server/api/endpoints/following/create.ts index ceaf32ccb2..042d7f119d 100644 --- a/packages/backend/src/server/api/endpoints/following/create.ts +++ b/packages/backend/src/server/api/endpoints/following/create.ts @@ -100,22 +100,11 @@ export default class extends Endpoint { // eslint- throw err; }); - // Check if already following - const exist = await this.followingsRepository.exists({ - where: { - followerId: follower.id, - followeeId: followee.id, - }, - }); - - if (exist) { - throw new ApiError(meta.errors.alreadyFollowing); - } - try { await this.userFollowingService.follow(follower, followee, { withReplies: ps.withReplies }); } catch (e) { if (e instanceof IdentifiableError) { + if (e.id === 'ec3f65c0-a9d1-47d9-8791-b2e7b9dcdced') throw new ApiError(meta.errors.alreadyFollowing); if (e.id === '710e8fb0-b8c3-4922-be49-d5d93d8e6a6e') throw new ApiError(meta.errors.blocking); if (e.id === '3338392a-f764-498d-8855-db939dcf8c48') throw new ApiError(meta.errors.blocked); } -- cgit v1.2.3-freya From 2c6f25b710b4f8095458fe88ddd56e6c6a41d006 Mon Sep 17 00:00:00 2001 From: tamaina Date: Sun, 25 Feb 2024 12:36:10 +0900 Subject: fix: 古いキャッシュを使うのを修正 (#13453) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- packages/backend/src/core/AccountMoveService.ts | 4 +--- packages/backend/src/core/CacheService.ts | 4 +++- packages/backend/src/core/GlobalEventService.ts | 1 + packages/backend/src/core/UserFollowingService.ts | 27 ++++++++-------------- packages/backend/src/misc/cache.ts | 8 +++++++ .../processors/RelationshipProcessorService.ts | 2 +- .../src/server/api/endpoints/following/create.ts | 2 +- .../backend/src/server/api/endpoints/i/update.ts | 6 ++--- 8 files changed, 27 insertions(+), 27 deletions(-) (limited to 'packages/backend/src/queue/processors') diff --git a/packages/backend/src/core/AccountMoveService.ts b/packages/backend/src/core/AccountMoveService.ts index b7796a5183..5bd885df40 100644 --- a/packages/backend/src/core/AccountMoveService.ts +++ b/packages/backend/src/core/AccountMoveService.ts @@ -20,7 +20,6 @@ import { ApPersonService } from '@/core/activitypub/models/ApPersonService.js'; import { ApDeliverManagerService } from '@/core/activitypub/ApDeliverManagerService.js'; import { ApRendererService } from '@/core/activitypub/ApRendererService.js'; import { UserEntityService } from '@/core/entities/UserEntityService.js'; -import { CacheService } from '@/core/CacheService.js'; import { ProxyAccountService } from '@/core/ProxyAccountService.js'; import { FederatedInstanceService } from '@/core/FederatedInstanceService.js'; import { MetaService } from '@/core/MetaService.js'; @@ -60,7 +59,6 @@ export class AccountMoveService { private instanceChart: InstanceChart, private metaService: MetaService, private relayService: RelayService, - private cacheService: CacheService, private queueService: QueueService, ) { } @@ -84,7 +82,7 @@ export class AccountMoveService { Object.assign(src, update); // Update cache - this.cacheService.uriPersonCache.set(srcUri, src); + this.globalEventService.publishInternalEvent('localUserUpdated', src); const srcPerson = await this.apRendererService.renderPerson(src); const updateAct = this.apRendererService.addContext(this.apRendererService.renderUpdate(srcPerson, src)); diff --git a/packages/backend/src/core/CacheService.ts b/packages/backend/src/core/CacheService.ts index 0fc47bf8ec..d008e7ec52 100644 --- a/packages/backend/src/core/CacheService.ts +++ b/packages/backend/src/core/CacheService.ts @@ -129,10 +129,12 @@ export class CacheService implements OnApplicationShutdown { switch (type) { case 'userChangeSuspendedState': case 'userChangeDeletedState': - case 'remoteUserUpdated': { + case 'remoteUserUpdated': + case 'localUserUpdated': { const user = await this.usersRepository.findOneBy({ id: body.id }); if (user == null) { this.userByIdCache.delete(body.id); + this.localUserByIdCache.delete(body.id); for (const [k, v] of this.uriPersonCache.cache.entries()) { if (v.value?.id === body.id) { this.uriPersonCache.delete(k); diff --git a/packages/backend/src/core/GlobalEventService.ts b/packages/backend/src/core/GlobalEventService.ts index a127a6df3b..7c1b34da05 100644 --- a/packages/backend/src/core/GlobalEventService.ts +++ b/packages/backend/src/core/GlobalEventService.ts @@ -212,6 +212,7 @@ export interface InternalEventTypes { userChangeDeletedState: { id: MiUser['id']; isDeleted: MiUser['isDeleted']; }; userTokenRegenerated: { id: MiUser['id']; oldToken: string; newToken: string; }; remoteUserUpdated: { id: MiUser['id']; }; + localUserUpdated: { id: MiUser['id']; }; follow: { followerId: MiUser['id']; followeeId: MiUser['id']; }; unfollow: { followerId: MiUser['id']; followeeId: MiUser['id']; }; blockingCreated: { blockerId: MiUser['id']; blockeeId: MiUser['id']; }; diff --git a/packages/backend/src/core/UserFollowingService.ts b/packages/backend/src/core/UserFollowingService.ts index d87cbacdcb..0a492c06e4 100644 --- a/packages/backend/src/core/UserFollowingService.ts +++ b/packages/backend/src/core/UserFollowingService.ts @@ -101,33 +101,24 @@ export class UserFollowingService implements OnModuleInit { this.queueService.deliver(followee, content, follower.inbox, false); } - /** - * ThinUserでなくともユーザーの情報が最新でない場合はこちらを使うべき - */ @bindThis - public async followByThinUser( + public async follow( _follower: ThinUser, _followee: ThinUser, - options: Parameters[2] = {}, - ) { - const [follower, followee] = await Promise.all([ - this.usersRepository.findOneByOrFail({ id: _follower.id }), - this.usersRepository.findOneByOrFail({ id: _followee.id }), - ]) as [MiLocalUser | MiRemoteUser, MiLocalUser | MiRemoteUser]; - - await this.follow(follower, followee, options); - } - - @bindThis - public async follow( - follower: MiLocalUser | MiRemoteUser, - followee: MiLocalUser | MiRemoteUser, { requestId, silent = false, withReplies }: { requestId?: string, silent?: boolean, withReplies?: boolean, } = {}, ): Promise { + /** + * 必ず最新のユーザー情報を取得する + */ + const [follower, followee] = await Promise.all([ + this.usersRepository.findOneByOrFail({ id: _follower.id }), + this.usersRepository.findOneByOrFail({ id: _followee.id }), + ]) as [MiLocalUser | MiRemoteUser, MiLocalUser | MiRemoteUser]; + if (this.userEntityService.isRemoteUser(follower) && this.userEntityService.isRemoteUser(followee)) { // What? throw new Error('Remote user cannot follow remote user.'); diff --git a/packages/backend/src/misc/cache.ts b/packages/backend/src/misc/cache.ts index 7f4d1521b5..bba64a06ef 100644 --- a/packages/backend/src/misc/cache.ts +++ b/packages/backend/src/misc/cache.ts @@ -187,6 +187,10 @@ export class RedisSingleCache { // TODO: メモリ節約のためあまり参照されないキーを定期的に削除できるようにする？ export class MemoryKVCache { + /** + * データを持つマップ + * @deprecated これを直接操作するべきではない + */ public cache: Map; private lifetime: number; private gcIntervalHandle: NodeJS.Timeout; @@ -201,6 +205,10 @@ export class MemoryKVCache { } @bindThis + /** + * Mapにキャッシュをセットします + * @deprecated これを直接呼び出すべきではない。InternalEventなどで変更を全てのプロセス/マシンに通知するべき + */ public set(key: string, value: T): void { this.cache.set(key, { date: Date.now(), diff --git a/packages/backend/src/queue/processors/RelationshipProcessorService.ts b/packages/backend/src/queue/processors/RelationshipProcessorService.ts index 53dbb42169..408b02fb38 100644 --- a/packages/backend/src/queue/processors/RelationshipProcessorService.ts +++ b/packages/backend/src/queue/processors/RelationshipProcessorService.ts @@ -35,7 +35,7 @@ export class RelationshipProcessorService { @bindThis public async processFollow(job: Bull.Job): Promise { this.logger.info(`${job.data.from.id} is trying to follow ${job.data.to.id} ${job.data.withReplies ? "with replies" : "without replies"}`); - await this.userFollowingService.followByThinUser(job.data.from, job.data.to, { + await this.userFollowingService.follow(job.data.from, job.data.to, { requestId: job.data.requestId, silent: job.data.silent, withReplies: job.data.withReplies, diff --git a/packages/backend/src/server/api/endpoints/following/create.ts b/packages/backend/src/server/api/endpoints/following/create.ts index 042d7f119d..db320e7129 100644 --- a/packages/backend/src/server/api/endpoints/following/create.ts +++ b/packages/backend/src/server/api/endpoints/following/create.ts @@ -71,7 +71,7 @@ export const paramDef = { type: 'object', properties: { userId: { type: 'string', format: 'misskey:id' }, - withReplies: { type: 'boolean' } + withReplies: { type: 'boolean' }, }, required: ['userId'], } as const; diff --git a/packages/backend/src/server/api/endpoints/i/update.ts b/packages/backend/src/server/api/endpoints/i/update.ts index bf6c53d8eb..84a1931a3d 100644 --- a/packages/backend/src/server/api/endpoints/i/update.ts +++ b/packages/backend/src/server/api/endpoints/i/update.ts @@ -456,9 +456,9 @@ export default class extends Endpoint { // eslint- this.hashtagService.updateUsertags(user, tags); //#endregion - if (Object.keys(updates).length > 0) await this.usersRepository.update(user.id, updates); - if (Object.keys(updates).includes('alsoKnownAs')) { - this.cacheService.uriPersonCache.set(this.userEntityService.genLocalUserUri(user.id), { ...user, ...updates }); + if (Object.keys(updates).length > 0) { + await this.usersRepository.update(user.id, updates); + this.globalEventService.publishInternalEvent('localUserUpdated', { id: user.id }); } await this.userProfilesRepository.update(user.id, { -- cgit v1.2.3-freya From b7d9d1620161a728e34ab20d8ea99160b3eb4196 Mon Sep 17 00:00:00 2001 From: okayurisotto <47853651+okayurisotto@users.noreply.github.com> Date: Wed, 28 Feb 2024 15:34:58 +0900 Subject: refactor(backend): ノートのエクスポート処理でStreams APIを使うように (#13465) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * refactor(backend): ノートのエクスポート処理でStreams APIを使うように * fixup! refactor(backend): ノートのエクスポート処理でStreams APIを使うように `await`忘れにより、ジョブがすぐに完了したことになり削除されてしまっていた。 それによって、`NoteStream`内での`updateProgress`メソッドの呼び出しで、`Missing key for job`のエラーが発生することがあった。 --------- Co-authored-by: syuilo <4439005+syuilo@users.noreply.github.com> --- packages/backend/src/misc/FileWriterStream.ts | 31 ++++ packages/backend/src/misc/JsonArrayStream.ts | 30 ++++ .../processors/ExportNotesProcessorService.ts | 164 +++++++++++---------- 3 files changed, 146 insertions(+), 79 deletions(-) create mode 100644 packages/backend/src/misc/FileWriterStream.ts create mode 100644 packages/backend/src/misc/JsonArrayStream.ts (limited to 'packages/backend/src/queue/processors') diff --git a/packages/backend/src/misc/FileWriterStream.ts b/packages/backend/src/misc/FileWriterStream.ts new file mode 100644 index 0000000000..828851df0e --- /dev/null +++ b/packages/backend/src/misc/FileWriterStream.ts @@ -0,0 +1,31 @@ +import * as fs from 'node:fs/promises'; +import type { PathLike } from 'node:fs'; + +/** + * `fs.createWriteStream()`相当のことを行う`WritableStream` (Web標準) + */ +export class FileWriterStream extends WritableStream { + constructor(path: PathLike) { + let file: fs.FileHandle | null = null; + + super({ + start: async () => { + file = await fs.open(path, 'a'); + }, + write: async (chunk, controller) => { + if (file === null) { + controller.error(); + throw new Error(); + } + + await file.write(chunk); + }, + close: async () => { + await file?.close(); + }, + abort: async () => { + await file?.close(); + }, + }); + } +} diff --git a/packages/backend/src/misc/JsonArrayStream.ts b/packages/backend/src/misc/JsonArrayStream.ts new file mode 100644 index 0000000000..ad35bb3a79 --- /dev/null +++ b/packages/backend/src/misc/JsonArrayStream.ts @@ -0,0 +1,30 @@ +import { TransformStream } from 'node:stream/web'; + +/** + * ストリームに流れてきた各データについて`JSON.stringify()`した上で、それらを一つの配列にまとめる + */ +export class JsonArrayStream extends TransformStream { + constructor() { + /** 最初の要素かどうかを変数に記録 */ + let isFirst = true; + + super({ + start(controller) { + controller.enqueue('['); + }, + flush(controller) { + controller.enqueue(']'); + }, + transform(chunk, controller) { + if (isFirst) { + isFirst = false; + } else { + // 妥当なJSON配列にするためには最初以外の要素の前に`,`を挿入しなければならない + controller.enqueue(',\n'); + } + + controller.enqueue(JSON.stringify(chunk)); + }, + }); + } +} diff --git a/packages/backend/src/queue/processors/ExportNotesProcessorService.ts b/packages/backend/src/queue/processors/ExportNotesProcessorService.ts index f2ae0ce4b4..c7611012d7 100644 --- a/packages/backend/src/queue/processors/ExportNotesProcessorService.ts +++ b/packages/backend/src/queue/processors/ExportNotesProcessorService.ts @@ -3,7 +3,7 @@ * SPDX-License-Identifier: AGPL-3.0-only */ -import * as fs from 'node:fs'; +import { ReadableStream, TextEncoderStream } from 'node:stream/web'; import { Inject, Injectable } from '@nestjs/common'; import { MoreThan } from 'typeorm'; import { format as dateFormat } from 'date-fns'; @@ -18,10 +18,82 @@ import { bindThis } from '@/decorators.js'; import { DriveFileEntityService } from '@/core/entities/DriveFileEntityService.js'; import { Packed } from '@/misc/json-schema.js'; import { IdService } from '@/core/IdService.js'; +import { JsonArrayStream } from '@/misc/JsonArrayStream.js'; +import { FileWriterStream } from '@/misc/FileWriterStream.js'; import { QueueLoggerService } from '../QueueLoggerService.js'; import type * as Bull from 'bullmq'; import type { DbJobDataWithUser } from '../types.js'; +class NoteStream extends ReadableStream> { + constructor( + job: Bull.Job, + notesRepository: NotesRepository, + pollsRepository: PollsRepository, + driveFileEntityService: DriveFileEntityService, + idService: IdService, + userId: string, + ) { + let exportedNotesCount = 0; + let cursor: MiNote['id'] | null = null; + + const serialize = ( + note: MiNote, + poll: MiPoll | null, + files: Packed<'DriveFile'>[], + ): Record => { + return { + id: note.id, + text: note.text, + createdAt: idService.parse(note.id).date.toISOString(), + fileIds: note.fileIds, + files: files, + replyId: note.replyId, + renoteId: note.renoteId, + poll: poll, + cw: note.cw, + visibility: note.visibility, + visibleUserIds: note.visibleUserIds, + localOnly: note.localOnly, + reactionAcceptance: note.reactionAcceptance, + }; + }; + + super({ + async pull(controller): Promise { + const notes = await notesRepository.find({ + where: { + userId, + ...(cursor !== null ? { id: MoreThan(cursor) } : {}), + }, + take: 100, // 100件ずつ取得 + order: { id: 1 }, + }); + + if (notes.length === 0) { + job.updateProgress(100); + controller.close(); + } + + cursor = notes.at(-1)?.id ?? null; + + for (const note of notes) { + const poll = note.hasPoll + ? await pollsRepository.findOneByOrFail({ noteId: note.id }) // N+1 + : null; + const files = await driveFileEntityService.packManyByIds(note.fileIds); // N+1 + const content = serialize(note, poll, files); + + controller.enqueue(content); + exportedNotesCount++; + } + + const total = await notesRepository.countBy({ userId }); + job.updateProgress(exportedNotesCount / total); + }, + }); + } +} + @Injectable() export class ExportNotesProcessorService { private logger: Logger; @@ -59,67 +131,19 @@ export class ExportNotesProcessorService { this.logger.info(`Temp file is ${path}`); try { - const stream = fs.createWriteStream(path, { flags: 'a' }); - - const write = (text: string): Promise => { - return new Promise((res, rej) => { - stream.write(text, err => { - if (err) { - this.logger.error(err); - rej(err); - } else { - res(); - } - }); - }); - }; - - await write('['); - - let exportedNotesCount = 0; - let cursor: MiNote['id'] | null = null; - - while (true) { - const notes = await this.notesRepository.find({ - where: { - userId: user.id, - ...(cursor ? { id: MoreThan(cursor) } : {}), - }, - take: 100, - order: { - id: 1, - }, - }) as MiNote[]; - - if (notes.length === 0) { - job.updateProgress(100); - break; - } - - cursor = notes.at(-1)?.id ?? null; + // メモリが足りなくならないようにストリームで処理する + await new NoteStream( + job, + this.notesRepository, + this.pollsRepository, + this.driveFileEntityService, + this.idService, + user.id, + ) + .pipeThrough(new JsonArrayStream()) + .pipeThrough(new TextEncoderStream()) + .pipeTo(new FileWriterStream(path)); - for (const note of notes) { - let poll: MiPoll | undefined; - if (note.hasPoll) { - poll = await this.pollsRepository.findOneByOrFail({ noteId: note.id }); - } - const files = await this.driveFileEntityService.packManyByIds(note.fileIds); - const content = JSON.stringify(this.serialize(note, poll, files)); - const isFirst = exportedNotesCount === 0; - await write(isFirst ? content : ',\n' + content); - exportedNotesCount++; - } - - const total = await this.notesRepository.countBy({ - userId: user.id, - }); - - job.updateProgress(exportedNotesCount / total); - } - - await write(']'); - - stream.end(); this.logger.succ(`Exported to: ${path}`); const fileName = 'notes-' + dateFormat(new Date(), 'yyyy-MM-dd-HH-mm-ss') + '.json'; @@ -130,22 +154,4 @@ export class ExportNotesProcessorService { cleanup(); } } - - private serialize(note: MiNote, poll: MiPoll | null = null, files: Packed<'DriveFile'>[]): Record { - return { - id: note.id, - text: note.text, - createdAt: this.idService.parse(note.id).date.toISOString(), - fileIds: note.fileIds, - files: files, - replyId: note.replyId, - renoteId: note.renoteId, - poll: poll, - cw: note.cw, - visibility: note.visibility, - visibleUserIds: note.visibleUserIds, - localOnly: note.localOnly, - reactionAcceptance: note.reactionAcceptance, - }; - } } -- cgit v1.2.3-freya From eb60460d28be24513b567d378cec6ecba5c158c7 Mon Sep 17 00:00:00 2001 From: tamaina Date: Fri, 1 Mar 2024 11:57:26 +0900 Subject: enhance: 禁止ワードチェック強化 (#27) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * enhance: 禁止ワードチェック強化 * リモートの禁止ワードチェックを添付ファイルとユーザーを登録する前に行うなど Resolve https://github.com/misskey-dev/misskey/issues/13374 * 禁止ワートの対象の見直し * performActivityで特定のエラーが出た際にDelayedに追加しないように * use IdentifiableError * NoteCreateService.checkProhibitedWords * https://github.com/misskey-dev/misskey-private/pull/27/files#r1507416135 * remove comment --- packages/backend/src/core/NoteCreateService.ts | 25 ++++++++- packages/backend/src/core/UtilityService.ts | 14 +++++ .../backend/src/core/activitypub/ApInboxService.ts | 1 - .../src/core/activitypub/models/ApNoteService.ts | 62 ++++++++++++++-------- .../src/queue/processors/InboxProcessorService.ts | 5 +- 5 files changed, 83 insertions(+), 24 deletions(-) (limited to 'packages/backend/src/queue/processors') diff --git a/packages/backend/src/core/NoteCreateService.ts b/packages/backend/src/core/NoteCreateService.ts index 727787f868..81ae2908d3 100644 --- a/packages/backend/src/core/NoteCreateService.ts +++ b/packages/backend/src/core/NoteCreateService.ts @@ -263,7 +263,13 @@ export class NoteCreateService implements OnApplicationShutdown { } } - if (this.utilityService.isKeyWordIncluded(data.cw ?? data.text ?? '', meta.prohibitedWords)) { + const hasProhibitedWords = await this.checkProhibitedWordsContain({ + cw: data.cw, + text: data.text, + pollChoices: data.poll?.choices, + }, meta.prohibitedWords); + + if (hasProhibitedWords) { throw new IdentifiableError('689ee33f-f97c-479a-ac49-1b9f8140af99', 'Note contains prohibited words'); } @@ -995,6 +1001,23 @@ export class NoteCreateService implements OnApplicationShutdown { } } + public async checkProhibitedWordsContain(content: Parameters[0], prohibitedWords?: string[]) { + if (prohibitedWords == null) { + prohibitedWords = (await this.metaService.fetch()).prohibitedWords; + } + + if ( + this.utilityService.isKeyWordIncluded( + this.utilityService.concatNoteContentsForKeyWordCheck(content), + prohibitedWords, + ) + ) { + return true; + } + + return false; + } + @bindThis public dispose(): void { this.#shutdownController.abort(); diff --git a/packages/backend/src/core/UtilityService.ts b/packages/backend/src/core/UtilityService.ts index 638a0c019e..652e8f7449 100644 --- a/packages/backend/src/core/UtilityService.ts +++ b/packages/backend/src/core/UtilityService.ts @@ -42,6 +42,20 @@ export class UtilityService { return silencedHosts.some(x => `.${host.toLowerCase()}`.endsWith(`.${x}`)); } + @bindThis + public concatNoteContentsForKeyWordCheck(content: { + cw?: string | null; + text?: string | null; + pollChoices?: string[] | null; + others?: string[] | null; + }): string { + /** + * ノートの内容を結合してキーワードチェック用の文字列を生成する + * cwとtextは内容が繋がっているかもしれないので間に何も入れずにチェックする + */ + return `${content.cw ?? ''}${content.text ?? ''}\n${(content.pollChoices ?? []).join('\n')}\n${(content.others ?? []).join('\n')}`; + } + @bindThis public isKeyWordIncluded(text: string, keyWords: string[]): boolean { if (keyWords.length === 0) return false; diff --git a/packages/backend/src/core/activitypub/ApInboxService.ts b/packages/backend/src/core/activitypub/ApInboxService.ts index b0f56a5d82..1621c41bcc 100644 --- a/packages/backend/src/core/activitypub/ApInboxService.ts +++ b/packages/backend/src/core/activitypub/ApInboxService.ts @@ -36,7 +36,6 @@ import { ApResolverService } from './ApResolverService.js'; import { ApAudienceService } from './ApAudienceService.js'; import { ApPersonService } from './models/ApPersonService.js'; import { ApQuestionService } from './models/ApQuestionService.js'; -import { CacheService } from '@/core/CacheService.js'; import { GlobalEventService } from '@/core/GlobalEventService.js'; import type { Resolver } from './ApResolverService.js'; import type { IAccept, IAdd, IAnnounce, IBlock, ICreate, IDelete, IFlag, IFollow, ILike, IObject, IReject, IRemove, IUndo, IUpdate, IMove } from './type.js'; diff --git a/packages/backend/src/core/activitypub/models/ApNoteService.ts b/packages/backend/src/core/activitypub/models/ApNoteService.ts index e201b88173..b2fd435f93 100644 --- a/packages/backend/src/core/activitypub/models/ApNoteService.ts +++ b/packages/backend/src/core/activitypub/models/ApNoteService.ts @@ -24,6 +24,8 @@ import { StatusError } from '@/misc/status-error.js'; import { UtilityService } from '@/core/UtilityService.js'; import { bindThis } from '@/decorators.js'; import { checkHttps } from '@/misc/check-https.js'; +import { IdentifiableError } from '@/misc/identifiable-error.js'; +import { isNotNull } from '@/misc/is-not-null.js'; import { getOneApId, getApId, getOneApHrefNullable, validPost, isEmoji, getApType } from '../type.js'; import { ApLoggerService } from '../ApLoggerService.js'; import { ApMfmService } from '../ApMfmService.js'; @@ -37,7 +39,6 @@ import { ApQuestionService } from './ApQuestionService.js'; import { ApImageService } from './ApImageService.js'; import type { Resolver } from '../ApResolverService.js'; import type { IObject, IPost } from '../type.js'; -import { isNotNull } from '@/misc/is-not-null.js'; @Injectable() export class ApNoteService { @@ -152,11 +153,47 @@ export class ApNoteService { throw new Error('invalid note.attributedTo: ' + note.attributedTo); } - const actor = await this.apPersonService.resolvePerson(getOneApId(note.attributedTo), resolver) as MiRemoteUser; + const uri = getOneApId(note.attributedTo); + + // ローカルで投稿者を検索し、もし凍結されていたらスキップ + const cachedActor = await this.apPersonService.fetchPerson(uri) as MiRemoteUser; + if (cachedActor && cachedActor.isSuspended) { + throw new IdentifiableError('85ab9bd7-3a41-4530-959d-f07073900109', 'actor has been suspended'); + } + + const apMentions = await this.apMentionService.extractApMentions(note.tag, resolver); + const apHashtags = extractApHashtags(note.tag); + + const cw = note.summary === '' ? null : note.summary; + + // テキストのパース + let text: string | null = null; + if (note.source?.mediaType === 'text/x.misskeymarkdown' && typeof note.source.content === 'string') { + text = note.source.content; + } else if (typeof note._misskey_content !== 'undefined') { + text = note._misskey_content; + } else if (typeof note.content === 'string') { + text = this.apMfmService.htmlToMfm(note.content, note.tag); + } + + const poll = await this.apQuestionService.extractPollFromQuestion(note, resolver).catch(() => undefined); - // 投稿者が凍結されていたらスキップ + //#region Contents Check + // 添付ファイルとユーザーをこのサーバーで登録する前に内容をチェックする + /** + * 禁止ワードチェック + */ + const hasProhibitedWords = await this.noteCreateService.checkProhibitedWordsContain({ cw, text, pollChoices: poll?.choices }); + if (hasProhibitedWords) { + throw new IdentifiableError('689ee33f-f97c-479a-ac49-1b9f8140af99', 'Note contains prohibited words'); + } + //#endregion + + const actor = cachedActor ?? await this.apPersonService.resolvePerson(uri, resolver) as MiRemoteUser; + + // 解決した投稿者が凍結されていたらスキップ if (actor.isSuspended) { - throw new Error('actor has been suspended'); + throw new IdentifiableError('85ab9bd7-3a41-4530-959d-f07073900109', 'actor has been suspended'); } const noteAudience = await this.apAudienceService.parseAudience(actor, note.to, note.cc, resolver); @@ -171,9 +208,6 @@ export class ApNoteService { } } - const apMentions = await this.apMentionService.extractApMentions(note.tag, resolver); - const apHashtags = extractApHashtags(note.tag); - // 添付ファイル // TODO: attachmentは必ずしもImageではない // TODO: attachmentは必ずしも配列ではない @@ -233,18 +267,6 @@ export class ApNoteService { } } - const cw = note.summary === '' ? null : note.summary; - - // テキストのパース - let text: string | null = null; - if (note.source?.mediaType === 'text/x.misskeymarkdown' && typeof note.source.content === 'string') { - text = note.source.content; - } else if (typeof note._misskey_content !== 'undefined') { - text = note._misskey_content; - } else if (typeof note.content === 'string') { - text = this.apMfmService.htmlToMfm(note.content, note.tag); - } - // vote if (reply && reply.hasPoll) { const poll = await this.pollsRepository.findOneByOrFail({ noteId: reply.id }); @@ -274,8 +296,6 @@ export class ApNoteService { const apEmojis = emojis.map(emoji => emoji.name); - const poll = await this.apQuestionService.extractPollFromQuestion(note, resolver).catch(() => undefined); - try { return await this.noteCreateService.create(actor, { createdAt: note.published ? new Date(note.published) : null, diff --git a/packages/backend/src/queue/processors/InboxProcessorService.ts b/packages/backend/src/queue/processors/InboxProcessorService.ts index 0a713149e5..3addead058 100644 --- a/packages/backend/src/queue/processors/InboxProcessorService.ts +++ b/packages/backend/src/queue/processors/InboxProcessorService.ts @@ -185,7 +185,10 @@ export class InboxProcessorService { await this.apInboxService.performActivity(authUser.user, activity); } catch (e) { if (e instanceof IdentifiableError) { - if (e.id === '689ee33f-f97c-479a-ac49-1b9f8140af99') return 'blocked notes with prohibited words'; + if (e.id === '689ee33f-f97c-479a-ac49-1b9f8140af99') { + return 'blocked notes with prohibited words'; + } + if (e.id === '85ab9bd7-3a41-4530-959d-f07073900109') return 'actor has been suspended'; } throw e; } -- cgit v1.2.3-freya