From bf3d493d049f30a93aecd8c39fbf433dcfbe959b Mon Sep 17 00:00:00 2001 From: Mar0xy Date: Tue, 3 Oct 2023 20:21:26 +0200 Subject: Revert "feat: improve tl performance" --- packages/backend/src/core/NoteCreateService.ts | 247 +++---------------------- 1 file changed, 29 insertions(+), 218 deletions(-) (limited to 'packages/backend/src/core/NoteCreateService.ts') diff --git a/packages/backend/src/core/NoteCreateService.ts b/packages/backend/src/core/NoteCreateService.ts index 8fb34fd637..f20727ce41 100644 --- a/packages/backend/src/core/NoteCreateService.ts +++ b/packages/backend/src/core/NoteCreateService.ts @@ -5,7 +5,7 @@ import { setImmediate } from 'node:timers/promises'; import * as mfm from 'mfm-js'; -import { In, DataSource, IsNull, LessThan } from 'typeorm'; +import { In, DataSource } from 'typeorm'; import * as Redis from 'ioredis'; import { Inject, Injectable, OnApplicationShutdown } from '@nestjs/common'; import RE2 from 're2'; @@ -14,7 +14,7 @@ import { extractCustomEmojisFromMfm } from '@/misc/extract-custom-emojis-from-mf import { extractHashtags } from '@/misc/extract-hashtags.js'; import type { IMentionedRemoteUsers } from '@/models/Note.js'; import { MiNote } from '@/models/Note.js'; -import type { ChannelFollowingsRepository, ChannelsRepository, FollowingsRepository, InstancesRepository, MiFollowing, MutingsRepository, NotesRepository, NoteThreadMutingsRepository, UserListMembershipsRepository, UserProfilesRepository, UsersRepository } from '@/models/_.js'; +import type { ChannelsRepository, FollowingsRepository, InstancesRepository, MutedNotesRepository, MutingsRepository, NotesRepository, NoteThreadMutingsRepository, UserProfilesRepository, UsersRepository } from '@/models/_.js'; import type { MiDriveFile } from '@/models/DriveFile.js'; import type { MiApp } from '@/models/App.js'; import { concat } from '@/misc/prelude/array.js'; @@ -54,6 +54,8 @@ import { RoleService } from '@/core/RoleService.js'; import { MetaService } from '@/core/MetaService.js'; import { SearchService } from '@/core/SearchService.js'; +const mutedWordsCache = new MemorySingleCache<{ userId: MiUserProfile['userId']; mutedWords: MiUserProfile['mutedWords']; }[]>(1000 * 60 * 5); + type NotificationType = 'reply' | 'renote' | 'quote' | 'mention'; class NotificationManager { @@ -155,8 +157,8 @@ export class NoteCreateService implements OnApplicationShutdown { @Inject(DI.db) private db: DataSource, - @Inject(DI.redisForTimelines) - private redisForTimelines: Redis.Redis, + @Inject(DI.redis) + private redisClient: Redis.Redis, @Inject(DI.usersRepository) private usersRepository: UsersRepository, @@ -173,8 +175,8 @@ export class NoteCreateService implements OnApplicationShutdown { @Inject(DI.userProfilesRepository) private userProfilesRepository: UserProfilesRepository, - @Inject(DI.userListMembershipsRepository) - private userListMembershipsRepository: UserListMembershipsRepository, + @Inject(DI.mutedNotesRepository) + private mutedNotesRepository: MutedNotesRepository, @Inject(DI.channelsRepository) private channelsRepository: ChannelsRepository, @@ -185,9 +187,6 @@ export class NoteCreateService implements OnApplicationShutdown { @Inject(DI.followingsRepository) private followingsRepository: FollowingsRepository, - @Inject(DI.channelFollowingsRepository) - private channelFollowingsRepository: ChannelFollowingsRepository, - private userEntityService: UserEntityService, private noteEntityService: NoteEntityService, private idService: IdService, @@ -335,7 +334,7 @@ export class NoteCreateService implements OnApplicationShutdown { const note = await this.insertNote(user, data, tags, emojis, mentionedUsers); if (data.channel) { - this.redisForTimelines.xadd( + this.redisClient.xadd( `channelTimeline:${data.channel.id}`, 'MAXLEN', '~', this.config.perChannelMaxNoteCacheCount.toString(), '*', @@ -481,13 +480,26 @@ export class NoteCreateService implements OnApplicationShutdown { // Increment notes count (user) this.incNotesCountOfUser(user); - if (data.visibility === 'public' || data.visibility === 'home') { - this.pushToTl(note, user); - } else if (data.visibility === 'followers') { - this.pushToTl(note, user); - } else if (data.visibility === 'specified') { - // TODO - } + // Word mute + mutedWordsCache.fetch(() => this.userProfilesRepository.find({ + where: { + enableWordMute: true, + }, + select: ['userId', 'mutedWords'], + })).then(us => { + for (const u of us) { + checkWordMute(note, { id: u.userId }, u.mutedWords).then(shouldMute => { + if (shouldMute) { + this.mutedNotesRepository.insert({ + id: this.idService.genId(), + userId: u.userId, + noteId: note.id, + reason: 'word', + }); + } + }); + } + }); this.antennaService.addNoteToAntennas(note, user); @@ -496,13 +508,11 @@ export class NoteCreateService implements OnApplicationShutdown { } if (data.reply == null) { - // TODO: キャッシュ this.followingsRepository.findBy({ followeeId: user.id, notify: 'normal', }).then(followings => { for (const following of followings) { - // TODO: ワードミュート考慮 this.notificationService.createNotification(following.followerId, 'note', { noteId: note.id, }, user.id); @@ -801,205 +811,6 @@ export class NoteCreateService implements OnApplicationShutdown { return mentionedUsers; } - @bindThis - private async pushToTl(note: MiNote, user: { id: MiUser['id']; host: MiUser['host']; }) { - const redisPipeline = this.redisForTimelines.pipeline(); - - if (note.channelId) { - const channelFollowings = await this.channelFollowingsRepository.find({ - where: { - followeeId: note.channelId, - }, - select: ['followerId'], - }); - - for (const channelFollowing of channelFollowings) { - redisPipeline.xadd( - `homeTimeline:${channelFollowing.followerId}`, - 'MAXLEN', '~', '200', - '*', - 'note', note.id); - - if (note.fileIds.length > 0) { - redisPipeline.xadd( - `homeTimelineWithFiles:${channelFollowing.followerId}`, - 'MAXLEN', '~', '100', - '*', - 'note', note.id); - } - } - } else { - // TODO: キャッシュ？ - const followings = await this.followingsRepository.find({ - where: { - followeeId: user.id, - followerHost: IsNull(), - isFollowerHibernated: false, - }, - select: ['followerId', 'withReplies'], - }); - - const userListMemberships = await this.userListMembershipsRepository.find({ - where: { - userId: user.id, - }, - select: ['userListId', 'withReplies'], - }); - - // TODO: あまりにも数が多いと redisPipeline.exec に失敗する(理由は不明)ため、3万件程度を目安に分割して実行するようにする - for (const following of followings) { - // 自分自身以外への返信 - if (note.replyId && note.replyUserId !== note.userId) { - if (!following.withReplies) continue; - } - - redisPipeline.xadd( - `homeTimeline:${following.followerId}`, - 'MAXLEN', '~', '200', - '*', - 'note', note.id); - - if (note.fileIds.length > 0) { - redisPipeline.xadd( - `homeTimelineWithFiles:${following.followerId}`, - 'MAXLEN', '~', '100', - '*', - 'note', note.id); - } - } - - // TODO - //if (note.visibility === 'followers') { - // // TODO: 重そうだから何とかしたい Set 使う？ - // userLists = userLists.filter(x => followings.some(f => f.followerId === x.userListUserId)); - //} - - for (const userListMembership of userListMemberships) { - // 自分自身以外への返信 - if (note.replyId && note.replyUserId !== note.userId) { - if (!userListMembership.withReplies) continue; - } - - redisPipeline.xadd( - `userListTimeline:${userListMembership.userListId}`, - 'MAXLEN', '~', '200', - '*', - 'note', note.id); - - if (note.fileIds.length > 0) { - redisPipeline.xadd( - `userListTimelineWithFiles:${userListMembership.userListId}`, - 'MAXLEN', '~', '100', - '*', - 'note', note.id); - } - } - - { // 自分自身のHTL - redisPipeline.xadd( - `homeTimeline:${user.id}`, - 'MAXLEN', '~', '200', - '*', - 'note', note.id); - - if (note.fileIds.length > 0) { - redisPipeline.xadd( - `homeTimelineWithFiles:${user.id}`, - 'MAXLEN', '~', '100', - '*', - 'note', note.id); - } - } - - if (note.visibility === 'public' || note.visibility === 'home') { - // 自分自身以外への返信 - if (note.replyId && note.replyUserId !== note.userId) { - redisPipeline.xadd( - `userTimelineWithReplies:${user.id}`, - 'MAXLEN', '~', '1000', - '*', - 'note', note.id); - } else { - redisPipeline.xadd( - `userTimeline:${user.id}`, - 'MAXLEN', '~', '1000', - '*', - 'note', note.id); - - if (note.fileIds.length > 0) { - redisPipeline.xadd( - `userTimelineWithFiles:${user.id}`, - 'MAXLEN', '~', '500', - '*', - 'note', note.id); - } - - if (note.visibility === 'public' && note.userHost == null) { - redisPipeline.xadd( - 'localTimeline', - 'MAXLEN', '~', '1000', - '*', - 'note', note.id); - - if (note.fileIds.length > 0) { - redisPipeline.xadd( - 'localTimelineWithFiles', - 'MAXLEN', '~', '500', - '*', - 'note', note.id); - } - } - } - } - - if (Math.random() < 0.1) { - process.nextTick(() => { - this.checkHibernation(followings); - }); - } - } - - redisPipeline.exec(); - } - - @bindThis - public async checkHibernation(followings: MiFollowing[]) { - if (followings.length === 0) return; - - const shuffle = (array: MiFollowing[]) => { - for (let i = array.length - 1; i > 0; i--) { - const j = Math.floor(Math.random() * (i + 1)); - [array[i], array[j]] = [array[j], array[i]]; - } - return array; - }; - - // ランダムに最大1000件サンプリング - const samples = shuffle(followings).slice(0, Math.min(followings.length, 1000)); - - const hibernatedUsers = await this.usersRepository.find({ - where: { - id: In(samples.map(x => x.followerId)), - lastActiveDate: LessThan(new Date(Date.now() - (1000 * 60 * 60 * 24 * 50))), - }, - select: ['id'], - }); - - if (hibernatedUsers.length > 0) { - this.usersRepository.update({ - id: In(hibernatedUsers.map(x => x.id)), - }, { - isHibernated: true, - }); - - this.followingsRepository.update({ - followerId: In(hibernatedUsers.map(x => x.id)), - }, { - isFollowerHibernated: true, - }); - } - } - @bindThis public dispose(): void { this.#shutdownController.abort(); -- cgit v1.2.3-freya