From 6277a5545c746fac15ee6b4fe58de2e354ed7fda Mon Sep 17 00:00:00 2001 From: syuilo Date: Tue, 3 Oct 2023 20:26:11 +0900 Subject: feat: improve tl performance (#11946) * wip * wip * wip * wip * wip * wip * Update NoteCreateService.ts * wip * wip * wip * wip * Update NoteCreateService.ts * wip * Update NoteCreateService.ts * wip * Update user-notes.ts * wip * wip * wip * Update NoteCreateService.ts * wip * Update timeline.ts * Update timeline.ts * Update timeline.ts * Update timeline.ts * Update timeline.ts * wip * Update timelines.ts * Update timelines.ts * Update timelines.ts * wip * wip * wip * Update timelines.ts * Update misskey-js.api.md * Update timelines.ts * Update timelines.ts * wip * wip * wip * Update timelines.ts * wip * Update timelines.ts * wip * test * Update activitypub.ts * refactor: UserListJoining -> UserListMembership * Update NoteCreateService.ts * wip --- packages/backend/src/core/NoteCreateService.ts | 247 ++++++++++++++++++++++--- 1 file changed, 218 insertions(+), 29 deletions(-) (limited to 'packages/backend/src/core/NoteCreateService.ts') diff --git a/packages/backend/src/core/NoteCreateService.ts b/packages/backend/src/core/NoteCreateService.ts index f20727ce41..8fb34fd637 100644 --- a/packages/backend/src/core/NoteCreateService.ts +++ b/packages/backend/src/core/NoteCreateService.ts @@ -5,7 +5,7 @@ import { setImmediate } from 'node:timers/promises'; import * as mfm from 'mfm-js'; -import { In, DataSource } from 'typeorm'; +import { In, DataSource, IsNull, LessThan } from 'typeorm'; import * as Redis from 'ioredis'; import { Inject, Injectable, OnApplicationShutdown } from '@nestjs/common'; import RE2 from 're2'; @@ -14,7 +14,7 @@ import { extractCustomEmojisFromMfm } from '@/misc/extract-custom-emojis-from-mf import { extractHashtags } from '@/misc/extract-hashtags.js'; import type { IMentionedRemoteUsers } from '@/models/Note.js'; import { MiNote } from '@/models/Note.js'; -import type { ChannelsRepository, FollowingsRepository, InstancesRepository, MutedNotesRepository, MutingsRepository, NotesRepository, NoteThreadMutingsRepository, UserProfilesRepository, UsersRepository } from '@/models/_.js'; +import type { ChannelFollowingsRepository, ChannelsRepository, FollowingsRepository, InstancesRepository, MiFollowing, MutingsRepository, NotesRepository, NoteThreadMutingsRepository, UserListMembershipsRepository, UserProfilesRepository, UsersRepository } from '@/models/_.js'; import type { MiDriveFile } from '@/models/DriveFile.js'; import type { MiApp } from '@/models/App.js'; import { concat } from '@/misc/prelude/array.js'; @@ -54,8 +54,6 @@ import { RoleService } from '@/core/RoleService.js'; import { MetaService } from '@/core/MetaService.js'; import { SearchService } from '@/core/SearchService.js'; -const mutedWordsCache = new MemorySingleCache<{ userId: MiUserProfile['userId']; mutedWords: MiUserProfile['mutedWords']; }[]>(1000 * 60 * 5); - type NotificationType = 'reply' | 'renote' | 'quote' | 'mention'; class NotificationManager { @@ -157,8 +155,8 @@ export class NoteCreateService implements OnApplicationShutdown { @Inject(DI.db) private db: DataSource, - @Inject(DI.redis) - private redisClient: Redis.Redis, + @Inject(DI.redisForTimelines) + private redisForTimelines: Redis.Redis, @Inject(DI.usersRepository) private usersRepository: UsersRepository, @@ -175,8 +173,8 @@ export class NoteCreateService implements OnApplicationShutdown { @Inject(DI.userProfilesRepository) private userProfilesRepository: UserProfilesRepository, - @Inject(DI.mutedNotesRepository) - private mutedNotesRepository: MutedNotesRepository, + @Inject(DI.userListMembershipsRepository) + private userListMembershipsRepository: UserListMembershipsRepository, @Inject(DI.channelsRepository) private channelsRepository: ChannelsRepository, @@ -187,6 +185,9 @@ export class NoteCreateService implements OnApplicationShutdown { @Inject(DI.followingsRepository) private followingsRepository: FollowingsRepository, + @Inject(DI.channelFollowingsRepository) + private channelFollowingsRepository: ChannelFollowingsRepository, + private userEntityService: UserEntityService, private noteEntityService: NoteEntityService, private idService: IdService, @@ -334,7 +335,7 @@ export class NoteCreateService implements OnApplicationShutdown { const note = await this.insertNote(user, data, tags, emojis, mentionedUsers); if (data.channel) { - this.redisClient.xadd( + this.redisForTimelines.xadd( `channelTimeline:${data.channel.id}`, 'MAXLEN', '~', this.config.perChannelMaxNoteCacheCount.toString(), '*', @@ -480,26 +481,13 @@ export class NoteCreateService implements OnApplicationShutdown { // Increment notes count (user) this.incNotesCountOfUser(user); - // Word mute - mutedWordsCache.fetch(() => this.userProfilesRepository.find({ - where: { - enableWordMute: true, - }, - select: ['userId', 'mutedWords'], - })).then(us => { - for (const u of us) { - checkWordMute(note, { id: u.userId }, u.mutedWords).then(shouldMute => { - if (shouldMute) { - this.mutedNotesRepository.insert({ - id: this.idService.genId(), - userId: u.userId, - noteId: note.id, - reason: 'word', - }); - } - }); - } - }); + if (data.visibility === 'public' || data.visibility === 'home') { + this.pushToTl(note, user); + } else if (data.visibility === 'followers') { + this.pushToTl(note, user); + } else if (data.visibility === 'specified') { + // TODO + } this.antennaService.addNoteToAntennas(note, user); @@ -508,11 +496,13 @@ export class NoteCreateService implements OnApplicationShutdown { } if (data.reply == null) { + // TODO: キャッシュ this.followingsRepository.findBy({ followeeId: user.id, notify: 'normal', }).then(followings => { for (const following of followings) { + // TODO: ワードミュート考慮 this.notificationService.createNotification(following.followerId, 'note', { noteId: note.id, }, user.id); @@ -811,6 +801,205 @@ export class NoteCreateService implements OnApplicationShutdown { return mentionedUsers; } + @bindThis + private async pushToTl(note: MiNote, user: { id: MiUser['id']; host: MiUser['host']; }) { + const redisPipeline = this.redisForTimelines.pipeline(); + + if (note.channelId) { + const channelFollowings = await this.channelFollowingsRepository.find({ + where: { + followeeId: note.channelId, + }, + select: ['followerId'], + }); + + for (const channelFollowing of channelFollowings) { + redisPipeline.xadd( + `homeTimeline:${channelFollowing.followerId}`, + 'MAXLEN', '~', '200', + '*', + 'note', note.id); + + if (note.fileIds.length > 0) { + redisPipeline.xadd( + `homeTimelineWithFiles:${channelFollowing.followerId}`, + 'MAXLEN', '~', '100', + '*', + 'note', note.id); + } + } + } else { + // TODO: キャッシュ？ + const followings = await this.followingsRepository.find({ + where: { + followeeId: user.id, + followerHost: IsNull(), + isFollowerHibernated: false, + }, + select: ['followerId', 'withReplies'], + }); + + const userListMemberships = await this.userListMembershipsRepository.find({ + where: { + userId: user.id, + }, + select: ['userListId', 'withReplies'], + }); + + // TODO: あまりにも数が多いと redisPipeline.exec に失敗する(理由は不明)ため、3万件程度を目安に分割して実行するようにする + for (const following of followings) { + // 自分自身以外への返信 + if (note.replyId && note.replyUserId !== note.userId) { + if (!following.withReplies) continue; + } + + redisPipeline.xadd( + `homeTimeline:${following.followerId}`, + 'MAXLEN', '~', '200', + '*', + 'note', note.id); + + if (note.fileIds.length > 0) { + redisPipeline.xadd( + `homeTimelineWithFiles:${following.followerId}`, + 'MAXLEN', '~', '100', + '*', + 'note', note.id); + } + } + + // TODO + //if (note.visibility === 'followers') { + // // TODO: 重そうだから何とかしたい Set 使う？ + // userLists = userLists.filter(x => followings.some(f => f.followerId === x.userListUserId)); + //} + + for (const userListMembership of userListMemberships) { + // 自分自身以外への返信 + if (note.replyId && note.replyUserId !== note.userId) { + if (!userListMembership.withReplies) continue; + } + + redisPipeline.xadd( + `userListTimeline:${userListMembership.userListId}`, + 'MAXLEN', '~', '200', + '*', + 'note', note.id); + + if (note.fileIds.length > 0) { + redisPipeline.xadd( + `userListTimelineWithFiles:${userListMembership.userListId}`, + 'MAXLEN', '~', '100', + '*', + 'note', note.id); + } + } + + { // 自分自身のHTL + redisPipeline.xadd( + `homeTimeline:${user.id}`, + 'MAXLEN', '~', '200', + '*', + 'note', note.id); + + if (note.fileIds.length > 0) { + redisPipeline.xadd( + `homeTimelineWithFiles:${user.id}`, + 'MAXLEN', '~', '100', + '*', + 'note', note.id); + } + } + + if (note.visibility === 'public' || note.visibility === 'home') { + // 自分自身以外への返信 + if (note.replyId && note.replyUserId !== note.userId) { + redisPipeline.xadd( + `userTimelineWithReplies:${user.id}`, + 'MAXLEN', '~', '1000', + '*', + 'note', note.id); + } else { + redisPipeline.xadd( + `userTimeline:${user.id}`, + 'MAXLEN', '~', '1000', + '*', + 'note', note.id); + + if (note.fileIds.length > 0) { + redisPipeline.xadd( + `userTimelineWithFiles:${user.id}`, + 'MAXLEN', '~', '500', + '*', + 'note', note.id); + } + + if (note.visibility === 'public' && note.userHost == null) { + redisPipeline.xadd( + 'localTimeline', + 'MAXLEN', '~', '1000', + '*', + 'note', note.id); + + if (note.fileIds.length > 0) { + redisPipeline.xadd( + 'localTimelineWithFiles', + 'MAXLEN', '~', '500', + '*', + 'note', note.id); + } + } + } + } + + if (Math.random() < 0.1) { + process.nextTick(() => { + this.checkHibernation(followings); + }); + } + } + + redisPipeline.exec(); + } + + @bindThis + public async checkHibernation(followings: MiFollowing[]) { + if (followings.length === 0) return; + + const shuffle = (array: MiFollowing[]) => { + for (let i = array.length - 1; i > 0; i--) { + const j = Math.floor(Math.random() * (i + 1)); + [array[i], array[j]] = [array[j], array[i]]; + } + return array; + }; + + // ランダムに最大1000件サンプリング + const samples = shuffle(followings).slice(0, Math.min(followings.length, 1000)); + + const hibernatedUsers = await this.usersRepository.find({ + where: { + id: In(samples.map(x => x.followerId)), + lastActiveDate: LessThan(new Date(Date.now() - (1000 * 60 * 60 * 24 * 50))), + }, + select: ['id'], + }); + + if (hibernatedUsers.length > 0) { + this.usersRepository.update({ + id: In(hibernatedUsers.map(x => x.id)), + }, { + isHibernated: true, + }); + + this.followingsRepository.update({ + followerId: In(hibernatedUsers.map(x => x.id)), + }, { + isFollowerHibernated: true, + }); + } + } + @bindThis public dispose(): void { this.#shutdownController.abort(); -- cgit v1.2.3-freya From 17b83ff4c13e873b63262c349ea9c7bade0d656a Mon Sep 17 00:00:00 2001 From: syuilo Date: Wed, 4 Oct 2023 08:46:27 +0900 Subject: enhance: TLキャッシュ容量を設定できるように MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- locales/index.d.ts | 1 + locales/ja-JP.yml | 1 + .../migration/1696373953614-meta-cache-settings.js | 22 +++++ packages/backend/src/core/NoteCreateService.ts | 24 ++--- packages/backend/src/models/Meta.ts | 20 ++++ .../backend/src/server/api/endpoints/admin/meta.ts | 103 +++++++++++---------- .../src/server/api/endpoints/admin/update-meta.ts | 20 ++++ packages/backend/src/server/api/endpoints/meta.ts | 4 +- .../frontend/src/pages/admin/external-services.vue | 81 ++++++++++++++++ packages/frontend/src/pages/admin/index.vue | 5 + packages/frontend/src/pages/admin/settings.vue | 40 +++++--- packages/frontend/src/router.ts | 4 + 12 files changed, 251 insertions(+), 74 deletions(-) create mode 100644 packages/backend/migration/1696373953614-meta-cache-settings.js create mode 100644 packages/frontend/src/pages/admin/external-services.vue (limited to 'packages/backend/src/core/NoteCreateService.ts') diff --git a/locales/index.d.ts b/locales/index.d.ts index 418e1c67ff..172cdcb754 100644 --- a/locales/index.d.ts +++ b/locales/index.d.ts @@ -1131,6 +1131,7 @@ export interface Locale { "fileAttachedOnly": string; "showRepliesToOthersInTimeline": string; "hideRepliesToOthersInTimeline": string; + "externalServices": string; "_announcement": { "forExistingUsers": string; "forExistingUsersDescription": string; diff --git a/locales/ja-JP.yml b/locales/ja-JP.yml index 80e4466a74..1136f67baf 100644 --- a/locales/ja-JP.yml +++ b/locales/ja-JP.yml @@ -1128,6 +1128,7 @@ mutualFollow: "相互フォロー" fileAttachedOnly: "ファイル付きのみ" showRepliesToOthersInTimeline: "TLに他の人への返信を含める" hideRepliesToOthersInTimeline: "TLに他の人への返信を含めない" +externalServices: "外部サービス" _announcement: forExistingUsers: "既存ユーザーのみ" diff --git a/packages/backend/migration/1696373953614-meta-cache-settings.js b/packages/backend/migration/1696373953614-meta-cache-settings.js new file mode 100644 index 0000000000..f994b76ef2 --- /dev/null +++ b/packages/backend/migration/1696373953614-meta-cache-settings.js @@ -0,0 +1,22 @@ +/* + * SPDX-FileCopyrightText: syuilo and other misskey contributors + * SPDX-License-Identifier: AGPL-3.0-only + */ + +export class MetaCacheSettings1696373953614 { + name = 'MetaCacheSettings1696373953614' + + async up(queryRunner) { + await queryRunner.query(`ALTER TABLE "meta" ADD "perLocalUserUserTimelineCacheMax" integer NOT NULL DEFAULT '300'`); + await queryRunner.query(`ALTER TABLE "meta" ADD "perRemoteUserUserTimelineCacheMax" integer NOT NULL DEFAULT '100'`); + await queryRunner.query(`ALTER TABLE "meta" ADD "perUserHomeTimelineCacheMax" integer NOT NULL DEFAULT '300'`); + await queryRunner.query(`ALTER TABLE "meta" ADD "perUserListTimelineCacheMax" integer NOT NULL DEFAULT '300'`); + } + + async down(queryRunner) { + await queryRunner.query(`ALTER TABLE "meta" DROP COLUMN "perUserListTimelineCacheMax"`); + await queryRunner.query(`ALTER TABLE "meta" DROP COLUMN "perUserHomeTimelineCacheMax"`); + await queryRunner.query(`ALTER TABLE "meta" DROP COLUMN "perRemoteUserUserTimelineCacheMax"`); + await queryRunner.query(`ALTER TABLE "meta" DROP COLUMN "perLocalUserUserTimelineCacheMax"`); + } +} diff --git a/packages/backend/src/core/NoteCreateService.ts b/packages/backend/src/core/NoteCreateService.ts index 8fb34fd637..7e1c0b5c22 100644 --- a/packages/backend/src/core/NoteCreateService.ts +++ b/packages/backend/src/core/NoteCreateService.ts @@ -803,6 +803,8 @@ export class NoteCreateService implements OnApplicationShutdown { @bindThis private async pushToTl(note: MiNote, user: { id: MiUser['id']; host: MiUser['host']; }) { + const meta = await this.metaService.fetch(); + const redisPipeline = this.redisForTimelines.pipeline(); if (note.channelId) { @@ -816,14 +818,14 @@ export class NoteCreateService implements OnApplicationShutdown { for (const channelFollowing of channelFollowings) { redisPipeline.xadd( `homeTimeline:${channelFollowing.followerId}`, - 'MAXLEN', '~', '200', + 'MAXLEN', '~', meta.perUserHomeTimelineCacheMax.toString(), '*', 'note', note.id); if (note.fileIds.length > 0) { redisPipeline.xadd( `homeTimelineWithFiles:${channelFollowing.followerId}`, - 'MAXLEN', '~', '100', + 'MAXLEN', '~', (meta.perUserHomeTimelineCacheMax / 2).toString(), '*', 'note', note.id); } @@ -855,14 +857,14 @@ export class NoteCreateService implements OnApplicationShutdown { redisPipeline.xadd( `homeTimeline:${following.followerId}`, - 'MAXLEN', '~', '200', + 'MAXLEN', '~', meta.perUserHomeTimelineCacheMax.toString(), '*', 'note', note.id); if (note.fileIds.length > 0) { redisPipeline.xadd( `homeTimelineWithFiles:${following.followerId}`, - 'MAXLEN', '~', '100', + 'MAXLEN', '~', (meta.perUserHomeTimelineCacheMax / 2).toString(), '*', 'note', note.id); } @@ -882,14 +884,14 @@ export class NoteCreateService implements OnApplicationShutdown { redisPipeline.xadd( `userListTimeline:${userListMembership.userListId}`, - 'MAXLEN', '~', '200', + 'MAXLEN', '~', meta.perUserListTimelineCacheMax.toString(), '*', 'note', note.id); if (note.fileIds.length > 0) { redisPipeline.xadd( `userListTimelineWithFiles:${userListMembership.userListId}`, - 'MAXLEN', '~', '100', + 'MAXLEN', '~', (meta.perUserListTimelineCacheMax / 2).toString(), '*', 'note', note.id); } @@ -898,14 +900,14 @@ export class NoteCreateService implements OnApplicationShutdown { { // 自分自身のHTL redisPipeline.xadd( `homeTimeline:${user.id}`, - 'MAXLEN', '~', '200', + 'MAXLEN', '~', meta.perUserHomeTimelineCacheMax.toString(), '*', 'note', note.id); if (note.fileIds.length > 0) { redisPipeline.xadd( `homeTimelineWithFiles:${user.id}`, - 'MAXLEN', '~', '100', + 'MAXLEN', '~', (meta.perUserHomeTimelineCacheMax / 2).toString(), '*', 'note', note.id); } @@ -916,20 +918,20 @@ export class NoteCreateService implements OnApplicationShutdown { if (note.replyId && note.replyUserId !== note.userId) { redisPipeline.xadd( `userTimelineWithReplies:${user.id}`, - 'MAXLEN', '~', '1000', + 'MAXLEN', '~', note.userHost == null ? meta.perLocalUserUserTimelineCacheMax.toString() : meta.perRemoteUserUserTimelineCacheMax.toString(), '*', 'note', note.id); } else { redisPipeline.xadd( `userTimeline:${user.id}`, - 'MAXLEN', '~', '1000', + 'MAXLEN', '~', note.userHost == null ? meta.perLocalUserUserTimelineCacheMax.toString() : meta.perRemoteUserUserTimelineCacheMax.toString(), '*', 'note', note.id); if (note.fileIds.length > 0) { redisPipeline.xadd( `userTimelineWithFiles:${user.id}`, - 'MAXLEN', '~', '500', + 'MAXLEN', '~', note.userHost == null ? (meta.perLocalUserUserTimelineCacheMax / 2).toString() : (meta.perRemoteUserUserTimelineCacheMax / 2).toString(), '*', 'note', note.id); } diff --git a/packages/backend/src/models/Meta.ts b/packages/backend/src/models/Meta.ts index e69bef8e98..491d446723 100644 --- a/packages/backend/src/models/Meta.ts +++ b/packages/backend/src/models/Meta.ts @@ -471,4 +471,24 @@ export class MiMeta { length: 1024, array: true, default: '{ "admin", "administrator", "root", "system", "maintainer", "host", "mod", "moderator", "owner", "superuser", "staff", "auth", "i", "me", "everyone", "all", "mention", "mentions", "example", "user", "users", "account", "accounts", "official", "help", "helps", "support", "supports", "info", "information", "informations", "announce", "announces", "announcement", "announcements", "notice", "notification", "notifications", "dev", "developer", "developers", "tech", "misskey" }', }) public preservedUsernames: string[]; + + @Column('integer', { + default: 300, + }) + public perLocalUserUserTimelineCacheMax: number; + + @Column('integer', { + default: 100, + }) + public perRemoteUserUserTimelineCacheMax: number; + + @Column('integer', { + default: 300, + }) + public perUserHomeTimelineCacheMax: number; + + @Column('integer', { + default: 300, + }) + public perUserListTimelineCacheMax: number; } diff --git a/packages/backend/src/server/api/endpoints/admin/meta.ts b/packages/backend/src/server/api/endpoints/admin/meta.ts index c3ba07cdd0..53e3672784 100644 --- a/packages/backend/src/server/api/endpoints/admin/meta.ts +++ b/packages/backend/src/server/api/endpoints/admin/meta.ts @@ -105,40 +105,32 @@ export const meta = { type: 'boolean', optional: false, nullable: false, }, - userStarForReactionFallback: { - type: 'boolean', - optional: true, nullable: false, - }, pinnedUsers: { type: 'array', - optional: true, nullable: false, + optional: false, nullable: false, items: { type: 'string', - optional: false, nullable: false, }, }, hiddenTags: { type: 'array', - optional: true, nullable: false, + optional: false, nullable: false, items: { type: 'string', - optional: false, nullable: false, }, }, blockedHosts: { type: 'array', - optional: true, nullable: false, + optional: false, nullable: false, items: { type: 'string', - optional: false, nullable: false, }, }, sensitiveWords: { type: 'array', - optional: true, nullable: false, + optional: false, nullable: false, items: { type: 'string', - optional: false, nullable: false, }, }, preservedUsernames: { @@ -146,129 +138,124 @@ export const meta = { optional: false, nullable: false, items: { type: 'string', - optional: false, nullable: false, }, }, hcaptchaSecretKey: { type: 'string', - optional: true, nullable: true, + optional: false, nullable: true, }, recaptchaSecretKey: { type: 'string', - optional: true, nullable: true, + optional: false, nullable: true, }, turnstileSecretKey: { type: 'string', - optional: true, nullable: true, + optional: false, nullable: true, }, sensitiveMediaDetection: { type: 'string', - optional: true, nullable: false, + optional: false, nullable: false, }, sensitiveMediaDetectionSensitivity: { type: 'string', - optional: true, nullable: false, + optional: false, nullable: false, }, setSensitiveFlagAutomatically: { type: 'boolean', - optional: true, nullable: false, + optional: false, nullable: false, }, enableSensitiveMediaDetectionForVideos: { type: 'boolean', - optional: true, nullable: false, + optional: false, nullable: false, }, proxyAccountId: { type: 'string', - optional: true, nullable: true, + optional: false, nullable: true, format: 'id', }, - summaryProxy: { - type: 'string', - optional: true, nullable: true, - }, email: { type: 'string', - optional: true, nullable: true, + optional: false, nullable: true, }, smtpSecure: { type: 'boolean', - optional: true, nullable: false, + optional: false, nullable: false, }, smtpHost: { type: 'string', - optional: true, nullable: true, + optional: false, nullable: true, }, smtpPort: { type: 'number', - optional: true, nullable: true, + optional: false, nullable: true, }, smtpUser: { type: 'string', - optional: true, nullable: true, + optional: false, nullable: true, }, smtpPass: { type: 'string', - optional: true, nullable: true, + optional: false, nullable: true, }, swPrivateKey: { type: 'string', - optional: true, nullable: true, + optional: false, nullable: true, }, useObjectStorage: { type: 'boolean', - optional: true, nullable: false, + optional: false, nullable: false, }, objectStorageBaseUrl: { type: 'string', - optional: true, nullable: true, + optional: false, nullable: true, }, objectStorageBucket: { type: 'string', - optional: true, nullable: true, + optional: false, nullable: true, }, objectStoragePrefix: { type: 'string', - optional: true, nullable: true, + optional: false, nullable: true, }, objectStorageEndpoint: { type: 'string', - optional: true, nullable: true, + optional: false, nullable: true, }, objectStorageRegion: { type: 'string', - optional: true, nullable: true, + optional: false, nullable: true, }, objectStoragePort: { type: 'number', - optional: true, nullable: true, + optional: false, nullable: true, }, objectStorageAccessKey: { type: 'string', - optional: true, nullable: true, + optional: false, nullable: true, }, objectStorageSecretKey: { type: 'string', - optional: true, nullable: true, + optional: false, nullable: true, }, objectStorageUseSSL: { type: 'boolean', - optional: true, nullable: false, + optional: false, nullable: false, }, objectStorageUseProxy: { type: 'boolean', - optional: true, nullable: false, + optional: false, nullable: false, }, objectStorageSetPublicRead: { type: 'boolean', - optional: true, nullable: false, + optional: false, nullable: false, }, enableIpLogging: { type: 'boolean', - optional: true, nullable: false, + optional: false, nullable: false, }, enableActiveEmailValidation: { type: 'boolean', - optional: true, nullable: false, + optional: false, nullable: false, }, enableChartsForRemoteUser: { type: 'boolean', @@ -288,12 +275,28 @@ export const meta = { }, manifestJsonOverride: { type: 'string', - optional: true, nullable: false, + optional: false, nullable: false, }, policies: { type: 'object', optional: false, nullable: false, }, + perLocalUserUserTimelineCacheMax: { + type: 'number', + optional: false, nullable: false, + }, + perRemoteUserUserTimelineCacheMax: { + type: 'number', + optional: false, nullable: false, + }, + perUserHomeTimelineCacheMax: { + type: 'number', + optional: false, nullable: false, + }, + perUserListTimelineCacheMax: { + type: 'number', + optional: false, nullable: false, + }, }, }, } as const; @@ -313,7 +316,7 @@ export default class extends Endpoint { // eslint- private metaService: MetaService, ) { - super(meta, paramDef, async (ps, me) => { + super(meta, paramDef, async () => { const instance = await this.metaService.fetch(true); return { @@ -399,6 +402,10 @@ export default class extends Endpoint { // eslint- enableIdenticonGeneration: instance.enableIdenticonGeneration, policies: { ...DEFAULT_POLICIES, ...instance.policies }, manifestJsonOverride: instance.manifestJsonOverride, + perLocalUserUserTimelineCacheMax: instance.perLocalUserUserTimelineCacheMax, + perRemoteUserUserTimelineCacheMax: instance.perRemoteUserUserTimelineCacheMax, + perUserHomeTimelineCacheMax: instance.perUserHomeTimelineCacheMax, + perUserListTimelineCacheMax: instance.perUserListTimelineCacheMax, }; }); } diff --git a/packages/backend/src/server/api/endpoints/admin/update-meta.ts b/packages/backend/src/server/api/endpoints/admin/update-meta.ts index ea6ebdd1fe..247d3ba4e0 100644 --- a/packages/backend/src/server/api/endpoints/admin/update-meta.ts +++ b/packages/backend/src/server/api/endpoints/admin/update-meta.ts @@ -108,6 +108,10 @@ export const paramDef = { serverRules: { type: 'array', items: { type: 'string' } }, preservedUsernames: { type: 'array', items: { type: 'string' } }, manifestJsonOverride: { type: 'string' }, + perLocalUserUserTimelineCacheMax: { type: 'integer' }, + perRemoteUserUserTimelineCacheMax: { type: 'integer' }, + perUserHomeTimelineCacheMax: { type: 'integer' }, + perUserListTimelineCacheMax: { type: 'integer' }, }, required: [], } as const; @@ -441,6 +445,22 @@ export default class extends Endpoint { // eslint- set.manifestJsonOverride = ps.manifestJsonOverride; } + if (ps.perLocalUserUserTimelineCacheMax !== undefined) { + set.perLocalUserUserTimelineCacheMax = ps.perLocalUserUserTimelineCacheMax; + } + + if (ps.perRemoteUserUserTimelineCacheMax !== undefined) { + set.perRemoteUserUserTimelineCacheMax = ps.perRemoteUserUserTimelineCacheMax; + } + + if (ps.perUserHomeTimelineCacheMax !== undefined) { + set.perUserHomeTimelineCacheMax = ps.perUserHomeTimelineCacheMax; + } + + if (ps.perUserListTimelineCacheMax !== undefined) { + set.perUserListTimelineCacheMax = ps.perUserListTimelineCacheMax; + } + const before = await this.metaService.fetch(true); await this.metaService.update(set); diff --git a/packages/backend/src/server/api/endpoints/meta.ts b/packages/backend/src/server/api/endpoints/meta.ts index fa6486ed18..271b3f6fb2 100644 --- a/packages/backend/src/server/api/endpoints/meta.ts +++ b/packages/backend/src/server/api/endpoints/meta.ts @@ -214,11 +214,11 @@ export const meta = { type: 'boolean', optional: false, nullable: false, }, - localTimeLine: { + localTimeline: { type: 'boolean', optional: false, nullable: false, }, - globalTimeLine: { + globalTimeline: { type: 'boolean', optional: false, nullable: false, }, diff --git a/packages/frontend/src/pages/admin/external-services.vue b/packages/frontend/src/pages/admin/external-services.vue new file mode 100644 index 0000000000..5944bf500a --- /dev/null +++ b/packages/frontend/src/pages/admin/external-services.vue @@ -0,0 +1,81 @@ + + + + + + + diff --git a/packages/frontend/src/pages/admin/index.vue b/packages/frontend/src/pages/admin/index.vue index 944ba7b950..a508c20cf3 100644 --- a/packages/frontend/src/pages/admin/index.vue +++ b/packages/frontend/src/pages/admin/index.vue @@ -198,6 +198,11 @@ const menuDef = $computed(() => [{ text: i18n.ts.proxyAccount, to: '/admin/proxy-account', active: currentPage?.route.name === 'proxy-account', + }, { + icon: 'ti ti-link', + text: i18n.ts.externalServices, + to: '/admin/external-services', + active: currentPage?.route.name === 'external-services', }, { icon: 'ti ti-adjustments', text: i18n.ts.other, diff --git a/packages/frontend/src/pages/admin/settings.vue b/packages/frontend/src/pages/admin/settings.vue index f93678d728..09a6cc7e2c 100644 --- a/packages/frontend/src/pages/admin/settings.vue +++ b/packages/frontend/src/pages/admin/settings.vue @@ -81,16 +81,24 @@ SPDX-License-Identifier: AGPL-3.0-only - +
- - - + + + + + + + + + + + + + + - - -
@@ -133,8 +141,10 @@ let cacheRemoteSensitiveFiles: boolean = $ref(false); let enableServiceWorker: boolean = $ref(false); let swPublicKey: any = $ref(null); let swPrivateKey: any = $ref(null); -let deeplAuthKey: string = $ref(''); -let deeplIsPro: boolean = $ref(false); +let perLocalUserUserTimelineCacheMax: number = $ref(0); +let perRemoteUserUserTimelineCacheMax: number = $ref(0); +let perUserHomeTimelineCacheMax: number = $ref(0); +let perUserListTimelineCacheMax: number = $ref(0); async function init(): Promise { const meta = await os.api('admin/meta'); @@ -149,8 +159,10 @@ async function init(): Promise { enableServiceWorker = meta.enableServiceWorker; swPublicKey = meta.swPublickey; swPrivateKey = meta.swPrivateKey; - deeplAuthKey = meta.deeplAuthKey; - deeplIsPro = meta.deeplIsPro; + perLocalUserUserTimelineCacheMax = meta.perLocalUserUserTimelineCacheMax; + perRemoteUserUserTimelineCacheMax = meta.perRemoteUserUserTimelineCacheMax; + perUserHomeTimelineCacheMax = meta.perUserHomeTimelineCacheMax; + perUserListTimelineCacheMax = meta.perUserListTimelineCacheMax; } function save(): void { @@ -166,8 +178,10 @@ function save(): void { enableServiceWorker, swPublicKey, swPrivateKey, - deeplAuthKey, - deeplIsPro, + perLocalUserUserTimelineCacheMax, + perRemoteUserUserTimelineCacheMax, + perUserHomeTimelineCacheMax, + perUserListTimelineCacheMax, }).then(() => { fetchInstance(); }); diff --git a/packages/frontend/src/router.ts b/packages/frontend/src/router.ts index 415d2f1974..20314711a4 100644 --- a/packages/frontend/src/router.ts +++ b/packages/frontend/src/router.ts @@ -435,6 +435,10 @@ export const routes = [{ path: '/proxy-account', name: 'proxy-account', component: page(() => import('./pages/admin/proxy-account.vue')), + }, { + path: '/external-services', + name: 'external-services', + component: page(() => import('./pages/admin/external-services.vue')), }, { path: '/other-settings', name: 'other-settings', -- cgit v1.2.3-freya From 3dd84f78240fc7fbeca9272f3941d90986f46c0a Mon Sep 17 00:00:00 2001 From: syuilo Date: Wed, 4 Oct 2023 11:48:51 +0900 Subject: tweak --- packages/backend/src/core/NoteCreateService.ts | 58 ++++++++++++-------------- packages/backend/test/e2e/timelines.ts | 27 ++++++++++++ 2 files changed, 54 insertions(+), 31 deletions(-) (limited to 'packages/backend/src/core/NoteCreateService.ts') diff --git a/packages/backend/src/core/NoteCreateService.ts b/packages/backend/src/core/NoteCreateService.ts index 7e1c0b5c22..3ea8af5cda 100644 --- a/packages/backend/src/core/NoteCreateService.ts +++ b/packages/backend/src/core/NoteCreateService.ts @@ -481,12 +481,10 @@ export class NoteCreateService implements OnApplicationShutdown { // Increment notes count (user) this.incNotesCountOfUser(user); - if (data.visibility === 'public' || data.visibility === 'home') { - this.pushToTl(note, user); - } else if (data.visibility === 'followers') { + if (data.visibility === 'specified') { + // TODO? + } else { this.pushToTl(note, user); - } else if (data.visibility === 'specified') { - // TODO } this.antennaService.addNoteToAntennas(note, user); @@ -913,43 +911,41 @@ export class NoteCreateService implements OnApplicationShutdown { } } - if (note.visibility === 'public' || note.visibility === 'home') { - // 自分自身以外への返信 - if (note.replyId && note.replyUserId !== note.userId) { + // 自分自身以外への返信 + if (note.replyId && note.replyUserId !== note.userId) { + redisPipeline.xadd( + `userTimelineWithReplies:${user.id}`, + 'MAXLEN', '~', note.userHost == null ? meta.perLocalUserUserTimelineCacheMax.toString() : meta.perRemoteUserUserTimelineCacheMax.toString(), + '*', + 'note', note.id); + } else { + redisPipeline.xadd( + `userTimeline:${user.id}`, + 'MAXLEN', '~', note.userHost == null ? meta.perLocalUserUserTimelineCacheMax.toString() : meta.perRemoteUserUserTimelineCacheMax.toString(), + '*', + 'note', note.id); + + if (note.fileIds.length > 0) { redisPipeline.xadd( - `userTimelineWithReplies:${user.id}`, - 'MAXLEN', '~', note.userHost == null ? meta.perLocalUserUserTimelineCacheMax.toString() : meta.perRemoteUserUserTimelineCacheMax.toString(), + `userTimelineWithFiles:${user.id}`, + 'MAXLEN', '~', note.userHost == null ? (meta.perLocalUserUserTimelineCacheMax / 2).toString() : (meta.perRemoteUserUserTimelineCacheMax / 2).toString(), '*', 'note', note.id); - } else { + } + + if (note.visibility === 'public' && note.userHost == null) { redisPipeline.xadd( - `userTimeline:${user.id}`, - 'MAXLEN', '~', note.userHost == null ? meta.perLocalUserUserTimelineCacheMax.toString() : meta.perRemoteUserUserTimelineCacheMax.toString(), + 'localTimeline', + 'MAXLEN', '~', '1000', '*', 'note', note.id); if (note.fileIds.length > 0) { redisPipeline.xadd( - `userTimelineWithFiles:${user.id}`, - 'MAXLEN', '~', note.userHost == null ? (meta.perLocalUserUserTimelineCacheMax / 2).toString() : (meta.perRemoteUserUserTimelineCacheMax / 2).toString(), - '*', - 'note', note.id); - } - - if (note.visibility === 'public' && note.userHost == null) { - redisPipeline.xadd( - 'localTimeline', - 'MAXLEN', '~', '1000', + 'localTimelineWithFiles', + 'MAXLEN', '~', '500', '*', 'note', note.id); - - if (note.fileIds.length > 0) { - redisPipeline.xadd( - 'localTimelineWithFiles', - 'MAXLEN', '~', '500', - '*', - 'note', note.id); - } } } } diff --git a/packages/backend/test/e2e/timelines.ts b/packages/backend/test/e2e/timelines.ts index b1e84a821c..10840b1336 100644 --- a/packages/backend/test/e2e/timelines.ts +++ b/packages/backend/test/e2e/timelines.ts @@ -701,6 +701,18 @@ describe('Timelines', () => { }); describe('User TL', () => { + test.concurrent('ノートが含まれる', async () => { + const [alice, bob] = await Promise.all([signup(), signup()]); + + const bobNote = await post(bob, { text: 'hi' }); + + await sleep(100); // redisに追加されるのを待つ + + const res = await api('/users/notes', { userId: bob.id }, alice); + + assert.strictEqual(res.body.some((note: any) => note.id === bobNote.id), true); + }); + test.concurrent('フォローしていないユーザーの visibility: followers なノートが含まれない', async () => { const [alice, bob] = await Promise.all([signup(), signup()]); @@ -756,6 +768,21 @@ describe('Timelines', () => { assert.strictEqual(res.body.some((note: any) => note.id === bobNote1.id), true); assert.strictEqual(res.body.some((note: any) => note.id === bobNote2.id), true); }); + + test.concurrent('[withFiles: true] ファイル付きノートのみ含まれる', async () => { + const [alice, bob] = await Promise.all([signup(), signup()]); + + const file = await uploadUrl(bob, 'https://raw.githubusercontent.com/misskey-dev/assets/main/icon.png'); + const bobNote1 = await post(bob, { text: 'hi' }); + const bobNote2 = await post(bob, { fileIds: [file.id] }); + + await sleep(100); // redisに追加されるのを待つ + + const res = await api('/users/notes', { userId: bob.id, withFiles: true }, alice); + + assert.strictEqual(res.body.some((note: any) => note.id === bobNote1.id), false); + assert.strictEqual(res.body.some((note: any) => note.id === bobNote2.id), true); + }, 1000 * 10); }); // TODO: リノートミュート済みユーザーのテスト -- cgit v1.2.3-freya From 5b00fa6f825d5cf498ecf14c1a005762f84b923d Mon Sep 17 00:00:00 2001 From: syuilo Date: Thu, 5 Oct 2023 09:48:45 +0900 Subject: enhance(backend): users/notesでチャンネル投稿を含めるオプション MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Resolve #11965 --- packages/backend/src/core/NoteCreateService.ts | 6 ++++++ packages/backend/src/server/api/endpoints/users/notes.ts | 12 +++++++++++- packages/backend/test/e2e/timelines.ts | 13 +++++++++++++ packages/frontend/src/pages/user/index.timeline.vue | 1 + 4 files changed, 31 insertions(+), 1 deletion(-) (limited to 'packages/backend/src/core/NoteCreateService.ts') diff --git a/packages/backend/src/core/NoteCreateService.ts b/packages/backend/src/core/NoteCreateService.ts index 3ea8af5cda..e8e9973b6e 100644 --- a/packages/backend/src/core/NoteCreateService.ts +++ b/packages/backend/src/core/NoteCreateService.ts @@ -806,6 +806,12 @@ export class NoteCreateService implements OnApplicationShutdown { const redisPipeline = this.redisForTimelines.pipeline(); if (note.channelId) { + redisPipeline.xadd( + `userTimelineWithChannel:${user.id}`, + 'MAXLEN', '~', note.userHost == null ? meta.perLocalUserUserTimelineCacheMax.toString() : meta.perRemoteUserUserTimelineCacheMax.toString(), + '*', + 'note', note.id); + const channelFollowings = await this.channelFollowingsRepository.find({ where: { followeeId: note.channelId, diff --git a/packages/backend/src/server/api/endpoints/users/notes.ts b/packages/backend/src/server/api/endpoints/users/notes.ts index 09adcf20aa..4374863f2b 100644 --- a/packages/backend/src/server/api/endpoints/users/notes.ts +++ b/packages/backend/src/server/api/endpoints/users/notes.ts @@ -44,6 +44,7 @@ export const paramDef = { userId: { type: 'string', format: 'misskey:id' }, withReplies: { type: 'boolean', default: false }, withRenotes: { type: 'boolean', default: true }, + withChannelNotes: { type: 'boolean', default: false }, limit: { type: 'integer', minimum: 1, maximum: 100, default: 10 }, sinceId: { type: 'string', format: 'misskey:id' }, untilId: { type: 'string', format: 'misskey:id' }, @@ -82,9 +83,10 @@ export default class extends Endpoint { // eslint- const limit = ps.limit + (ps.untilId ? 1 : 0); // untilIdに指定したものも含まれるため+1 let noteIdsRes: [string, string[]][] = []; let repliesNoteIdsRes: [string, string[]][] = []; + let channelNoteIdsRes: [string, string[]][] = []; if (!ps.sinceId && !ps.sinceDate) { - [noteIdsRes, repliesNoteIdsRes] = await Promise.all([ + [noteIdsRes, repliesNoteIdsRes, channelNoteIdsRes] = await Promise.all([ this.redisForTimelines.xrevrange( ps.withFiles ? `userTimelineWithFiles:${ps.userId}` : `userTimeline:${ps.userId}`, ps.untilId ? this.idService.parse(ps.untilId).date.getTime() : ps.untilDate ?? '+', @@ -97,12 +99,20 @@ export default class extends Endpoint { // eslint- '-', 'COUNT', limit) : Promise.resolve([]), + ps.withChannelNotes + ? this.redisForTimelines.xrevrange( + `userTimelineWithChannel:${ps.userId}`, + ps.untilId ? this.idService.parse(ps.untilId).date.getTime() : ps.untilDate ?? '+', + '-', + 'COUNT', limit) + : Promise.resolve([]), ]); } let noteIds = Array.from(new Set([ ...noteIdsRes.map(x => x[1][1]).filter(x => x !== ps.untilId), ...repliesNoteIdsRes.map(x => x[1][1]).filter(x => x !== ps.untilId), + ...channelNoteIdsRes.map(x => x[1][1]).filter(x => x !== ps.untilId), ])); noteIds.sort((a, b) => a > b ? -1 : 1); noteIds = noteIds.slice(0, ps.limit); diff --git a/packages/backend/test/e2e/timelines.ts b/packages/backend/test/e2e/timelines.ts index 8bd6f0d195..81301502d4 100644 --- a/packages/backend/test/e2e/timelines.ts +++ b/packages/backend/test/e2e/timelines.ts @@ -877,6 +877,19 @@ describe('Timelines', () => { assert.strictEqual(res.body.some((note: any) => note.id === bobNote2.id), true); }, 1000 * 10); + test.concurrent('[withChannelNotes: true] チャンネル投稿が含まれる', async () => { + const [alice, bob] = await Promise.all([signup(), signup()]); + + const channel = await api('/channels/create', { name: 'channel' }, bob).then(x => x.body); + const bobNote = await post(bob, { text: 'hi', channelId: channel.id }); + + await waitForPushToTl(); + + const res = await api('/users/notes', { userId: bob.id, withChannelNotes: true }, alice); + + assert.strictEqual(res.body.some((note: any) => note.id === bobNote.id), true); + }); + test.concurrent('ミュートしているユーザーに関連する投稿が含まれない', async () => { const [alice, bob, carol] = await Promise.all([signup(), signup(), signup()]); diff --git a/packages/frontend/src/pages/user/index.timeline.vue b/packages/frontend/src/pages/user/index.timeline.vue index 42040f5304..2e0455df65 100644 --- a/packages/frontend/src/pages/user/index.timeline.vue +++ b/packages/frontend/src/pages/user/index.timeline.vue @@ -38,6 +38,7 @@ const pagination = { userId: props.user.id, withRenotes: include.value === 'all', withReplies: include.value === 'all' || include.value === 'files', + withChannelNotes: include.value === 'all', withFiles: include.value === 'files', })), }; -- cgit v1.2.3-freya From ee483f2deee56ecac38ccc5554594c6ecad1cc96 Mon Sep 17 00:00:00 2001 From: anatawa12 Date: Thu, 5 Oct 2023 17:03:50 +0900 Subject: Disallow renote of direct note (#11970) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * chore: renoteに関するチェックをまとめる * fix: ダイレクト投稿をrenoteできる * fix(frontend): 自分のダイレクト投稿をrenoteできる * docs(changelog): ダイレクト投稿をリノートできてしまう * fix lint * chore(backend): visibilityに関するエラーをApi Errorとして返す --- CHANGELOG.md | 1 + packages/backend/src/core/NoteCreateService.ts | 35 ++++++++++++++-------- .../src/server/api/endpoints/notes/create.ts | 14 +++++++++ packages/frontend/src/components/MkNote.vue | 2 +- 4 files changed, 39 insertions(+), 13 deletions(-) (limited to 'packages/backend/src/core/NoteCreateService.ts') diff --git a/CHANGELOG.md b/CHANGELOG.md index 6959c8577c..2780ebb86a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -27,6 +27,7 @@ - Enhance: ソフトワードミュートとハードワードミュートは統合されました - Enhance: モデレーションログ機能の強化 - Enhance: ローカリゼーションの更新 +- Fix: ダイレクト投稿をリノートできてしまう ### Client - Enhance: 二要素認証のバックアップコード一覧をテキストファイルでダウンロード可能に diff --git a/packages/backend/src/core/NoteCreateService.ts b/packages/backend/src/core/NoteCreateService.ts index e8e9973b6e..34d103df77 100644 --- a/packages/backend/src/core/NoteCreateService.ts +++ b/packages/backend/src/core/NoteCreateService.ts @@ -252,19 +252,30 @@ export class NoteCreateService implements OnApplicationShutdown { } } - // Renote対象が「ホームまたは全体」以外の公開範囲ならreject - if (data.renote && data.renote.visibility !== 'public' && data.renote.visibility !== 'home' && data.renote.userId !== user.id) { - throw new Error('Renote target is not public or home'); - } - - // Renote対象がpublicではないならhomeにする - if (data.renote && data.renote.visibility !== 'public' && data.visibility === 'public') { - data.visibility = 'home'; - } + if (data.renote) { + switch (data.renote.visibility) { + case 'public': + // public noteは無条件にrenote可能 + break; + case 'home': + // home noteはhome以下にrenote可能 + if (data.visibility === 'public') { + data.visibility = 'home'; + } + break; + case 'followers': + // 他人のfollowers noteはreject + if (data.renote.userId !== user.id) { + throw new Error('Renote target is not public or home'); + } - // Renote対象がfollowersならfollowersにする - if (data.renote && data.renote.visibility === 'followers') { - data.visibility = 'followers'; + // Renote対象がfollowersならfollowersにする + data.visibility = 'followers'; + break; + case 'specified': + // specified / direct noteはreject + throw new Error('Renote target is not public or home'); + } } // 返信対象がpublicではないならhomeにする diff --git a/packages/backend/src/server/api/endpoints/notes/create.ts b/packages/backend/src/server/api/endpoints/notes/create.ts index 37a0525e25..3ae4ac044a 100644 --- a/packages/backend/src/server/api/endpoints/notes/create.ts +++ b/packages/backend/src/server/api/endpoints/notes/create.ts @@ -57,6 +57,12 @@ export const meta = { id: 'fd4cc33e-2a37-48dd-99cc-9b806eb2031a', }, + cannotRenoteDueToVisibility: { + message: 'You can not Renote due to target visibility.', + code: 'CANNOT_RENOTE_DUE_TO_VISIBILITY', + id: 'be9529e9-fe72-4de0-ae43-0b363c4938af', + }, + noSuchReplyTarget: { message: 'No such reply target.', code: 'NO_SUCH_REPLY_TARGET', @@ -231,6 +237,14 @@ export default class extends Endpoint { // eslint- throw new ApiError(meta.errors.youHaveBeenBlocked); } } + + if (renote.visibility === 'followers' && renote.userId !== me.id) { + // 他人のfollowers noteはreject + throw new ApiError(meta.errors.cannotRenoteDueToVisibility); + } else if (renote.visibility === 'specified') { + // specified / direct noteはreject + throw new ApiError(meta.errors.cannotRenoteDueToVisibility); + } } let reply: MiNote | null = null; diff --git a/packages/frontend/src/components/MkNote.vue b/packages/frontend/src/components/MkNote.vue index 4860f42cdc..62deefc67d 100644 --- a/packages/frontend/src/components/MkNote.vue +++ b/packages/frontend/src/components/MkNote.vue @@ -215,7 +215,7 @@ const muted = ref($i ? checkWordMute(appearNote, $i, $i.mutedWords) : false); const translation = ref(null); const translating = ref(false); const showTicker = (defaultStore.state.instanceTicker === 'always') || (defaultStore.state.instanceTicker === 'remote' && appearNote.user.instance); -const canRenote = computed(() => ['public', 'home'].includes(appearNote.visibility) || appearNote.userId === $i.id); +const canRenote = computed(() => ['public', 'home'].includes(appearNote.visibility) || (appearNote.visibility === 'followers' && appearNote.userId === $i.id)); let renoteCollapsed = $ref(defaultStore.state.collapseRenotes && isRenote && (($i && ($i.id === note.userId || $i.id === appearNote.userId)) || (appearNote.myReaction != null))); const keymap = { -- cgit v1.2.3-freya From dab205edb87ea9cdaee5b6564aa11dfcea245d7b Mon Sep 17 00:00:00 2001 From: syuilo Date: Fri, 6 Oct 2023 14:24:25 +0900 Subject: enhance(backend): improve featured system --- CHANGELOG.md | 3 +- .../backend/migration/1696569742153-clean-up.js | 18 +++ packages/backend/src/core/CoreModule.ts | 6 + packages/backend/src/core/FeaturedService.ts | 126 +++++++++++++++++++++ packages/backend/src/core/NoteCreateService.ts | 12 +- packages/backend/src/core/NoteDeleteService.ts | 1 - packages/backend/src/core/ReactionService.ts | 18 ++- packages/backend/src/models/Note.ts | 5 - packages/backend/src/models/NoteReaction.ts | 1 - .../src/server/api/endpoints/notes/featured.ts | 51 +++++---- 10 files changed, 208 insertions(+), 33 deletions(-) create mode 100644 packages/backend/migration/1696569742153-clean-up.js create mode 100644 packages/backend/src/core/FeaturedService.ts (limited to 'packages/backend/src/core/NoteCreateService.ts') diff --git a/CHANGELOG.md b/CHANGELOG.md index 2780ebb86a..c95ad4dd65 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -34,7 +34,8 @@ - Fix: リアクションしたユーザ一覧のUIが稀に左上に残ってしまう不具合を修正 ### Server -- Enhance: タイムライン取得時のパフォーマンスを改善 +- Enhance: タイムライン取得時のパフォーマンスを大幅に向上 +- Enhance: ハイライト取得時のパフォーマンスを大幅に向上 ## 2023.9.3 ### General diff --git a/packages/backend/migration/1696569742153-clean-up.js b/packages/backend/migration/1696569742153-clean-up.js new file mode 100644 index 0000000000..de48fab5aa --- /dev/null +++ b/packages/backend/migration/1696569742153-clean-up.js @@ -0,0 +1,18 @@ +/* + * SPDX-FileCopyrightText: syuilo and other misskey contributors + * SPDX-License-Identifier: AGPL-3.0-only + */ + +export class CleanUp1696569742153 { + name = 'CleanUp1696569742153' + + async up(queryRunner) { + await queryRunner.query(`DROP INDEX "public"."IDX_01f4581f114e0ebd2bbb876f0b"`); + await queryRunner.query(`ALTER TABLE "note" DROP COLUMN "score"`); + } + + async down(queryRunner) { + await queryRunner.query(`ALTER TABLE "note" ADD "score" integer NOT NULL DEFAULT '0'`); + await queryRunner.query(`CREATE INDEX "IDX_01f4581f114e0ebd2bbb876f0b" ON "note_reaction" ("createdAt") `); + } +} diff --git a/packages/backend/src/core/CoreModule.ts b/packages/backend/src/core/CoreModule.ts index cd66d1a81c..1984d9e6c2 100644 --- a/packages/backend/src/core/CoreModule.ts +++ b/packages/backend/src/core/CoreModule.ts @@ -60,6 +60,7 @@ import { UtilityService } from './UtilityService.js'; import { FileInfoService } from './FileInfoService.js'; import { SearchService } from './SearchService.js'; import { ClipService } from './ClipService.js'; +import { FeaturedService } from './FeaturedService.js'; import { ChartLoggerService } from './chart/ChartLoggerService.js'; import FederationChart from './chart/charts/federation.js'; import NotesChart from './chart/charts/notes.js'; @@ -187,6 +188,7 @@ const $UtilityService: Provider = { provide: 'UtilityService', useExisting: Util const $FileInfoService: Provider = { provide: 'FileInfoService', useExisting: FileInfoService }; const $SearchService: Provider = { provide: 'SearchService', useExisting: SearchService }; const $ClipService: Provider = { provide: 'ClipService', useExisting: ClipService }; +const $FeaturedService: Provider = { provide: 'FeaturedService', useExisting: FeaturedService }; const $ChartLoggerService: Provider = { provide: 'ChartLoggerService', useExisting: ChartLoggerService }; const $FederationChart: Provider = { provide: 'FederationChart', useExisting: FederationChart }; @@ -318,6 +320,7 @@ const $ApQuestionService: Provider = { provide: 'ApQuestionService', useExisting FileInfoService, SearchService, ClipService, + FeaturedService, ChartLoggerService, FederationChart, NotesChart, @@ -442,6 +445,7 @@ const $ApQuestionService: Provider = { provide: 'ApQuestionService', useExisting $FileInfoService, $SearchService, $ClipService, + $FeaturedService, $ChartLoggerService, $FederationChart, $NotesChart, @@ -567,6 +571,7 @@ const $ApQuestionService: Provider = { provide: 'ApQuestionService', useExisting FileInfoService, SearchService, ClipService, + FeaturedService, FederationChart, NotesChart, UsersChart, @@ -690,6 +695,7 @@ const $ApQuestionService: Provider = { provide: 'ApQuestionService', useExisting $FileInfoService, $SearchService, $ClipService, + $FeaturedService, $FederationChart, $NotesChart, $UsersChart, diff --git a/packages/backend/src/core/FeaturedService.ts b/packages/backend/src/core/FeaturedService.ts new file mode 100644 index 0000000000..89b86b2e30 --- /dev/null +++ b/packages/backend/src/core/FeaturedService.ts @@ -0,0 +1,126 @@ +/* + * SPDX-FileCopyrightText: syuilo and other misskey contributors + * SPDX-License-Identifier: AGPL-3.0-only + */ + +import { Inject, Injectable } from '@nestjs/common'; +import * as Redis from 'ioredis'; +import type { MiNote } from '@/models/_.js'; +import { DI } from '@/di-symbols.js'; +import { bindThis } from '@/decorators.js'; + +@Injectable() +export class FeaturedService { + constructor( + @Inject(DI.redis) + private redisClient: Redis.Redis, + ) { + } + + @bindThis + private getCurrentPerUserFriendRankingWindow(): number { + const passed = new Date().getTime() - new Date(new Date().getFullYear(), 0, 1).getTime(); + return Math.floor(passed / (1000 * 60 * 60 * 24 * 7)); // 1週間ごと + } + + @bindThis + private getCurrentGlobalNotesRankingWindow(): number { + const passed = new Date().getTime() - new Date(new Date().getFullYear(), 0, 1).getTime(); + return Math.floor(passed / (1000 * 60 * 60 * 24 * 3)); // 3日ごと + } + + @bindThis + public async updateGlobalNotesRanking(noteId: MiNote['id'], score = 1): Promise { + // TODO: フォロワー数の多い人が常にランキング上位になるのを防ぎたい + const currentWindow = this.getCurrentGlobalNotesRankingWindow(); + const redisTransaction = this.redisClient.multi(); + redisTransaction.zincrby( + `featuredGlobalNotesRanking:${currentWindow}`, + score.toString(), + noteId); + redisTransaction.expire( + `featuredGlobalNotesRanking:${currentWindow}`, + 60 * 60 * 24 * 9, // 9日間保持 + 'NX'); // "NX -- Set expiry only when the key has no expiry" = 有効期限がないときだけ設定 + await redisTransaction.exec(); + } + + @bindThis + public async updateInChannelNotesRanking(noteId: MiNote['id'], channelId: MiNote['channelId'], score = 1): Promise { + const currentWindow = this.getCurrentGlobalNotesRankingWindow(); + const redisTransaction = this.redisClient.multi(); + redisTransaction.zincrby( + `featuredInChannelNotesRanking:${channelId}:${currentWindow}`, + score.toString(), + noteId); + redisTransaction.expire( + `featuredInChannelNotesRanking:${channelId}:${currentWindow}`, + 60 * 60 * 24 * 9, // 9日間保持 + 'NX'); // "NX -- Set expiry only when the key has no expiry" = 有効期限がないときだけ設定 + await redisTransaction.exec(); + } + + @bindThis + public async getGlobalNotesRanking(limit: number): Promise { + const currentWindow = this.getCurrentGlobalNotesRankingWindow(); + const previousWindow = currentWindow - 1; + + const [currentRankingResult, previousRankingResult] = await Promise.all([ + this.redisClient.zrange( + `featuredGlobalNotesRanking:${currentWindow}`, 0, limit, 'REV', 'WITHSCORES'), + this.redisClient.zrange( + `featuredGlobalNotesRanking:${previousWindow}`, 0, limit, 'REV', 'WITHSCORES'), + ]); + + const ranking = new Map(); + for (let i = 0; i < currentRankingResult.length; i += 2) { + const noteId = currentRankingResult[i]; + const score = parseInt(currentRankingResult[i + 1], 10); + ranking.set(noteId, score); + } + for (let i = 0; i < previousRankingResult.length; i += 2) { + const noteId = previousRankingResult[i]; + const score = parseInt(previousRankingResult[i + 1], 10); + const exist = ranking.get(noteId); + if (exist != null) { + ranking.set(noteId, (exist + score) / 2); + } else { + ranking.set(noteId, score); + } + } + + return Array.from(ranking.keys()); + } + + @bindThis + public async getInChannelNotesRanking(channelId: MiNote['channelId'], limit: number): Promise { + const currentWindow = this.getCurrentGlobalNotesRankingWindow(); + const previousWindow = currentWindow - 1; + + const [currentRankingResult, previousRankingResult] = await Promise.all([ + this.redisClient.zrange( + `featuredInChannelNotesRanking:${channelId}:${currentWindow}`, 0, limit, 'REV', 'WITHSCORES'), + this.redisClient.zrange( + `featuredInChannelNotesRanking:${channelId}:${previousWindow}`, 0, limit, 'REV', 'WITHSCORES'), + ]); + + const ranking = new Map(); + for (let i = 0; i < currentRankingResult.length; i += 2) { + const noteId = currentRankingResult[i]; + const score = parseInt(currentRankingResult[i + 1], 10); + ranking.set(noteId, score); + } + for (let i = 0; i < previousRankingResult.length; i += 2) { + const noteId = previousRankingResult[i]; + const score = parseInt(previousRankingResult[i + 1], 10); + const exist = ranking.get(noteId); + if (exist != null) { + ranking.set(noteId, (exist + score) / 2); + } else { + ranking.set(noteId, score); + } + } + + return Array.from(ranking.keys()); + } +} diff --git a/packages/backend/src/core/NoteCreateService.ts b/packages/backend/src/core/NoteCreateService.ts index 34d103df77..ca9dbfa642 100644 --- a/packages/backend/src/core/NoteCreateService.ts +++ b/packages/backend/src/core/NoteCreateService.ts @@ -53,6 +53,7 @@ import { DB_MAX_NOTE_TEXT_LENGTH } from '@/const.js'; import { RoleService } from '@/core/RoleService.js'; import { MetaService } from '@/core/MetaService.js'; import { SearchService } from '@/core/SearchService.js'; +import { FeaturedService } from '@/core/FeaturedService.js'; type NotificationType = 'reply' | 'renote' | 'quote' | 'mention'; @@ -200,6 +201,7 @@ export class NoteCreateService implements OnApplicationShutdown { private hashtagService: HashtagService, private antennaService: AntennaService, private webhookService: WebhookService, + private featuredService: FeaturedService, private remoteUserResolveService: RemoteUserResolveService, private apDeliverManagerService: ApDeliverManagerService, private apRendererService: ApRendererService, @@ -721,10 +723,18 @@ export class NoteCreateService implements OnApplicationShutdown { this.notesRepository.createQueryBuilder().update() .set({ renoteCount: () => '"renoteCount" + 1', - score: () => '"score" + 1', }) .where('id = :id', { id: renote.id }) .execute(); + + // 30%の確率でハイライト用ランキング更新 + if (Math.random() < 0.3) { + if (renote.channelId != null) { + this.featuredService.updateInChannelNotesRanking(renote.id, renote.channelId, 1); + } else if (renote.visibility === 'public' && renote.userHost == null) { + this.featuredService.updateGlobalNotesRanking(renote.id, 1); + } + } } @bindThis diff --git a/packages/backend/src/core/NoteDeleteService.ts b/packages/backend/src/core/NoteDeleteService.ts index 87979f22ac..3d443b4a06 100644 --- a/packages/backend/src/core/NoteDeleteService.ts +++ b/packages/backend/src/core/NoteDeleteService.ts @@ -67,7 +67,6 @@ export class NoteDeleteService { // この投稿を除く指定したユーザーによる指定したノートのリノートが存在しないとき if (note.renoteId && (await this.noteEntityService.countSameRenotes(user.id, note.renoteId, note.id)) === 0) { this.notesRepository.decrement({ id: note.renoteId }, 'renoteCount', 1); - if (!user.isBot) this.notesRepository.decrement({ id: note.renoteId }, 'score', 1); } if (note.replyId) { diff --git a/packages/backend/src/core/ReactionService.ts b/packages/backend/src/core/ReactionService.ts index 25464b19a8..298a62ffd9 100644 --- a/packages/backend/src/core/ReactionService.ts +++ b/packages/backend/src/core/ReactionService.ts @@ -4,6 +4,7 @@ */ import { Inject, Injectable } from '@nestjs/common'; +import * as Redis from 'ioredis'; import { DI } from '@/di-symbols.js'; import type { EmojisRepository, NoteReactionsRepository, UsersRepository, NotesRepository } from '@/models/_.js'; import { IdentifiableError } from '@/misc/identifiable-error.js'; @@ -26,6 +27,7 @@ import { UtilityService } from '@/core/UtilityService.js'; import { UserBlockingService } from '@/core/UserBlockingService.js'; import { CustomEmojiService } from '@/core/CustomEmojiService.js'; import { RoleService } from '@/core/RoleService.js'; +import { FeaturedService } from '@/core/FeaturedService.js'; const FALLBACK = '❤'; @@ -66,6 +68,9 @@ const decodeCustomEmojiRegexp = /^:([\w+-]+)(?:@([\w.-]+))?:$/; @Injectable() export class ReactionService { constructor( + @Inject(DI.redis) + private redisClient: Redis.Redis, + @Inject(DI.usersRepository) private usersRepository: UsersRepository, @@ -86,6 +91,7 @@ export class ReactionService { private noteEntityService: NoteEntityService, private userBlockingService: UserBlockingService, private idService: IdService, + private featuredService: FeaturedService, private globalEventService: GlobalEventService, private apRendererService: ApRendererService, private apDeliverManagerService: ApDeliverManagerService, @@ -182,11 +188,19 @@ export class ReactionService { await this.notesRepository.createQueryBuilder().update() .set({ reactions: () => sql, - ... (!user.isBot ? { score: () => '"score" + 1' } : {}), }) .where('id = :id', { id: note.id }) .execute(); + // 30%の確率でハイライト用ランキング更新 + if (Math.random() < 0.3) { + if (note.channelId != null) { + this.featuredService.updateInChannelNotesRanking(note.id, note.channelId, 1); + } else if (note.visibility === 'public' && note.userHost == null) { + this.featuredService.updateGlobalNotesRanking(note.id, 1); + } + } + const meta = await this.metaService.fetch(); if (meta.enableChartsForRemoteUser || (user.host == null)) { @@ -275,8 +289,6 @@ export class ReactionService { .where('id = :id', { id: note.id }) .execute(); - if (!user.isBot) this.notesRepository.decrement({ id: note.id }, 'score', 1); - this.globalEventService.publishNoteStream(note.id, 'unreacted', { reaction: this.decodeReaction(exist.reaction).reaction, userId: user.id, diff --git a/packages/backend/src/models/Note.ts b/packages/backend/src/models/Note.ts index 0d2422c4f3..3e2adf4d82 100644 --- a/packages/backend/src/models/Note.ts +++ b/packages/backend/src/models/Note.ts @@ -138,11 +138,6 @@ export class MiNote { }) public url: string | null; - @Column('integer', { - default: 0, select: false, - }) - public score: number; - @Index() @Column({ ...id(), diff --git a/packages/backend/src/models/NoteReaction.ts b/packages/backend/src/models/NoteReaction.ts index 7c08d31c6d..43323f8a43 100644 --- a/packages/backend/src/models/NoteReaction.ts +++ b/packages/backend/src/models/NoteReaction.ts @@ -14,7 +14,6 @@ export class MiNoteReaction { @PrimaryColumn(id()) public id: string; - @Index() @Column('timestamp with time zone', { comment: 'The created date of the NoteReaction.', }) diff --git a/packages/backend/src/server/api/endpoints/notes/featured.ts b/packages/backend/src/server/api/endpoints/notes/featured.ts index 5283b0e0bc..bf4ad1deb6 100644 --- a/packages/backend/src/server/api/endpoints/notes/featured.ts +++ b/packages/backend/src/server/api/endpoints/notes/featured.ts @@ -6,9 +6,9 @@ import { Inject, Injectable } from '@nestjs/common'; import type { NotesRepository } from '@/models/_.js'; import { Endpoint } from '@/server/api/endpoint-base.js'; -import { QueryService } from '@/core/QueryService.js'; import { NoteEntityService } from '@/core/entities/NoteEntityService.js'; import { DI } from '@/di-symbols.js'; +import { FeaturedService } from '@/core/FeaturedService.js'; export const meta = { tags: ['notes'], @@ -40,41 +40,50 @@ export const paramDef = { @Injectable() export default class extends Endpoint { // eslint-disable-line import/no-default-export + private globalNotesRankingCache: string[] = []; + private globalNotesRankingCacheLastFetchedAt = 0; + constructor( @Inject(DI.notesRepository) private notesRepository: NotesRepository, private noteEntityService: NoteEntityService, - private queryService: QueryService, + private featuredService: FeaturedService, ) { super(meta, paramDef, async (ps, me) => { - const day = 1000 * 60 * 60 * 24 * 3; // 3日前まで + let noteIds: string[]; + if (ps.channelId) { + noteIds = await this.featuredService.getInChannelNotesRanking(ps.channelId, 50); + } else { + if (this.globalNotesRankingCacheLastFetchedAt !== 0 && (Date.now() - this.globalNotesRankingCacheLastFetchedAt < 1000 * 60 * 30)) { + noteIds = this.globalNotesRankingCache; + } else { + noteIds = await this.featuredService.getGlobalNotesRanking(100); + this.globalNotesRankingCache = noteIds; + this.globalNotesRankingCacheLastFetchedAt = Date.now(); + } + } + + if (noteIds.length === 0) { + return []; + } + + noteIds.sort((a, b) => a > b ? -1 : 1); + noteIds.slice(ps.offset, ps.offset + ps.limit); const query = this.notesRepository.createQueryBuilder('note') - .addSelect('note.score') - .where('note.userHost IS NULL') - .andWhere('note.score > 0') - .andWhere('note.createdAt > :date', { date: new Date(Date.now() - day) }) - .andWhere('note.visibility = \'public\'') + .where('note.id IN (:...noteIds)', { noteIds: noteIds }) .innerJoinAndSelect('note.user', 'user') .leftJoinAndSelect('note.reply', 'reply') .leftJoinAndSelect('note.renote', 'renote') .leftJoinAndSelect('reply.user', 'replyUser') - .leftJoinAndSelect('renote.user', 'renoteUser'); - - if (ps.channelId) query.andWhere('note.channelId = :channelId', { channelId: ps.channelId }); - - if (me) this.queryService.generateMutedUserQuery(query, me); - if (me) this.queryService.generateBlockedUserQuery(query, me); - - let notes = await query - .orderBy('note.score', 'DESC') - .limit(100) - .getMany(); + .leftJoinAndSelect('renote.user', 'renoteUser') + .leftJoinAndSelect('note.channel', 'channel'); - notes.sort((a, b) => new Date(b.createdAt).getTime() - new Date(a.createdAt).getTime()); + const notes = await query.getMany(); + notes.sort((a, b) => a.id > b.id ? -1 : 1); - notes = notes.slice(ps.offset, ps.offset + ps.limit); + // TODO: ミュート等考慮 return await this.noteEntityService.packMany(notes, me); }); -- cgit v1.2.3-freya From 87416710c323966aa267442acea419e62c56f325 Mon Sep 17 00:00:00 2001 From: syuilo Date: Fri, 6 Oct 2023 16:17:29 +0900 Subject: enhance(backend): some tweaks --- packages/backend/src/core/NoteCreateService.ts | 5 ++--- packages/backend/src/core/NoteDeleteService.ts | 5 ----- packages/backend/src/core/ReactionService.ts | 2 +- packages/backend/src/core/entities/NoteEntityService.ts | 15 --------------- 4 files changed, 3 insertions(+), 24 deletions(-) (limited to 'packages/backend/src/core/NoteCreateService.ts') diff --git a/packages/backend/src/core/NoteCreateService.ts b/packages/backend/src/core/NoteCreateService.ts index ca9dbfa642..32f1af4528 100644 --- a/packages/backend/src/core/NoteCreateService.ts +++ b/packages/backend/src/core/NoteCreateService.ts @@ -521,9 +521,8 @@ export class NoteCreateService implements OnApplicationShutdown { }); } - // この投稿を除く指定したユーザーによる指定したノートのリノートが存在しないとき - if (data.renote && (await this.noteEntityService.countSameRenotes(user.id, data.renote.id, note.id) === 0)) { - if (!user.isBot) this.incRenoteCount(data.renote); + if (data.renote && data.renote.userId !== user.id && !user.isBot) { + this.incRenoteCount(data.renote); } if (data.poll && data.poll.expiresAt) { diff --git a/packages/backend/src/core/NoteDeleteService.ts b/packages/backend/src/core/NoteDeleteService.ts index 3d443b4a06..9a817ffd76 100644 --- a/packages/backend/src/core/NoteDeleteService.ts +++ b/packages/backend/src/core/NoteDeleteService.ts @@ -64,11 +64,6 @@ export class NoteDeleteService { const deletedAt = new Date(); const cascadingNotes = await this.findCascadingNotes(note); - // この投稿を除く指定したユーザーによる指定したノートのリノートが存在しないとき - if (note.renoteId && (await this.noteEntityService.countSameRenotes(user.id, note.renoteId, note.id)) === 0) { - this.notesRepository.decrement({ id: note.renoteId }, 'renoteCount', 1); - } - if (note.replyId) { await this.notesRepository.decrement({ id: note.replyId }, 'repliesCount', 1); } diff --git a/packages/backend/src/core/ReactionService.ts b/packages/backend/src/core/ReactionService.ts index 298a62ffd9..e409495de5 100644 --- a/packages/backend/src/core/ReactionService.ts +++ b/packages/backend/src/core/ReactionService.ts @@ -193,7 +193,7 @@ export class ReactionService { .execute(); // 30%の確率でハイライト用ランキング更新 - if (Math.random() < 0.3) { + if (Math.random() < 0.3 && note.userId !== user.id) { if (note.channelId != null) { this.featuredService.updateInChannelNotesRanking(note.id, note.channelId, 1); } else if (note.visibility === 'public' && note.userHost == null) { diff --git a/packages/backend/src/core/entities/NoteEntityService.ts b/packages/backend/src/core/entities/NoteEntityService.ts index 824f8fa71d..e45a7992bb 100644 --- a/packages/backend/src/core/entities/NoteEntityService.ts +++ b/packages/backend/src/core/entities/NoteEntityService.ts @@ -450,19 +450,4 @@ export class NoteEntityService implements OnModuleInit { } return emojis.filter(x => x.name != null && x.host != null) as { name: string; host: string; }[]; } - - @bindThis - public async countSameRenotes(userId: string, renoteId: string, excludeNoteId: string | undefined): Promise { - // 指定したユーザーの指定したノートのリノートがいくつあるか数える - const query = this.notesRepository.createQueryBuilder('note') - .where('note.userId = :userId', { userId }) - .andWhere('note.renoteId = :renoteId', { renoteId }); - - // 指定した投稿を除く - if (excludeNoteId) { - query.andWhere('note.id != :excludeNoteId', { excludeNoteId }); - } - - return await query.getCount(); - } } -- cgit v1.2.3-freya From a5b6e807bb6aaf3212f88b4ec4f96c285a80e390 Mon Sep 17 00:00:00 2001 From: syuilo Date: Fri, 6 Oct 2023 18:30:08 +0900 Subject: feat: per user featured notes --- CHANGELOG.md | 2 + packages/backend/src/core/FeaturedService.ts | 15 +++- packages/backend/src/core/NoteCreateService.ts | 5 +- packages/backend/src/core/ReactionService.ts | 3 +- packages/backend/src/server/api/EndpointsModule.ts | 4 ++ packages/backend/src/server/api/endpoints.ts | 2 + .../src/server/api/endpoints/notes/featured.ts | 7 +- .../server/api/endpoints/users/featured-notes.ts | 80 ++++++++++++++++++++++ packages/frontend/src/pages/channel.vue | 1 - packages/frontend/src/pages/explore.featured.vue | 1 - packages/frontend/src/pages/user/home.vue | 7 +- 11 files changed, 116 insertions(+), 11 deletions(-) create mode 100644 packages/backend/src/server/api/endpoints/users/featured-notes.ts (limited to 'packages/backend/src/core/NoteCreateService.ts') diff --git a/CHANGELOG.md b/CHANGELOG.md index f7e8b820ec..f9ccaa9f1e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -21,10 +21,12 @@ ### Changes - API: users/notes, notes/local-timeline で fileType 指定はできなくなりました - API: notes/global-timeline は現在常に `[]` を返します +- API: notes/featured でページネーションは他APIと同様 untilId を使って行うようになりました ### General - Feat: ユーザーごとに他ユーザーへの返信をタイムラインに含めるか設定可能になりました - Feat: ユーザーリスト内のメンバーごとに他ユーザーへの返信をユーザーリストタイムラインに含めるか設定可能になりました +- Feat: ユーザーごとのハイライト - Enhance: ソフトワードミュートとハードワードミュートは統合されました - Enhance: モデレーションログ機能の強化 - Enhance: ローカリゼーションの更新 diff --git a/packages/backend/src/core/FeaturedService.ts b/packages/backend/src/core/FeaturedService.ts index e8a08cd866..945c23b0e2 100644 --- a/packages/backend/src/core/FeaturedService.ts +++ b/packages/backend/src/core/FeaturedService.ts @@ -5,11 +5,12 @@ import { Inject, Injectable } from '@nestjs/common'; import * as Redis from 'ioredis'; -import type { MiNote } from '@/models/_.js'; +import type { MiNote, MiUser } from '@/models/_.js'; import { DI } from '@/di-symbols.js'; import { bindThis } from '@/decorators.js'; const GLOBAL_NOTES_RANKING_WINDOW = 1000 * 60 * 60 * 24 * 3; // 3日ごと +const PER_USER_NOTES_RANKING_WINDOW = 1000 * 60 * 60 * 24 * 7; // 1週間ごと @Injectable() export class FeaturedService { @@ -78,10 +79,15 @@ export class FeaturedService { } @bindThis - public updateInChannelNotesRanking(noteId: MiNote['id'], channelId: MiNote['channelId'], score = 1): Promise { + public updateInChannelNotesRanking(channelId: MiNote['channelId'], noteId: MiNote['id'], score = 1): Promise { return this.updateRankingOf(`featuredInChannelNotesRanking:${channelId}`, GLOBAL_NOTES_RANKING_WINDOW, noteId, score); } + @bindThis + public updatePerUserNotesRanking(userId: MiUser['id'], noteId: MiNote['id'], score = 1): Promise { + return this.updateRankingOf(`featuredPerUserNotesRanking:${userId}`, PER_USER_NOTES_RANKING_WINDOW, noteId, score); + } + @bindThis public getGlobalNotesRanking(limit: number): Promise { return this.getRankingOf('featuredGlobalNotesRanking', GLOBAL_NOTES_RANKING_WINDOW, limit); @@ -91,4 +97,9 @@ export class FeaturedService { public getInChannelNotesRanking(channelId: MiNote['channelId'], limit: number): Promise { return this.getRankingOf(`featuredInChannelNotesRanking:${channelId}`, GLOBAL_NOTES_RANKING_WINDOW, limit); } + + @bindThis + public getPerUserNotesRanking(userId: MiUser['id'], limit: number): Promise { + return this.getRankingOf(`featuredPerUserNotesRanking:${userId}`, PER_USER_NOTES_RANKING_WINDOW, limit); + } } diff --git a/packages/backend/src/core/NoteCreateService.ts b/packages/backend/src/core/NoteCreateService.ts index 32f1af4528..b6fc4b3c49 100644 --- a/packages/backend/src/core/NoteCreateService.ts +++ b/packages/backend/src/core/NoteCreateService.ts @@ -729,9 +729,10 @@ export class NoteCreateService implements OnApplicationShutdown { // 30%の確率でハイライト用ランキング更新 if (Math.random() < 0.3) { if (renote.channelId != null) { - this.featuredService.updateInChannelNotesRanking(renote.id, renote.channelId, 1); + this.featuredService.updateInChannelNotesRanking(renote.channelId, renote.id, 5); } else if (renote.visibility === 'public' && renote.userHost == null) { - this.featuredService.updateGlobalNotesRanking(renote.id, 1); + this.featuredService.updateGlobalNotesRanking(renote.id, 5); + this.featuredService.updatePerUserNotesRanking(renote.userId, renote.id, 5); } } } diff --git a/packages/backend/src/core/ReactionService.ts b/packages/backend/src/core/ReactionService.ts index e409495de5..63cf4be322 100644 --- a/packages/backend/src/core/ReactionService.ts +++ b/packages/backend/src/core/ReactionService.ts @@ -195,9 +195,10 @@ export class ReactionService { // 30%の確率でハイライト用ランキング更新 if (Math.random() < 0.3 && note.userId !== user.id) { if (note.channelId != null) { - this.featuredService.updateInChannelNotesRanking(note.id, note.channelId, 1); + this.featuredService.updateInChannelNotesRanking(note.channelId, note.id, 1); } else if (note.visibility === 'public' && note.userHost == null) { this.featuredService.updateGlobalNotesRanking(note.id, 1); + this.featuredService.updatePerUserNotesRanking(note.userId, note.id, 1); } } diff --git a/packages/backend/src/server/api/EndpointsModule.ts b/packages/backend/src/server/api/EndpointsModule.ts index 3c4adafdbd..f834561456 100644 --- a/packages/backend/src/server/api/EndpointsModule.ts +++ b/packages/backend/src/server/api/EndpointsModule.ts @@ -325,6 +325,7 @@ import * as ep___users_followers from './endpoints/users/followers.js'; import * as ep___users_following from './endpoints/users/following.js'; import * as ep___users_gallery_posts from './endpoints/users/gallery/posts.js'; import * as ep___users_getFrequentlyRepliedUsers from './endpoints/users/get-frequently-replied-users.js'; +import * as ep___users_featuredNotes from './endpoints/users/featured-notes.js'; import * as ep___users_lists_create from './endpoints/users/lists/create.js'; import * as ep___users_lists_delete from './endpoints/users/lists/delete.js'; import * as ep___users_lists_list from './endpoints/users/lists/list.js'; @@ -674,6 +675,7 @@ const $users_followers: Provider = { provide: 'ep:users/followers', useClass: ep const $users_following: Provider = { provide: 'ep:users/following', useClass: ep___users_following.default }; const $users_gallery_posts: Provider = { provide: 'ep:users/gallery/posts', useClass: ep___users_gallery_posts.default }; const $users_getFrequentlyRepliedUsers: Provider = { provide: 'ep:users/get-frequently-replied-users', useClass: ep___users_getFrequentlyRepliedUsers.default }; +const $users_featuredNotes: Provider = { provide: 'ep:users/featured-notes', useClass: ep___users_featuredNotes.default }; const $users_lists_create: Provider = { provide: 'ep:users/lists/create', useClass: ep___users_lists_create.default }; const $users_lists_delete: Provider = { provide: 'ep:users/lists/delete', useClass: ep___users_lists_delete.default }; const $users_lists_list: Provider = { provide: 'ep:users/lists/list', useClass: ep___users_lists_list.default }; @@ -1027,6 +1029,7 @@ const $retention: Provider = { provide: 'ep:retention', useClass: ep___retention $users_following, $users_gallery_posts, $users_getFrequentlyRepliedUsers, + $users_featuredNotes, $users_lists_create, $users_lists_delete, $users_lists_list, @@ -1371,6 +1374,7 @@ const $retention: Provider = { provide: 'ep:retention', useClass: ep___retention $users_following, $users_gallery_posts, $users_getFrequentlyRepliedUsers, + $users_featuredNotes, $users_lists_create, $users_lists_delete, $users_lists_list, diff --git a/packages/backend/src/server/api/endpoints.ts b/packages/backend/src/server/api/endpoints.ts index 199d910fc4..d12a035afa 100644 --- a/packages/backend/src/server/api/endpoints.ts +++ b/packages/backend/src/server/api/endpoints.ts @@ -325,6 +325,7 @@ import * as ep___users_followers from './endpoints/users/followers.js'; import * as ep___users_following from './endpoints/users/following.js'; import * as ep___users_gallery_posts from './endpoints/users/gallery/posts.js'; import * as ep___users_getFrequentlyRepliedUsers from './endpoints/users/get-frequently-replied-users.js'; +import * as ep___users_featuredNotes from './endpoints/users/featured-notes.js'; import * as ep___users_lists_create from './endpoints/users/lists/create.js'; import * as ep___users_lists_delete from './endpoints/users/lists/delete.js'; import * as ep___users_lists_list from './endpoints/users/lists/list.js'; @@ -672,6 +673,7 @@ const eps = [ ['users/following', ep___users_following], ['users/gallery/posts', ep___users_gallery_posts], ['users/get-frequently-replied-users', ep___users_getFrequentlyRepliedUsers], + ['users/featured-notes', ep___users_featuredNotes], ['users/lists/create', ep___users_lists_create], ['users/lists/delete', ep___users_lists_delete], ['users/lists/list', ep___users_lists_list], diff --git a/packages/backend/src/server/api/endpoints/notes/featured.ts b/packages/backend/src/server/api/endpoints/notes/featured.ts index bf4ad1deb6..c456874309 100644 --- a/packages/backend/src/server/api/endpoints/notes/featured.ts +++ b/packages/backend/src/server/api/endpoints/notes/featured.ts @@ -32,7 +32,7 @@ export const paramDef = { type: 'object', properties: { limit: { type: 'integer', minimum: 1, maximum: 100, default: 10 }, - offset: { type: 'integer', default: 0 }, + untilId: { type: 'string', format: 'misskey:id' }, channelId: { type: 'string', nullable: true, format: 'misskey:id' }, }, required: [], @@ -69,7 +69,10 @@ export default class extends Endpoint { // eslint- } noteIds.sort((a, b) => a > b ? -1 : 1); - noteIds.slice(ps.offset, ps.offset + ps.limit); + if (ps.untilId) { + noteIds = noteIds.filter(id => id < ps.untilId!); + } + noteIds = noteIds.slice(0, ps.limit); const query = this.notesRepository.createQueryBuilder('note') .where('note.id IN (:...noteIds)', { noteIds: noteIds }) diff --git a/packages/backend/src/server/api/endpoints/users/featured-notes.ts b/packages/backend/src/server/api/endpoints/users/featured-notes.ts new file mode 100644 index 0000000000..fdf36a6ae0 --- /dev/null +++ b/packages/backend/src/server/api/endpoints/users/featured-notes.ts @@ -0,0 +1,80 @@ +/* + * SPDX-FileCopyrightText: syuilo and other misskey contributors + * SPDX-License-Identifier: AGPL-3.0-only + */ + +import { Inject, Injectable } from '@nestjs/common'; +import type { NotesRepository } from '@/models/_.js'; +import { Endpoint } from '@/server/api/endpoint-base.js'; +import { NoteEntityService } from '@/core/entities/NoteEntityService.js'; +import { DI } from '@/di-symbols.js'; +import { FeaturedService } from '@/core/FeaturedService.js'; + +export const meta = { + tags: ['notes'], + + requireCredential: false, + allowGet: true, + cacheSec: 3600, + + res: { + type: 'array', + optional: false, nullable: false, + items: { + type: 'object', + optional: false, nullable: false, + ref: 'Note', + }, + }, +} as const; + +export const paramDef = { + type: 'object', + properties: { + limit: { type: 'integer', minimum: 1, maximum: 100, default: 10 }, + untilId: { type: 'string', format: 'misskey:id' }, + userId: { type: 'string', format: 'misskey:id' }, + }, + required: ['userId'], +} as const; + +@Injectable() +export default class extends Endpoint { // eslint-disable-line import/no-default-export + constructor( + @Inject(DI.notesRepository) + private notesRepository: NotesRepository, + + private noteEntityService: NoteEntityService, + private featuredService: FeaturedService, + ) { + super(meta, paramDef, async (ps, me) => { + let noteIds = await this.featuredService.getPerUserNotesRanking(ps.userId, 50); + + if (noteIds.length === 0) { + return []; + } + + noteIds.sort((a, b) => a > b ? -1 : 1); + if (ps.untilId) { + noteIds = noteIds.filter(id => id < ps.untilId!); + } + noteIds = noteIds.slice(0, ps.limit); + + const query = this.notesRepository.createQueryBuilder('note') + .where('note.id IN (:...noteIds)', { noteIds: noteIds }) + .innerJoinAndSelect('note.user', 'user') + .leftJoinAndSelect('note.reply', 'reply') + .leftJoinAndSelect('note.renote', 'renote') + .leftJoinAndSelect('reply.user', 'replyUser') + .leftJoinAndSelect('renote.user', 'renoteUser') + .leftJoinAndSelect('note.channel', 'channel'); + + const notes = await query.getMany(); + notes.sort((a, b) => a.id > b.id ? -1 : 1); + + // TODO: ミュート等考慮 + + return await this.noteEntityService.packMany(notes, me); + }); + } +} diff --git a/packages/frontend/src/pages/channel.vue b/packages/frontend/src/pages/channel.vue index 00e88cbbfb..911f4e95d2 100644 --- a/packages/frontend/src/pages/channel.vue +++ b/packages/frontend/src/pages/channel.vue @@ -102,7 +102,6 @@ let searchKey = $ref(''); const featuredPagination = $computed(() => ({ endpoint: 'notes/featured' as const, limit: 10, - offsetMode: true, params: { channelId: props.channelId, }, diff --git a/packages/frontend/src/pages/explore.featured.vue b/packages/frontend/src/pages/explore.featured.vue index 0558faec16..a36d1b3bda 100644 --- a/packages/frontend/src/pages/explore.featured.vue +++ b/packages/frontend/src/pages/explore.featured.vue @@ -22,7 +22,6 @@ import { i18n } from '@/i18n.js'; const paginationForNotes = { endpoint: 'notes/featured' as const, limit: 10, - offsetMode: true, }; const paginationForPolls = { diff --git a/packages/frontend/src/pages/user/home.vue b/packages/frontend/src/pages/user/home.vue index 71eec0aa26..605e9fbb76 100644 --- a/packages/frontend/src/pages/user/home.vue +++ b/packages/frontend/src/pages/user/home.vue @@ -131,7 +131,10 @@ SPDX-License-Identifier: AGPL-3.0-only - +
+
{{ i18n.ts.featured }}
+ +
@@ -210,7 +213,7 @@ watch($$(moderationNote), async () => { }); const pagination = { - endpoint: 'users/notes' as const, + endpoint: 'users/featured-notes' as const, limit: 10, params: computed(() => ({ userId: props.user.id, -- cgit v1.2.3-freya From 4a595153dc3a82206c5ee1c05a12deab663ba544 Mon Sep 17 00:00:00 2001 From: syuilo Date: Sat, 7 Oct 2023 07:53:14 +0900 Subject: enhance(backend): 返信はハイライトに載らないように MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- packages/backend/src/core/NoteCreateService.ts | 12 ++++++++---- packages/backend/src/core/ReactionService.ts | 12 ++++++++---- 2 files changed, 16 insertions(+), 8 deletions(-) (limited to 'packages/backend/src/core/NoteCreateService.ts') diff --git a/packages/backend/src/core/NoteCreateService.ts b/packages/backend/src/core/NoteCreateService.ts index b6fc4b3c49..5e86443945 100644 --- a/packages/backend/src/core/NoteCreateService.ts +++ b/packages/backend/src/core/NoteCreateService.ts @@ -729,10 +729,14 @@ export class NoteCreateService implements OnApplicationShutdown { // 30%の確率でハイライト用ランキング更新 if (Math.random() < 0.3) { if (renote.channelId != null) { - this.featuredService.updateInChannelNotesRanking(renote.channelId, renote.id, 5); - } else if (renote.visibility === 'public' && renote.userHost == null) { - this.featuredService.updateGlobalNotesRanking(renote.id, 5); - this.featuredService.updatePerUserNotesRanking(renote.userId, renote.id, 5); + if (renote.replyId == null) { + this.featuredService.updateInChannelNotesRanking(renote.channelId, renote.id, 5); + } + } else { + if (renote.visibility === 'public' && renote.userHost == null && renote.replyId == null) { + this.featuredService.updateGlobalNotesRanking(renote.id, 5); + this.featuredService.updatePerUserNotesRanking(renote.userId, renote.id, 5); + } } } } diff --git a/packages/backend/src/core/ReactionService.ts b/packages/backend/src/core/ReactionService.ts index 63cf4be322..86213e4576 100644 --- a/packages/backend/src/core/ReactionService.ts +++ b/packages/backend/src/core/ReactionService.ts @@ -195,10 +195,14 @@ export class ReactionService { // 30%の確率でハイライト用ランキング更新 if (Math.random() < 0.3 && note.userId !== user.id) { if (note.channelId != null) { - this.featuredService.updateInChannelNotesRanking(note.channelId, note.id, 1); - } else if (note.visibility === 'public' && note.userHost == null) { - this.featuredService.updateGlobalNotesRanking(note.id, 1); - this.featuredService.updatePerUserNotesRanking(note.userId, note.id, 1); + if (note.replyId == null) { + this.featuredService.updateInChannelNotesRanking(note.channelId, note.id, 1); + } + } else { + if (note.visibility === 'public' && note.userHost == null && note.replyId == null) { + this.featuredService.updateGlobalNotesRanking(note.id, 1); + this.featuredService.updatePerUserNotesRanking(note.userId, note.id, 1); + } } } -- cgit v1.2.3-freya From 3a7558f36c4c035c96e4b9a96c1ae0889b8ae308 Mon Sep 17 00:00:00 2001 From: syuilo Date: Sat, 7 Oct 2023 07:59:46 +0900 Subject: enhance(backend): 3日以内に投稿されたノートのみハイライト用スコア加算 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- packages/backend/src/core/NoteCreateService.ts | 4 ++-- packages/backend/src/core/ReactionService.ts | 8 ++++++-- 2 files changed, 8 insertions(+), 4 deletions(-) (limited to 'packages/backend/src/core/NoteCreateService.ts') diff --git a/packages/backend/src/core/NoteCreateService.ts b/packages/backend/src/core/NoteCreateService.ts index 5e86443945..4a454e79e7 100644 --- a/packages/backend/src/core/NoteCreateService.ts +++ b/packages/backend/src/core/NoteCreateService.ts @@ -726,8 +726,8 @@ export class NoteCreateService implements OnApplicationShutdown { .where('id = :id', { id: renote.id }) .execute(); - // 30%の確率でハイライト用ランキング更新 - if (Math.random() < 0.3) { + // 30%の確率、3日以内に投稿されたノートの場合ハイライト用ランキング更新 + if (Math.random() < 0.3 && (Date.now() - this.idService.parse(renote.id).date.getTime()) < 1000 * 60 * 60 * 24 * 3) { if (renote.channelId != null) { if (renote.replyId == null) { this.featuredService.updateInChannelNotesRanking(renote.channelId, renote.id, 5); diff --git a/packages/backend/src/core/ReactionService.ts b/packages/backend/src/core/ReactionService.ts index 86213e4576..e7bbd44926 100644 --- a/packages/backend/src/core/ReactionService.ts +++ b/packages/backend/src/core/ReactionService.ts @@ -192,8 +192,12 @@ export class ReactionService { .where('id = :id', { id: note.id }) .execute(); - // 30%の確率でハイライト用ランキング更新 - if (Math.random() < 0.3 && note.userId !== user.id) { + // 30%の確率、セルフではない、3日以内に投稿されたノートの場合ハイライト用ランキング更新 + if ( + Math.random() < 0.3 && + note.userId !== user.id && + (Date.now() - this.idService.parse(note.id).date.getTime()) < 1000 * 60 * 60 * 24 * 3 + ) { if (note.channelId != null) { if (note.replyId == null) { this.featuredService.updateInChannelNotesRanking(note.channelId, note.id, 1); -- cgit v1.2.3-freya From ca07459f5e80d7602c7f68df431bed4df9a8fd81 Mon Sep 17 00:00:00 2001 From: _ Date: Mon, 9 Oct 2023 12:36:25 +0900 Subject: fix(backend): ダイレクト投稿がタイムライン上に正常に表示されない問題を修正 (#11993) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * DMをredisにpushするように * add test * add CHANGELOG * Update NoteCreateService.ts * lint * :v: * 前のバージョンから発生した問題ではないため不要 --------- Co-authored-by: syuilo --- packages/backend/src/core/NoteCreateService.ts | 60 +++---- .../src/server/api/endpoints/users/notes.ts | 1 + packages/backend/test/e2e/timelines.ts | 175 ++++++++++++++++++--- packages/shared/.eslintrc.js | 3 + 4 files changed, 194 insertions(+), 45 deletions(-) (limited to 'packages/backend/src/core/NoteCreateService.ts') diff --git a/packages/backend/src/core/NoteCreateService.ts b/packages/backend/src/core/NoteCreateService.ts index 4a454e79e7..277875a19f 100644 --- a/packages/backend/src/core/NoteCreateService.ts +++ b/packages/backend/src/core/NoteCreateService.ts @@ -494,11 +494,7 @@ export class NoteCreateService implements OnApplicationShutdown { // Increment notes count (user) this.incNotesCountOfUser(user); - if (data.visibility === 'specified') { - // TODO? - } else { - this.pushToTl(note, user); - } + this.pushToTl(note, user); this.antennaService.addNoteToAntennas(note, user); @@ -861,24 +857,34 @@ export class NoteCreateService implements OnApplicationShutdown { } } else { // TODO: キャッシュ？ - const followings = await this.followingsRepository.find({ - where: { - followeeId: user.id, - followerHost: IsNull(), - isFollowerHibernated: false, - }, - select: ['followerId', 'withReplies'], - }); - - const userListMemberships = await this.userListMembershipsRepository.find({ - where: { - userId: user.id, - }, - select: ['userListId', 'withReplies'], - }); + // eslint-disable-next-line prefer-const + let [followings, userListMemberships] = await Promise.all([ + this.followingsRepository.find({ + where: { + followeeId: user.id, + followerHost: IsNull(), + isFollowerHibernated: false, + }, + select: ['followerId', 'withReplies'], + }), + this.userListMembershipsRepository.find({ + where: { + userId: user.id, + }, + select: ['userListId', 'userListUserId', 'withReplies'], + }), + ]); + + if (note.visibility === 'followers') { + // TODO: 重そうだから何とかしたい Set 使う？ + userListMemberships = userListMemberships.filter(x => followings.some(f => f.followerId === x.userListUserId)); + } // TODO: あまりにも数が多いと redisPipeline.exec に失敗する(理由は不明)ため、3万件程度を目安に分割して実行するようにする for (const following of followings) { + // 基本的にvisibleUserIdsには自身のidが含まれている前提であること + if (note.visibility === 'specified' && !note.visibleUserIds.some(v => v === following.followerId)) continue; + // 自分自身以外への返信 if (note.replyId && note.replyUserId !== note.userId) { if (!following.withReplies) continue; @@ -899,13 +905,13 @@ export class NoteCreateService implements OnApplicationShutdown { } } - // TODO - //if (note.visibility === 'followers') { - // // TODO: 重そうだから何とかしたい Set 使う？ - // userLists = userLists.filter(x => followings.some(f => f.followerId === x.userListUserId)); - //} - for (const userListMembership of userListMemberships) { + // ダイレクトのとき、そのリストが対象外のユーザーの場合 + if ( + note.visibility === 'specified' && + !note.visibleUserIds.some(v => v === userListMembership.userListUserId) + ) continue; + // 自分自身以外への返信 if (note.replyId && note.replyUserId !== note.userId) { if (!userListMembership.withReplies) continue; @@ -926,7 +932,7 @@ export class NoteCreateService implements OnApplicationShutdown { } } - { // 自分自身のHTL + if (note.visibility !== 'specified' || !note.visibleUserIds.some(v => v === user.id)) { // 自分自身のHTL redisPipeline.xadd( `homeTimeline:${user.id}`, 'MAXLEN', '~', meta.perUserHomeTimelineCacheMax.toString(), diff --git a/packages/backend/src/server/api/endpoints/users/notes.ts b/packages/backend/src/server/api/endpoints/users/notes.ts index d4f9186e3a..5736aad5f9 100644 --- a/packages/backend/src/server/api/endpoints/users/notes.ts +++ b/packages/backend/src/server/api/endpoints/users/notes.ts @@ -136,6 +136,7 @@ export default class extends Endpoint { // eslint- } } + if (note.visibility === 'specified' && (!me || (me.id !== note.userId && !note.visibleUserIds.some(v => v === me.id)))) return false; if (note.visibility === 'followers' && !isFollowing) return false; return true; diff --git a/packages/backend/test/e2e/timelines.ts b/packages/backend/test/e2e/timelines.ts index 6de58f0146..05209c9024 100644 --- a/packages/backend/test/e2e/timelines.ts +++ b/packages/backend/test/e2e/timelines.ts @@ -3,6 +3,9 @@ * SPDX-License-Identifier: AGPL-3.0-only */ +// How to run: +// pnpm jest -- e2e/timelines.ts + process.env.NODE_ENV = 'test'; process.env.FORCE_FOLLOW_REMOTE_USER_FOR_TESTING = 'true'; @@ -378,6 +381,104 @@ describe('Timelines', () => { assert.strictEqual(res.body.some((note: any) => note.id === bobNote.id), false); }); + + test.concurrent('自分の visibility: specified なノートが含まれる', async () => { + const [alice] = await Promise.all([signup()]); + + const aliceNote = await post(alice, { text: 'hi', visibility: 'specified' }); + + await waitForPushToTl(); + + const res = await api('/notes/timeline', {}, alice); + + assert.strictEqual(res.body.some((note: any) => note.id === aliceNote.id), true); + assert.strictEqual(res.body.find((note: any) => note.id === aliceNote.id).text, 'hi'); + }); + + test.concurrent('フォローしているユーザーの自身を visibleUserIds に指定した visibility: specified なノートが含まれる', async () => { + const [alice, bob] = await Promise.all([signup(), signup()]); + + await api('/following/create', { userId: bob.id }, alice); + await sleep(1000); + const bobNote = await post(bob, { text: 'hi', visibility: 'specified', visibleUserIds: [alice.id] }); + + await waitForPushToTl(); + + const res = await api('/notes/timeline', {}, alice); + + assert.strictEqual(res.body.some((note: any) => note.id === bobNote.id), true); + assert.strictEqual(res.body.find((note: any) => note.id === bobNote.id).text, 'hi'); + }); + + test.concurrent('フォローしていないユーザーの自身を visibleUserIds に指定した visibility: specified なノートが含まれない', async () => { + const [alice, bob] = await Promise.all([signup(), signup()]); + + const bobNote = await post(bob, { text: 'hi', visibility: 'specified', visibleUserIds: [alice.id] }); + + await waitForPushToTl(); + + const res = await api('/notes/timeline', {}, alice); + + assert.strictEqual(res.body.some((note: any) => note.id === bobNote.id), false); + }); + + test.concurrent('フォローしているユーザーの自身を visibleUserIds に指定していない visibility: specified なノートが含まれない', async () => { + const [alice, bob, carol] = await Promise.all([signup(), signup(), signup()]); + + await api('/following/create', { userId: bob.id }, alice); + await sleep(1000); + const bobNote = await post(bob, { text: 'hi', visibility: 'specified', visibleUserIds: [carol.id] }); + + await waitForPushToTl(); + + const res = await api('/notes/timeline', {}, alice); + + assert.strictEqual(res.body.some((note: any) => note.id === bobNote.id), false); + }); + + test.concurrent('フォローしていないユーザーからの visibility: specified なノートに返信したときの自身のノートが含まれる', async () => { + const [alice, bob] = await Promise.all([signup(), signup()]); + + const bobNote = await post(bob, { text: 'hi', visibility: 'specified', visibleUserIds: [alice.id] }); + const aliceNote = await post(alice, { text: 'ok', visibility: 'specified', visibleUserIds: [bob.id], replyId: bobNote.id }); + + await waitForPushToTl(); + + const res = await api('/notes/timeline', {}, alice); + + assert.strictEqual(res.body.some((note: any) => note.id === aliceNote.id), true); + assert.strictEqual(res.body.find((note: any) => note.id === aliceNote.id).text, 'ok'); + }); + + /* TODO + test.concurrent('自身の visibility: specified なノートへのフォローしていないユーザーからの返信が含まれる', async () => { + const [alice, bob] = await Promise.all([signup(), signup()]); + + const aliceNote = await post(alice, { text: 'hi', visibility: 'specified', visibleUserIds: [bob.id] }); + const bobNote = await post(bob, { text: 'ok', visibility: 'specified', visibleUserIds: [alice.id], replyId: aliceNote.id }); + + await waitForPushToTl(); + + const res = await api('/notes/timeline', {}, alice); + + assert.strictEqual(res.body.some((note: any) => note.id === bobNote.id), true); + assert.strictEqual(res.body.find((note: any) => note.id === bobNote.id).text, 'ok'); + }); + */ + + // ↑の挙動が理想だけど実装が面倒かも + test.concurrent('自身の visibility: specified なノートへのフォローしていないユーザーからの返信が含まれない', async () => { + const [alice, bob] = await Promise.all([signup(), signup()]); + + const aliceNote = await post(alice, { text: 'hi', visibility: 'specified', visibleUserIds: [bob.id] }); + const bobNote = await post(bob, { text: 'ok', visibility: 'specified', visibleUserIds: [alice.id], replyId: aliceNote.id }); + + await waitForPushToTl(); + + const res = await api('/notes/timeline', {}, alice); + + assert.strictEqual(res.body.some((note: any) => note.id === bobNote.id), false); + }); }); describe('Local TL', () => { @@ -630,7 +731,6 @@ describe('Timelines', () => { assert.strictEqual(res.body.some((note: any) => note.id === bobNote.id), true); }); - /* 未実装 test.concurrent('リスインしているフォローしていないユーザーの visibility: followers なノートが含まれない', async () => { const [alice, bob] = await Promise.all([signup(), signup()]); @@ -645,23 +745,6 @@ describe('Timelines', () => { assert.strictEqual(res.body.some((note: any) => note.id === bobNote.id), false); }); - */ - - test.concurrent('リスインしているフォローしていないユーザーの visibility: followers なノートが含まれるが隠される', async () => { - const [alice, bob] = await Promise.all([signup(), signup()]); - - const list = await api('/users/lists/create', { name: 'list' }, alice).then(res => res.body); - await api('/users/lists/push', { listId: list.id, userId: bob.id }, alice); - await sleep(1000); - const bobNote = await post(bob, { text: 'hi', visibility: 'followers' }); - - await waitForPushToTl(); - - const res = await api('/notes/user-list-timeline', { listId: list.id }, alice); - - assert.strictEqual(res.body.some((note: any) => note.id === bobNote.id), true); - assert.strictEqual(res.body.find((note: any) => note.id === bobNote.id).text, null); - }); test.concurrent('リスインしているフォローしていないユーザーの他人への返信が含まれない', async () => { const [alice, bob, carol] = await Promise.all([signup(), signup(), signup()]); @@ -778,6 +861,38 @@ describe('Timelines', () => { assert.strictEqual(res.body.some((note: any) => note.id === bobNote1.id), false); assert.strictEqual(res.body.some((note: any) => note.id === bobNote2.id), true); }, 1000 * 10); + + test.concurrent('リスインしているユーザーの自身宛ての visibility: specified なノートが含まれる', async () => { + const [alice, bob] = await Promise.all([signup(), signup()]); + + const list = await api('/users/lists/create', { name: 'list' }, alice).then(res => res.body); + await api('/users/lists/push', { listId: list.id, userId: bob.id }, alice); + await sleep(1000); + const bobNote = await post(bob, { text: 'hi', visibility: 'specified', visibleUserIds: [alice.id] }); + + await waitForPushToTl(); + + const res = await api('/notes/user-list-timeline', { listId: list.id }, alice); + + assert.strictEqual(res.body.some((note: any) => note.id === bobNote.id), true); + assert.strictEqual(res.body.find((note: any) => note.id === bobNote.id).text, 'hi'); + }); + + test.concurrent('リスインしているユーザーの自身宛てではない visibility: specified なノートが含まれない', async () => { + const [alice, bob, carol] = await Promise.all([signup(), signup(), signup()]); + + const list = await api('/users/lists/create', { name: 'list' }, alice).then(res => res.body); + await api('/users/lists/push', { listId: list.id, userId: bob.id }, alice); + await api('/users/lists/push', { listId: list.id, userId: carol.id }, alice); + await sleep(1000); + const bobNote = await post(bob, { text: 'hi', visibility: 'specified', visibleUserIds: [carol.id] }); + + await waitForPushToTl(); + + const res = await api('/notes/user-list-timeline', { listId: list.id }, alice); + + assert.strictEqual(res.body.some((note: any) => note.id === bobNote.id), false); + }); }); describe('User TL', () => { @@ -951,6 +1066,30 @@ describe('Timelines', () => { assert.strictEqual(res.body.some((note: any) => note.id === bobNote2.id), true); assert.strictEqual(res.body.some((note: any) => note.id === bobNote3.id), true); }); + + test.concurrent('自身の visibility: specified なノートが含まれる', async () => { + const [alice] = await Promise.all([signup()]); + + const aliceNote = await post(alice, { text: 'hi', visibility: 'specified' }); + + await waitForPushToTl(); + + const res = await api('/users/notes', { userId: alice.id, withReplies: true }, alice); + + assert.strictEqual(res.body.some((note: any) => note.id === aliceNote.id), true); + }); + + test.concurrent('visibleUserIds に指定されてない visibility: specified なノートが含まれない', async () => { + const [alice, bob] = await Promise.all([signup(), signup()]); + + const bobNote = await post(bob, { text: 'hi', visibility: 'specified' }); + + await waitForPushToTl(); + + const res = await api('/users/notes', { userId: bob.id, withReplies: true }, alice); + + assert.strictEqual(res.body.some((note: any) => note.id === bobNote.id), false); + }); }); // TODO: リノートミュート済みユーザーのテスト diff --git a/packages/shared/.eslintrc.js b/packages/shared/.eslintrc.js index 1ecad7ab75..c578894f60 100644 --- a/packages/shared/.eslintrc.js +++ b/packages/shared/.eslintrc.js @@ -38,6 +38,9 @@ module.exports = { 'before': true, 'after': true, }], + 'brace-style': ['error', '1tbs', { + 'allowSingleLine': true, + }], 'padded-blocks': ['error', 'never'], /* TODO: path aliasを使わないとwarnする 'no-restricted-imports': ['warn', { -- cgit v1.2.3-freya From 0f367da84b65c4ca2b3bb5af152df261d6de1260 Mon Sep 17 00:00:00 2001 From: syuilo Date: Mon, 9 Oct 2023 16:47:46 +0900 Subject: fix(backend): TLを途中までしかページネーションできなくなることがある問題を修正 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fix #11404 --- CHANGELOG.md | 1 + packages/backend/src/core/AntennaService.ts | 3 +++ packages/backend/src/core/NoteCreateService.ts | 4 ++++ 3 files changed, 8 insertions(+) (limited to 'packages/backend/src/core/NoteCreateService.ts') diff --git a/CHANGELOG.md b/CHANGELOG.md index e655bef8b7..ba34554ba2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -52,6 +52,7 @@ - Fix: 連合なしアンケートに投票をするとUpdateがリモートに配信されてしまうのを修正 - Fix: nodeinfoにおいてCORS用のヘッダーが設定されていないのを修正 - Fix: 同じ種類のTLのストリーミングを複数接続できない問題を修正 +- Fix: アンテナTLを途中までしかページネーションできなくなることがある問題を修正 ## 2023.9.3 ### General diff --git a/packages/backend/src/core/AntennaService.ts b/packages/backend/src/core/AntennaService.ts index 95712b35b7..b64120772c 100644 --- a/packages/backend/src/core/AntennaService.ts +++ b/packages/backend/src/core/AntennaService.ts @@ -77,6 +77,9 @@ export class AntennaService implements OnApplicationShutdown { @bindThis public async addNoteToAntennas(note: MiNote, noteUser: { id: MiUser['id']; username: string; host: string | null; }): Promise { + // リモートから遅れて届いた(もしくは後から追加された)投稿日時が古い投稿が追加されるとページネーション時に問題を引き起こすため、3分以内に投稿されたもののみを追加する + if (Date.now() - note.createdAt.getTime() > 1000 * 60 * 3) return; + const antennas = await this.getAntennas(); const antennasWithMatchResult = await Promise.all(antennas.map(antenna => this.checkHitAntenna(antenna, note, noteUser).then(hit => [antenna, hit] as const))); const matchedAntennas = antennasWithMatchResult.filter(([, hit]) => hit).map(([antenna]) => antenna); diff --git a/packages/backend/src/core/NoteCreateService.ts b/packages/backend/src/core/NoteCreateService.ts index 277875a19f..65beb9f970 100644 --- a/packages/backend/src/core/NoteCreateService.ts +++ b/packages/backend/src/core/NoteCreateService.ts @@ -822,6 +822,10 @@ export class NoteCreateService implements OnApplicationShutdown { @bindThis private async pushToTl(note: MiNote, user: { id: MiUser['id']; host: MiUser['host']; }) { + // リモートから遅れて届いた(もしくは後から追加された)投稿日時が古い投稿が追加されるとページネーション時に問題を引き起こすため、3分以内に投稿されたもののみを追加する + // TODO: https://github.com/misskey-dev/misskey/issues/11404#issuecomment-1752480890 をやる + if (note.userHost != null && (Date.now() - note.createdAt.getTime()) > 1000 * 60 * 3) return; + const meta = await this.metaService.fetch(); const redisPipeline = this.redisForTimelines.pipeline(); -- cgit v1.2.3-freya From 0bb0c329087149355a9968e7c142b9ecdf1023ec Mon Sep 17 00:00:00 2001 From: syuilo Date: Mon, 9 Oct 2023 20:31:39 +0900 Subject: enhance(backend): RedisへのTLの構築をListで行うように MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit #11404 --- packages/backend/src/core/AntennaService.ts | 12 +-- packages/backend/src/core/CoreModule.ts | 6 ++ packages/backend/src/core/NoteCreateService.ts | 110 ++++----------------- packages/backend/src/core/RedisTimelineService.ts | 80 +++++++++++++++ packages/backend/src/core/RoleService.ts | 9 +- .../src/server/api/endpoints/antennas/notes.ts | 16 ++- .../src/server/api/endpoints/channels/timeline.ts | 107 ++++++++++---------- .../server/api/endpoints/notes/hybrid-timeline.ts | 28 ++---- .../server/api/endpoints/notes/local-timeline.ts | 21 ++-- .../src/server/api/endpoints/notes/timeline.ts | 19 ++-- .../api/endpoints/notes/user-list-timeline.ts | 19 ++-- .../src/server/api/endpoints/roles/notes.ts | 16 +-- .../src/server/api/endpoints/users/notes.ts | 43 +++----- 13 files changed, 231 insertions(+), 255 deletions(-) create mode 100644 packages/backend/src/core/RedisTimelineService.ts (limited to 'packages/backend/src/core/NoteCreateService.ts') diff --git a/packages/backend/src/core/AntennaService.ts b/packages/backend/src/core/AntennaService.ts index b64120772c..ca7624b1d4 100644 --- a/packages/backend/src/core/AntennaService.ts +++ b/packages/backend/src/core/AntennaService.ts @@ -16,6 +16,7 @@ import type { AntennasRepository, UserListMembershipsRepository } from '@/models import { UtilityService } from '@/core/UtilityService.js'; import { bindThis } from '@/decorators.js'; import type { GlobalEvents } from '@/core/GlobalEventService.js'; +import { RedisTimelineService } from '@/core/RedisTimelineService.js'; import type { OnApplicationShutdown } from '@nestjs/common'; @Injectable() @@ -38,6 +39,7 @@ export class AntennaService implements OnApplicationShutdown { private utilityService: UtilityService, private globalEventService: GlobalEventService, + private redisTimelineService: RedisTimelineService, ) { this.antennasFetched = false; this.antennas = []; @@ -77,9 +79,6 @@ export class AntennaService implements OnApplicationShutdown { @bindThis public async addNoteToAntennas(note: MiNote, noteUser: { id: MiUser['id']; username: string; host: string | null; }): Promise { - // リモートから遅れて届いた(もしくは後から追加された)投稿日時が古い投稿が追加されるとページネーション時に問題を引き起こすため、3分以内に投稿されたもののみを追加する - if (Date.now() - note.createdAt.getTime() > 1000 * 60 * 3) return; - const antennas = await this.getAntennas(); const antennasWithMatchResult = await Promise.all(antennas.map(antenna => this.checkHitAntenna(antenna, note, noteUser).then(hit => [antenna, hit] as const))); const matchedAntennas = antennasWithMatchResult.filter(([, hit]) => hit).map(([antenna]) => antenna); @@ -87,12 +86,7 @@ export class AntennaService implements OnApplicationShutdown { const redisPipeline = this.redisForTimelines.pipeline(); for (const antenna of matchedAntennas) { - redisPipeline.xadd( - `antennaTimeline:${antenna.id}`, - 'MAXLEN', '~', '200', - '*', - 'note', note.id); - + this.redisTimelineService.push(`antennaTimeline:${antenna.id}`, note.id, 200, redisPipeline); this.globalEventService.publishAntennaStream(antenna.id, 'note', note); } diff --git a/packages/backend/src/core/CoreModule.ts b/packages/backend/src/core/CoreModule.ts index 1984d9e6c2..0dc025d998 100644 --- a/packages/backend/src/core/CoreModule.ts +++ b/packages/backend/src/core/CoreModule.ts @@ -61,6 +61,7 @@ import { FileInfoService } from './FileInfoService.js'; import { SearchService } from './SearchService.js'; import { ClipService } from './ClipService.js'; import { FeaturedService } from './FeaturedService.js'; +import { RedisTimelineService } from './RedisTimelineService.js'; import { ChartLoggerService } from './chart/ChartLoggerService.js'; import FederationChart from './chart/charts/federation.js'; import NotesChart from './chart/charts/notes.js'; @@ -189,6 +190,7 @@ const $FileInfoService: Provider = { provide: 'FileInfoService', useExisting: Fi const $SearchService: Provider = { provide: 'SearchService', useExisting: SearchService }; const $ClipService: Provider = { provide: 'ClipService', useExisting: ClipService }; const $FeaturedService: Provider = { provide: 'FeaturedService', useExisting: FeaturedService }; +const $RedisTimelineService: Provider = { provide: 'RedisTimelineService', useExisting: RedisTimelineService }; const $ChartLoggerService: Provider = { provide: 'ChartLoggerService', useExisting: ChartLoggerService }; const $FederationChart: Provider = { provide: 'FederationChart', useExisting: FederationChart }; @@ -321,6 +323,7 @@ const $ApQuestionService: Provider = { provide: 'ApQuestionService', useExisting SearchService, ClipService, FeaturedService, + RedisTimelineService, ChartLoggerService, FederationChart, NotesChart, @@ -446,6 +449,7 @@ const $ApQuestionService: Provider = { provide: 'ApQuestionService', useExisting $SearchService, $ClipService, $FeaturedService, + $RedisTimelineService, $ChartLoggerService, $FederationChart, $NotesChart, @@ -572,6 +576,7 @@ const $ApQuestionService: Provider = { provide: 'ApQuestionService', useExisting SearchService, ClipService, FeaturedService, + RedisTimelineService, FederationChart, NotesChart, UsersChart, @@ -696,6 +701,7 @@ const $ApQuestionService: Provider = { provide: 'ApQuestionService', useExisting $SearchService, $ClipService, $FeaturedService, + $RedisTimelineService, $FederationChart, $NotesChart, $UsersChart, diff --git a/packages/backend/src/core/NoteCreateService.ts b/packages/backend/src/core/NoteCreateService.ts index 65beb9f970..2a73467122 100644 --- a/packages/backend/src/core/NoteCreateService.ts +++ b/packages/backend/src/core/NoteCreateService.ts @@ -54,6 +54,7 @@ import { RoleService } from '@/core/RoleService.js'; import { MetaService } from '@/core/MetaService.js'; import { SearchService } from '@/core/SearchService.js'; import { FeaturedService } from '@/core/FeaturedService.js'; +import { RedisTimelineService } from '@/core/RedisTimelineService.js'; type NotificationType = 'reply' | 'renote' | 'quote' | 'mention'; @@ -194,6 +195,7 @@ export class NoteCreateService implements OnApplicationShutdown { private idService: IdService, private globalEventService: GlobalEventService, private queueService: QueueService, + private redisTimelineService: RedisTimelineService, private noteReadService: NoteReadService, private notificationService: NotificationService, private relayService: RelayService, @@ -347,14 +349,6 @@ export class NoteCreateService implements OnApplicationShutdown { const note = await this.insertNote(user, data, tags, emojis, mentionedUsers); - if (data.channel) { - this.redisForTimelines.xadd( - `channelTimeline:${data.channel.id}`, - 'MAXLEN', '~', this.config.perChannelMaxNoteCacheCount.toString(), - '*', - 'note', note.id); - } - setImmediate('post created', { signal: this.#shutdownController.signal }).then( () => this.postNoteCreated(note, user, data, silent, tags!, mentionedUsers!), () => { /* aborted, ignore this */ }, @@ -822,20 +816,14 @@ export class NoteCreateService implements OnApplicationShutdown { @bindThis private async pushToTl(note: MiNote, user: { id: MiUser['id']; host: MiUser['host']; }) { - // リモートから遅れて届いた(もしくは後から追加された)投稿日時が古い投稿が追加されるとページネーション時に問題を引き起こすため、3分以内に投稿されたもののみを追加する - // TODO: https://github.com/misskey-dev/misskey/issues/11404#issuecomment-1752480890 をやる - if (note.userHost != null && (Date.now() - note.createdAt.getTime()) > 1000 * 60 * 3) return; - const meta = await this.metaService.fetch(); - const redisPipeline = this.redisForTimelines.pipeline(); + const r = this.redisForTimelines.pipeline(); if (note.channelId) { - redisPipeline.xadd( - `userTimelineWithChannel:${user.id}`, - 'MAXLEN', '~', note.userHost == null ? meta.perLocalUserUserTimelineCacheMax.toString() : meta.perRemoteUserUserTimelineCacheMax.toString(), - '*', - 'note', note.id); + this.redisTimelineService.push(`channelTimeline:${note.channelId}`, note.id, this.config.perChannelMaxNoteCacheCount, r); + + this.redisTimelineService.push(`userTimelineWithChannel:${user.id}`, note.id, note.userHost == null ? meta.perLocalUserUserTimelineCacheMax : meta.perRemoteUserUserTimelineCacheMax, r); const channelFollowings = await this.channelFollowingsRepository.find({ where: { @@ -845,18 +833,9 @@ export class NoteCreateService implements OnApplicationShutdown { }); for (const channelFollowing of channelFollowings) { - redisPipeline.xadd( - `homeTimeline:${channelFollowing.followerId}`, - 'MAXLEN', '~', meta.perUserHomeTimelineCacheMax.toString(), - '*', - 'note', note.id); - + this.redisTimelineService.push(`homeTimeline:${channelFollowing.followerId}`, note.id, meta.perUserHomeTimelineCacheMax, r); if (note.fileIds.length > 0) { - redisPipeline.xadd( - `homeTimelineWithFiles:${channelFollowing.followerId}`, - 'MAXLEN', '~', (meta.perUserHomeTimelineCacheMax / 2).toString(), - '*', - 'note', note.id); + this.redisTimelineService.push(`homeTimelineWithFiles:${channelFollowing.followerId}`, note.id, meta.perUserHomeTimelineCacheMax / 2, r); } } } else { @@ -894,18 +873,9 @@ export class NoteCreateService implements OnApplicationShutdown { if (!following.withReplies) continue; } - redisPipeline.xadd( - `homeTimeline:${following.followerId}`, - 'MAXLEN', '~', meta.perUserHomeTimelineCacheMax.toString(), - '*', - 'note', note.id); - + this.redisTimelineService.push(`homeTimeline:${following.followerId}`, note.id, meta.perUserHomeTimelineCacheMax, r); if (note.fileIds.length > 0) { - redisPipeline.xadd( - `homeTimelineWithFiles:${following.followerId}`, - 'MAXLEN', '~', (meta.perUserHomeTimelineCacheMax / 2).toString(), - '*', - 'note', note.id); + this.redisTimelineService.push(`homeTimelineWithFiles:${following.followerId}`, note.id, meta.perUserHomeTimelineCacheMax / 2, r); } } @@ -921,72 +891,32 @@ export class NoteCreateService implements OnApplicationShutdown { if (!userListMembership.withReplies) continue; } - redisPipeline.xadd( - `userListTimeline:${userListMembership.userListId}`, - 'MAXLEN', '~', meta.perUserListTimelineCacheMax.toString(), - '*', - 'note', note.id); - + this.redisTimelineService.push(`userListTimeline:${userListMembership.userListId}`, note.id, meta.perUserListTimelineCacheMax, r); if (note.fileIds.length > 0) { - redisPipeline.xadd( - `userListTimelineWithFiles:${userListMembership.userListId}`, - 'MAXLEN', '~', (meta.perUserListTimelineCacheMax / 2).toString(), - '*', - 'note', note.id); + this.redisTimelineService.push(`userListTimelineWithFiles:${userListMembership.userListId}`, note.id, meta.perUserListTimelineCacheMax / 2, r); } } if (note.visibility !== 'specified' || !note.visibleUserIds.some(v => v === user.id)) { // 自分自身のHTL - redisPipeline.xadd( - `homeTimeline:${user.id}`, - 'MAXLEN', '~', meta.perUserHomeTimelineCacheMax.toString(), - '*', - 'note', note.id); - + this.redisTimelineService.push(`homeTimeline:${user.id}`, note.id, meta.perUserHomeTimelineCacheMax, r); if (note.fileIds.length > 0) { - redisPipeline.xadd( - `homeTimelineWithFiles:${user.id}`, - 'MAXLEN', '~', (meta.perUserHomeTimelineCacheMax / 2).toString(), - '*', - 'note', note.id); + this.redisTimelineService.push(`homeTimelineWithFiles:${user.id}`, note.id, meta.perUserHomeTimelineCacheMax / 2, r); } } // 自分自身以外への返信 if (note.replyId && note.replyUserId !== note.userId) { - redisPipeline.xadd( - `userTimelineWithReplies:${user.id}`, - 'MAXLEN', '~', note.userHost == null ? meta.perLocalUserUserTimelineCacheMax.toString() : meta.perRemoteUserUserTimelineCacheMax.toString(), - '*', - 'note', note.id); + this.redisTimelineService.push(`userTimelineWithReplies:${user.id}`, note.id, note.userHost == null ? meta.perLocalUserUserTimelineCacheMax : meta.perRemoteUserUserTimelineCacheMax, r); } else { - redisPipeline.xadd( - `userTimeline:${user.id}`, - 'MAXLEN', '~', note.userHost == null ? meta.perLocalUserUserTimelineCacheMax.toString() : meta.perRemoteUserUserTimelineCacheMax.toString(), - '*', - 'note', note.id); - + this.redisTimelineService.push(`userTimeline:${user.id}`, note.id, note.userHost == null ? meta.perLocalUserUserTimelineCacheMax : meta.perRemoteUserUserTimelineCacheMax, r); if (note.fileIds.length > 0) { - redisPipeline.xadd( - `userTimelineWithFiles:${user.id}`, - 'MAXLEN', '~', note.userHost == null ? (meta.perLocalUserUserTimelineCacheMax / 2).toString() : (meta.perRemoteUserUserTimelineCacheMax / 2).toString(), - '*', - 'note', note.id); + this.redisTimelineService.push(`userTimelineWithFiles:${user.id}`, note.id, note.userHost == null ? meta.perLocalUserUserTimelineCacheMax / 2 : meta.perRemoteUserUserTimelineCacheMax / 2, r); } if (note.visibility === 'public' && note.userHost == null) { - redisPipeline.xadd( - 'localTimeline', - 'MAXLEN', '~', '1000', - '*', - 'note', note.id); - + this.redisTimelineService.push('localTimeline', note.id, 1000, r); if (note.fileIds.length > 0) { - redisPipeline.xadd( - 'localTimelineWithFiles', - 'MAXLEN', '~', '500', - '*', - 'note', note.id); + this.redisTimelineService.push('localTimelineWithFiles', note.id, 500, r); } } } @@ -998,7 +928,7 @@ export class NoteCreateService implements OnApplicationShutdown { } } - redisPipeline.exec(); + r.exec(); } @bindThis diff --git a/packages/backend/src/core/RedisTimelineService.ts b/packages/backend/src/core/RedisTimelineService.ts new file mode 100644 index 0000000000..f0ca2726d6 --- /dev/null +++ b/packages/backend/src/core/RedisTimelineService.ts @@ -0,0 +1,80 @@ +/* + * SPDX-FileCopyrightText: syuilo and other misskey contributors + * SPDX-License-Identifier: AGPL-3.0-only + */ + +import { Inject, Injectable } from '@nestjs/common'; +import * as Redis from 'ioredis'; +import { DI } from '@/di-symbols.js'; +import { bindThis } from '@/decorators.js'; +import { IdService } from '@/core/IdService.js'; + +@Injectable() +export class RedisTimelineService { + constructor( + @Inject(DI.redisForTimelines) + private redisForTimelines: Redis.Redis, + + private idService: IdService, + ) { + } + + @bindThis + public push(tl: string, id: string, maxlen: number, pipeline: Redis.ChainableCommander) { + // リモートから遅れて届いた(もしくは後から追加された)投稿日時が古い投稿が追加されるとページネーション時に問題を引き起こすため、 + // 3分以内に投稿されたものでない場合、Redisにある最古のIDより新しい場合のみ追加する + if (this.idService.parse(id).date.getTime() > Date.now() - 1000 * 60 * 3) { + pipeline.lpush('list:' + tl, id); + if (Math.random() < 0.1) { // 10%の確率でトリム + pipeline.ltrim('list:' + tl, 0, maxlen - 1); + } + } else { + // 末尾のIDを取得 + this.redisForTimelines.lindex('list:' + tl, -1).then(lastId => { + if (lastId == null || (this.idService.parse(id).date.getTime() > this.idService.parse(lastId).date.getTime())) { + this.redisForTimelines.lpush('list:' + tl, id); + } else { + Promise.resolve(); + } + }); + } + } + + @bindThis + public get(name: string, untilId?: string | null, sinceId?: string | null) { + if (untilId && sinceId) { + return this.redisForTimelines.lrange('list:' + name, 0, -1) + .then(ids => ids.filter(id => id > untilId && id < sinceId).sort((a, b) => a > b ? -1 : 1)); + } else if (untilId) { + return this.redisForTimelines.lrange('list:' + name, 0, -1) + .then(ids => ids.filter(id => id > untilId).sort((a, b) => a > b ? -1 : 1)); + } else if (sinceId) { + return this.redisForTimelines.lrange('list:' + name, 0, -1) + .then(ids => ids.filter(id => id < sinceId).sort((a, b) => a < b ? -1 : 1)); + } else { + return this.redisForTimelines.lrange('list:' + name, 0, -1) + .then(ids => ids.sort((a, b) => a > b ? -1 : 1)); + } + } + + @bindThis + public getMulti(name: string[], untilId?: string | null, sinceId?: string | null): Promise { + const pipeline = this.redisForTimelines.pipeline(); + for (const n of name) { + pipeline.lrange('list:' + n, 0, -1); + } + return pipeline.exec().then(res => { + if (res == null) return []; + const tls = res.map(r => r[1] as string[]); + return tls.map(ids => + (untilId && sinceId) + ? ids.filter(id => id > untilId && id < sinceId).sort((a, b) => a > b ? -1 : 1) + : untilId + ? ids.filter(id => id > untilId).sort((a, b) => a > b ? -1 : 1) + : sinceId + ? ids.filter(id => id < sinceId).sort((a, b) => a < b ? -1 : 1) + : ids.sort((a, b) => a > b ? -1 : 1), + ); + }); + } +} diff --git a/packages/backend/src/core/RoleService.ts b/packages/backend/src/core/RoleService.ts index f2bd9de5ee..2c3547e4ac 100644 --- a/packages/backend/src/core/RoleService.ts +++ b/packages/backend/src/core/RoleService.ts @@ -20,6 +20,7 @@ import { IdService } from '@/core/IdService.js'; import { GlobalEventService } from '@/core/GlobalEventService.js'; import { ModerationLogService } from '@/core/ModerationLogService.js'; import type { Packed } from '@/misc/json-schema.js'; +import { RedisTimelineService } from '@/core/RedisTimelineService.js'; import type { OnApplicationShutdown } from '@nestjs/common'; export type RolePolicies = { @@ -102,6 +103,7 @@ export class RoleService implements OnApplicationShutdown { private globalEventService: GlobalEventService, private idService: IdService, private moderationLogService: ModerationLogService, + private redisTimelineService: RedisTimelineService, ) { //this.onMessage = this.onMessage.bind(this); @@ -472,12 +474,7 @@ export class RoleService implements OnApplicationShutdown { const redisPipeline = this.redisClient.pipeline(); for (const role of roles) { - redisPipeline.xadd( - `roleTimeline:${role.id}`, - 'MAXLEN', '~', '1000', - '*', - 'note', note.id); - + this.redisTimelineService.push(`roleTimeline:${role.id}`, note.id, 1000, redisPipeline); this.globalEventService.publishRoleTimelineStream(role.id, 'note', note); } diff --git a/packages/backend/src/server/api/endpoints/antennas/notes.ts b/packages/backend/src/server/api/endpoints/antennas/notes.ts index b563f704a8..7d6841563a 100644 --- a/packages/backend/src/server/api/endpoints/antennas/notes.ts +++ b/packages/backend/src/server/api/endpoints/antennas/notes.ts @@ -12,6 +12,7 @@ import { NoteReadService } from '@/core/NoteReadService.js'; import { DI } from '@/di-symbols.js'; import { NoteEntityService } from '@/core/entities/NoteEntityService.js'; import { IdService } from '@/core/IdService.js'; +import { RedisTimelineService } from '@/core/RedisTimelineService.js'; import { ApiError } from '../../error.js'; export const meta = { @@ -69,8 +70,12 @@ export default class extends Endpoint { // eslint- private noteEntityService: NoteEntityService, private queryService: QueryService, private noteReadService: NoteReadService, + private redisTimelineService: RedisTimelineService, ) { super(meta, paramDef, async (ps, me) => { + const untilId = ps.untilId ?? ps.untilDate ? this.idService.genId(new Date(ps.untilDate!)) : null; + const sinceId = ps.sinceId ?? ps.sinceDate ? this.idService.genId(new Date(ps.sinceDate!)) : null; + const antenna = await this.antennasRepository.findOneBy({ id: ps.antennaId, userId: me.id, @@ -85,15 +90,8 @@ export default class extends Endpoint { // eslint- lastUsedAt: new Date(), }); - const limit = ps.limit + (ps.untilId ? 1 : 0) + (ps.sinceId ? 1 : 0); // untilIdに指定したものも含まれるため+1 - - const noteIds = await this.redisForTimelines.xrevrange( - `antennaTimeline:${antenna.id}`, - ps.untilId ? this.idService.parse(ps.untilId).date.getTime() : ps.untilDate ?? '+', - ps.sinceId ? this.idService.parse(ps.sinceId).date.getTime() : ps.sinceDate ?? '-', - 'COUNT', limit, - ).then(res => res.map(x => x[1][1]).filter(x => x !== ps.untilId && x !== ps.sinceId)); - + let noteIds = await this.redisTimelineService.get(`antennaTimeline:${antenna.id}`, untilId, sinceId); + noteIds = noteIds.slice(0, ps.limit); if (noteIds.length === 0) { return []; } diff --git a/packages/backend/src/server/api/endpoints/channels/timeline.ts b/packages/backend/src/server/api/endpoints/channels/timeline.ts index f0b14d4fd8..e063e0e2fc 100644 --- a/packages/backend/src/server/api/endpoints/channels/timeline.ts +++ b/packages/backend/src/server/api/endpoints/channels/timeline.ts @@ -12,6 +12,9 @@ import { NoteEntityService } from '@/core/entities/NoteEntityService.js'; import ActiveUsersChart from '@/core/chart/charts/active-users.js'; import { DI } from '@/di-symbols.js'; import { IdService } from '@/core/IdService.js'; +import { RedisTimelineService } from '@/core/RedisTimelineService.js'; +import { isUserRelated } from '@/misc/is-user-related.js'; +import { CacheService } from '@/core/CacheService.js'; import { ApiError } from '../../error.js'; export const meta = { @@ -66,9 +69,15 @@ export default class extends Endpoint { // eslint- private idService: IdService, private noteEntityService: NoteEntityService, private queryService: QueryService, + private redisTimelineService: RedisTimelineService, + private cacheService: CacheService, private activeUsersChart: ActiveUsersChart, ) { super(meta, paramDef, async (ps, me) => { + const untilId = ps.untilId ?? ps.untilDate ? this.idService.genId(new Date(ps.untilDate!)) : null; + const sinceId = ps.sinceId ?? ps.sinceDate ? this.idService.genId(new Date(ps.sinceDate!)) : null; + const isRangeSpecified = untilId != null && sinceId != null; + const channel = await this.channelsRepository.findOneBy({ id: ps.channelId, }); @@ -77,68 +86,66 @@ export default class extends Endpoint { // eslint- throw new ApiError(meta.errors.noSuchChannel); } - let timeline: MiNote[] = []; + if (me) this.activeUsersChart.read(me); - const limit = ps.limit + (ps.untilId ? 1 : 0) + (ps.sinceId ? 1 : 0); // untilIdに指定したものも含まれるため+1 - let noteIdsRes: [string, string[]][] = []; + if (isRangeSpecified || sinceId == null) { + const [ + userIdsWhoMeMuting, + ] = me ? await Promise.all([ + this.cacheService.userMutingsCache.fetch(me.id), + ]) : [new Set()]; - if (!ps.sinceId && !ps.sinceDate) { - noteIdsRes = await this.redisForTimelines.xrevrange( - `channelTimeline:${channel.id}`, - ps.untilId ? this.idService.parse(ps.untilId).date.getTime() : ps.untilDate ?? '+', - ps.sinceId ? this.idService.parse(ps.sinceId).date.getTime() : ps.sinceDate ?? '-', - 'COUNT', limit); - } + let noteIds = await this.redisTimelineService.get(`channelTimeline:${channel.id}`, untilId, sinceId); + noteIds = noteIds.slice(0, ps.limit); - // redis から取得していないとき・取得数が足りないとき - if (noteIdsRes.length < limit) { - //#region Construct query - const query = this.queryService.makePaginationQuery(this.notesRepository.createQueryBuilder('note'), ps.sinceId, ps.untilId, ps.sinceDate, ps.untilDate) - .andWhere('note.channelId = :channelId', { channelId: channel.id }) - .innerJoinAndSelect('note.user', 'user') - .leftJoinAndSelect('note.reply', 'reply') - .leftJoinAndSelect('note.renote', 'renote') - .leftJoinAndSelect('reply.user', 'replyUser') - .leftJoinAndSelect('renote.user', 'renoteUser') - .leftJoinAndSelect('note.channel', 'channel'); - - if (me) { - this.queryService.generateMutedUserQuery(query, me); - this.queryService.generateBlockedUserQuery(query, me); - } - //#endregion + if (noteIds.length > 0) { + const query = this.notesRepository.createQueryBuilder('note') + .where('note.id IN (:...noteIds)', { noteIds: noteIds }) + .innerJoinAndSelect('note.user', 'user') + .leftJoinAndSelect('note.reply', 'reply') + .leftJoinAndSelect('note.renote', 'renote') + .leftJoinAndSelect('reply.user', 'replyUser') + .leftJoinAndSelect('renote.user', 'renoteUser') + .leftJoinAndSelect('note.channel', 'channel'); - timeline = await query.limit(ps.limit).getMany(); - } else { - const noteIds = noteIdsRes.map(x => x[1][1]).filter(x => x !== ps.untilId && x !== ps.sinceId); + let timeline = await query.getMany(); - if (noteIds.length === 0) { - return []; - } + timeline = timeline.filter(note => { + if (me && isUserRelated(note, userIdsWhoMeMuting, true)) return false; - //#region Construct query - const query = this.notesRepository.createQueryBuilder('note') - .where('note.id IN (:...noteIds)', { noteIds: noteIds }) - .innerJoinAndSelect('note.user', 'user') - .leftJoinAndSelect('note.reply', 'reply') - .leftJoinAndSelect('note.renote', 'renote') - .leftJoinAndSelect('reply.user', 'replyUser') - .leftJoinAndSelect('renote.user', 'renoteUser') - .leftJoinAndSelect('note.channel', 'channel'); - - if (me) { - this.queryService.generateMutedUserQuery(query, me); - this.queryService.generateBlockedUserQuery(query, me); + return true; + }); + + // TODO: フィルタで件数が減った場合の埋め合わせ処理 + + timeline.sort((a, b) => a.id > b.id ? -1 : 1); + + if (timeline.length > 0) { + return await this.noteEntityService.packMany(timeline, me); + } } - //#endregion + } - timeline = await query.getMany(); - timeline.sort((a, b) => a.id > b.id ? -1 : 1); + //#region fallback to database + const query = this.queryService.makePaginationQuery(this.notesRepository.createQueryBuilder('note'), ps.sinceId, ps.untilId, ps.sinceDate, ps.untilDate) + .andWhere('note.channelId = :channelId', { channelId: channel.id }) + .innerJoinAndSelect('note.user', 'user') + .leftJoinAndSelect('note.reply', 'reply') + .leftJoinAndSelect('note.renote', 'renote') + .leftJoinAndSelect('reply.user', 'replyUser') + .leftJoinAndSelect('renote.user', 'renoteUser') + .leftJoinAndSelect('note.channel', 'channel'); + + if (me) { + this.queryService.generateMutedUserQuery(query, me); + this.queryService.generateBlockedUserQuery(query, me); } + //#endregion - if (me) this.activeUsersChart.read(me); + const timeline = await query.limit(ps.limit).getMany(); return await this.noteEntityService.packMany(timeline, me); + //#endregion }); } } diff --git a/packages/backend/src/server/api/endpoints/notes/hybrid-timeline.ts b/packages/backend/src/server/api/endpoints/notes/hybrid-timeline.ts index d5b706e543..a05269794c 100644 --- a/packages/backend/src/server/api/endpoints/notes/hybrid-timeline.ts +++ b/packages/backend/src/server/api/endpoints/notes/hybrid-timeline.ts @@ -15,6 +15,7 @@ import { RoleService } from '@/core/RoleService.js'; import { IdService } from '@/core/IdService.js'; import { isUserRelated } from '@/misc/is-user-related.js'; import { CacheService } from '@/core/CacheService.js'; +import { RedisTimelineService } from '@/core/RedisTimelineService.js'; import { ApiError } from '../../error.js'; export const meta = { @@ -72,8 +73,12 @@ export default class extends Endpoint { // eslint- private activeUsersChart: ActiveUsersChart, private idService: IdService, private cacheService: CacheService, + private redisTimelineService: RedisTimelineService, ) { super(meta, paramDef, async (ps, me) => { + const untilId = ps.untilId ?? ps.untilDate ? this.idService.genId(new Date(ps.untilDate!)) : null; + const sinceId = ps.sinceId ?? ps.sinceDate ? this.idService.genId(new Date(ps.sinceDate!)) : null; + const policies = await this.roleService.getUserPolicies(me.id); if (!policies.ltlAvailable) { throw new ApiError(meta.errors.stlDisabled); @@ -89,27 +94,10 @@ export default class extends Endpoint { // eslint- this.cacheService.userBlockedCache.fetch(me.id), ]); - let timeline: MiNote[] = []; - - const limit = ps.limit + (ps.untilId ? 1 : 0) + (ps.sinceId ? 1 : 0); // untilIdに指定したものも含まれるため+1 - - const redisPipeline = this.redisForTimelines.pipeline(); - redisPipeline.xrevrange( + const [htlNoteIds, ltlNoteIds] = await this.redisTimelineService.getMulti([ ps.withFiles ? `homeTimelineWithFiles:${me.id}` : `homeTimeline:${me.id}`, - ps.untilId ? this.idService.parse(ps.untilId).date.getTime() : ps.untilDate ?? '+', - ps.sinceId ? this.idService.parse(ps.sinceId).date.getTime() : ps.sinceDate ?? '-', - 'COUNT', limit, - ); - redisPipeline.xrevrange( ps.withFiles ? 'localTimelineWithFiles' : 'localTimeline', - ps.untilId ? this.idService.parse(ps.untilId).date.getTime() : ps.untilDate ?? '+', - ps.sinceId ? this.idService.parse(ps.sinceId).date.getTime() : ps.sinceDate ?? '-', - 'COUNT', limit, - ); - const [htlNoteIds, ltlNoteIds] = await redisPipeline.exec().then(res => res ? [ - (res[0][1] as string[][]).map(x => x[1][1]).filter(x => x !== ps.untilId && x !== ps.sinceId), - (res[1][1] as string[][]).map(x => x[1][1]).filter(x => x !== ps.untilId && x !== ps.sinceId), - ] : []); + ], untilId, sinceId); let noteIds = Array.from(new Set([...htlNoteIds, ...ltlNoteIds])); noteIds.sort((a, b) => a > b ? -1 : 1); @@ -128,7 +116,7 @@ export default class extends Endpoint { // eslint- .leftJoinAndSelect('renote.user', 'renoteUser') .leftJoinAndSelect('note.channel', 'channel'); - timeline = await query.getMany(); + let timeline = await query.getMany(); timeline = timeline.filter(note => { if (note.userId === me.id) { diff --git a/packages/backend/src/server/api/endpoints/notes/local-timeline.ts b/packages/backend/src/server/api/endpoints/notes/local-timeline.ts index b9305e8e26..c8ea003ba6 100644 --- a/packages/backend/src/server/api/endpoints/notes/local-timeline.ts +++ b/packages/backend/src/server/api/endpoints/notes/local-timeline.ts @@ -15,6 +15,7 @@ import { RoleService } from '@/core/RoleService.js'; import { IdService } from '@/core/IdService.js'; import { CacheService } from '@/core/CacheService.js'; import { isUserRelated } from '@/misc/is-user-related.js'; +import { RedisTimelineService } from '@/core/RedisTimelineService.js'; import { ApiError } from '../../error.js'; export const meta = { @@ -68,8 +69,12 @@ export default class extends Endpoint { // eslint- private activeUsersChart: ActiveUsersChart, private idService: IdService, private cacheService: CacheService, + private redisTimelineService: RedisTimelineService, ) { super(meta, paramDef, async (ps, me) => { + const untilId = ps.untilId ?? ps.untilDate ? this.idService.genId(new Date(ps.untilDate!)) : null; + const sinceId = ps.sinceId ?? ps.sinceDate ? this.idService.genId(new Date(ps.sinceDate!)) : null; + const policies = await this.roleService.getUserPolicies(me ? me.id : null); if (!policies.ltlAvailable) { throw new ApiError(meta.errors.ltlDisabled); @@ -85,16 +90,8 @@ export default class extends Endpoint { // eslint- this.cacheService.userBlockedCache.fetch(me.id), ]) : [new Set(), new Set(), new Set()]; - let timeline: MiNote[] = []; - - const limit = ps.limit + (ps.untilId ? 1 : 0) + (ps.sinceId ? 1 : 0); // untilIdに指定したものも含まれるため+1 - - const noteIds = await this.redisForTimelines.xrevrange( - ps.withFiles ? 'localTimelineWithFiles' : 'localTimeline', - ps.untilId ? this.idService.parse(ps.untilId).date.getTime() : ps.untilDate ?? '+', - ps.sinceId ? this.idService.parse(ps.sinceId).date.getTime() : ps.sinceDate ?? '-', - 'COUNT', limit, - ).then(res => res.map(x => x[1][1]).filter(x => x !== ps.untilId && x !== ps.sinceId)); + let noteIds = await this.redisTimelineService.get(ps.withFiles ? 'localTimelineWithFiles' : 'localTimeline', untilId, sinceId); + noteIds = noteIds.slice(0, ps.limit); if (noteIds.length === 0) { return []; @@ -109,7 +106,7 @@ export default class extends Endpoint { // eslint- .leftJoinAndSelect('renote.user', 'renoteUser') .leftJoinAndSelect('note.channel', 'channel'); - timeline = await query.getMany(); + let timeline = await query.getMany(); timeline = timeline.filter(note => { if (me && (note.userId === me.id)) { @@ -127,6 +124,8 @@ export default class extends Endpoint { // eslint- return true; }); + // TODO: フィルタした結果件数が足りなかった場合の対応 + timeline.sort((a, b) => a.id > b.id ? -1 : 1); process.nextTick(() => { diff --git a/packages/backend/src/server/api/endpoints/notes/timeline.ts b/packages/backend/src/server/api/endpoints/notes/timeline.ts index cd2ec8fe29..cd45135f99 100644 --- a/packages/backend/src/server/api/endpoints/notes/timeline.ts +++ b/packages/backend/src/server/api/endpoints/notes/timeline.ts @@ -15,6 +15,7 @@ import { DI } from '@/di-symbols.js'; import { IdService } from '@/core/IdService.js'; import { CacheService } from '@/core/CacheService.js'; import { isUserRelated } from '@/misc/is-user-related.js'; +import { RedisTimelineService } from '@/core/RedisTimelineService.js'; export const meta = { tags: ['notes'], @@ -62,8 +63,12 @@ export default class extends Endpoint { // eslint- private activeUsersChart: ActiveUsersChart, private idService: IdService, private cacheService: CacheService, + private redisTimelineService: RedisTimelineService, ) { super(meta, paramDef, async (ps, me) => { + const untilId = ps.untilId ?? ps.untilDate ? this.idService.genId(new Date(ps.untilDate!)) : null; + const sinceId = ps.sinceId ?? ps.sinceDate ? this.idService.genId(new Date(ps.sinceDate!)) : null; + const [ followings, userIdsWhoMeMuting, @@ -76,16 +81,8 @@ export default class extends Endpoint { // eslint- this.cacheService.userBlockedCache.fetch(me.id), ]); - let timeline: MiNote[] = []; - - const limit = ps.limit + (ps.untilId ? 1 : 0) + (ps.sinceId ? 1 : 0); // untilIdに指定したものも含まれるため+1 - - const noteIds = await this.redisForTimelines.xrevrange( - ps.withFiles ? `homeTimelineWithFiles:${me.id}` : `homeTimeline:${me.id}`, - ps.untilId ? this.idService.parse(ps.untilId).date.getTime() : ps.untilDate ?? '+', - ps.sinceId ? this.idService.parse(ps.sinceId).date.getTime() : ps.sinceDate ?? '-', - 'COUNT', limit, - ).then(res => res.map(x => x[1][1]).filter(x => x !== ps.untilId && x !== ps.sinceId)); + let noteIds = await this.redisTimelineService.get(ps.withFiles ? `homeTimelineWithFiles:${me.id}` : `homeTimeline:${me.id}`, untilId, sinceId); + noteIds = noteIds.slice(0, ps.limit); if (noteIds.length === 0) { return []; @@ -100,7 +97,7 @@ export default class extends Endpoint { // eslint- .leftJoinAndSelect('renote.user', 'renoteUser') .leftJoinAndSelect('note.channel', 'channel'); - timeline = await query.getMany(); + let timeline = await query.getMany(); timeline = timeline.filter(note => { if (note.userId === me.id) { diff --git a/packages/backend/src/server/api/endpoints/notes/user-list-timeline.ts b/packages/backend/src/server/api/endpoints/notes/user-list-timeline.ts index 2eec6297c0..96c8ff5b1b 100644 --- a/packages/backend/src/server/api/endpoints/notes/user-list-timeline.ts +++ b/packages/backend/src/server/api/endpoints/notes/user-list-timeline.ts @@ -15,6 +15,7 @@ import { DI } from '@/di-symbols.js'; import { CacheService } from '@/core/CacheService.js'; import { IdService } from '@/core/IdService.js'; import { isUserRelated } from '@/misc/is-user-related.js'; +import { RedisTimelineService } from '@/core/RedisTimelineService.js'; import { ApiError } from '../../error.js'; export const meta = { @@ -79,8 +80,12 @@ export default class extends Endpoint { // eslint- private activeUsersChart: ActiveUsersChart, private cacheService: CacheService, private idService: IdService, + private redisTimelineService: RedisTimelineService, ) { super(meta, paramDef, async (ps, me) => { + const untilId = ps.untilId ?? ps.untilDate ? this.idService.genId(new Date(ps.untilDate!)) : null; + const sinceId = ps.sinceId ?? ps.sinceDate ? this.idService.genId(new Date(ps.sinceDate!)) : null; + const list = await this.userListsRepository.findOneBy({ id: ps.listId, userId: me.id, @@ -100,16 +105,8 @@ export default class extends Endpoint { // eslint- this.cacheService.userBlockedCache.fetch(me.id), ]); - let timeline: MiNote[] = []; - - const limit = ps.limit + (ps.untilId ? 1 : 0) + (ps.sinceId ? 1 : 0); // untilIdに指定したものも含まれるため+1 - - const noteIds = await this.redisForTimelines.xrevrange( - ps.withFiles ? `userListTimelineWithFiles:${list.id}` : `userListTimeline:${list.id}`, - ps.untilId ? this.idService.parse(ps.untilId).date.getTime() : ps.untilDate ?? '+', - ps.sinceId ? this.idService.parse(ps.sinceId).date.getTime() : ps.sinceDate ?? '-', - 'COUNT', limit, - ).then(res => res.map(x => x[1][1]).filter(x => x !== ps.untilId && x !== ps.sinceId)); + let noteIds = await this.redisTimelineService.get(ps.withFiles ? `userListTimelineWithFiles:${list.id}` : `userListTimeline:${list.id}`, untilId, sinceId); + noteIds = noteIds.slice(0, ps.limit); if (noteIds.length === 0) { return []; @@ -124,7 +121,7 @@ export default class extends Endpoint { // eslint- .leftJoinAndSelect('renote.user', 'renoteUser') .leftJoinAndSelect('note.channel', 'channel'); - timeline = await query.getMany(); + let timeline = await query.getMany(); timeline = timeline.filter(note => { if (note.userId === me.id) { diff --git a/packages/backend/src/server/api/endpoints/roles/notes.ts b/packages/backend/src/server/api/endpoints/roles/notes.ts index 3f2b31c02f..366c3a7cc1 100644 --- a/packages/backend/src/server/api/endpoints/roles/notes.ts +++ b/packages/backend/src/server/api/endpoints/roles/notes.ts @@ -11,6 +11,7 @@ import { QueryService } from '@/core/QueryService.js'; import { DI } from '@/di-symbols.js'; import { NoteEntityService } from '@/core/entities/NoteEntityService.js'; import { IdService } from '@/core/IdService.js'; +import { RedisTimelineService } from '@/core/RedisTimelineService.js'; import { ApiError } from '../../error.js'; export const meta = { @@ -65,8 +66,12 @@ export default class extends Endpoint { // eslint- private idService: IdService, private noteEntityService: NoteEntityService, private queryService: QueryService, + private redisTimelineService: RedisTimelineService, ) { super(meta, paramDef, async (ps, me) => { + const untilId = ps.untilId ?? ps.untilDate ? this.idService.genId(new Date(ps.untilDate!)) : null; + const sinceId = ps.sinceId ?? ps.sinceDate ? this.idService.genId(new Date(ps.sinceDate!)) : null; + const role = await this.rolesRepository.findOneBy({ id: ps.roleId, isPublic: true, @@ -78,14 +83,9 @@ export default class extends Endpoint { // eslint- if (!role.isExplorable) { return []; } - const limit = ps.limit + (ps.untilId ? 1 : 0) + (ps.sinceId ? 1 : 0); // untilIdに指定したものも含まれるため+1 - - const noteIds = await this.redisForTimelines.xrevrange( - `roleTimeline:${role.id}`, - ps.untilId ? this.idService.parse(ps.untilId).date.getTime() : ps.untilDate ?? '+', - ps.sinceId ? this.idService.parse(ps.sinceId).date.getTime() : ps.sinceDate ?? '-', - 'COUNT', limit, - ).then(res => res.map(x => x[1][1]).filter(x => x !== ps.untilId && x !== ps.sinceId)); + + let noteIds = await this.redisTimelineService.get(`roleTimeline:${role.id}`, untilId, sinceId); + noteIds = noteIds.slice(0, ps.limit); if (noteIds.length === 0) { return []; diff --git a/packages/backend/src/server/api/endpoints/users/notes.ts b/packages/backend/src/server/api/endpoints/users/notes.ts index 4a3d2e5be4..30cd5b791f 100644 --- a/packages/backend/src/server/api/endpoints/users/notes.ts +++ b/packages/backend/src/server/api/endpoints/users/notes.ts @@ -14,6 +14,7 @@ import { CacheService } from '@/core/CacheService.js'; import { IdService } from '@/core/IdService.js'; import { isUserRelated } from '@/misc/is-user-related.js'; import { QueryService } from '@/core/QueryService.js'; +import { RedisTimelineService } from '@/core/RedisTimelineService.js'; import { ApiError } from '../../error.js'; export const meta = { @@ -70,42 +71,24 @@ export default class extends Endpoint { // eslint- private queryService: QueryService, private cacheService: CacheService, private idService: IdService, + private redisTimelineService: RedisTimelineService, ) { super(meta, paramDef, async (ps, me) => { - const isRangeSpecified = (ps.sinceId != null || ps.sinceDate != null) && (ps.untilId != null || ps.untilDate != null); + const untilId = ps.untilId ?? ps.untilDate ? this.idService.genId(new Date(ps.untilDate!)) : null; + const sinceId = ps.sinceId ?? ps.sinceDate ? this.idService.genId(new Date(ps.sinceDate!)) : null; + const isRangeSpecified = untilId != null && sinceId != null; - if (isRangeSpecified || !(ps.sinceId != null || ps.sinceDate != null)) { + if (isRangeSpecified || sinceId == null) { const [ userIdsWhoMeMuting, ] = me ? await Promise.all([ this.cacheService.userMutingsCache.fetch(me.id), ]) : [new Set()]; - const limit = ps.limit + (ps.untilId ? 1 : 0) + (ps.sinceId ? 1 : 0); // untilIdに指定したものも含まれるため+1 - const [noteIdsRes, repliesNoteIdsRes, channelNoteIdsRes] = await Promise.all([ - this.redisForTimelines.xrevrange( - ps.withFiles ? `userTimelineWithFiles:${ps.userId}` : `userTimeline:${ps.userId}`, - ps.untilId ? this.idService.parse(ps.untilId).date.getTime() : ps.untilDate ?? '+', - ps.sinceId ? this.idService.parse(ps.sinceId).date.getTime() : ps.sinceDate ?? '-', - 'COUNT', limit, - ).then(res => res.map(x => x[1][1]).filter(x => x !== ps.untilId && x !== ps.sinceId)), - ps.withReplies - ? this.redisForTimelines.xrevrange( - `userTimelineWithReplies:${ps.userId}`, - ps.untilId ? this.idService.parse(ps.untilId).date.getTime() : ps.untilDate ?? '+', - ps.sinceId ? this.idService.parse(ps.sinceId).date.getTime() : ps.sinceDate ?? '-', - 'COUNT', limit, - ).then(res => res.map(x => x[1][1]).filter(x => x !== ps.untilId && x !== ps.sinceId)) - : Promise.resolve([]), - ps.withChannelNotes - ? this.redisForTimelines.xrevrange( - `userTimelineWithChannel:${ps.userId}`, - ps.untilId ? this.idService.parse(ps.untilId).date.getTime() : ps.untilDate ?? '+', - ps.sinceId ? this.idService.parse(ps.sinceId).date.getTime() : ps.sinceDate ?? '-', - 'COUNT', limit, - ).then(res => res.map(x => x[1][1]).filter(x => x !== ps.untilId && x !== ps.sinceId)) - : Promise.resolve([]), + this.redisTimelineService.get(ps.withFiles ? `userTimelineWithFiles:${ps.userId}` : `userTimeline:${ps.userId}`, untilId, sinceId), + ps.withReplies ? this.redisTimelineService.get(`userTimelineWithReplies:${ps.userId}`, untilId, sinceId) : Promise.resolve([]), + ps.withChannelNotes ? this.redisTimelineService.get(`userTimelineWithChannel:${ps.userId}`, untilId, sinceId) : Promise.resolve([]), ]); let noteIds = Array.from(new Set([ @@ -145,6 +128,8 @@ export default class extends Endpoint { // eslint- return true; }); + // TODO: フィルタで件数が減った場合の埋め合わせ処理 + timeline.sort((a, b) => a.id > b.id ? -1 : 1); if (timeline.length > 0) { @@ -153,9 +138,7 @@ export default class extends Endpoint { // eslint- } } - // fallback to database - - //#region Construct query + //#region fallback to database const query = this.queryService.makePaginationQuery(this.notesRepository.createQueryBuilder('note'), ps.sinceId, ps.untilId, ps.sinceDate, ps.untilDate) .andWhere('note.userId = :userId', { userId: ps.userId }) .innerJoinAndSelect('note.user', 'user') @@ -188,11 +171,11 @@ export default class extends Endpoint { // eslint- qb.orWhere('0 < (SELECT COUNT(*) FROM poll WHERE poll."noteId" = note.id)'); })); } - //#endregion const timeline = await query.limit(ps.limit).getMany(); return await this.noteEntityService.packMany(timeline, me); + //#endregion }); } } -- cgit v1.2.3-freya