From 7a494b2aa7e0337a220b8988122839e9a7b75538 Mon Sep 17 00:00:00 2001 From: zyoshoka <107108195+zyoshoka@users.noreply.github.com> Date: Sun, 26 Nov 2023 10:02:22 +0900 Subject: fix(backend): rename FunoutTimelineService to FanoutTimelineService (#12453) --- .../backend/src/server/api/endpoints/notes/hybrid-timeline.ts | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) (limited to 'packages/backend/src/server/api/endpoints/notes/hybrid-timeline.ts') diff --git a/packages/backend/src/server/api/endpoints/notes/hybrid-timeline.ts b/packages/backend/src/server/api/endpoints/notes/hybrid-timeline.ts index 408c2fa371..372199844d 100644 --- a/packages/backend/src/server/api/endpoints/notes/hybrid-timeline.ts +++ b/packages/backend/src/server/api/endpoints/notes/hybrid-timeline.ts @@ -14,7 +14,7 @@ import { RoleService } from '@/core/RoleService.js'; import { IdService } from '@/core/IdService.js'; import { isUserRelated } from '@/misc/is-user-related.js'; import { CacheService } from '@/core/CacheService.js'; -import { FunoutTimelineService } from '@/core/FunoutTimelineService.js'; +import { FanoutTimelineService } from '@/core/FanoutTimelineService.js'; import { QueryService } from '@/core/QueryService.js'; import { UserFollowingService } from '@/core/UserFollowingService.js'; import { MetaService } from '@/core/MetaService.js'; @@ -77,7 +77,7 @@ export default class extends Endpoint { // eslint- private activeUsersChart: ActiveUsersChart, private idService: IdService, private cacheService: CacheService, - private funoutTimelineService: FunoutTimelineService, + private fanoutTimelineService: FanoutTimelineService, private queryService: QueryService, private userFollowingService: UserFollowingService, private metaService: MetaService, @@ -120,20 +120,20 @@ export default class extends Endpoint { // eslint- let shouldFallbackToDb = false; if (ps.withFiles) { - const [htlNoteIds, ltlNoteIds] = await this.funoutTimelineService.getMulti([ + const [htlNoteIds, ltlNoteIds] = await this.fanoutTimelineService.getMulti([ `homeTimelineWithFiles:${me.id}`, 'localTimelineWithFiles', ], untilId, sinceId); noteIds = Array.from(new Set([...htlNoteIds, ...ltlNoteIds])); } else if (ps.withReplies) { - const [htlNoteIds, ltlNoteIds, ltlReplyNoteIds] = await this.funoutTimelineService.getMulti([ + const [htlNoteIds, ltlNoteIds, ltlReplyNoteIds] = await this.fanoutTimelineService.getMulti([ `homeTimeline:${me.id}`, 'localTimeline', 'localTimelineWithReplies', ], untilId, sinceId); noteIds = Array.from(new Set([...htlNoteIds, ...ltlNoteIds, ...ltlReplyNoteIds])); } else { - const [htlNoteIds, ltlNoteIds] = await this.funoutTimelineService.getMulti([ + const [htlNoteIds, ltlNoteIds] = await this.fanoutTimelineService.getMulti([ `homeTimeline:${me.id}`, 'localTimeline', ], untilId, sinceId); -- cgit v1.2.3-freya From a631b976c99a4b3977079a4bafc8a7b7de0bf269 Mon Sep 17 00:00:00 2001 From: anatawa12 Date: Sat, 2 Dec 2023 18:25:07 +0900 Subject: Refine fanout timeline (#12507) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * chore(endpoints/hybrid-timeline): don't pack inside getFromDb * chore(endpoints/hybrid-timeline): Redisから取得する部分のうちSTLに依存しなそうなところを別のServiceに切り出し * chore(endpoints/local-timeline): FanoutTimelineEndpointServiceで再実装 * chore(endpoints/channels/timeline): FanoutTimelineEndpointServiceで再実装 * chore(endpoints/timeline): FanoutTimelineEndpointServiceで再実装 * chore(endpoints/user-list-timeline): FanoutTimelineEndpointServiceで再実装 * chore(endpoints/users/notes): FanoutTimelineEndpointServiceで再実装 * chore: add useDbFallback to FanoutTimelineEndpointService.timeline and always true for channel / user note list * style: fix lint error * chore: split logic to multiple functions * chore: implement redis fallback * chore: 成功率を上げる * fix: db fallback not working * feat: allowPartial * chore(frontend): set allowPartial * chore(backend): remove fallbackIfEmpty HTL will never be purged so it's no longer required * fix: missing allowPartial in channel timeline * fix: type of timelineConfig in hybrid-timeline --------- Co-authored-by: syuilo --- packages/backend/src/core/CoreModule.ts | 6 + .../src/core/FanoutTimelineEndpointService.ts | 123 +++++++++++ .../src/server/api/endpoints/channels/timeline.ts | 116 +++++------ .../server/api/endpoints/notes/hybrid-timeline.ts | 110 ++++------ .../server/api/endpoints/notes/local-timeline.ts | 103 ++++------ .../src/server/api/endpoints/notes/timeline.ts | 91 ++++----- .../api/endpoints/notes/user-list-timeline.ts | 78 +++---- .../src/server/api/endpoints/users/notes.ts | 226 +++++++++++---------- packages/frontend/src/components/MkPagination.vue | 1 + 9 files changed, 449 insertions(+), 405 deletions(-) create mode 100644 packages/backend/src/core/FanoutTimelineEndpointService.ts (limited to 'packages/backend/src/server/api/endpoints/notes/hybrid-timeline.ts') diff --git a/packages/backend/src/core/CoreModule.ts b/packages/backend/src/core/CoreModule.ts index bf6f0ef879..bc6d24b951 100644 --- a/packages/backend/src/core/CoreModule.ts +++ b/packages/backend/src/core/CoreModule.ts @@ -4,6 +4,7 @@ */ import { Module } from '@nestjs/common'; +import { FanoutTimelineEndpointService } from '@/core/FanoutTimelineEndpointService.js'; import { AccountMoveService } from './AccountMoveService.js'; import { AccountUpdateService } from './AccountUpdateService.js'; import { AiService } from './AiService.js'; @@ -195,6 +196,7 @@ const $SearchService: Provider = { provide: 'SearchService', useExisting: Search const $ClipService: Provider = { provide: 'ClipService', useExisting: ClipService }; const $FeaturedService: Provider = { provide: 'FeaturedService', useExisting: FeaturedService }; const $FanoutTimelineService: Provider = { provide: 'FanoutTimelineService', useExisting: FanoutTimelineService }; +const $FanoutTimelineEndpointService: Provider = { provide: 'FanoutTimelineEndpointService', useExisting: FanoutTimelineEndpointService }; const $ChannelFollowingService: Provider = { provide: 'ChannelFollowingService', useExisting: ChannelFollowingService }; const $RegistryApiService: Provider = { provide: 'RegistryApiService', useExisting: RegistryApiService }; @@ -331,6 +333,7 @@ const $ApQuestionService: Provider = { provide: 'ApQuestionService', useExisting ClipService, FeaturedService, FanoutTimelineService, + FanoutTimelineEndpointService, ChannelFollowingService, RegistryApiService, ChartLoggerService, @@ -460,6 +463,7 @@ const $ApQuestionService: Provider = { provide: 'ApQuestionService', useExisting $ClipService, $FeaturedService, $FanoutTimelineService, + $FanoutTimelineEndpointService, $ChannelFollowingService, $RegistryApiService, $ChartLoggerService, @@ -590,6 +594,7 @@ const $ApQuestionService: Provider = { provide: 'ApQuestionService', useExisting ClipService, FeaturedService, FanoutTimelineService, + FanoutTimelineEndpointService, ChannelFollowingService, RegistryApiService, FederationChart, @@ -718,6 +723,7 @@ const $ApQuestionService: Provider = { provide: 'ApQuestionService', useExisting $ClipService, $FeaturedService, $FanoutTimelineService, + $FanoutTimelineEndpointService, $ChannelFollowingService, $RegistryApiService, $FederationChart, diff --git a/packages/backend/src/core/FanoutTimelineEndpointService.ts b/packages/backend/src/core/FanoutTimelineEndpointService.ts new file mode 100644 index 0000000000..157fcbe877 --- /dev/null +++ b/packages/backend/src/core/FanoutTimelineEndpointService.ts @@ -0,0 +1,123 @@ +/* + * SPDX-FileCopyrightText: syuilo and other misskey contributors + * SPDX-License-Identifier: AGPL-3.0-only + */ + +import { Inject, Injectable } from '@nestjs/common'; +import { DI } from '@/di-symbols.js'; +import { bindThis } from '@/decorators.js'; +import type { MiUser } from '@/models/User.js'; +import type { MiNote } from '@/models/Note.js'; +import { Packed } from '@/misc/json-schema.js'; +import type { NotesRepository } from '@/models/_.js'; +import { NoteEntityService } from '@/core/entities/NoteEntityService.js'; +import { FanoutTimelineService } from '@/core/FanoutTimelineService.js'; + +@Injectable() +export class FanoutTimelineEndpointService { + constructor( + @Inject(DI.notesRepository) + private notesRepository: NotesRepository, + + private noteEntityService: NoteEntityService, + private fanoutTimelineService: FanoutTimelineService, + ) { + } + + @bindThis + async timeline(ps: { + untilId: string | null, + sinceId: string | null, + limit: number, + allowPartial: boolean, + me?: { id: MiUser['id'] } | undefined | null, + useDbFallback: boolean, + redisTimelines: string[], + noteFilter: (note: MiNote) => boolean, + dbFallback: (untilId: string | null, sinceId: string | null, limit: number) => Promise, + }): Promise[]> { + return await this.noteEntityService.packMany(await this.getMiNotes(ps), ps.me); + } + + @bindThis + private async getMiNotes(ps: { + untilId: string | null, + sinceId: string | null, + limit: number, + allowPartial: boolean, + me?: { id: MiUser['id'] } | undefined | null, + useDbFallback: boolean, + redisTimelines: string[], + noteFilter: (note: MiNote) => boolean, + dbFallback: (untilId: string | null, sinceId: string | null, limit: number) => Promise, + }): Promise { + let noteIds: string[]; + let shouldFallbackToDb = false; + + // 呼び出し元と以下の処理をシンプルにするためにdbFallbackを置き換える + if (!ps.useDbFallback) ps.dbFallback = () => Promise.resolve([]); + + const redisResult = await this.fanoutTimelineService.getMulti(ps.redisTimelines, ps.untilId, ps.sinceId); + + const redisResultIds = Array.from(new Set(redisResult.flat(1))); + + redisResultIds.sort((a, b) => a > b ? -1 : 1); + noteIds = redisResultIds.slice(0, ps.limit); + + shouldFallbackToDb = shouldFallbackToDb || (noteIds.length === 0); + + if (!shouldFallbackToDb) { + const redisTimeline: MiNote[] = []; + let readFromRedis = 0; + let lastSuccessfulRate = 1; // rateをキャッシュする？ + let trialCount = 1; + + while ((redisResultIds.length - readFromRedis) !== 0) { + const remainingToRead = ps.limit - redisTimeline.length; + + // DBからの取り直しを減らす初回と同じ割合以上で成功すると仮定するが、クエリの長さを考えて三倍まで + const countToGet = remainingToRead * Math.ceil(Math.min(1.1 / lastSuccessfulRate, 3)); + noteIds = redisResultIds.slice(readFromRedis, readFromRedis + countToGet); + + readFromRedis += noteIds.length; + + const gotFromDb = await this.getAndFilterFromDb(noteIds, ps.noteFilter); + redisTimeline.push(...gotFromDb); + lastSuccessfulRate = gotFromDb.length / noteIds.length; + + console.log(`fanoutTimelineTrial#${trialCount++}: req: ${ps.limit}, tried: ${noteIds.length}, got: ${gotFromDb.length}, rate: ${lastSuccessfulRate}, total: ${redisTimeline.length}, fromRedis: ${redisResultIds.length}`); + + if (ps.allowPartial ? redisTimeline.length !== 0 : redisTimeline.length >= ps.limit) { + // 十分Redisからとれた + return redisTimeline.slice(0, ps.limit); + } + } + + // まだ足りない分はDBにフォールバック + const remainingToRead = ps.limit - redisTimeline.length; + const gotFromDb = await ps.dbFallback(noteIds[noteIds.length - 1], ps.sinceId, remainingToRead); + redisTimeline.push(...gotFromDb); + console.log(`fanoutTimelineTrial#db: req: ${ps.limit}, tried: ${remainingToRead}, got: ${gotFromDb.length}, since: ${noteIds[noteIds.length - 1]}, until: ${ps.untilId}, total: ${redisTimeline.length}`); + return redisTimeline; + } + + return await ps.dbFallback(ps.untilId, ps.sinceId, ps.limit); + } + + private async getAndFilterFromDb(noteIds: string[], noteFilter: (note: MiNote) => boolean): Promise { + const query = this.notesRepository.createQueryBuilder('note') + .where('note.id IN (:...noteIds)', { noteIds: noteIds }) + .innerJoinAndSelect('note.user', 'user') + .leftJoinAndSelect('note.reply', 'reply') + .leftJoinAndSelect('note.renote', 'renote') + .leftJoinAndSelect('reply.user', 'replyUser') + .leftJoinAndSelect('renote.user', 'renoteUser') + .leftJoinAndSelect('note.channel', 'channel'); + + const notes = (await query.getMany()).filter(noteFilter); + + notes.sort((a, b) => a.id > b.id ? -1 : 1); + + return notes; + } +} diff --git a/packages/backend/src/server/api/endpoints/channels/timeline.ts b/packages/backend/src/server/api/endpoints/channels/timeline.ts index f9207199d6..9ef494d6d8 100644 --- a/packages/backend/src/server/api/endpoints/channels/timeline.ts +++ b/packages/backend/src/server/api/endpoints/channels/timeline.ts @@ -12,10 +12,11 @@ import { NoteEntityService } from '@/core/entities/NoteEntityService.js'; import ActiveUsersChart from '@/core/chart/charts/active-users.js'; import { DI } from '@/di-symbols.js'; import { IdService } from '@/core/IdService.js'; -import { FanoutTimelineService } from '@/core/FanoutTimelineService.js'; import { isUserRelated } from '@/misc/is-user-related.js'; import { CacheService } from '@/core/CacheService.js'; import { MetaService } from '@/core/MetaService.js'; +import { FanoutTimelineEndpointService } from '@/core/FanoutTimelineEndpointService.js'; +import { MiLocalUser } from '@/models/User.js'; import { ApiError } from '../../error.js'; export const meta = { @@ -51,6 +52,7 @@ export const paramDef = { untilId: { type: 'string', format: 'misskey:id' }, sinceDate: { type: 'integer' }, untilDate: { type: 'integer' }, + allowPartial: { type: 'boolean', default: false }, // true is recommended but for compatibility false by default }, required: ['channelId'], } as const; @@ -58,9 +60,6 @@ export const paramDef = { @Injectable() export default class extends Endpoint { // eslint-disable-line import/no-default-export constructor( - @Inject(DI.redisForTimelines) - private redisForTimelines: Redis.Redis, - @Inject(DI.notesRepository) private notesRepository: NotesRepository, @@ -70,7 +69,7 @@ export default class extends Endpoint { // eslint- private idService: IdService, private noteEntityService: NoteEntityService, private queryService: QueryService, - private fanoutTimelineService: FanoutTimelineService, + private fanoutTimelineEndpointService: FanoutTimelineEndpointService, private cacheService: CacheService, private activeUsersChart: ActiveUsersChart, private metaService: MetaService, @@ -78,7 +77,6 @@ export default class extends Endpoint { // eslint- super(meta, paramDef, async (ps, me) => { const untilId = ps.untilId ?? (ps.untilDate ? this.idService.gen(ps.untilDate!) : null); const sinceId = ps.sinceId ?? (ps.sinceDate ? this.idService.gen(ps.sinceDate!) : null); - const isRangeSpecified = untilId != null && sinceId != null; const serverSettings = await this.metaService.fetch(); @@ -92,64 +90,58 @@ export default class extends Endpoint { // eslint- if (me) this.activeUsersChart.read(me); - if (serverSettings.enableFanoutTimeline && (isRangeSpecified || sinceId == null)) { - const [ - userIdsWhoMeMuting, - ] = me ? await Promise.all([ - this.cacheService.userMutingsCache.fetch(me.id), - ]) : [new Set()]; - - let noteIds = await this.fanoutTimelineService.get(`channelTimeline:${channel.id}`, untilId, sinceId); - noteIds = noteIds.slice(0, ps.limit); - - if (noteIds.length > 0) { - const query = this.notesRepository.createQueryBuilder('note') - .where('note.id IN (:...noteIds)', { noteIds: noteIds }) - .innerJoinAndSelect('note.user', 'user') - .leftJoinAndSelect('note.reply', 'reply') - .leftJoinAndSelect('note.renote', 'renote') - .leftJoinAndSelect('reply.user', 'replyUser') - .leftJoinAndSelect('renote.user', 'renoteUser') - .leftJoinAndSelect('note.channel', 'channel'); - - let timeline = await query.getMany(); - - timeline = timeline.filter(note => { - if (me && isUserRelated(note, userIdsWhoMeMuting)) return false; - - return true; - }); - - // TODO: フィルタで件数が減った場合の埋め合わせ処理 - - timeline.sort((a, b) => a.id > b.id ? -1 : 1); - - if (timeline.length > 0) { - return await this.noteEntityService.packMany(timeline, me); - } - } - } - - //#region fallback to database - const query = this.queryService.makePaginationQuery(this.notesRepository.createQueryBuilder('note'), ps.sinceId, ps.untilId, ps.sinceDate, ps.untilDate) - .andWhere('note.channelId = :channelId', { channelId: channel.id }) - .innerJoinAndSelect('note.user', 'user') - .leftJoinAndSelect('note.reply', 'reply') - .leftJoinAndSelect('note.renote', 'renote') - .leftJoinAndSelect('reply.user', 'replyUser') - .leftJoinAndSelect('renote.user', 'renoteUser') - .leftJoinAndSelect('note.channel', 'channel'); - - if (me) { - this.queryService.generateMutedUserQuery(query, me); - this.queryService.generateBlockedUserQuery(query, me); + if (!serverSettings.enableFanoutTimeline) { + return await this.noteEntityService.packMany(await this.getFromDb({ untilId, sinceId, limit: ps.limit, channelId: channel.id }, me), me); } - //#endregion - - const timeline = await query.limit(ps.limit).getMany(); - return await this.noteEntityService.packMany(timeline, me); - //#endregion + const [ + userIdsWhoMeMuting, + ] = me ? await Promise.all([ + this.cacheService.userMutingsCache.fetch(me.id), + ]) : [new Set()]; + + return await this.fanoutTimelineEndpointService.timeline({ + untilId, + sinceId, + limit: ps.limit, + allowPartial: ps.allowPartial, + me, + useDbFallback: true, + redisTimelines: [`channelTimeline:${channel.id}`], + noteFilter: note => { + if (me && isUserRelated(note, userIdsWhoMeMuting)) return false; + + return true; + }, + dbFallback: async (untilId, sinceId, limit) => { + return await this.getFromDb({ untilId, sinceId, limit, channelId: channel.id }, me); + }, + }); }); } + + private async getFromDb(ps: { + untilId: string | null, + sinceId: string | null, + limit: number, + channelId: string + }, me: MiLocalUser | null) { + //#region fallback to database + const query = this.queryService.makePaginationQuery(this.notesRepository.createQueryBuilder('note'), ps.sinceId, ps.untilId) + .andWhere('note.channelId = :channelId', { channelId: ps.channelId }) + .innerJoinAndSelect('note.user', 'user') + .leftJoinAndSelect('note.reply', 'reply') + .leftJoinAndSelect('note.renote', 'renote') + .leftJoinAndSelect('reply.user', 'replyUser') + .leftJoinAndSelect('renote.user', 'renoteUser') + .leftJoinAndSelect('note.channel', 'channel'); + + if (me) { + this.queryService.generateMutedUserQuery(query, me); + this.queryService.generateBlockedUserQuery(query, me); + } + //#endregion + + return await query.limit(ps.limit).getMany(); + } } diff --git a/packages/backend/src/server/api/endpoints/notes/hybrid-timeline.ts b/packages/backend/src/server/api/endpoints/notes/hybrid-timeline.ts index 372199844d..820692626c 100644 --- a/packages/backend/src/server/api/endpoints/notes/hybrid-timeline.ts +++ b/packages/backend/src/server/api/endpoints/notes/hybrid-timeline.ts @@ -5,7 +5,7 @@ import { Brackets } from 'typeorm'; import { Inject, Injectable } from '@nestjs/common'; -import type { NotesRepository, FollowingsRepository, MiNote, ChannelFollowingsRepository } from '@/models/_.js'; +import type { NotesRepository, ChannelFollowingsRepository } from '@/models/_.js'; import { Endpoint } from '@/server/api/endpoint-base.js'; import ActiveUsersChart from '@/core/chart/charts/active-users.js'; import { NoteEntityService } from '@/core/entities/NoteEntityService.js'; @@ -19,6 +19,7 @@ import { QueryService } from '@/core/QueryService.js'; import { UserFollowingService } from '@/core/UserFollowingService.js'; import { MetaService } from '@/core/MetaService.js'; import { MiLocalUser } from '@/models/User.js'; +import { FanoutTimelineEndpointService } from '@/core/FanoutTimelineEndpointService.js'; import { ApiError } from '../../error.js'; export const meta = { @@ -53,6 +54,7 @@ export const paramDef = { untilId: { type: 'string', format: 'misskey:id' }, sinceDate: { type: 'integer' }, untilDate: { type: 'integer' }, + allowPartial: { type: 'boolean', default: false }, // true is recommended but for compatibility false by default includeMyRenotes: { type: 'boolean', default: true }, includeRenotedMyNotes: { type: 'boolean', default: true }, includeLocalRenotes: { type: 'boolean', default: true }, @@ -77,10 +79,10 @@ export default class extends Endpoint { // eslint- private activeUsersChart: ActiveUsersChart, private idService: IdService, private cacheService: CacheService, - private fanoutTimelineService: FanoutTimelineService, private queryService: QueryService, private userFollowingService: UserFollowingService, private metaService: MetaService, + private fanoutTimelineEndpointService: FanoutTimelineEndpointService, ) { super(meta, paramDef, async (ps, me) => { const untilId = ps.untilId ?? (ps.untilDate ? this.idService.gen(ps.untilDate!) : null); @@ -94,7 +96,7 @@ export default class extends Endpoint { // eslint- const serverSettings = await this.metaService.fetch(); if (!serverSettings.enableFanoutTimeline) { - return await this.getFromDb({ + const timeline = await this.getFromDb({ untilId, sinceId, limit: ps.limit, @@ -104,6 +106,12 @@ export default class extends Endpoint { // eslint- withFiles: ps.withFiles, withReplies: ps.withReplies, }, me); + + process.nextTick(() => { + this.activeUsersChart.read(me); + }); + + return await this.noteEntityService.packMany(timeline, me); } const [ @@ -116,51 +124,34 @@ export default class extends Endpoint { // eslint- this.cacheService.userBlockedCache.fetch(me.id), ]); - let noteIds: string[]; - let shouldFallbackToDb = false; + let timelineConfig: string[]; if (ps.withFiles) { - const [htlNoteIds, ltlNoteIds] = await this.fanoutTimelineService.getMulti([ + timelineConfig = [ `homeTimelineWithFiles:${me.id}`, 'localTimelineWithFiles', - ], untilId, sinceId); - noteIds = Array.from(new Set([...htlNoteIds, ...ltlNoteIds])); + ]; } else if (ps.withReplies) { - const [htlNoteIds, ltlNoteIds, ltlReplyNoteIds] = await this.fanoutTimelineService.getMulti([ + timelineConfig = [ `homeTimeline:${me.id}`, 'localTimeline', 'localTimelineWithReplies', - ], untilId, sinceId); - noteIds = Array.from(new Set([...htlNoteIds, ...ltlNoteIds, ...ltlReplyNoteIds])); + ]; } else { - const [htlNoteIds, ltlNoteIds] = await this.fanoutTimelineService.getMulti([ + timelineConfig = [ `homeTimeline:${me.id}`, 'localTimeline', - ], untilId, sinceId); - noteIds = Array.from(new Set([...htlNoteIds, ...ltlNoteIds])); - shouldFallbackToDb = htlNoteIds.length === 0; + ]; } - noteIds.sort((a, b) => a > b ? -1 : 1); - noteIds = noteIds.slice(0, ps.limit); - - shouldFallbackToDb = shouldFallbackToDb || (noteIds.length === 0); - - let redisTimeline: MiNote[] = []; - - if (!shouldFallbackToDb) { - const query = this.notesRepository.createQueryBuilder('note') - .where('note.id IN (:...noteIds)', { noteIds: noteIds }) - .innerJoinAndSelect('note.user', 'user') - .leftJoinAndSelect('note.reply', 'reply') - .leftJoinAndSelect('note.renote', 'renote') - .leftJoinAndSelect('reply.user', 'replyUser') - .leftJoinAndSelect('renote.user', 'renoteUser') - .leftJoinAndSelect('note.channel', 'channel'); - - redisTimeline = await query.getMany(); - - redisTimeline = redisTimeline.filter(note => { + const redisTimeline = await this.fanoutTimelineEndpointService.timeline({ + untilId, + sinceId, + limit: ps.limit, + allowPartial: ps.allowPartial, + redisTimelines: timelineConfig, + useDbFallback: serverSettings.enableFanoutTimelineDbFallback, + noteFilter: (note) => { if (note.userId === me.id) { return true; } @@ -174,33 +165,24 @@ export default class extends Endpoint { // eslint- } return true; - }); - - redisTimeline.sort((a, b) => a.id > b.id ? -1 : 1); - } + }, + dbFallback: async (untilId, sinceId, limit) => await this.getFromDb({ + untilId, + sinceId, + limit, + includeMyRenotes: ps.includeMyRenotes, + includeRenotedMyNotes: ps.includeRenotedMyNotes, + includeLocalRenotes: ps.includeLocalRenotes, + withFiles: ps.withFiles, + withReplies: ps.withReplies, + }, me), + }); - if (redisTimeline.length > 0) { - process.nextTick(() => { - this.activeUsersChart.read(me); - }); + process.nextTick(() => { + this.activeUsersChart.read(me); + }); - return await this.noteEntityService.packMany(redisTimeline, me); - } else { - if (serverSettings.enableFanoutTimelineDbFallback) { // fallback to db - return await this.getFromDb({ - untilId, - sinceId, - limit: ps.limit, - includeMyRenotes: ps.includeMyRenotes, - includeRenotedMyNotes: ps.includeRenotedMyNotes, - includeLocalRenotes: ps.includeLocalRenotes, - withFiles: ps.withFiles, - withReplies: ps.withReplies, - }, me); - } else { - return []; - } - } + return redisTimeline; }); } @@ -301,12 +283,6 @@ export default class extends Endpoint { // eslint- } //#endregion - const timeline = await query.limit(ps.limit).getMany(); - - process.nextTick(() => { - this.activeUsersChart.read(me); - }); - - return await this.noteEntityService.packMany(timeline, me); + return await query.limit(ps.limit).getMany(); } } diff --git a/packages/backend/src/server/api/endpoints/notes/local-timeline.ts b/packages/backend/src/server/api/endpoints/notes/local-timeline.ts index 8867070055..97b05016ec 100644 --- a/packages/backend/src/server/api/endpoints/notes/local-timeline.ts +++ b/packages/backend/src/server/api/endpoints/notes/local-timeline.ts @@ -14,10 +14,10 @@ import { RoleService } from '@/core/RoleService.js'; import { IdService } from '@/core/IdService.js'; import { CacheService } from '@/core/CacheService.js'; import { isUserRelated } from '@/misc/is-user-related.js'; -import { FanoutTimelineService } from '@/core/FanoutTimelineService.js'; import { QueryService } from '@/core/QueryService.js'; import { MetaService } from '@/core/MetaService.js'; import { MiLocalUser } from '@/models/User.js'; +import { FanoutTimelineEndpointService } from '@/core/FanoutTimelineEndpointService.js'; import { ApiError } from '../../error.js'; export const meta = { @@ -51,6 +51,7 @@ export const paramDef = { limit: { type: 'integer', minimum: 1, maximum: 100, default: 10 }, sinceId: { type: 'string', format: 'misskey:id' }, untilId: { type: 'string', format: 'misskey:id' }, + allowPartial: { type: 'boolean', default: false }, // true is recommended but for compatibility false by default sinceDate: { type: 'integer' }, untilDate: { type: 'integer' }, }, @@ -68,7 +69,7 @@ export default class extends Endpoint { // eslint- private activeUsersChart: ActiveUsersChart, private idService: IdService, private cacheService: CacheService, - private fanoutTimelineService: FanoutTimelineService, + private fanoutTimelineEndpointService: FanoutTimelineEndpointService, private queryService: QueryService, private metaService: MetaService, ) { @@ -84,13 +85,21 @@ export default class extends Endpoint { // eslint- const serverSettings = await this.metaService.fetch(); if (!serverSettings.enableFanoutTimeline) { - return await this.getFromDb({ + const timeline = await this.getFromDb({ untilId, sinceId, limit: ps.limit, withFiles: ps.withFiles, withReplies: ps.withReplies, }, me); + + process.nextTick(() => { + if (me) { + this.activeUsersChart.read(me); + } + }); + + return await this.noteEntityService.packMany(timeline, me); } const [ @@ -103,36 +112,15 @@ export default class extends Endpoint { // eslint- this.cacheService.userBlockedCache.fetch(me.id), ]) : [new Set(), new Set(), new Set()]; - let noteIds: string[]; - - if (ps.withFiles) { - noteIds = await this.fanoutTimelineService.get('localTimelineWithFiles', untilId, sinceId); - } else { - const [nonReplyNoteIds, replyNoteIds] = await this.fanoutTimelineService.getMulti([ - 'localTimeline', - 'localTimelineWithReplies', - ], untilId, sinceId); - noteIds = Array.from(new Set([...nonReplyNoteIds, ...replyNoteIds])); - noteIds.sort((a, b) => a > b ? -1 : 1); - } - - noteIds = noteIds.slice(0, ps.limit); - - let redisTimeline: MiNote[] = []; - - if (noteIds.length > 0) { - const query = this.notesRepository.createQueryBuilder('note') - .where('note.id IN (:...noteIds)', { noteIds: noteIds }) - .innerJoinAndSelect('note.user', 'user') - .leftJoinAndSelect('note.reply', 'reply') - .leftJoinAndSelect('note.renote', 'renote') - .leftJoinAndSelect('reply.user', 'replyUser') - .leftJoinAndSelect('renote.user', 'renoteUser') - .leftJoinAndSelect('note.channel', 'channel'); - - redisTimeline = await query.getMany(); - - redisTimeline = redisTimeline.filter(note => { + const timeline = await this.fanoutTimelineEndpointService.timeline({ + untilId, + sinceId, + limit: ps.limit, + allowPartial: ps.allowPartial, + me, + useDbFallback: serverSettings.enableFanoutTimelineDbFallback, + redisTimelines: ps.withFiles ? ['localTimelineWithFiles'] : ['localTimeline', 'localTimelineWithReplies'], + noteFilter: note => { if (me && (note.userId === me.id)) { return true; } @@ -147,32 +135,23 @@ export default class extends Endpoint { // eslint- } return true; - }); - - redisTimeline.sort((a, b) => a.id > b.id ? -1 : 1); - } - - if (redisTimeline.length > 0) { - process.nextTick(() => { - if (me) { - this.activeUsersChart.read(me); - } - }); + }, + dbFallback: async (untilId, sinceId, limit) => await this.getFromDb({ + untilId, + sinceId, + limit, + withFiles: ps.withFiles, + withReplies: ps.withReplies, + }, me), + }); - return await this.noteEntityService.packMany(redisTimeline, me); - } else { - if (serverSettings.enableFanoutTimelineDbFallback) { // fallback to db - return await this.getFromDb({ - untilId, - sinceId, - limit: ps.limit, - withFiles: ps.withFiles, - withReplies: ps.withReplies, - }, me); - } else { - return []; + process.nextTick(() => { + if (me) { + this.activeUsersChart.read(me); } - } + }); + + return timeline; }); } @@ -213,14 +192,6 @@ export default class extends Endpoint { // eslint- })); } - const timeline = await query.limit(ps.limit).getMany(); - - process.nextTick(() => { - if (me) { - this.activeUsersChart.read(me); - } - }); - - return await this.noteEntityService.packMany(timeline, me); + return await query.limit(ps.limit).getMany(); } } diff --git a/packages/backend/src/server/api/endpoints/notes/timeline.ts b/packages/backend/src/server/api/endpoints/notes/timeline.ts index 470abe0b14..74d0a6e0c0 100644 --- a/packages/backend/src/server/api/endpoints/notes/timeline.ts +++ b/packages/backend/src/server/api/endpoints/notes/timeline.ts @@ -5,7 +5,7 @@ import { Brackets } from 'typeorm'; import { Inject, Injectable } from '@nestjs/common'; -import type { MiNote, NotesRepository, ChannelFollowingsRepository } from '@/models/_.js'; +import type { NotesRepository, ChannelFollowingsRepository } from '@/models/_.js'; import { Endpoint } from '@/server/api/endpoint-base.js'; import { QueryService } from '@/core/QueryService.js'; import ActiveUsersChart from '@/core/chart/charts/active-users.js'; @@ -14,10 +14,10 @@ import { DI } from '@/di-symbols.js'; import { IdService } from '@/core/IdService.js'; import { CacheService } from '@/core/CacheService.js'; import { isUserRelated } from '@/misc/is-user-related.js'; -import { FanoutTimelineService } from '@/core/FanoutTimelineService.js'; import { UserFollowingService } from '@/core/UserFollowingService.js'; import { MiLocalUser } from '@/models/User.js'; import { MetaService } from '@/core/MetaService.js'; +import { FanoutTimelineEndpointService } from '@/core/FanoutTimelineEndpointService.js'; export const meta = { tags: ['notes'], @@ -43,6 +43,7 @@ export const paramDef = { untilId: { type: 'string', format: 'misskey:id' }, sinceDate: { type: 'integer' }, untilDate: { type: 'integer' }, + allowPartial: { type: 'boolean', default: false }, // true is recommended but for compatibility false by default includeMyRenotes: { type: 'boolean', default: true }, includeRenotedMyNotes: { type: 'boolean', default: true }, includeLocalRenotes: { type: 'boolean', default: true }, @@ -65,7 +66,7 @@ export default class extends Endpoint { // eslint- private activeUsersChart: ActiveUsersChart, private idService: IdService, private cacheService: CacheService, - private fanoutTimelineService: FanoutTimelineService, + private fanoutTimelineEndpointService: FanoutTimelineEndpointService, private userFollowingService: UserFollowingService, private queryService: QueryService, private metaService: MetaService, @@ -77,7 +78,7 @@ export default class extends Endpoint { // eslint- const serverSettings = await this.metaService.fetch(); if (!serverSettings.enableFanoutTimeline) { - return await this.getFromDb({ + const timeline = await this.getFromDb({ untilId, sinceId, limit: ps.limit, @@ -87,6 +88,12 @@ export default class extends Endpoint { // eslint- withFiles: ps.withFiles, withRenotes: ps.withRenotes, }, me); + + process.nextTick(() => { + this.activeUsersChart.read(me); + }); + + return await this.noteEntityService.packMany(timeline, me); } const [ @@ -101,24 +108,15 @@ export default class extends Endpoint { // eslint- this.cacheService.userBlockedCache.fetch(me.id), ]); - let noteIds = await this.fanoutTimelineService.get(ps.withFiles ? `homeTimelineWithFiles:${me.id}` : `homeTimeline:${me.id}`, untilId, sinceId); - noteIds = noteIds.slice(0, ps.limit); - - let redisTimeline: MiNote[] = []; - - if (noteIds.length > 0) { - const query = this.notesRepository.createQueryBuilder('note') - .where('note.id IN (:...noteIds)', { noteIds: noteIds }) - .innerJoinAndSelect('note.user', 'user') - .leftJoinAndSelect('note.reply', 'reply') - .leftJoinAndSelect('note.renote', 'renote') - .leftJoinAndSelect('reply.user', 'replyUser') - .leftJoinAndSelect('renote.user', 'renoteUser') - .leftJoinAndSelect('note.channel', 'channel'); - - redisTimeline = await query.getMany(); - - redisTimeline = redisTimeline.filter(note => { + const timeline = this.fanoutTimelineEndpointService.timeline({ + untilId, + sinceId, + limit: ps.limit, + allowPartial: ps.allowPartial, + me, + useDbFallback: serverSettings.enableFanoutTimelineDbFallback, + redisTimelines: ps.withFiles ? [`homeTimelineWithFiles:${me.id}`] : [`homeTimeline:${me.id}`], + noteFilter: note => { if (note.userId === me.id) { return true; } @@ -135,33 +133,24 @@ export default class extends Endpoint { // eslint- } return true; - }); - - redisTimeline.sort((a, b) => a.id > b.id ? -1 : 1); - } + }, + dbFallback: async (untilId, sinceId, limit) => await this.getFromDb({ + untilId, + sinceId, + limit, + includeMyRenotes: ps.includeMyRenotes, + includeRenotedMyNotes: ps.includeRenotedMyNotes, + includeLocalRenotes: ps.includeLocalRenotes, + withFiles: ps.withFiles, + withRenotes: ps.withRenotes, + }, me), + }); - if (redisTimeline.length > 0) { - process.nextTick(() => { - this.activeUsersChart.read(me); - }); + process.nextTick(() => { + this.activeUsersChart.read(me); + }); - return await this.noteEntityService.packMany(redisTimeline, me); - } else { - if (serverSettings.enableFanoutTimelineDbFallback) { // fallback to db - return await this.getFromDb({ - untilId, - sinceId, - limit: ps.limit, - includeMyRenotes: ps.includeMyRenotes, - includeRenotedMyNotes: ps.includeRenotedMyNotes, - includeLocalRenotes: ps.includeLocalRenotes, - withFiles: ps.withFiles, - withRenotes: ps.withRenotes, - }, me); - } else { - return []; - } - } + return timeline; }); } @@ -269,12 +258,6 @@ export default class extends Endpoint { // eslint- } //#endregion - const timeline = await query.limit(ps.limit).getMany(); - - process.nextTick(() => { - this.activeUsersChart.read(me); - }); - - return await this.noteEntityService.packMany(timeline, me); + return await query.limit(ps.limit).getMany(); } } diff --git a/packages/backend/src/server/api/endpoints/notes/user-list-timeline.ts b/packages/backend/src/server/api/endpoints/notes/user-list-timeline.ts index 1ac1d37f48..f39cac5c3e 100644 --- a/packages/backend/src/server/api/endpoints/notes/user-list-timeline.ts +++ b/packages/backend/src/server/api/endpoints/notes/user-list-timeline.ts @@ -17,6 +17,7 @@ import { FanoutTimelineService } from '@/core/FanoutTimelineService.js'; import { QueryService } from '@/core/QueryService.js'; import { MiLocalUser } from '@/models/User.js'; import { MetaService } from '@/core/MetaService.js'; +import { FanoutTimelineEndpointService } from '@/core/FanoutTimelineEndpointService.js'; import { ApiError } from '../../error.js'; export const meta = { @@ -52,6 +53,7 @@ export const paramDef = { untilId: { type: 'string', format: 'misskey:id' }, sinceDate: { type: 'integer' }, untilDate: { type: 'integer' }, + allowPartial: { type: 'boolean', default: false }, // true is recommended but for compatibility false by default includeMyRenotes: { type: 'boolean', default: true }, includeRenotedMyNotes: { type: 'boolean', default: true }, includeLocalRenotes: { type: 'boolean', default: true }, @@ -82,6 +84,7 @@ export default class extends Endpoint { // eslint- private cacheService: CacheService, private idService: IdService, private fanoutTimelineService: FanoutTimelineService, + private fanoutTimelineEndpointService: FanoutTimelineEndpointService, private queryService: QueryService, private metaService: MetaService, ) { @@ -101,7 +104,7 @@ export default class extends Endpoint { // eslint- const serverSettings = await this.metaService.fetch(); if (!serverSettings.enableFanoutTimeline) { - return await this.getFromDb(list, { + const timeline = await this.getFromDb(list, { untilId, sinceId, limit: ps.limit, @@ -111,6 +114,10 @@ export default class extends Endpoint { // eslint- withFiles: ps.withFiles, withRenotes: ps.withRenotes, }, me); + + this.activeUsersChart.read(me); + + await this.noteEntityService.packMany(timeline, me); } const [ @@ -123,24 +130,15 @@ export default class extends Endpoint { // eslint- this.cacheService.userBlockedCache.fetch(me.id), ]); - let noteIds = await this.fanoutTimelineService.get(ps.withFiles ? `userListTimelineWithFiles:${list.id}` : `userListTimeline:${list.id}`, untilId, sinceId); - noteIds = noteIds.slice(0, ps.limit); - - let redisTimeline: MiNote[] = []; - - if (noteIds.length > 0) { - const query = this.notesRepository.createQueryBuilder('note') - .where('note.id IN (:...noteIds)', { noteIds: noteIds }) - .innerJoinAndSelect('note.user', 'user') - .leftJoinAndSelect('note.reply', 'reply') - .leftJoinAndSelect('note.renote', 'renote') - .leftJoinAndSelect('reply.user', 'replyUser') - .leftJoinAndSelect('renote.user', 'renoteUser') - .leftJoinAndSelect('note.channel', 'channel'); - - redisTimeline = await query.getMany(); - - redisTimeline = redisTimeline.filter(note => { + const timeline = await this.fanoutTimelineEndpointService.timeline({ + untilId, + sinceId, + limit: ps.limit, + allowPartial: ps.allowPartial, + me, + useDbFallback: serverSettings.enableFanoutTimelineDbFallback, + redisTimelines: ps.withFiles ? [`userListTimelineWithFiles:${list.id}`] : [`userListTimeline:${list.id}`], + noteFilter: note => { if (note.userId === me.id) { return true; } @@ -154,30 +152,22 @@ export default class extends Endpoint { // eslint- } return true; - }); + }, + dbFallback: async (untilId, sinceId, limit) => await this.getFromDb(list, { + untilId, + sinceId, + limit, + includeMyRenotes: ps.includeMyRenotes, + includeRenotedMyNotes: ps.includeRenotedMyNotes, + includeLocalRenotes: ps.includeLocalRenotes, + withFiles: ps.withFiles, + withRenotes: ps.withRenotes, + }, me), + }); - redisTimeline.sort((a, b) => a.id > b.id ? -1 : 1); - } + this.activeUsersChart.read(me); - if (redisTimeline.length > 0) { - this.activeUsersChart.read(me); - return await this.noteEntityService.packMany(redisTimeline, me); - } else { - if (serverSettings.enableFanoutTimelineDbFallback) { // fallback to db - return await this.getFromDb(list, { - untilId, - sinceId, - limit: ps.limit, - includeMyRenotes: ps.includeMyRenotes, - includeRenotedMyNotes: ps.includeRenotedMyNotes, - includeLocalRenotes: ps.includeLocalRenotes, - withFiles: ps.withFiles, - withRenotes: ps.withRenotes, - }, me); - } else { - return []; - } - } + return timeline; }); } @@ -271,10 +261,6 @@ export default class extends Endpoint { // eslint- } //#endregion - const timeline = await query.limit(ps.limit).getMany(); - - this.activeUsersChart.read(me); - - return await this.noteEntityService.packMany(timeline, me); + return await query.limit(ps.limit).getMany(); } } diff --git a/packages/backend/src/server/api/endpoints/users/notes.ts b/packages/backend/src/server/api/endpoints/users/notes.ts index 76033ddb06..56983f7bc4 100644 --- a/packages/backend/src/server/api/endpoints/users/notes.ts +++ b/packages/backend/src/server/api/endpoints/users/notes.ts @@ -5,8 +5,7 @@ import { Brackets } from 'typeorm'; import { Inject, Injectable } from '@nestjs/common'; -import * as Redis from 'ioredis'; -import type { MiNote, NotesRepository } from '@/models/_.js'; +import type { NotesRepository } from '@/models/_.js'; import { Endpoint } from '@/server/api/endpoint-base.js'; import { NoteEntityService } from '@/core/entities/NoteEntityService.js'; import { DI } from '@/di-symbols.js'; @@ -14,9 +13,9 @@ import { CacheService } from '@/core/CacheService.js'; import { IdService } from '@/core/IdService.js'; import { isUserRelated } from '@/misc/is-user-related.js'; import { QueryService } from '@/core/QueryService.js'; -import { FanoutTimelineService } from '@/core/FanoutTimelineService.js'; import { MetaService } from '@/core/MetaService.js'; -import { ApiError } from '../../error.js'; +import { MiLocalUser } from '@/models/User.js'; +import { FanoutTimelineEndpointService } from '@/core/FanoutTimelineEndpointService.js'; export const meta = { tags: ['users', 'notes'], @@ -52,6 +51,7 @@ export const paramDef = { untilId: { type: 'string', format: 'misskey:id' }, sinceDate: { type: 'integer' }, untilDate: { type: 'integer' }, + allowPartial: { type: 'boolean', default: false }, // true is recommended but for compatibility false by default withFiles: { type: 'boolean', default: false }, }, required: ['userId'], @@ -60,9 +60,6 @@ export const paramDef = { @Injectable() export default class extends Endpoint { // eslint-disable-line import/no-default-export constructor( - @Inject(DI.redisForTimelines) - private redisForTimelines: Redis.Redis, - @Inject(DI.notesRepository) private notesRepository: NotesRepository, @@ -70,121 +67,130 @@ export default class extends Endpoint { // eslint- private queryService: QueryService, private cacheService: CacheService, private idService: IdService, - private fanoutTimelineService: FanoutTimelineService, + private fanoutTimelineEndpointService: FanoutTimelineEndpointService, private metaService: MetaService, ) { super(meta, paramDef, async (ps, me) => { const untilId = ps.untilId ?? (ps.untilDate ? this.idService.gen(ps.untilDate!) : null); const sinceId = ps.sinceId ?? (ps.sinceDate ? this.idService.gen(ps.sinceDate!) : null); - const isRangeSpecified = untilId != null && sinceId != null; const isSelf = me && (me.id === ps.userId); const serverSettings = await this.metaService.fetch(); - if (serverSettings.enableFanoutTimeline && (isRangeSpecified || sinceId == null)) { - const [ - userIdsWhoMeMuting, - ] = me ? await Promise.all([ - this.cacheService.userMutingsCache.fetch(me.id), - ]) : [new Set()]; - - const [noteIdsRes, repliesNoteIdsRes, channelNoteIdsRes] = await Promise.all([ - this.fanoutTimelineService.get(ps.withFiles ? `userTimelineWithFiles:${ps.userId}` : `userTimeline:${ps.userId}`, untilId, sinceId), - ps.withReplies ? this.fanoutTimelineService.get(`userTimelineWithReplies:${ps.userId}`, untilId, sinceId) : Promise.resolve([]), - ps.withChannelNotes ? this.fanoutTimelineService.get(`userTimelineWithChannel:${ps.userId}`, untilId, sinceId) : Promise.resolve([]), - ]); - - let noteIds = Array.from(new Set([ - ...noteIdsRes, - ...repliesNoteIdsRes, - ...channelNoteIdsRes, - ])); - noteIds.sort((a, b) => a > b ? -1 : 1); - noteIds = noteIds.slice(0, ps.limit); - - if (noteIds.length > 0) { - const isFollowing = me && Object.hasOwn(await this.cacheService.userFollowingsCache.fetch(me.id), ps.userId); - - const query = this.notesRepository.createQueryBuilder('note') - .where('note.id IN (:...noteIds)', { noteIds: noteIds }) - .innerJoinAndSelect('note.user', 'user') - .leftJoinAndSelect('note.reply', 'reply') - .leftJoinAndSelect('note.renote', 'renote') - .leftJoinAndSelect('reply.user', 'replyUser') - .leftJoinAndSelect('renote.user', 'renoteUser') - .leftJoinAndSelect('note.channel', 'channel'); - - let timeline = await query.getMany(); - - timeline = timeline.filter(note => { - if (me && isUserRelated(note, userIdsWhoMeMuting, true)) return false; - - if (note.renoteId) { - if (note.text == null && note.fileIds.length === 0 && !note.hasPoll) { - if (ps.withRenotes === false) return false; - } - } - - if (note.channel?.isSensitive && !isSelf) return false; - if (note.visibility === 'specified' && (!me || (me.id !== note.userId && !note.visibleUserIds.some(v => v === me.id)))) return false; - if (note.visibility === 'followers' && !isFollowing && !isSelf) return false; - - return true; - }); - - // TODO: フィルタで件数が減った場合の埋め合わせ処理 - - timeline.sort((a, b) => a.id > b.id ? -1 : 1); - - if (timeline.length > 0) { - return await this.noteEntityService.packMany(timeline, me); - } - } - } - - //#region fallback to database - const query = this.queryService.makePaginationQuery(this.notesRepository.createQueryBuilder('note'), ps.sinceId, ps.untilId, ps.sinceDate, ps.untilDate) - .andWhere('note.userId = :userId', { userId: ps.userId }) - .innerJoinAndSelect('note.user', 'user') - .leftJoinAndSelect('note.reply', 'reply') - .leftJoinAndSelect('note.renote', 'renote') - .leftJoinAndSelect('note.channel', 'channel') - .leftJoinAndSelect('reply.user', 'replyUser') - .leftJoinAndSelect('renote.user', 'renoteUser'); - - if (ps.withChannelNotes) { - if (!isSelf) query.andWhere(new Brackets(qb => { - qb.orWhere('note.channelId IS NULL'); - qb.orWhere('channel.isSensitive = false'); - })); - } else { - query.andWhere('note.channelId IS NULL'); + if (!serverSettings.enableFanoutTimeline) { + const timeline = await this.getFromDb({ + untilId, + sinceId, + limit: ps.limit, + userId: ps.userId, + withChannelNotes: ps.withChannelNotes, + withFiles: ps.withFiles, + withRenotes: ps.withRenotes, + }, me); + + return await this.noteEntityService.packMany(timeline, me); } - this.queryService.generateVisibilityQuery(query, me); - if (me) { - this.queryService.generateMutedUserQuery(query, me, { id: ps.userId }); - this.queryService.generateBlockedUserQuery(query, me); - } - - if (ps.withFiles) { - query.andWhere('note.fileIds != \'{}\''); - } - - if (ps.withRenotes === false) { - query.andWhere(new Brackets(qb => { - qb.orWhere('note.userId != :userId', { userId: ps.userId }); - qb.orWhere('note.renoteId IS NULL'); - qb.orWhere('note.text IS NOT NULL'); - qb.orWhere('note.fileIds != \'{}\''); - qb.orWhere('0 < (SELECT COUNT(*) FROM poll WHERE poll."noteId" = note.id)'); - })); - } - - const timeline = await query.limit(ps.limit).getMany(); + const [ + userIdsWhoMeMuting, + ] = me ? await Promise.all([ + this.cacheService.userMutingsCache.fetch(me.id), + ]) : [new Set()]; + + const redisTimelines = [ps.withFiles ? `userTimelineWithFiles:${ps.userId}` : `userTimeline:${ps.userId}`]; + + if (ps.withReplies) redisTimelines.push(`userTimelineWithReplies:${ps.userId}`); + if (ps.withChannelNotes) redisTimelines.push(`userTimelineWithChannel:${ps.userId}`); + + const isFollowing = me && Object.hasOwn(await this.cacheService.userFollowingsCache.fetch(me.id), ps.userId); + + const timeline = await this.fanoutTimelineEndpointService.timeline({ + untilId, + sinceId, + limit: ps.limit, + allowPartial: ps.allowPartial, + me, + redisTimelines, + useDbFallback: true, + noteFilter: note => { + if (me && isUserRelated(note, userIdsWhoMeMuting, true)) return false; + + if (note.renoteId) { + if (note.text == null && note.fileIds.length === 0 && !note.hasPoll) { + if (ps.withRenotes === false) return false; + } + } - return await this.noteEntityService.packMany(timeline, me); - //#endregion + if (note.channel?.isSensitive && !isSelf) return false; + if (note.visibility === 'specified' && (!me || (me.id !== note.userId && !note.visibleUserIds.some(v => v === me.id)))) return false; + if (note.visibility === 'followers' && !isFollowing && !isSelf) return false; + + return true; + }, + dbFallback: async (untilId, sinceId, limit) => await this.getFromDb({ + untilId, + sinceId, + limit, + userId: ps.userId, + withChannelNotes: ps.withChannelNotes, + withFiles: ps.withFiles, + withRenotes: ps.withRenotes, + }, me), + }); + + return timeline; }); } + + private async getFromDb(ps: { + untilId: string | null, + sinceId: string | null, + limit: number, + userId: string, + withChannelNotes: boolean, + withFiles: boolean, + withRenotes: boolean, + }, me: MiLocalUser | null) { + const isSelf = me && (me.id === ps.userId); + + const query = this.queryService.makePaginationQuery(this.notesRepository.createQueryBuilder('note'), ps.sinceId, ps.untilId) + .andWhere('note.userId = :userId', { userId: ps.userId }) + .innerJoinAndSelect('note.user', 'user') + .leftJoinAndSelect('note.reply', 'reply') + .leftJoinAndSelect('note.renote', 'renote') + .leftJoinAndSelect('note.channel', 'channel') + .leftJoinAndSelect('reply.user', 'replyUser') + .leftJoinAndSelect('renote.user', 'renoteUser'); + + if (ps.withChannelNotes) { + if (!isSelf) query.andWhere(new Brackets(qb => { + qb.orWhere('note.channelId IS NULL'); + qb.orWhere('channel.isSensitive = false'); + })); + } else { + query.andWhere('note.channelId IS NULL'); + } + + this.queryService.generateVisibilityQuery(query, me); + if (me) { + this.queryService.generateMutedUserQuery(query, me, { id: ps.userId }); + this.queryService.generateBlockedUserQuery(query, me); + } + + if (ps.withFiles) { + query.andWhere('note.fileIds != \'{}\''); + } + + if (ps.withRenotes === false) { + query.andWhere(new Brackets(qb => { + qb.orWhere('note.userId != :userId', { userId: ps.userId }); + qb.orWhere('note.renoteId IS NULL'); + qb.orWhere('note.text IS NOT NULL'); + qb.orWhere('note.fileIds != \'{}\''); + qb.orWhere('0 < (SELECT COUNT(*) FROM poll WHERE poll."noteId" = note.id)'); + })); + } + + return await query.limit(ps.limit).getMany(); + } } diff --git a/packages/frontend/src/components/MkPagination.vue b/packages/frontend/src/components/MkPagination.vue index 2c59e6d4e8..57348cde53 100644 --- a/packages/frontend/src/components/MkPagination.vue +++ b/packages/frontend/src/components/MkPagination.vue @@ -206,6 +206,7 @@ async function init(): Promise { await os.api(props.pagination.endpoint, { ...params, limit: props.pagination.limit ?? 10, + allowPartial: true, }).then(res => { for (let i = 0; i < res.length; i++) { const item = res[i]; -- cgit v1.2.3-freya From 92029ac325c8cf74d2227beffb763ec3e6c01aea Mon Sep 17 00:00:00 2001 From: おさむのひと <46447427+samunohito@users.noreply.github.com> Date: Sat, 2 Dec 2023 20:11:31 +0900 Subject: fix: #12544 (#12545) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * meを渡し忘れている * fix CHANGELOG.md * Revert "fix CHANGELOG.md" This reverts commit aaee4e9b8a6abf510f393bc02282f6ac016d2124. --- packages/backend/src/server/api/endpoints/notes/hybrid-timeline.ts | 1 + 1 file changed, 1 insertion(+) (limited to 'packages/backend/src/server/api/endpoints/notes/hybrid-timeline.ts') diff --git a/packages/backend/src/server/api/endpoints/notes/hybrid-timeline.ts b/packages/backend/src/server/api/endpoints/notes/hybrid-timeline.ts index 820692626c..deb9e014c4 100644 --- a/packages/backend/src/server/api/endpoints/notes/hybrid-timeline.ts +++ b/packages/backend/src/server/api/endpoints/notes/hybrid-timeline.ts @@ -149,6 +149,7 @@ export default class extends Endpoint { // eslint- sinceId, limit: ps.limit, allowPartial: ps.allowPartial, + me, redisTimelines: timelineConfig, useDbFallback: serverSettings.enableFanoutTimelineDbFallback, noteFilter: (note) => { -- cgit v1.2.3-freya From 18109fcef760dee8171364fd0382375c4047b8e7 Mon Sep 17 00:00:00 2001 From: anatawa12 Date: Mon, 4 Dec 2023 14:38:21 +0900 Subject: Filter User / Instance Mutes in FanoutTimelineEndpointService (#12565) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix: unnecessary logging in FanoutTimelineEndpointService * chore: TimelineOptions * chore: add FanoutTimelineName type * chore: forbid specifying both withReplies and withFiles since it's not implemented correctly * chore: filter mutes, replies, renotes, files in FanoutTimelineEndpointService * revert unintended changes * use isReply in NoteCreateService * fix: excludePureRenotes is not implemented * fix: replies to me is excluded from local timeline * chore(frontend): forbid enabling both withReplies and withFiles * docs(changelog): インスタンスミュートが効かない問題の修正について言及 --- CHANGELOG.md | 3 + .../src/core/FanoutTimelineEndpointService.ts | 103 +++++++++++++++------ packages/backend/src/core/FanoutTimelineService.ts | 36 ++++++- packages/backend/src/core/NoteCreateService.ts | 7 +- packages/backend/src/misc/is-reply.ts | 10 ++ .../src/server/api/endpoints/channels/timeline.ts | 16 +--- .../server/api/endpoints/notes/hybrid-timeline.ts | 40 +++----- .../server/api/endpoints/notes/local-timeline.ts | 40 +++----- .../src/server/api/endpoints/notes/timeline.ts | 20 +--- .../api/endpoints/notes/user-list-timeline.ts | 36 +------ .../src/server/api/endpoints/users/notes.ts | 34 +++---- packages/frontend/src/components/MkMenu.vue | 2 +- packages/frontend/src/pages/timeline.vue | 4 +- packages/frontend/src/types/menu.ts | 2 +- packages/frontend/src/ui/deck/tl-column.vue | 2 + 15 files changed, 176 insertions(+), 179 deletions(-) create mode 100644 packages/backend/src/misc/is-reply.ts (limited to 'packages/backend/src/server/api/endpoints/notes/hybrid-timeline.ts') diff --git a/CHANGELOG.md b/CHANGELOG.md index 453bdeff59..51eb5400ba 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -55,6 +55,9 @@ - Fix: 何もノートしていないユーザーのフィードにアクセスするとエラーになる問題を修正 - Fix: リストタイムラインにてミュートが機能しないケースがある問題と、チャンネル投稿がストリーミングで流れてきてしまう問題を修正 #10443 - Fix: 「みつける」のなかにミュートしたユーザが現れてしまう問題を修正 #12383 +- Fix: Social/Local/Home Timelineにてインスタンスミュートが効かない問題 +- Fix: ユーザのノート一覧にてインスタンスミュートが効かない問題 +- Fix: チャンネルのノート一覧にてインスタンスミュートが効かない問題 ## 2023.11.1 diff --git a/packages/backend/src/core/FanoutTimelineEndpointService.ts b/packages/backend/src/core/FanoutTimelineEndpointService.ts index 157fcbe877..6775f0051a 100644 --- a/packages/backend/src/core/FanoutTimelineEndpointService.ts +++ b/packages/backend/src/core/FanoutTimelineEndpointService.ts @@ -11,7 +11,29 @@ import type { MiNote } from '@/models/Note.js'; import { Packed } from '@/misc/json-schema.js'; import type { NotesRepository } from '@/models/_.js'; import { NoteEntityService } from '@/core/entities/NoteEntityService.js'; -import { FanoutTimelineService } from '@/core/FanoutTimelineService.js'; +import { FanoutTimelineName, FanoutTimelineService } from '@/core/FanoutTimelineService.js'; +import { isUserRelated } from '@/misc/is-user-related.js'; +import { isPureRenote } from '@/misc/is-pure-renote.js'; +import { CacheService } from '@/core/CacheService.js'; +import { isReply } from '@/misc/is-reply.js'; +import { isInstanceMuted } from '@/misc/is-instance-muted.js'; + +type TimelineOptions = { + untilId: string | null, + sinceId: string | null, + limit: number, + allowPartial: boolean, + me?: { id: MiUser['id'] } | undefined | null, + useDbFallback: boolean, + redisTimelines: FanoutTimelineName[], + noteFilter?: (note: MiNote) => boolean, + alwaysIncludeMyNotes?: boolean; + ignoreAuthorFromMute?: boolean; + excludeNoFiles?: boolean; + excludeReplies?: boolean; + excludePureRenotes: boolean; + dbFallback: (untilId: string | null, sinceId: string | null, limit: number) => Promise, +}; @Injectable() export class FanoutTimelineEndpointService { @@ -20,37 +42,18 @@ export class FanoutTimelineEndpointService { private notesRepository: NotesRepository, private noteEntityService: NoteEntityService, + private cacheService: CacheService, private fanoutTimelineService: FanoutTimelineService, ) { } @bindThis - async timeline(ps: { - untilId: string | null, - sinceId: string | null, - limit: number, - allowPartial: boolean, - me?: { id: MiUser['id'] } | undefined | null, - useDbFallback: boolean, - redisTimelines: string[], - noteFilter: (note: MiNote) => boolean, - dbFallback: (untilId: string | null, sinceId: string | null, limit: number) => Promise, - }): Promise[]> { + async timeline(ps: TimelineOptions): Promise[]> { return await this.noteEntityService.packMany(await this.getMiNotes(ps), ps.me); } @bindThis - private async getMiNotes(ps: { - untilId: string | null, - sinceId: string | null, - limit: number, - allowPartial: boolean, - me?: { id: MiUser['id'] } | undefined | null, - useDbFallback: boolean, - redisTimelines: string[], - noteFilter: (note: MiNote) => boolean, - dbFallback: (untilId: string | null, sinceId: string | null, limit: number) => Promise, - }): Promise { + private async getMiNotes(ps: TimelineOptions): Promise { let noteIds: string[]; let shouldFallbackToDb = false; @@ -67,10 +70,57 @@ export class FanoutTimelineEndpointService { shouldFallbackToDb = shouldFallbackToDb || (noteIds.length === 0); if (!shouldFallbackToDb) { + let filter = ps.noteFilter ?? (_note => true); + + if (ps.alwaysIncludeMyNotes && ps.me) { + const me = ps.me; + const parentFilter = filter; + filter = (note) => note.userId === me.id || parentFilter(note); + } + + if (ps.excludeNoFiles) { + const parentFilter = filter; + filter = (note) => note.fileIds.length !== 0 && parentFilter(note); + } + + if (ps.excludeReplies) { + const parentFilter = filter; + filter = (note) => !isReply(note, ps.me?.id) && parentFilter(note); + } + + if (ps.excludePureRenotes) { + const parentFilter = filter; + filter = (note) => !isPureRenote(note) && parentFilter(note); + } + + if (ps.me) { + const me = ps.me; + const [ + userIdsWhoMeMuting, + userIdsWhoMeMutingRenotes, + userIdsWhoBlockingMe, + userMutedInstances, + ] = await Promise.all([ + this.cacheService.userMutingsCache.fetch(ps.me.id), + this.cacheService.renoteMutingsCache.fetch(ps.me.id), + this.cacheService.userBlockedCache.fetch(ps.me.id), + this.cacheService.userProfileCache.fetch(me.id).then(p => new Set(p.mutedInstances)), + ]); + + const parentFilter = filter; + filter = (note) => { + if (isUserRelated(note, userIdsWhoBlockingMe, ps.ignoreAuthorFromMute)) return false; + if (isUserRelated(note, userIdsWhoMeMuting, ps.ignoreAuthorFromMute)) return false; + if (isPureRenote(note) && isUserRelated(note, userIdsWhoMeMutingRenotes, ps.ignoreAuthorFromMute)) return false; + if (isInstanceMuted(note, userMutedInstances)) return false; + + return parentFilter(note); + }; + } + const redisTimeline: MiNote[] = []; let readFromRedis = 0; let lastSuccessfulRate = 1; // rateをキャッシュする？ - let trialCount = 1; while ((redisResultIds.length - readFromRedis) !== 0) { const remainingToRead = ps.limit - redisTimeline.length; @@ -81,12 +131,10 @@ export class FanoutTimelineEndpointService { readFromRedis += noteIds.length; - const gotFromDb = await this.getAndFilterFromDb(noteIds, ps.noteFilter); + const gotFromDb = await this.getAndFilterFromDb(noteIds, filter); redisTimeline.push(...gotFromDb); lastSuccessfulRate = gotFromDb.length / noteIds.length; - console.log(`fanoutTimelineTrial#${trialCount++}: req: ${ps.limit}, tried: ${noteIds.length}, got: ${gotFromDb.length}, rate: ${lastSuccessfulRate}, total: ${redisTimeline.length}, fromRedis: ${redisResultIds.length}`); - if (ps.allowPartial ? redisTimeline.length !== 0 : redisTimeline.length >= ps.limit) { // 十分Redisからとれた return redisTimeline.slice(0, ps.limit); @@ -97,7 +145,6 @@ export class FanoutTimelineEndpointService { const remainingToRead = ps.limit - redisTimeline.length; const gotFromDb = await ps.dbFallback(noteIds[noteIds.length - 1], ps.sinceId, remainingToRead); redisTimeline.push(...gotFromDb); - console.log(`fanoutTimelineTrial#db: req: ${ps.limit}, tried: ${remainingToRead}, got: ${gotFromDb.length}, since: ${noteIds[noteIds.length - 1]}, until: ${ps.untilId}, total: ${redisTimeline.length}`); return redisTimeline; } diff --git a/packages/backend/src/core/FanoutTimelineService.ts b/packages/backend/src/core/FanoutTimelineService.ts index 6a1b0aa879..654a035a5f 100644 --- a/packages/backend/src/core/FanoutTimelineService.ts +++ b/packages/backend/src/core/FanoutTimelineService.ts @@ -9,6 +9,34 @@ import { DI } from '@/di-symbols.js'; import { bindThis } from '@/decorators.js'; import { IdService } from '@/core/IdService.js'; +export type FanoutTimelineName = + // home timeline + | `homeTimeline:${string}` + | `homeTimelineWithFiles:${string}` // only notes with files are included + // local timeline + | `localTimeline` // replies are not included + | `localTimelineWithFiles` // only non-reply notes with files are included + | `localTimelineWithReplies` // only replies are included + + // antenna + | `antennaTimeline:${string}` + + // user timeline + | `userTimeline:${string}` // replies are not included + | `userTimelineWithFiles:${string}` // only non-reply notes with files are included + | `userTimelineWithReplies:${string}` // only replies are included + | `userTimelineWithChannel:${string}` // only channel notes are included, replies are included + + // user list timelines + | `userListTimeline:${string}` + | `userListTimelineWithFiles:${string}` // only notes with files are included + + // channel timelines + | `channelTimeline:${string}` // replies are included + + // role timelines + | `roleTimeline:${string}` // any notes are included + @Injectable() export class FanoutTimelineService { constructor( @@ -20,7 +48,7 @@ export class FanoutTimelineService { } @bindThis - public push(tl: string, id: string, maxlen: number, pipeline: Redis.ChainableCommander) { + public push(tl: FanoutTimelineName, id: string, maxlen: number, pipeline: Redis.ChainableCommander) { // リモートから遅れて届いた(もしくは後から追加された)投稿日時が古い投稿が追加されるとページネーション時に問題を引き起こすため、 // 3分以内に投稿されたものでない場合、Redisにある最古のIDより新しい場合のみ追加する if (this.idService.parse(id).date.getTime() > Date.now() - 1000 * 60 * 3) { @@ -41,7 +69,7 @@ export class FanoutTimelineService { } @bindThis - public get(name: string, untilId?: string | null, sinceId?: string | null) { + public get(name: FanoutTimelineName, untilId?: string | null, sinceId?: string | null) { if (untilId && sinceId) { return this.redisForTimelines.lrange('list:' + name, 0, -1) .then(ids => ids.filter(id => id < untilId && id > sinceId).sort((a, b) => a > b ? -1 : 1)); @@ -58,7 +86,7 @@ export class FanoutTimelineService { } @bindThis - public getMulti(name: string[], untilId?: string | null, sinceId?: string | null): Promise { + public getMulti(name: FanoutTimelineName[], untilId?: string | null, sinceId?: string | null): Promise { const pipeline = this.redisForTimelines.pipeline(); for (const n of name) { pipeline.lrange('list:' + n, 0, -1); @@ -79,7 +107,7 @@ export class FanoutTimelineService { } @bindThis - public purge(name: string) { + public purge(name: FanoutTimelineName) { return this.redisForTimelines.del('list:' + name); } } diff --git a/packages/backend/src/core/NoteCreateService.ts b/packages/backend/src/core/NoteCreateService.ts index fd87edc28e..0110ebaf5e 100644 --- a/packages/backend/src/core/NoteCreateService.ts +++ b/packages/backend/src/core/NoteCreateService.ts @@ -57,6 +57,7 @@ import { FeaturedService } from '@/core/FeaturedService.js'; import { FanoutTimelineService } from '@/core/FanoutTimelineService.js'; import { UtilityService } from '@/core/UtilityService.js'; import { UserBlockingService } from '@/core/UserBlockingService.js'; +import { isReply } from '@/misc/is-reply.js'; type NotificationType = 'reply' | 'renote' | 'quote' | 'mention'; @@ -891,7 +892,7 @@ export class NoteCreateService implements OnApplicationShutdown { if (note.visibility === 'specified' && !note.visibleUserIds.some(v => v === following.followerId)) continue; // 「自分自身への返信 or そのフォロワーへの返信」のどちらでもない場合 - if (note.replyId && !(note.replyUserId === note.userId || note.replyUserId === following.followerId)) { + if (isReply(note, following.followerId)) { if (!following.withReplies) continue; } @@ -909,7 +910,7 @@ export class NoteCreateService implements OnApplicationShutdown { ) continue; // 「自分自身への返信 or そのリストの作成者への返信」のどちらでもない場合 - if (note.replyId && !(note.replyUserId === note.userId || note.replyUserId === userListMembership.userListUserId)) { + if (isReply(note, userListMembership.userListUserId)) { if (!userListMembership.withReplies) continue; } @@ -927,7 +928,7 @@ export class NoteCreateService implements OnApplicationShutdown { } // 自分自身以外への返信 - if (note.replyId && note.replyUserId !== note.userId) { + if (isReply(note)) { this.fanoutTimelineService.push(`userTimelineWithReplies:${user.id}`, note.id, note.userHost == null ? meta.perLocalUserUserTimelineCacheMax : meta.perRemoteUserUserTimelineCacheMax, r); if (note.visibility === 'public' && note.userHost == null) { diff --git a/packages/backend/src/misc/is-reply.ts b/packages/backend/src/misc/is-reply.ts new file mode 100644 index 0000000000..964c2aa153 --- /dev/null +++ b/packages/backend/src/misc/is-reply.ts @@ -0,0 +1,10 @@ +/* + * SPDX-FileCopyrightText: syuilo and other misskey contributors + * SPDX-License-Identifier: AGPL-3.0-only + */ + +import { MiUser } from '@/models/User.js'; + +export function isReply(note: any, viewerId?: MiUser['id'] | undefined | null): boolean { + return note.replyId && note.replyUserId !== note.userId && note.replyUserId !== viewerId; +} diff --git a/packages/backend/src/server/api/endpoints/channels/timeline.ts b/packages/backend/src/server/api/endpoints/channels/timeline.ts index 9ef494d6d8..006228ceee 100644 --- a/packages/backend/src/server/api/endpoints/channels/timeline.ts +++ b/packages/backend/src/server/api/endpoints/channels/timeline.ts @@ -4,15 +4,13 @@ */ import { Inject, Injectable } from '@nestjs/common'; -import * as Redis from 'ioredis'; import { Endpoint } from '@/server/api/endpoint-base.js'; -import type { ChannelsRepository, MiNote, NotesRepository } from '@/models/_.js'; +import type { ChannelsRepository, NotesRepository } from '@/models/_.js'; import { QueryService } from '@/core/QueryService.js'; import { NoteEntityService } from '@/core/entities/NoteEntityService.js'; import ActiveUsersChart from '@/core/chart/charts/active-users.js'; import { DI } from '@/di-symbols.js'; import { IdService } from '@/core/IdService.js'; -import { isUserRelated } from '@/misc/is-user-related.js'; import { CacheService } from '@/core/CacheService.js'; import { MetaService } from '@/core/MetaService.js'; import { FanoutTimelineEndpointService } from '@/core/FanoutTimelineEndpointService.js'; @@ -94,12 +92,6 @@ export default class extends Endpoint { // eslint- return await this.noteEntityService.packMany(await this.getFromDb({ untilId, sinceId, limit: ps.limit, channelId: channel.id }, me), me); } - const [ - userIdsWhoMeMuting, - ] = me ? await Promise.all([ - this.cacheService.userMutingsCache.fetch(me.id), - ]) : [new Set()]; - return await this.fanoutTimelineEndpointService.timeline({ untilId, sinceId, @@ -108,11 +100,7 @@ export default class extends Endpoint { // eslint- me, useDbFallback: true, redisTimelines: [`channelTimeline:${channel.id}`], - noteFilter: note => { - if (me && isUserRelated(note, userIdsWhoMeMuting)) return false; - - return true; - }, + excludePureRenotes: false, dbFallback: async (untilId, sinceId, limit) => { return await this.getFromDb({ untilId, sinceId, limit, channelId: channel.id }, me); }, diff --git a/packages/backend/src/server/api/endpoints/notes/hybrid-timeline.ts b/packages/backend/src/server/api/endpoints/notes/hybrid-timeline.ts index deb9e014c4..effcbaf2ee 100644 --- a/packages/backend/src/server/api/endpoints/notes/hybrid-timeline.ts +++ b/packages/backend/src/server/api/endpoints/notes/hybrid-timeline.ts @@ -12,9 +12,8 @@ import { NoteEntityService } from '@/core/entities/NoteEntityService.js'; import { DI } from '@/di-symbols.js'; import { RoleService } from '@/core/RoleService.js'; import { IdService } from '@/core/IdService.js'; -import { isUserRelated } from '@/misc/is-user-related.js'; import { CacheService } from '@/core/CacheService.js'; -import { FanoutTimelineService } from '@/core/FanoutTimelineService.js'; +import { FanoutTimelineName } from '@/core/FanoutTimelineService.js'; import { QueryService } from '@/core/QueryService.js'; import { UserFollowingService } from '@/core/UserFollowingService.js'; import { MetaService } from '@/core/MetaService.js'; @@ -43,6 +42,12 @@ export const meta = { code: 'STL_DISABLED', id: '620763f4-f621-4533-ab33-0577a1a3c342', }, + + bothWithRepliesAndWithFiles: { + message: 'Specifying both withReplies and withFiles is not supported', + code: 'BOTH_WITH_REPLIES_AND_WITH_FILES', + id: 'dfaa3eb7-8002-4cb7-bcc4-1095df46656f' + }, }, } as const; @@ -93,6 +98,8 @@ export default class extends Endpoint { // eslint- throw new ApiError(meta.errors.stlDisabled); } + if (ps.withReplies && ps.withFiles) throw new ApiError(meta.errors.bothWithRepliesAndWithFiles); + const serverSettings = await this.metaService.fetch(); if (!serverSettings.enableFanoutTimeline) { @@ -114,17 +121,7 @@ export default class extends Endpoint { // eslint- return await this.noteEntityService.packMany(timeline, me); } - const [ - userIdsWhoMeMuting, - userIdsWhoMeMutingRenotes, - userIdsWhoBlockingMe, - ] = await Promise.all([ - this.cacheService.userMutingsCache.fetch(me.id), - this.cacheService.renoteMutingsCache.fetch(me.id), - this.cacheService.userBlockedCache.fetch(me.id), - ]); - - let timelineConfig: string[]; + let timelineConfig: FanoutTimelineName[]; if (ps.withFiles) { timelineConfig = [ @@ -152,21 +149,8 @@ export default class extends Endpoint { // eslint- me, redisTimelines: timelineConfig, useDbFallback: serverSettings.enableFanoutTimelineDbFallback, - noteFilter: (note) => { - if (note.userId === me.id) { - return true; - } - if (isUserRelated(note, userIdsWhoBlockingMe)) return false; - if (isUserRelated(note, userIdsWhoMeMuting)) return false; - if (note.renoteId) { - if (note.text == null && note.fileIds.length === 0 && !note.hasPoll) { - if (isUserRelated(note, userIdsWhoMeMutingRenotes)) return false; - if (ps.withRenotes === false) return false; - } - } - - return true; - }, + alwaysIncludeMyNotes: true, + excludePureRenotes: !ps.withRenotes, dbFallback: async (untilId, sinceId, limit) => await this.getFromDb({ untilId, sinceId, diff --git a/packages/backend/src/server/api/endpoints/notes/local-timeline.ts b/packages/backend/src/server/api/endpoints/notes/local-timeline.ts index 97b05016ec..e8ba39bbf0 100644 --- a/packages/backend/src/server/api/endpoints/notes/local-timeline.ts +++ b/packages/backend/src/server/api/endpoints/notes/local-timeline.ts @@ -5,7 +5,7 @@ import { Brackets } from 'typeorm'; import { Inject, Injectable } from '@nestjs/common'; -import type { MiNote, NotesRepository } from '@/models/_.js'; +import type { NotesRepository } from '@/models/_.js'; import { Endpoint } from '@/server/api/endpoint-base.js'; import { NoteEntityService } from '@/core/entities/NoteEntityService.js'; import ActiveUsersChart from '@/core/chart/charts/active-users.js'; @@ -13,7 +13,6 @@ import { DI } from '@/di-symbols.js'; import { RoleService } from '@/core/RoleService.js'; import { IdService } from '@/core/IdService.js'; import { CacheService } from '@/core/CacheService.js'; -import { isUserRelated } from '@/misc/is-user-related.js'; import { QueryService } from '@/core/QueryService.js'; import { MetaService } from '@/core/MetaService.js'; import { MiLocalUser } from '@/models/User.js'; @@ -39,6 +38,12 @@ export const meta = { code: 'LTL_DISABLED', id: '45a6eb02-7695-4393-b023-dd3be9aaaefd', }, + + bothWithRepliesAndWithFiles: { + message: 'Specifying both withReplies and withFiles is not supported', + code: 'BOTH_WITH_REPLIES_AND_WITH_FILES', + id: 'dd9c8400-1cb5-4eef-8a31-200c5f933793' + }, }, } as const; @@ -82,6 +87,8 @@ export default class extends Endpoint { // eslint- throw new ApiError(meta.errors.ltlDisabled); } + if (ps.withReplies && ps.withFiles) throw new ApiError(meta.errors.bothWithRepliesAndWithFiles); + const serverSettings = await this.metaService.fetch(); if (!serverSettings.enableFanoutTimeline) { @@ -102,16 +109,6 @@ export default class extends Endpoint { // eslint- return await this.noteEntityService.packMany(timeline, me); } - const [ - userIdsWhoMeMuting, - userIdsWhoMeMutingRenotes, - userIdsWhoBlockingMe, - ] = me ? await Promise.all([ - this.cacheService.userMutingsCache.fetch(me.id), - this.cacheService.renoteMutingsCache.fetch(me.id), - this.cacheService.userBlockedCache.fetch(me.id), - ]) : [new Set(), new Set(), new Set()]; - const timeline = await this.fanoutTimelineEndpointService.timeline({ untilId, sinceId, @@ -120,22 +117,9 @@ export default class extends Endpoint { // eslint- me, useDbFallback: serverSettings.enableFanoutTimelineDbFallback, redisTimelines: ps.withFiles ? ['localTimelineWithFiles'] : ['localTimeline', 'localTimelineWithReplies'], - noteFilter: note => { - if (me && (note.userId === me.id)) { - return true; - } - if (!ps.withReplies && note.replyId && note.replyUserId !== note.userId && (me == null || note.replyUserId !== me.id)) return false; - if (me && isUserRelated(note, userIdsWhoBlockingMe)) return false; - if (me && isUserRelated(note, userIdsWhoMeMuting)) return false; - if (note.renoteId) { - if (note.text == null && note.fileIds.length === 0 && !note.hasPoll) { - if (me && isUserRelated(note, userIdsWhoMeMutingRenotes)) return false; - if (ps.withRenotes === false) return false; - } - } - - return true; - }, + alwaysIncludeMyNotes: true, + excludeReplies: !ps.withReplies, + excludePureRenotes: !ps.withRenotes, dbFallback: async (untilId, sinceId, limit) => await this.getFromDb({ untilId, sinceId, diff --git a/packages/backend/src/server/api/endpoints/notes/timeline.ts b/packages/backend/src/server/api/endpoints/notes/timeline.ts index 74d0a6e0c0..790bcbe151 100644 --- a/packages/backend/src/server/api/endpoints/notes/timeline.ts +++ b/packages/backend/src/server/api/endpoints/notes/timeline.ts @@ -13,7 +13,6 @@ import { NoteEntityService } from '@/core/entities/NoteEntityService.js'; import { DI } from '@/di-symbols.js'; import { IdService } from '@/core/IdService.js'; import { CacheService } from '@/core/CacheService.js'; -import { isUserRelated } from '@/misc/is-user-related.js'; import { UserFollowingService } from '@/core/UserFollowingService.js'; import { MiLocalUser } from '@/models/User.js'; import { MetaService } from '@/core/MetaService.js'; @@ -98,14 +97,8 @@ export default class extends Endpoint { // eslint- const [ followings, - userIdsWhoMeMuting, - userIdsWhoMeMutingRenotes, - userIdsWhoBlockingMe, ] = await Promise.all([ this.cacheService.userFollowingsCache.fetch(me.id), - this.cacheService.userMutingsCache.fetch(me.id), - this.cacheService.renoteMutingsCache.fetch(me.id), - this.cacheService.userBlockedCache.fetch(me.id), ]); const timeline = this.fanoutTimelineEndpointService.timeline({ @@ -116,18 +109,9 @@ export default class extends Endpoint { // eslint- me, useDbFallback: serverSettings.enableFanoutTimelineDbFallback, redisTimelines: ps.withFiles ? [`homeTimelineWithFiles:${me.id}`] : [`homeTimeline:${me.id}`], + alwaysIncludeMyNotes: true, + excludePureRenotes: !ps.withRenotes, noteFilter: note => { - if (note.userId === me.id) { - return true; - } - if (isUserRelated(note, userIdsWhoBlockingMe)) return false; - if (isUserRelated(note, userIdsWhoMeMuting)) return false; - if (note.renoteId) { - if (note.text == null && note.fileIds.length === 0 && !note.hasPoll) { - if (isUserRelated(note, userIdsWhoMeMutingRenotes)) return false; - if (ps.withRenotes === false) return false; - } - } if (note.reply && note.reply.visibility === 'followers') { if (!Object.hasOwn(followings, note.reply.userId)) return false; } diff --git a/packages/backend/src/server/api/endpoints/notes/user-list-timeline.ts b/packages/backend/src/server/api/endpoints/notes/user-list-timeline.ts index f8f64738fe..10d3a7a697 100644 --- a/packages/backend/src/server/api/endpoints/notes/user-list-timeline.ts +++ b/packages/backend/src/server/api/endpoints/notes/user-list-timeline.ts @@ -5,20 +5,17 @@ import { Inject, Injectable } from '@nestjs/common'; import { Brackets } from 'typeorm'; -import type { MiNote, MiUserList, NotesRepository, UserListMembershipsRepository, UserListsRepository } from '@/models/_.js'; +import type { MiUserList, NotesRepository, UserListMembershipsRepository, UserListsRepository } from '@/models/_.js'; import { Endpoint } from '@/server/api/endpoint-base.js'; import { NoteEntityService } from '@/core/entities/NoteEntityService.js'; import ActiveUsersChart from '@/core/chart/charts/active-users.js'; import { DI } from '@/di-symbols.js'; import { CacheService } from '@/core/CacheService.js'; import { IdService } from '@/core/IdService.js'; -import { isUserRelated } from '@/misc/is-user-related.js'; -import { FanoutTimelineService } from '@/core/FanoutTimelineService.js'; import { QueryService } from '@/core/QueryService.js'; import { MiLocalUser } from '@/models/User.js'; import { MetaService } from '@/core/MetaService.js'; import { FanoutTimelineEndpointService } from '@/core/FanoutTimelineEndpointService.js'; -import { isInstanceMuted } from '@/misc/is-instance-muted.js'; import { ApiError } from '../../error.js'; export const meta = { @@ -84,7 +81,6 @@ export default class extends Endpoint { // eslint- private activeUsersChart: ActiveUsersChart, private cacheService: CacheService, private idService: IdService, - private fanoutTimelineService: FanoutTimelineService, private fanoutTimelineEndpointService: FanoutTimelineEndpointService, private queryService: QueryService, private metaService: MetaService, @@ -121,18 +117,6 @@ export default class extends Endpoint { // eslint- await this.noteEntityService.packMany(timeline, me); } - const [ - userIdsWhoMeMuting, - userIdsWhoMeMutingRenotes, - userIdsWhoBlockingMe, - userMutedInstances, - ] = await Promise.all([ - this.cacheService.userMutingsCache.fetch(me.id), - this.cacheService.renoteMutingsCache.fetch(me.id), - this.cacheService.userBlockedCache.fetch(me.id), - this.cacheService.userProfileCache.fetch(me.id).then(p => new Set(p.mutedInstances)), - ]); - const timeline = await this.fanoutTimelineEndpointService.timeline({ untilId, sinceId, @@ -141,22 +125,8 @@ export default class extends Endpoint { // eslint- me, useDbFallback: serverSettings.enableFanoutTimelineDbFallback, redisTimelines: ps.withFiles ? [`userListTimelineWithFiles:${list.id}`] : [`userListTimeline:${list.id}`], - noteFilter: note => { - if (note.userId === me.id) { - return true; - } - if (isUserRelated(note, userIdsWhoBlockingMe)) return false; - if (isUserRelated(note, userIdsWhoMeMuting)) return false; - if (note.renoteId) { - if (note.text == null && note.fileIds.length === 0 && !note.hasPoll) { - if (isUserRelated(note, userIdsWhoMeMutingRenotes)) return false; - if (ps.withRenotes === false) return false; - } - } - if (isInstanceMuted(note, userMutedInstances)) return false; - - return true; - }, + alwaysIncludeMyNotes: true, + excludePureRenotes: !ps.withRenotes, dbFallback: async (untilId, sinceId, limit) => await this.getFromDb(list, { untilId, sinceId, diff --git a/packages/backend/src/server/api/endpoints/users/notes.ts b/packages/backend/src/server/api/endpoints/users/notes.ts index 4a358b39cb..b32128a8aa 100644 --- a/packages/backend/src/server/api/endpoints/users/notes.ts +++ b/packages/backend/src/server/api/endpoints/users/notes.ts @@ -11,11 +11,12 @@ import { NoteEntityService } from '@/core/entities/NoteEntityService.js'; import { DI } from '@/di-symbols.js'; import { CacheService } from '@/core/CacheService.js'; import { IdService } from '@/core/IdService.js'; -import { isUserRelated } from '@/misc/is-user-related.js'; import { QueryService } from '@/core/QueryService.js'; import { MetaService } from '@/core/MetaService.js'; import { MiLocalUser } from '@/models/User.js'; import { FanoutTimelineEndpointService } from '@/core/FanoutTimelineEndpointService.js'; +import { FanoutTimelineName } from '@/core/FanoutTimelineService.js'; +import { ApiError } from '@/server/api/error.js'; export const meta = { tags: ['users', 'notes'], @@ -36,6 +37,12 @@ export const meta = { code: 'NO_SUCH_USER', id: '27e494ba-2ac2-48e8-893b-10d4d8c2387b', }, + + bothWithRepliesAndWithFiles: { + message: 'Specifying both withReplies and withFiles is not supported', + code: 'BOTH_WITH_REPLIES_AND_WITH_FILES', + id: '91c8cb9f-36ed-46e7-9ca2-7df96ed6e222', + }, }, } as const; @@ -77,6 +84,8 @@ export default class extends Endpoint { // eslint- const serverSettings = await this.metaService.fetch(); + if (ps.withReplies && ps.withFiles) throw new ApiError(meta.errors.bothWithRepliesAndWithFiles); + if (!serverSettings.enableFanoutTimeline) { const timeline = await this.getFromDb({ untilId, @@ -91,13 +100,7 @@ export default class extends Endpoint { // eslint- return await this.noteEntityService.packMany(timeline, me); } - const [ - userIdsWhoMeMuting, - ] = me ? await Promise.all([ - this.cacheService.userMutingsCache.fetch(me.id), - ]) : [new Set()]; - - const redisTimelines = [ps.withFiles ? `userTimelineWithFiles:${ps.userId}` : `userTimeline:${ps.userId}`]; + const redisTimelines: FanoutTimelineName[] = [ps.withFiles ? `userTimelineWithFiles:${ps.userId}` : `userTimeline:${ps.userId}`]; if (ps.withReplies) redisTimelines.push(`userTimelineWithReplies:${ps.userId}`); if (ps.withChannelNotes) redisTimelines.push(`userTimelineWithChannel:${ps.userId}`); @@ -112,18 +115,11 @@ export default class extends Endpoint { // eslint- me, redisTimelines, useDbFallback: true, + ignoreAuthorFromMute: true, + excludeReplies: ps.withChannelNotes && !ps.withReplies, // userTimelineWithChannel may include replies + excludeNoFiles: ps.withChannelNotes && ps.withFiles, // userTimelineWithChannel may include notes without files + excludePureRenotes: !ps.withRenotes, noteFilter: note => { - if (ps.withFiles && note.fileIds.length === 0) { - return false; - } - if (me && isUserRelated(note, userIdsWhoMeMuting, true)) return false; - - if (note.renoteId) { - if (note.text == null && note.fileIds.length === 0 && !note.hasPoll) { - if (ps.withRenotes === false) return false; - } - } - if (note.channel?.isSensitive && !isSelf) return false; if (note.visibility === 'specified' && (!me || (me.id !== note.userId && !note.visibleUserIds.some(v => v === me.id)))) return false; if (note.visibility === 'followers' && !isFollowing && !isSelf) return false; diff --git a/packages/frontend/src/components/MkMenu.vue b/packages/frontend/src/components/MkMenu.vue index 9457bf385f..4fafd35f72 100644 --- a/packages/frontend/src/components/MkMenu.vue +++ b/packages/frontend/src/components/MkMenu.vue @@ -202,7 +202,7 @@ function focusDown() { } function switchItem(item: MenuSwitch & { ref: any }) { - if (item.disabled) return; + if (typeof item.disabled === 'boolean' ? item.disabled : item.disabled.value) return; item.ref = !item.ref; } diff --git a/packages/frontend/src/pages/timeline.vue b/packages/frontend/src/pages/timeline.vue index cfe270aefb..b390d1931d 100644 --- a/packages/frontend/src/pages/timeline.vue +++ b/packages/frontend/src/pages/timeline.vue @@ -157,17 +157,17 @@ const headerActions = $computed(() => { os.popupMenu([{ type: 'switch', text: i18n.ts.showRenotes, - icon: 'ti ti-repeat', ref: $$(withRenotes), }, src === 'local' || src === 'social' ? { type: 'switch', text: i18n.ts.showRepliesToOthersInTimeline, ref: $$(withReplies), + disabled: $$(onlyFiles), } : undefined, { type: 'switch', text: i18n.ts.fileAttachedOnly, - icon: 'ti ti-photo', ref: $$(onlyFiles), + disabled: src === 'local' || src === 'social' ? $$(withReplies) : false, }], ev.currentTarget ?? ev.target); }, }, diff --git a/packages/frontend/src/types/menu.ts b/packages/frontend/src/types/menu.ts index 66061fcd70..fbe627176b 100644 --- a/packages/frontend/src/types/menu.ts +++ b/packages/frontend/src/types/menu.ts @@ -14,7 +14,7 @@ export type MenuLabel = { type: 'label', text: string }; export type MenuLink = { type: 'link', to: string, text: string, icon?: string, indicate?: boolean, avatar?: Misskey.entities.User }; export type MenuA = { type: 'a', href: string, target?: string, download?: string, text: string, icon?: string, indicate?: boolean }; export type MenuUser = { type: 'user', user: Misskey.entities.User, active?: boolean, indicate?: boolean, action: MenuAction }; -export type MenuSwitch = { type: 'switch', ref: Ref, text: string, disabled?: boolean }; +export type MenuSwitch = { type: 'switch', ref: Ref, text: string, disabled?: boolean | Ref }; export type MenuButton = { type?: 'button', text: string, icon?: string, indicate?: boolean, danger?: boolean, active?: boolean, avatar?: Misskey.entities.User; action: MenuAction }; export type MenuParent = { type: 'parent', text: string, icon?: string, children: MenuItem[] | (() => Promise | MenuItem[]) }; diff --git a/packages/frontend/src/ui/deck/tl-column.vue b/packages/frontend/src/ui/deck/tl-column.vue index 9f24ea31ed..41582bbfe3 100644 --- a/packages/frontend/src/ui/deck/tl-column.vue +++ b/packages/frontend/src/ui/deck/tl-column.vue @@ -120,10 +120,12 @@ const menu = [{ type: 'switch', text: i18n.ts.showRepliesToOthersInTimeline, ref: $$(withReplies), + disabled: $$(onlyFiles), } : undefined, { type: 'switch', text: i18n.ts.fileAttachedOnly, ref: $$(onlyFiles), + disabled: props.column.tl === 'local' || props.column.tl === 'social' ? $$(withReplies) : false, }]; -- cgit v1.2.3-freya