From 2a88d8ee2d153bc7dbfcd027154c2c658fc06142 Mon Sep 17 00:00:00 2001 From: syuilo Date: Wed, 18 Oct 2023 12:07:18 +0900 Subject: refactor(backend): rename service --- .../backend/src/server/api/endpoints/notes/hybrid-timeline.ts | 10 +++++----- .../backend/src/server/api/endpoints/notes/local-timeline.ts | 8 ++++---- packages/backend/src/server/api/endpoints/notes/timeline.ts | 6 +++--- .../src/server/api/endpoints/notes/user-list-timeline.ts | 6 +++--- 4 files changed, 15 insertions(+), 15 deletions(-) (limited to 'packages/backend/src/server/api/endpoints/notes') diff --git a/packages/backend/src/server/api/endpoints/notes/hybrid-timeline.ts b/packages/backend/src/server/api/endpoints/notes/hybrid-timeline.ts index 378529e30d..adb0f87bba 100644 --- a/packages/backend/src/server/api/endpoints/notes/hybrid-timeline.ts +++ b/packages/backend/src/server/api/endpoints/notes/hybrid-timeline.ts @@ -15,7 +15,7 @@ import { RoleService } from '@/core/RoleService.js'; import { IdService } from '@/core/IdService.js'; import { isUserRelated } from '@/misc/is-user-related.js'; import { CacheService } from '@/core/CacheService.js'; -import { RedisTimelineService } from '@/core/RedisTimelineService.js'; +import { FunoutTimelineService } from '@/core/FunoutTimelineService.js'; import { ApiError } from '../../error.js'; export const meta = { @@ -74,7 +74,7 @@ export default class extends Endpoint { // eslint- private activeUsersChart: ActiveUsersChart, private idService: IdService, private cacheService: CacheService, - private redisTimelineService: RedisTimelineService, + private funoutTimelineService: FunoutTimelineService, ) { super(meta, paramDef, async (ps, me) => { const untilId = ps.untilId ?? (ps.untilDate ? this.idService.gen(ps.untilDate!) : null); @@ -98,20 +98,20 @@ export default class extends Endpoint { // eslint- let noteIds: string[]; if (ps.withFiles) { - const [htlNoteIds, ltlNoteIds] = await this.redisTimelineService.getMulti([ + const [htlNoteIds, ltlNoteIds] = await this.funoutTimelineService.getMulti([ `homeTimelineWithFiles:${me.id}`, 'localTimelineWithFiles', ], untilId, sinceId); noteIds = Array.from(new Set([...htlNoteIds, ...ltlNoteIds])); } else if (ps.withReplies) { - const [htlNoteIds, ltlNoteIds, ltlReplyNoteIds] = await this.redisTimelineService.getMulti([ + const [htlNoteIds, ltlNoteIds, ltlReplyNoteIds] = await this.funoutTimelineService.getMulti([ `homeTimeline:${me.id}`, 'localTimeline', 'localTimelineWithReplies', ], untilId, sinceId); noteIds = Array.from(new Set([...htlNoteIds, ...ltlNoteIds, ...ltlReplyNoteIds])); } else { - const [htlNoteIds, ltlNoteIds] = await this.redisTimelineService.getMulti([ + const [htlNoteIds, ltlNoteIds] = await this.funoutTimelineService.getMulti([ `homeTimeline:${me.id}`, 'localTimeline', ], untilId, sinceId); diff --git a/packages/backend/src/server/api/endpoints/notes/local-timeline.ts b/packages/backend/src/server/api/endpoints/notes/local-timeline.ts index f69e60ab54..d39b0358f5 100644 --- a/packages/backend/src/server/api/endpoints/notes/local-timeline.ts +++ b/packages/backend/src/server/api/endpoints/notes/local-timeline.ts @@ -15,7 +15,7 @@ import { RoleService } from '@/core/RoleService.js'; import { IdService } from '@/core/IdService.js'; import { CacheService } from '@/core/CacheService.js'; import { isUserRelated } from '@/misc/is-user-related.js'; -import { RedisTimelineService } from '@/core/RedisTimelineService.js'; +import { FunoutTimelineService } from '@/core/FunoutTimelineService.js'; import { ApiError } from '../../error.js'; export const meta = { @@ -70,7 +70,7 @@ export default class extends Endpoint { // eslint- private activeUsersChart: ActiveUsersChart, private idService: IdService, private cacheService: CacheService, - private redisTimelineService: RedisTimelineService, + private funoutTimelineService: FunoutTimelineService, ) { super(meta, paramDef, async (ps, me) => { const untilId = ps.untilId ?? (ps.untilDate ? this.idService.gen(ps.untilDate!) : null); @@ -94,9 +94,9 @@ export default class extends Endpoint { // eslint- let noteIds: string[]; if (ps.withFiles) { - noteIds = await this.redisTimelineService.get('localTimelineWithFiles', untilId, sinceId); + noteIds = await this.funoutTimelineService.get('localTimelineWithFiles', untilId, sinceId); } else { - const [nonReplyNoteIds, replyNoteIds] = await this.redisTimelineService.getMulti([ + const [nonReplyNoteIds, replyNoteIds] = await this.funoutTimelineService.getMulti([ 'localTimeline', 'localTimelineWithReplies', ], untilId, sinceId); diff --git a/packages/backend/src/server/api/endpoints/notes/timeline.ts b/packages/backend/src/server/api/endpoints/notes/timeline.ts index 8f13b3a4ba..c0b78b8ee9 100644 --- a/packages/backend/src/server/api/endpoints/notes/timeline.ts +++ b/packages/backend/src/server/api/endpoints/notes/timeline.ts @@ -15,7 +15,7 @@ import { DI } from '@/di-symbols.js'; import { IdService } from '@/core/IdService.js'; import { CacheService } from '@/core/CacheService.js'; import { isUserRelated } from '@/misc/is-user-related.js'; -import { RedisTimelineService } from '@/core/RedisTimelineService.js'; +import { FunoutTimelineService } from '@/core/FunoutTimelineService.js'; export const meta = { tags: ['notes'], @@ -63,7 +63,7 @@ export default class extends Endpoint { // eslint- private activeUsersChart: ActiveUsersChart, private idService: IdService, private cacheService: CacheService, - private redisTimelineService: RedisTimelineService, + private funoutTimelineService: FunoutTimelineService, ) { super(meta, paramDef, async (ps, me) => { const untilId = ps.untilId ?? (ps.untilDate ? this.idService.gen(ps.untilDate!) : null); @@ -81,7 +81,7 @@ export default class extends Endpoint { // eslint- this.cacheService.userBlockedCache.fetch(me.id), ]); - let noteIds = await this.redisTimelineService.get(ps.withFiles ? `homeTimelineWithFiles:${me.id}` : `homeTimeline:${me.id}`, untilId, sinceId); + let noteIds = await this.funoutTimelineService.get(ps.withFiles ? `homeTimelineWithFiles:${me.id}` : `homeTimeline:${me.id}`, untilId, sinceId); noteIds = noteIds.slice(0, ps.limit); if (noteIds.length === 0) { diff --git a/packages/backend/src/server/api/endpoints/notes/user-list-timeline.ts b/packages/backend/src/server/api/endpoints/notes/user-list-timeline.ts index b8007e78fd..2b31e6169c 100644 --- a/packages/backend/src/server/api/endpoints/notes/user-list-timeline.ts +++ b/packages/backend/src/server/api/endpoints/notes/user-list-timeline.ts @@ -15,7 +15,7 @@ import { DI } from '@/di-symbols.js'; import { CacheService } from '@/core/CacheService.js'; import { IdService } from '@/core/IdService.js'; import { isUserRelated } from '@/misc/is-user-related.js'; -import { RedisTimelineService } from '@/core/RedisTimelineService.js'; +import { FunoutTimelineService } from '@/core/FunoutTimelineService.js'; import { ApiError } from '../../error.js'; export const meta = { @@ -80,7 +80,7 @@ export default class extends Endpoint { // eslint- private activeUsersChart: ActiveUsersChart, private cacheService: CacheService, private idService: IdService, - private redisTimelineService: RedisTimelineService, + private funoutTimelineService: FunoutTimelineService, ) { super(meta, paramDef, async (ps, me) => { const untilId = ps.untilId ?? (ps.untilDate ? this.idService.gen(ps.untilDate!) : null); @@ -105,7 +105,7 @@ export default class extends Endpoint { // eslint- this.cacheService.userBlockedCache.fetch(me.id), ]); - let noteIds = await this.redisTimelineService.get(ps.withFiles ? `userListTimelineWithFiles:${list.id}` : `userListTimeline:${list.id}`, untilId, sinceId); + let noteIds = await this.funoutTimelineService.get(ps.withFiles ? `userListTimelineWithFiles:${list.id}` : `userListTimeline:${list.id}`, untilId, sinceId); noteIds = noteIds.slice(0, ps.limit); if (noteIds.length === 0) { -- cgit v1.2.3-freya From 6cc02fee99d11d3866c6f5df8879d857d31f7f4c Mon Sep 17 00:00:00 2001 From: syuilo Date: Wed, 18 Oct 2023 12:26:16 +0900 Subject: enhance(backend): improve fanout tl Resolve #11958 Resolve #12061 --- CHANGELOG.md | 2 + packages/backend/src/core/FunoutTimelineService.ts | 5 + packages/backend/src/core/UserFollowingService.ts | 18 ++- .../server/api/endpoints/notes/local-timeline.ts | 102 +++++++++------ .../src/server/api/endpoints/notes/timeline.ts | 138 +++++++++++++++------ 5 files changed, 187 insertions(+), 78 deletions(-) (limited to 'packages/backend/src/server/api/endpoints/notes') diff --git a/CHANGELOG.md b/CHANGELOG.md index b33f0e3df1..faaac57de7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,6 +19,8 @@ - Feat: サーバーサイレンス機能が追加されました - Enhance: 依存関係の更新 - Enhance: 新規にフォローした人のをデフォルトでTLに追加できるように +- Enhance: HTLとLTLを2023.10.0アップデート以前まで遡れるように +- Enhance: フォロー/フォロー解除したときに過去分のHTLにも含まれる投稿が反映されるように ### Client - Enhance: TLの返信表示オプションを記憶するように diff --git a/packages/backend/src/core/FunoutTimelineService.ts b/packages/backend/src/core/FunoutTimelineService.ts index 512b2d9833..c633c329e5 100644 --- a/packages/backend/src/core/FunoutTimelineService.ts +++ b/packages/backend/src/core/FunoutTimelineService.ts @@ -77,4 +77,9 @@ export class FunoutTimelineService { ); }); } + + @bindThis + public purge(name: string) { + return this.redisForTimelines.del('list:' + name); + } } diff --git a/packages/backend/src/core/UserFollowingService.ts b/packages/backend/src/core/UserFollowingService.ts index 7548e3d28f..4d7e14f683 100644 --- a/packages/backend/src/core/UserFollowingService.ts +++ b/packages/backend/src/core/UserFollowingService.ts @@ -29,6 +29,7 @@ import { CacheService } from '@/core/CacheService.js'; import type { Config } from '@/config.js'; import { AccountMoveService } from '@/core/AccountMoveService.js'; import { UtilityService } from '@/core/UtilityService.js'; +import { FunoutTimelineService } from '@/core/FunoutTimelineService.js'; import Logger from '../logger.js'; const logger = new Logger('following/create'); @@ -83,6 +84,7 @@ export class UserFollowingService implements OnModuleInit { private webhookService: WebhookService, private apRendererService: ApRendererService, private accountMoveService: AccountMoveService, + private funoutTimelineService: FunoutTimelineService, private perUserFollowingChart: PerUserFollowingChart, private instanceChart: InstanceChart, ) { @@ -288,8 +290,8 @@ export class UserFollowingService implements OnModuleInit { this.perUserFollowingChart.update(follower, followee, true); } - // Publish follow event if (this.userEntityService.isLocalUser(follower) && !silent) { + // Publish follow event this.userEntityService.pack(followee.id, follower, { detail: true, }).then(async packed => { @@ -302,6 +304,8 @@ export class UserFollowingService implements OnModuleInit { }); } }); + + this.funoutTimelineService.purge(`homeTimeline:${follower.id}`); } // Publish followed event @@ -355,8 +359,8 @@ export class UserFollowingService implements OnModuleInit { this.decrementFollowing(following.follower, following.followee); - // Publish unfollow event if (!silent && this.userEntityService.isLocalUser(follower)) { + // Publish unfollow event this.userEntityService.pack(followee.id, follower, { detail: true, }).then(async packed => { @@ -369,6 +373,8 @@ export class UserFollowingService implements OnModuleInit { }); } }); + + this.funoutTimelineService.purge(`homeTimeline:${follower.id}`); } if (this.userEntityService.isLocalUser(follower) && this.userEntityService.isRemoteUser(followee)) { @@ -707,4 +713,12 @@ export class UserFollowingService implements OnModuleInit { }); } } + + @bindThis + public getFollowees(userId: MiUser['id']) { + return this.followingsRepository.createQueryBuilder('following') + .select('following.followeeId') + .where('following.followerId = :followerId', { followerId: userId }) + .getMany(); + } } diff --git a/packages/backend/src/server/api/endpoints/notes/local-timeline.ts b/packages/backend/src/server/api/endpoints/notes/local-timeline.ts index d39b0358f5..9d5688f96f 100644 --- a/packages/backend/src/server/api/endpoints/notes/local-timeline.ts +++ b/packages/backend/src/server/api/endpoints/notes/local-timeline.ts @@ -5,7 +5,6 @@ import { Brackets } from 'typeorm'; import { Inject, Injectable } from '@nestjs/common'; -import * as Redis from 'ioredis'; import type { MiNote, NotesRepository } from '@/models/_.js'; import { Endpoint } from '@/server/api/endpoint-base.js'; import { NoteEntityService } from '@/core/entities/NoteEntityService.js'; @@ -16,6 +15,7 @@ import { IdService } from '@/core/IdService.js'; import { CacheService } from '@/core/CacheService.js'; import { isUserRelated } from '@/misc/is-user-related.js'; import { FunoutTimelineService } from '@/core/FunoutTimelineService.js'; +import { QueryService } from '@/core/QueryService.js'; import { ApiError } from '../../error.js'; export const meta = { @@ -59,9 +59,6 @@ export const paramDef = { @Injectable() export default class extends Endpoint { // eslint-disable-line import/no-default-export constructor( - @Inject(DI.redisForTimelines) - private redisForTimelines: Redis.Redis, - @Inject(DI.notesRepository) private notesRepository: NotesRepository, @@ -71,6 +68,7 @@ export default class extends Endpoint { // eslint- private idService: IdService, private cacheService: CacheService, private funoutTimelineService: FunoutTimelineService, + private queryService: QueryService, ) { super(meta, paramDef, async (ps, me) => { const untilId = ps.untilId ?? (ps.untilDate ? this.idService.gen(ps.untilDate!) : null); @@ -106,49 +104,75 @@ export default class extends Endpoint { // eslint- noteIds = noteIds.slice(0, ps.limit); - if (noteIds.length === 0) { - return []; - } + if (noteIds.length > 0) { + const query = this.notesRepository.createQueryBuilder('note') + .where('note.id IN (:...noteIds)', { noteIds: noteIds }) + .innerJoinAndSelect('note.user', 'user') + .leftJoinAndSelect('note.reply', 'reply') + .leftJoinAndSelect('note.renote', 'renote') + .leftJoinAndSelect('reply.user', 'replyUser') + .leftJoinAndSelect('renote.user', 'renoteUser') + .leftJoinAndSelect('note.channel', 'channel'); + + let timeline = await query.getMany(); + + timeline = timeline.filter(note => { + if (me && (note.userId === me.id)) { + return true; + } + if (!ps.withReplies && note.replyId && (me == null || note.replyUserId !== me.id)) return false; + if (me && isUserRelated(note, userIdsWhoBlockingMe)) return false; + if (me && isUserRelated(note, userIdsWhoMeMuting)) return false; + if (note.renoteId) { + if (note.text == null && note.fileIds.length === 0 && !note.hasPoll) { + if (me && isUserRelated(note, userIdsWhoMeMutingRenotes)) return false; + if (ps.withRenotes === false) return false; + } + } - const query = this.notesRepository.createQueryBuilder('note') - .where('note.id IN (:...noteIds)', { noteIds: noteIds }) - .innerJoinAndSelect('note.user', 'user') - .leftJoinAndSelect('note.reply', 'reply') - .leftJoinAndSelect('note.renote', 'renote') - .leftJoinAndSelect('reply.user', 'replyUser') - .leftJoinAndSelect('renote.user', 'renoteUser') - .leftJoinAndSelect('note.channel', 'channel'); + return true; + }); - let timeline = await query.getMany(); + // TODO: フィルタした結果件数が足りなかった場合の対応 - timeline = timeline.filter(note => { - if (me && (note.userId === me.id)) { - return true; - } - if (!ps.withReplies && note.replyId && (me == null || note.replyUserId !== me.id)) return false; - if (me && isUserRelated(note, userIdsWhoBlockingMe)) return false; - if (me && isUserRelated(note, userIdsWhoMeMuting)) return false; - if (note.renoteId) { - if (note.text == null && note.fileIds.length === 0 && !note.hasPoll) { - if (me && isUserRelated(note, userIdsWhoMeMutingRenotes)) return false; - if (ps.withRenotes === false) return false; + timeline.sort((a, b) => a.id > b.id ? -1 : 1); + + process.nextTick(() => { + if (me) { + this.activeUsersChart.read(me); } + }); + + return await this.noteEntityService.packMany(timeline, me); + } else { // fallback to db + const query = this.queryService.makePaginationQuery(this.notesRepository.createQueryBuilder('note'), + ps.sinceId, ps.untilId, ps.sinceDate, ps.untilDate) + .andWhere('(note.visibility = \'public\') AND (note.userHost IS NULL)') + .innerJoinAndSelect('note.user', 'user') + .leftJoinAndSelect('note.reply', 'reply') + .leftJoinAndSelect('note.renote', 'renote') + .leftJoinAndSelect('reply.user', 'replyUser') + .leftJoinAndSelect('renote.user', 'renoteUser'); + + this.queryService.generateVisibilityQuery(query, me); + if (me) this.queryService.generateMutedUserQuery(query, me); + if (me) this.queryService.generateBlockedUserQuery(query, me); + if (me) this.queryService.generateMutedUserRenotesQueryForNotes(query, me); + + if (ps.withFiles) { + query.andWhere('note.fileIds != \'{}\''); } - return true; - }); - - // TODO: フィルタした結果件数が足りなかった場合の対応 - - timeline.sort((a, b) => a.id > b.id ? -1 : 1); + const timeline = await query.limit(ps.limit).getMany(); - process.nextTick(() => { - if (me) { - this.activeUsersChart.read(me); - } - }); + process.nextTick(() => { + if (me) { + this.activeUsersChart.read(me); + } + }); - return await this.noteEntityService.packMany(timeline, me); + return await this.noteEntityService.packMany(timeline, me); + } }); } } diff --git a/packages/backend/src/server/api/endpoints/notes/timeline.ts b/packages/backend/src/server/api/endpoints/notes/timeline.ts index c0b78b8ee9..8660e1602b 100644 --- a/packages/backend/src/server/api/endpoints/notes/timeline.ts +++ b/packages/backend/src/server/api/endpoints/notes/timeline.ts @@ -5,8 +5,7 @@ import { Brackets } from 'typeorm'; import { Inject, Injectable } from '@nestjs/common'; -import * as Redis from 'ioredis'; -import type { NotesRepository, FollowingsRepository, MiNote } from '@/models/_.js'; +import type { NotesRepository } from '@/models/_.js'; import { Endpoint } from '@/server/api/endpoint-base.js'; import { QueryService } from '@/core/QueryService.js'; import ActiveUsersChart from '@/core/chart/charts/active-users.js'; @@ -16,6 +15,7 @@ import { IdService } from '@/core/IdService.js'; import { CacheService } from '@/core/CacheService.js'; import { isUserRelated } from '@/misc/is-user-related.js'; import { FunoutTimelineService } from '@/core/FunoutTimelineService.js'; +import { UserFollowingService } from '@/core/UserFollowingService.js'; export const meta = { tags: ['notes'], @@ -53,9 +53,6 @@ export const paramDef = { @Injectable() export default class extends Endpoint { // eslint-disable-line import/no-default-export constructor( - @Inject(DI.redisForTimelines) - private redisForTimelines: Redis.Redis, - @Inject(DI.notesRepository) private notesRepository: NotesRepository, @@ -64,6 +61,8 @@ export default class extends Endpoint { // eslint- private idService: IdService, private cacheService: CacheService, private funoutTimelineService: FunoutTimelineService, + private userFollowingService: UserFollowingService, + private queryService: QueryService, ) { super(meta, paramDef, async (ps, me) => { const untilId = ps.untilId ?? (ps.untilDate ? this.idService.gen(ps.untilDate!) : null); @@ -84,49 +83,114 @@ export default class extends Endpoint { // eslint- let noteIds = await this.funoutTimelineService.get(ps.withFiles ? `homeTimelineWithFiles:${me.id}` : `homeTimeline:${me.id}`, untilId, sinceId); noteIds = noteIds.slice(0, ps.limit); - if (noteIds.length === 0) { - return []; - } + if (noteIds.length > 0) { + const query = this.notesRepository.createQueryBuilder('note') + .where('note.id IN (:...noteIds)', { noteIds: noteIds }) + .innerJoinAndSelect('note.user', 'user') + .leftJoinAndSelect('note.reply', 'reply') + .leftJoinAndSelect('note.renote', 'renote') + .leftJoinAndSelect('reply.user', 'replyUser') + .leftJoinAndSelect('renote.user', 'renoteUser') + .leftJoinAndSelect('note.channel', 'channel'); + + let timeline = await query.getMany(); + + timeline = timeline.filter(note => { + if (note.userId === me.id) { + return true; + } + if (isUserRelated(note, userIdsWhoBlockingMe)) return false; + if (isUserRelated(note, userIdsWhoMeMuting)) return false; + if (note.renoteId) { + if (note.text == null && note.fileIds.length === 0 && !note.hasPoll) { + if (isUserRelated(note, userIdsWhoMeMutingRenotes)) return false; + if (ps.withRenotes === false) return false; + } + } + if (note.reply && note.reply.visibility === 'followers') { + if (!Object.hasOwn(followings, note.reply.userId)) return false; + } - const query = this.notesRepository.createQueryBuilder('note') - .where('note.id IN (:...noteIds)', { noteIds: noteIds }) - .innerJoinAndSelect('note.user', 'user') - .leftJoinAndSelect('note.reply', 'reply') - .leftJoinAndSelect('note.renote', 'renote') - .leftJoinAndSelect('reply.user', 'replyUser') - .leftJoinAndSelect('renote.user', 'renoteUser') - .leftJoinAndSelect('note.channel', 'channel'); + return true; + }); - let timeline = await query.getMany(); + // TODO: フィルタした結果件数が足りなかった場合の対応 - timeline = timeline.filter(note => { - if (note.userId === me.id) { - return true; + timeline.sort((a, b) => a.id > b.id ? -1 : 1); + + process.nextTick(() => { + this.activeUsersChart.read(me); + }); + + return await this.noteEntityService.packMany(timeline, me); + } else { // fallback to db + const followees = await this.userFollowingService.getFollowees(me.id); + + //#region Construct query + const query = this.queryService.makePaginationQuery(this.notesRepository.createQueryBuilder('note'), + ps.sinceId, ps.untilId, ps.sinceDate, ps.untilDate) + .innerJoinAndSelect('note.user', 'user') + .leftJoinAndSelect('note.reply', 'reply') + .leftJoinAndSelect('note.renote', 'renote') + .leftJoinAndSelect('reply.user', 'replyUser') + .leftJoinAndSelect('renote.user', 'renoteUser'); + + if (followees.length > 0) { + const meOrFolloweeIds = [me.id, ...followees.map(f => f.followeeId)]; + + query.andWhere('note.userId IN (:...meOrFolloweeIds)', { meOrFolloweeIds: meOrFolloweeIds }); + } else { + query.andWhere('note.userId = :meId', { meId: me.id }); } - if (isUserRelated(note, userIdsWhoBlockingMe)) return false; - if (isUserRelated(note, userIdsWhoMeMuting)) return false; - if (note.renoteId) { - if (note.text == null && note.fileIds.length === 0 && !note.hasPoll) { - if (isUserRelated(note, userIdsWhoMeMutingRenotes)) return false; - if (ps.withRenotes === false) return false; - } + + this.queryService.generateVisibilityQuery(query, me); + this.queryService.generateMutedUserQuery(query, me); + this.queryService.generateBlockedUserQuery(query, me); + this.queryService.generateMutedUserRenotesQueryForNotes(query, me); + + if (ps.includeMyRenotes === false) { + query.andWhere(new Brackets(qb => { + qb.orWhere('note.userId != :meId', { meId: me.id }); + qb.orWhere('note.renoteId IS NULL'); + qb.orWhere('note.text IS NOT NULL'); + qb.orWhere('note.fileIds != \'{}\''); + qb.orWhere('0 < (SELECT COUNT(*) FROM poll WHERE poll."noteId" = note.id)'); + })); } - if (note.reply && note.reply.visibility === 'followers') { - if (!Object.hasOwn(followings, note.reply.userId)) return false; + + if (ps.includeRenotedMyNotes === false) { + query.andWhere(new Brackets(qb => { + qb.orWhere('note.renoteUserId != :meId', { meId: me.id }); + qb.orWhere('note.renoteId IS NULL'); + qb.orWhere('note.text IS NOT NULL'); + qb.orWhere('note.fileIds != \'{}\''); + qb.orWhere('0 < (SELECT COUNT(*) FROM poll WHERE poll."noteId" = note.id)'); + })); } - return true; - }); + if (ps.includeLocalRenotes === false) { + query.andWhere(new Brackets(qb => { + qb.orWhere('note.renoteUserHost IS NOT NULL'); + qb.orWhere('note.renoteId IS NULL'); + qb.orWhere('note.text IS NOT NULL'); + qb.orWhere('note.fileIds != \'{}\''); + qb.orWhere('0 < (SELECT COUNT(*) FROM poll WHERE poll."noteId" = note.id)'); + })); + } - // TODO: フィルタした結果件数が足りなかった場合の対応 + if (ps.withFiles) { + query.andWhere('note.fileIds != \'{}\''); + } + //#endregion - timeline.sort((a, b) => a.id > b.id ? -1 : 1); + const timeline = await query.limit(ps.limit).getMany(); - process.nextTick(() => { - this.activeUsersChart.read(me); - }); + process.nextTick(() => { + this.activeUsersChart.read(me); + }); - return await this.noteEntityService.packMany(timeline, me); + return await this.noteEntityService.packMany(timeline, me); + } }); } } -- cgit v1.2.3-freya From 6b5ee43800a8e4679f15c2ae7f1f79fbe177b950 Mon Sep 17 00:00:00 2001 From: syuilo Date: Wed, 18 Oct 2023 15:51:50 +0900 Subject: enhance(backend): improve fanout tl for stl --- .../server/api/endpoints/notes/hybrid-timeline.ts | 138 +++++++++++++++------ 1 file changed, 103 insertions(+), 35 deletions(-) (limited to 'packages/backend/src/server/api/endpoints/notes') diff --git a/packages/backend/src/server/api/endpoints/notes/hybrid-timeline.ts b/packages/backend/src/server/api/endpoints/notes/hybrid-timeline.ts index adb0f87bba..faaeb32c79 100644 --- a/packages/backend/src/server/api/endpoints/notes/hybrid-timeline.ts +++ b/packages/backend/src/server/api/endpoints/notes/hybrid-timeline.ts @@ -5,7 +5,6 @@ import { Brackets } from 'typeorm'; import { Inject, Injectable } from '@nestjs/common'; -import * as Redis from 'ioredis'; import type { NotesRepository, FollowingsRepository, MiNote } from '@/models/_.js'; import { Endpoint } from '@/server/api/endpoint-base.js'; import ActiveUsersChart from '@/core/chart/charts/active-users.js'; @@ -16,6 +15,8 @@ import { IdService } from '@/core/IdService.js'; import { isUserRelated } from '@/misc/is-user-related.js'; import { CacheService } from '@/core/CacheService.js'; import { FunoutTimelineService } from '@/core/FunoutTimelineService.js'; +import { QueryService } from '@/core/QueryService.js'; +import { UserFollowingService } from '@/core/UserFollowingService.js'; import { ApiError } from '../../error.js'; export const meta = { @@ -63,9 +64,6 @@ export const paramDef = { @Injectable() export default class extends Endpoint { // eslint-disable-line import/no-default-export constructor( - @Inject(DI.redisForTimelines) - private redisForTimelines: Redis.Redis, - @Inject(DI.notesRepository) private notesRepository: NotesRepository, @@ -75,6 +73,8 @@ export default class extends Endpoint { // eslint- private idService: IdService, private cacheService: CacheService, private funoutTimelineService: FunoutTimelineService, + private queryService: QueryService, + private userFollowingService: UserFollowingService, ) { super(meta, paramDef, async (ps, me) => { const untilId = ps.untilId ?? (ps.untilDate ? this.idService.gen(ps.untilDate!) : null); @@ -96,6 +96,7 @@ export default class extends Endpoint { // eslint- ]); let noteIds: string[]; + let shouldFallbackToDb = false; if (ps.withFiles) { const [htlNoteIds, ltlNoteIds] = await this.funoutTimelineService.getMulti([ @@ -116,51 +117,118 @@ export default class extends Endpoint { // eslint- 'localTimeline', ], untilId, sinceId); noteIds = Array.from(new Set([...htlNoteIds, ...ltlNoteIds])); + shouldFallbackToDb = htlNoteIds.length === 0; } noteIds.sort((a, b) => a > b ? -1 : 1); noteIds = noteIds.slice(0, ps.limit); - if (noteIds.length === 0) { - return []; - } - - const query = this.notesRepository.createQueryBuilder('note') - .where('note.id IN (:...noteIds)', { noteIds: noteIds }) - .innerJoinAndSelect('note.user', 'user') - .leftJoinAndSelect('note.reply', 'reply') - .leftJoinAndSelect('note.renote', 'renote') - .leftJoinAndSelect('reply.user', 'replyUser') - .leftJoinAndSelect('renote.user', 'renoteUser') - .leftJoinAndSelect('note.channel', 'channel'); - - let timeline = await query.getMany(); + if (!shouldFallbackToDb) { + const query = this.notesRepository.createQueryBuilder('note') + .where('note.id IN (:...noteIds)', { noteIds: noteIds }) + .innerJoinAndSelect('note.user', 'user') + .leftJoinAndSelect('note.reply', 'reply') + .leftJoinAndSelect('note.renote', 'renote') + .leftJoinAndSelect('reply.user', 'replyUser') + .leftJoinAndSelect('renote.user', 'renoteUser') + .leftJoinAndSelect('note.channel', 'channel'); + + let timeline = await query.getMany(); + + timeline = timeline.filter(note => { + if (note.userId === me.id) { + return true; + } + if (isUserRelated(note, userIdsWhoBlockingMe)) return false; + if (isUserRelated(note, userIdsWhoMeMuting)) return false; + if (note.renoteId) { + if (note.text == null && note.fileIds.length === 0 && !note.hasPoll) { + if (isUserRelated(note, userIdsWhoMeMutingRenotes)) return false; + if (ps.withRenotes === false) return false; + } + } - timeline = timeline.filter(note => { - if (note.userId === me.id) { return true; + }); + + // TODO: フィルタした結果件数が足りなかった場合の対応 + + timeline.sort((a, b) => a.id > b.id ? -1 : 1); + + process.nextTick(() => { + this.activeUsersChart.read(me); + }); + + return await this.noteEntityService.packMany(timeline, me); + } else { // fallback to db + const followees = await this.userFollowingService.getFollowees(me.id); + + const query = this.queryService.makePaginationQuery(this.notesRepository.createQueryBuilder('note'), + ps.sinceId, ps.untilId, ps.sinceDate, ps.untilDate) + .andWhere(new Brackets(qb => { + if (followees.length > 0) { + const meOrFolloweeIds = [me.id, ...followees.map(f => f.followeeId)]; + qb.where('note.userId IN (:...meOrFolloweeIds)', { meOrFolloweeIds: meOrFolloweeIds }); + qb.orWhere('(note.visibility = \'public\') AND (note.userHost IS NULL)'); + } else { + qb.where('note.userId = :meId', { meId: me.id }); + qb.orWhere('(note.visibility = \'public\') AND (note.userHost IS NULL)'); + } + })) + .innerJoinAndSelect('note.user', 'user') + .leftJoinAndSelect('note.reply', 'reply') + .leftJoinAndSelect('note.renote', 'renote') + .leftJoinAndSelect('reply.user', 'replyUser') + .leftJoinAndSelect('renote.user', 'renoteUser'); + + this.queryService.generateVisibilityQuery(query, me); + this.queryService.generateMutedUserQuery(query, me); + this.queryService.generateBlockedUserQuery(query, me); + this.queryService.generateMutedUserRenotesQueryForNotes(query, me); + + if (ps.includeMyRenotes === false) { + query.andWhere(new Brackets(qb => { + qb.orWhere('note.userId != :meId', { meId: me.id }); + qb.orWhere('note.renoteId IS NULL'); + qb.orWhere('note.text IS NOT NULL'); + qb.orWhere('note.fileIds != \'{}\''); + qb.orWhere('0 < (SELECT COUNT(*) FROM poll WHERE poll."noteId" = note.id)'); + })); } - if (isUserRelated(note, userIdsWhoBlockingMe)) return false; - if (isUserRelated(note, userIdsWhoMeMuting)) return false; - if (note.renoteId) { - if (note.text == null && note.fileIds.length === 0 && !note.hasPoll) { - if (isUserRelated(note, userIdsWhoMeMutingRenotes)) return false; - if (ps.withRenotes === false) return false; - } + + if (ps.includeRenotedMyNotes === false) { + query.andWhere(new Brackets(qb => { + qb.orWhere('note.renoteUserId != :meId', { meId: me.id }); + qb.orWhere('note.renoteId IS NULL'); + qb.orWhere('note.text IS NOT NULL'); + qb.orWhere('note.fileIds != \'{}\''); + qb.orWhere('0 < (SELECT COUNT(*) FROM poll WHERE poll."noteId" = note.id)'); + })); } - return true; - }); + if (ps.includeLocalRenotes === false) { + query.andWhere(new Brackets(qb => { + qb.orWhere('note.renoteUserHost IS NOT NULL'); + qb.orWhere('note.renoteId IS NULL'); + qb.orWhere('note.text IS NOT NULL'); + qb.orWhere('note.fileIds != \'{}\''); + qb.orWhere('0 < (SELECT COUNT(*) FROM poll WHERE poll."noteId" = note.id)'); + })); + } - // TODO: フィルタした結果件数が足りなかった場合の対応 + if (ps.withFiles) { + query.andWhere('note.fileIds != \'{}\''); + } + //#endregion - timeline.sort((a, b) => a.id > b.id ? -1 : 1); + const timeline = await query.limit(ps.limit).getMany(); - process.nextTick(() => { - this.activeUsersChart.read(me); - }); + process.nextTick(() => { + this.activeUsersChart.read(me); + }); - return await this.noteEntityService.packMany(timeline, me); + return await this.noteEntityService.packMany(timeline, me); + } }); } } -- cgit v1.2.3-freya From ae1e276085b00f34f0f2dce8d25271c3e7246fa3 Mon Sep 17 00:00:00 2001 From: syuilo Date: Wed, 18 Oct 2023 17:17:56 +0900 Subject: fix of 6cc02fee99 --- .../src/server/api/endpoints/notes/hybrid-timeline.ts | 13 +++++++++++-- .../backend/src/server/api/endpoints/notes/timeline.ts | 14 ++++++++++++-- 2 files changed, 23 insertions(+), 4 deletions(-) (limited to 'packages/backend/src/server/api/endpoints/notes') diff --git a/packages/backend/src/server/api/endpoints/notes/hybrid-timeline.ts b/packages/backend/src/server/api/endpoints/notes/hybrid-timeline.ts index faaeb32c79..9986c6e436 100644 --- a/packages/backend/src/server/api/endpoints/notes/hybrid-timeline.ts +++ b/packages/backend/src/server/api/endpoints/notes/hybrid-timeline.ts @@ -163,8 +163,7 @@ export default class extends Endpoint { // eslint- } else { // fallback to db const followees = await this.userFollowingService.getFollowees(me.id); - const query = this.queryService.makePaginationQuery(this.notesRepository.createQueryBuilder('note'), - ps.sinceId, ps.untilId, ps.sinceDate, ps.untilDate) + const query = this.queryService.makePaginationQuery(this.notesRepository.createQueryBuilder('note'), ps.sinceId, ps.untilId, ps.sinceDate, ps.untilDate) .andWhere(new Brackets(qb => { if (followees.length > 0) { const meOrFolloweeIds = [me.id, ...followees.map(f => f.followeeId)]; @@ -181,6 +180,16 @@ export default class extends Endpoint { // eslint- .leftJoinAndSelect('reply.user', 'replyUser') .leftJoinAndSelect('renote.user', 'renoteUser'); + query.andWhere(new Brackets(qb => { + qb + .where('note.replyId IS NULL') // 返信ではない + .orWhere(new Brackets(qb => { + qb // 返信だけど投稿者自身への返信 + .where('note.replyId IS NOT NULL') + .andWhere('note.replyUserId = note.userId'); + })); + })); + this.queryService.generateVisibilityQuery(query, me); this.queryService.generateMutedUserQuery(query, me); this.queryService.generateBlockedUserQuery(query, me); diff --git a/packages/backend/src/server/api/endpoints/notes/timeline.ts b/packages/backend/src/server/api/endpoints/notes/timeline.ts index 8660e1602b..3b597107ae 100644 --- a/packages/backend/src/server/api/endpoints/notes/timeline.ts +++ b/packages/backend/src/server/api/endpoints/notes/timeline.ts @@ -127,8 +127,8 @@ export default class extends Endpoint { // eslint- const followees = await this.userFollowingService.getFollowees(me.id); //#region Construct query - const query = this.queryService.makePaginationQuery(this.notesRepository.createQueryBuilder('note'), - ps.sinceId, ps.untilId, ps.sinceDate, ps.untilDate) + const query = this.queryService.makePaginationQuery(this.notesRepository.createQueryBuilder('note'), ps.sinceId, ps.untilId, ps.sinceDate, ps.untilDate) + .andWhere('note.channelId IS NULL') .innerJoinAndSelect('note.user', 'user') .leftJoinAndSelect('note.reply', 'reply') .leftJoinAndSelect('note.renote', 'renote') @@ -143,6 +143,16 @@ export default class extends Endpoint { // eslint- query.andWhere('note.userId = :meId', { meId: me.id }); } + query.andWhere(new Brackets(qb => { + qb + .where('note.replyId IS NULL') // 返信ではない + .orWhere(new Brackets(qb => { + qb // 返信だけど投稿者自身への返信 + .where('note.replyId IS NOT NULL') + .andWhere('note.replyUserId = note.userId'); + })); + })); + this.queryService.generateVisibilityQuery(query, me); this.queryService.generateMutedUserQuery(query, me); this.queryService.generateBlockedUserQuery(query, me); -- cgit v1.2.3-freya