merge: Add filter options to following feed (resolves #726) (!671)

View MR for information: https://activitypub.software/TransFem-org/Sharkey/-/merge_requests/671

Closes #726

Approved-by: dakkar <dakkar@thenautilus.net>
Approved-by: Marie <github@yuugi.dev>

9 files changed, 178 insertions, 32 deletions

diff --git a/packages/backend/src/core/NoteCreateService.ts b/packages/backend/src/core/NoteCreateService.ts
index ef0047ca90..cbc9dcaf8f 100644
--- a/packages/backend/src/core/NoteCreateService.ts
+++ b/packages/backend/src/core/NoteCreateService.ts
@@ -14,7 +14,7 @@ import { extractCustomEmojisFromMfm } from '@/misc/extract-custom-emojis-from-mf
 import { extractHashtags } from '@/misc/extract-hashtags.js';
 import type { IMentionedRemoteUsers } from '@/models/Note.js';
 import { MiNote } from '@/models/Note.js';
-import { LatestNote } from '@/models/LatestNote.js';
+import { SkLatestNote } from '@/models/LatestNote.js';
 import type { ChannelFollowingsRepository, ChannelsRepository, FollowingsRepository, InstancesRepository, LatestNotesRepository, MiFollowing, MutingsRepository, NotesRepository, NoteThreadMutingsRepository, UserListMembershipsRepository, UserProfilesRepository, UsersRepository } from '@/models/_.js';
 import type { MiDriveFile } from '@/models/DriveFile.js';
 import type { MiApp } from '@/models/App.js';
@@ -63,7 +63,7 @@ import { isReply } from '@/misc/is-reply.js';
 import { trackPromise } from '@/misc/promise-tracker.js';
 import { isUserRelated } from '@/misc/is-user-related.js';
 import { IdentifiableError } from '@/misc/identifiable-error.js';
-import { isQuote, isRenote } from '@/misc/is-renote.js';
+import { isPureRenote } from '@/misc/is-renote.js';
 
 type NotificationType = 'reply' | 'renote' | 'quote' | 'mention';
 
@@ -1151,18 +1151,21 @@ export class NoteCreateService implements OnApplicationShutdown {
 		if (note.visibility === 'specified') return;
 
 		// Ignore pure renotes
-		if (isRenote(note) && !isQuote(note)) return;
+		if (isPureRenote(note)) return;
+
+		// Compute the compound key of the entry to check
+		const key = SkLatestNote.keyFor(note);
 
 		// Make sure that this isn't an *older* post.
 		// We can get older posts through replies, lookups, etc.
-		const currentLatest = await this.latestNotesRepository.findOneBy({ userId: note.userId });
+		const currentLatest = await this.latestNotesRepository.findOneBy(key);
 		if (currentLatest != null && currentLatest.noteId >= note.id) return;
 
 		// Record this as the latest note for the given user
-		const latestNote = new LatestNote({
-			userId: note.userId,
+		const latestNote = new SkLatestNote({
+			...key,
 			noteId: note.id,
 		});
-		await this.latestNotesRepository.upsert(latestNote, ['userId']);
+		await this.latestNotesRepository.upsert(latestNote, ['userId', 'isPublic', 'isReply', 'isQuote']);
 	}
 }
diff --git a/packages/backend/src/core/NoteDeleteService.ts b/packages/backend/src/core/NoteDeleteService.ts
index 3f86f41942..fa77caabd1 100644
--- a/packages/backend/src/core/NoteDeleteService.ts
+++ b/packages/backend/src/core/NoteDeleteService.ts
@@ -6,8 +6,8 @@
 import { Brackets, In, Not } from 'typeorm';
 import { Injectable, Inject } from '@nestjs/common';
 import type { MiUser, MiLocalUser, MiRemoteUser } from '@/models/User.js';
-import type { MiNote, IMentionedRemoteUsers } from '@/models/Note.js';
-import { LatestNote } from '@/models/LatestNote.js';
+import { MiNote, IMentionedRemoteUsers } from '@/models/Note.js';
+import { SkLatestNote } from '@/models/LatestNote.js';
 import type { InstancesRepository, LatestNotesRepository, NotesRepository, UsersRepository } from '@/models/_.js';
 import { RelayService } from '@/core/RelayService.js';
 import { FederatedInstanceService } from '@/core/FederatedInstanceService.js';
@@ -25,7 +25,7 @@ import { bindThis } from '@/decorators.js';
 import { MetaService } from '@/core/MetaService.js';
 import { SearchService } from '@/core/SearchService.js';
 import { ModerationLogService } from '@/core/ModerationLogService.js';
-import { isQuote, isRenote } from '@/misc/is-renote.js';
+import { isPureRenote, isQuote, isRenote } from '@/misc/is-renote.js';
 
 @Injectable()
 export class NoteDeleteService {
@@ -240,8 +240,14 @@ export class NoteDeleteService {
 		// If it's a DM, then it can't possibly be the latest note so we can safely skip this.
 		if (note.visibility === 'specified') return;
 
+		// If it's a pure renote, then it can't possibly be the latest note so we can safely skip this.
+		if (isPureRenote(note)) return;
+
+		// Compute the compound key of the entry to check
+		const key = SkLatestNote.keyFor(note);
+
 		// Check if the deleted note was possibly the latest for the user
-		const hasLatestNote = await this.latestNotesRepository.existsBy({ userId: note.userId });
+		const hasLatestNote = await this.latestNotesRepository.existsBy(key);
 		if (hasLatestNote) return;
 
 		// Find the newest remaining note for the user.
@@ -250,8 +256,16 @@ export class NoteDeleteService {
 			.createQueryBuilder('note')
 			.select()
 			.where({
-				userId: note.userId,
-				visibility: Not('specified'),
+				userId: key.userId,
+				visibility: key.isPublic
+					? 'public'
+					: Not('specified'),
+				replyId: key.isReply
+					? Not(null)
+					: null,
+				renoteId: key.isQuote
+					? Not(null)
+					: null,
 			})
 			.andWhere(`
 				(
@@ -268,8 +282,8 @@ export class NoteDeleteService {
 		if (!nextLatest) return;
 
 		// Record it as the latest
-		const latestNote = new LatestNote({
-			userId: note.userId,
+		const latestNote = new SkLatestNote({
+			...key,
 			noteId: nextLatest.id,
 		});
 
@@ -278,7 +292,7 @@ export class NoteDeleteService {
 		await this.latestNotesRepository
 			.createQueryBuilder('latest')
 			.insert()
-			.into(LatestNote)
+			.into(SkLatestNote)
 			.values(latestNote)
 			.orIgnore()
 			.execute();
diff --git a/packages/backend/src/misc/is-renote.ts b/packages/backend/src/misc/is-renote.ts
index 48f821806c..c128fded14 100644
--- a/packages/backend/src/misc/is-renote.ts
+++ b/packages/backend/src/misc/is-renote.ts
@@ -23,6 +23,17 @@ type Quote =
 		hasPoll: true
 	});
 
+type PureRenote =
+	Renote & {
+		text: null,
+		cw: null,
+		replyId: null,
+		hasPoll: false,
+		fileIds: {
+			length: 0,
+		},
+	};
+
 export function isRenote(note: MiNote): note is Renote {
 	return note.renoteId != null;
 }
@@ -36,6 +47,10 @@ export function isQuote(note: Renote): note is Quote {
 		note.fileIds.length > 0;
 }
 
+export function isPureRenote(note: MiNote): note is PureRenote {
+	return isRenote(note) && !isQuote(note);
+}
+
 type PackedRenote =
 	Packed<'Note'> & {
 		renoteId: NonNullable<Packed<'Note'>['renoteId']>
diff --git a/packages/backend/src/models/LatestNote.ts b/packages/backend/src/models/LatestNote.ts
index 1163ff3bc0..d36a4d568a 100644
--- a/packages/backend/src/models/LatestNote.ts
+++ b/packages/backend/src/models/LatestNote.ts
@@ -6,6 +6,7 @@
 import { PrimaryColumn, Entity, JoinColumn, Column, ManyToOne } from 'typeorm';
 import { MiUser } from '@/models/User.js';
 import { MiNote } from '@/models/Note.js';
+import { isQuote, isRenote } from '@/misc/is-renote.js';
 
 /**
  * Maps a user to the most recent post by that user.
@@ -13,7 +14,7 @@ import { MiNote } from '@/models/Note.js';
  * DMs are not counted.
  */
 @Entity('latest_note')
-export class LatestNote {
+export class SkLatestNote {
 	@PrimaryColumn({
 		name: 'user_id',
 		type: 'varchar' as const,
@@ -21,6 +22,24 @@ export class LatestNote {
 	})
 	public userId: string;
 
+	@PrimaryColumn('boolean', {
+		name: 'is_public',
+		default: false,
+	})
+	public isPublic: boolean;
+
+	@PrimaryColumn('boolean', {
+		name: 'is_reply',
+		default: false,
+	})
+	public isReply: boolean;
+
+	@PrimaryColumn('boolean', {
+		name: 'is_quote',
+		default: false,
+	})
+	public isQuote: boolean;
+
 	@ManyToOne(() => MiUser, {
 		onDelete: 'CASCADE',
 	})
@@ -44,11 +63,23 @@ export class LatestNote {
 	})
 	public note: MiNote | null;
 
-	constructor(data?: Partial<LatestNote>) {
+	constructor(data?: Partial<SkLatestNote>) {
 		if (!data) return;
 
 		for (const [k, v] of Object.entries(data)) {
 			(this as Record<string, unknown>)[k] = v;
 		}
 	}
+
+	/**
+	 * Generates a compound key matching a provided note.
+	 */
+	static keyFor(note: MiNote) {
+		return {
+			userId: note.userId,
+			isPublic: note.visibility === 'public',
+			isReply: note.replyId != null,
+			isQuote: isRenote(note) && isQuote(note),
+		};
+	}
 }
diff --git a/packages/backend/src/models/RepositoryModule.ts b/packages/backend/src/models/RepositoryModule.ts
index f44334d84e..eb45b9a631 100644
--- a/packages/backend/src/models/RepositoryModule.ts
+++ b/packages/backend/src/models/RepositoryModule.ts
@@ -7,7 +7,7 @@ import type { Provider } from '@nestjs/common';
 import { Module } from '@nestjs/common';
 import { DI } from '@/di-symbols.js';
 import {
-	LatestNote,
+	SkLatestNote,
 	MiAbuseReportNotificationRecipient,
 	MiAbuseUserReport,
 	MiAccessToken,
@@ -121,7 +121,7 @@ const $avatarDecorationsRepository: Provider = {
 
 const $latestNotesRepository: Provider = {
 	provide: DI.latestNotesRepository,
-	useFactory: (db: DataSource) => db.getRepository(LatestNote).extend(miRepository as MiRepository<LatestNote>),
+	useFactory: (db: DataSource) => db.getRepository(SkLatestNote).extend(miRepository as MiRepository<SkLatestNote>),
 	inject: [DI.db],
 };
 
diff --git a/packages/backend/src/models/_.ts b/packages/backend/src/models/_.ts
index 9e01f4b6d7..ac2dd62aa2 100644
--- a/packages/backend/src/models/_.ts
+++ b/packages/backend/src/models/_.ts
@@ -10,7 +10,7 @@ import { RelationIdLoader } from 'typeorm/query-builder/relation-id/RelationIdLo
 import { RawSqlResultsToEntityTransformer } from 'typeorm/query-builder/transformer/RawSqlResultsToEntityTransformer.js';
 import { ObjectUtils } from 'typeorm/util/ObjectUtils.js';
 import { OrmUtils } from 'typeorm/util/OrmUtils.js';
-import { LatestNote } from '@/models/LatestNote.js';
+import { SkLatestNote } from '@/models/LatestNote.js';
 import { MiAbuseUserReport } from '@/models/AbuseUserReport.js';
 import { MiAbuseReportNotificationRecipient } from '@/models/AbuseReportNotificationRecipient.js';
 import { MiAccessToken } from '@/models/AccessToken.js';
@@ -127,7 +127,7 @@ export const miRepository = {
 } satisfies MiRepository<ObjectLiteral>;
 
 export {
-	LatestNote,
+	SkLatestNote,
 	MiAbuseUserReport,
 	MiAbuseReportNotificationRecipient,
 	MiAccessToken,
@@ -226,7 +226,7 @@ export type GalleryPostsRepository = Repository<MiGalleryPost> & MiRepository<Mi
 export type HashtagsRepository = Repository<MiHashtag> & MiRepository<MiHashtag>;
 export type InstancesRepository = Repository<MiInstance> & MiRepository<MiInstance>;
 export type MetasRepository = Repository<MiMeta> & MiRepository<MiMeta>;
-export type LatestNotesRepository = Repository<LatestNote> & MiRepository<LatestNote>;
+export type LatestNotesRepository = Repository<SkLatestNote> & MiRepository<SkLatestNote>;
 export type ModerationLogsRepository = Repository<MiModerationLog> & MiRepository<MiModerationLog>;
 export type MutingsRepository = Repository<MiMuting> & MiRepository<MiMuting>;
 export type RenoteMutingsRepository = Repository<MiRenoteMuting> & MiRepository<MiRenoteMuting>;
diff --git a/packages/backend/src/postgres.ts b/packages/backend/src/postgres.ts
index 0d17b3d046..2d66e6e445 100644
--- a/packages/backend/src/postgres.ts
+++ b/packages/backend/src/postgres.ts
@@ -83,7 +83,7 @@ import { MiReversiGame } from '@/models/ReversiGame.js';
 import { Config } from '@/config.js';
 import MisskeyLogger from '@/logger.js';
 import { bindThis } from '@/decorators.js';
-import { LatestNote } from '@/models/LatestNote.js';
+import { SkLatestNote } from '@/models/LatestNote.js';
 
 pg.types.setTypeParser(20, Number);
 
@@ -131,7 +131,7 @@ class MyCustomLogger implements Logger {
 }
 
 export const entities = [
-	LatestNote,
+	SkLatestNote,
 	MiAnnouncement,
 	MiAnnouncementRead,
 	MiMeta,
diff --git a/packages/backend/src/server/api/endpoints/notes/following.ts b/packages/backend/src/server/api/endpoints/notes/following.ts
index 436160f250..83e8f404e9 100644
--- a/packages/backend/src/server/api/endpoints/notes/following.ts
+++ b/packages/backend/src/server/api/endpoints/notes/following.ts
@@ -4,7 +4,7 @@
  */
 
 import { Inject, Injectable } from '@nestjs/common';
-import { LatestNote, MiFollowing } from '@/models/_.js';
+import { SkLatestNote, MiFollowing } from '@/models/_.js';
 import type { NotesRepository } from '@/models/_.js';
 import { Endpoint } from '@/server/api/endpoint-base.js';
 import { NoteEntityService } from '@/core/entities/NoteEntityService.js';
@@ -33,6 +33,12 @@ export const paramDef = {
 	type: 'object',
 	properties: {
 		mutualsOnly: { type: 'boolean', default: false },
+		filesOnly: { type: 'boolean', default: false },
+		includeNonPublic: { type: 'boolean', default: false },
+		includeReplies: { type: 'boolean', default: false },
+		includeQuotes: { type: 'boolean', default: false },
+		includeBots: { type: 'boolean', default: true },
+
 		limit: { type: 'integer', minimum: 1, maximum: 100, default: 10 },
 		sinceId: { type: 'string', format: 'misskey:id' },
 		untilId: { type: 'string', format: 'misskey:id' },
@@ -52,12 +58,12 @@ export default class extends Endpoint<typeof meta, typeof paramDef> { // eslint-
 		private queryService: QueryService,
 	) {
 		super(meta, paramDef, async (ps, me) => {
-			let query = this.notesRepository
+			const query = this.notesRepository
 				.createQueryBuilder('note')
 				.setParameter('me', me.id)
 
 				// Limit to latest notes
-				.innerJoin(LatestNote, 'latest', 'note.id = latest.note_id')
+				.innerJoin(SkLatestNote, 'latest', 'note.id = latest.note_id')
 
 				// Avoid N+1 queries from the "pack" method
 				.innerJoinAndSelect('note.user', 'user')
@@ -73,8 +79,28 @@ export default class extends Endpoint<typeof meta, typeof paramDef> { // eslint-
 
 			// Limit to mutuals, if requested
 			if (ps.mutualsOnly) {
-				query = query
-					.innerJoin(MiFollowing, 'mutuals', 'latest.user_id = mutuals."followerId" AND mutuals."followeeId" = :me');
+				query.innerJoin(MiFollowing, 'mutuals', 'latest.user_id = mutuals."followerId" AND mutuals."followeeId" = :me');
+			}
+
+			// Limit to files, if requested
+			if (ps.filesOnly) {
+				query.andWhere('note."fileIds" != \'{}\'');
+			}
+
+			// Match selected note types.
+			if (!ps.includeNonPublic) {
+				query.andWhere('latest.is_public');
+			}
+			if (!ps.includeReplies) {
+				query.andWhere('latest.is_reply = false');
+			}
+			if (!ps.includeQuotes) {
+				query.andWhere('latest.is_quote = false');
+			}
+
+			// Match selected user types.
+			if (!ps.includeBots) {
+				query.andWhere('"user"."isBot" = false');
 			}
 
 			// Respect blocks and mutes
@@ -82,7 +108,7 @@ export default class extends Endpoint<typeof meta, typeof paramDef> { // eslint-
 			this.queryService.generateMutedUserQuery(query, me);
 
 			// Support pagination
-			query = this.queryService
+			this.queryService
 				.makePaginationQuery(query, ps.sinceId, ps.untilId, ps.sinceDate, ps.untilDate)
 				.orderBy('note.id', 'DESC')
 				.take(ps.limit);
diff --git a/packages/backend/src/server/api/endpoints/users/notes.ts b/packages/backend/src/server/api/endpoints/users/notes.ts
index cc76c12f1d..efea15ca80 100644
--- a/packages/backend/src/server/api/endpoints/users/notes.ts
+++ b/packages/backend/src/server/api/endpoints/users/notes.ts
@@ -17,6 +17,7 @@ import { MiLocalUser } from '@/models/User.js';
 import { FanoutTimelineEndpointService } from '@/core/FanoutTimelineEndpointService.js';
 import { FanoutTimelineName } from '@/core/FanoutTimelineService.js';
 import { ApiError } from '@/server/api/error.js';
+import { isQuote, isRenote } from '@/misc/is-renote.js';
 
 export const meta = {
 	tags: ['users', 'notes'],
@@ -51,7 +52,11 @@ export const paramDef = {
 	properties: {
 		userId: { type: 'string', format: 'misskey:id' },
 		withReplies: { type: 'boolean', default: false },
+		withRepliesToSelf: { type: 'boolean', default: true },
+		withQuotes: { type: 'boolean', default: true },
 		withRenotes: { type: 'boolean', default: true },
+		withBots: { type: 'boolean', default: true },
+		withNonPublic: { type: 'boolean', default: true },
 		withChannelNotes: { type: 'boolean', default: false },
 		limit: { type: 'integer', minimum: 1, maximum: 100, default: 10 },
 		sinceId: { type: 'string', format: 'misskey:id' },
@@ -103,6 +108,11 @@ export default class extends Endpoint<typeof meta, typeof paramDef> { // eslint-
 					withChannelNotes: ps.withChannelNotes,
 					withFiles: ps.withFiles,
 					withRenotes: ps.withRenotes,
+					withQuotes: ps.withQuotes,
+					withBots: ps.withBots,
+					withNonPublic: ps.withNonPublic,
+					withRepliesToOthers: ps.withReplies,
+					withRepliesToSelf: ps.withRepliesToSelf,
 				}, me);
 
 				return await this.noteEntityService.packMany(timeline, me);
@@ -127,11 +137,17 @@ export default class extends Endpoint<typeof meta, typeof paramDef> { // eslint-
 				excludeReplies: ps.withChannelNotes && !ps.withReplies, // userTimelineWithChannel may include replies
 				excludeNoFiles: ps.withChannelNotes && ps.withFiles, // userTimelineWithChannel may include notes without files
 				excludePureRenotes: !ps.withRenotes,
+				excludeBots: !ps.withBots,
 				noteFilter: note => {
 					if (note.channel?.isSensitive && !isSelf) return false;
 					if (note.visibility === 'specified' && (!me || (me.id !== note.userId && !note.visibleUserIds.some(v => v === me.id)))) return false;
 					if (note.visibility === 'followers' && !isFollowing && !isSelf) return false;
 
+					// These are handled by DB fallback, but we duplicate them here in case a timeline was already populated with notes
+					if (!ps.withRepliesToSelf && note.reply?.userId === note.userId) return false;
+					if (!ps.withQuotes && isRenote(note) && isQuote(note)) return false;
+					if (!ps.withNonPublic && note.visibility !== 'public') return false;
+
 					return true;
 				},
 				dbFallback: async (untilId, sinceId, limit) => await this.getFromDb({
@@ -142,6 +158,11 @@ export default class extends Endpoint<typeof meta, typeof paramDef> { // eslint-
 					withChannelNotes: ps.withChannelNotes,
 					withFiles: ps.withFiles,
 					withRenotes: ps.withRenotes,
+					withQuotes: ps.withQuotes,
+					withBots: ps.withBots,
+					withNonPublic: ps.withNonPublic,
+					withRepliesToOthers: ps.withReplies,
+					withRepliesToSelf: ps.withRepliesToSelf,
 				}, me),
 			});
 
@@ -157,6 +178,11 @@ export default class extends Endpoint<typeof meta, typeof paramDef> { // eslint-
 		withChannelNotes: boolean,
 		withFiles: boolean,
 		withRenotes: boolean,
+		withQuotes: boolean,
+		withBots: boolean,
+		withNonPublic: boolean,
+		withRepliesToOthers: boolean,
+		withRepliesToSelf: boolean,
 	}, me: MiLocalUser | null) {
 		const isSelf = me && (me.id === ps.userId);
 
@@ -188,7 +214,9 @@ export default class extends Endpoint<typeof meta, typeof paramDef> { // eslint-
 			query.andWhere('note.fileIds != \'{}\'');
 		}
 
-		if (ps.withRenotes === false) {
+		if (!ps.withRenotes && !ps.withQuotes) {
+			query.andWhere('note.renoteId IS NULL');
+		} else if (!ps.withRenotes) {
 			query.andWhere(new Brackets(qb => {
 				qb.orWhere('note.userId != :userId', { userId: ps.userId });
 				qb.orWhere('note.renoteId IS NULL');
@@ -196,6 +224,35 @@ export default class extends Endpoint<typeof meta, typeof paramDef> { // eslint-
 				qb.orWhere('note.fileIds != \'{}\'');
 				qb.orWhere('0 < (SELECT COUNT(*) FROM poll WHERE poll."noteId" = note.id)');
 			}));
+		} else if (!ps.withQuotes) {
+			query.andWhere(`
+				(
+					note."renoteId" IS NULL
+					OR (
+						note.text IS NULL
+						AND note.cw IS NULL
+						AND note."replyId" IS NULL
+						AND note."hasPoll" IS FALSE
+						AND note."fileIds" = '{}'
+					)
+				)
+			`);
+		}
+
+		if (!ps.withRepliesToOthers && !ps.withRepliesToSelf) {
+			query.andWhere('reply.id IS NULL');
+		} else if (!ps.withRepliesToOthers) {
+			query.andWhere('(reply.id IS NULL OR reply."userId" = note."userId")');
+		} else if (!ps.withRepliesToSelf) {
+			query.andWhere('(reply.id IS NULL OR reply."userId" != note."userId")');
+		}
+
+		if (!ps.withNonPublic) {
+			query.andWhere('note.visibility = \'public\'');
+		}
+
+		if (!ps.withBots) {
+			query.andWhere('"user"."isBot" = false');
 		}
 
 		return await query.limit(ps.limit).getMany();