Merge branch 'develop' into future

9 files changed, 186 insertions, 11 deletions

diff --git a/packages/backend/src/boot/common.ts b/packages/backend/src/boot/common.ts
index 268c07582d..18a2ab149a 100644
--- a/packages/backend/src/boot/common.ts
+++ b/packages/backend/src/boot/common.ts
@@ -36,7 +36,6 @@ export async function jobQueue() {
 	});
 
 	jobQueue.get(QueueProcessorService).start();
-	jobQueue.get(ChartManagementService).start();
 
 	return jobQueue;
 }
diff --git a/packages/backend/src/boot/entry.ts b/packages/backend/src/boot/entry.ts
index ae74a43c84..3882686fdc 100644
--- a/packages/backend/src/boot/entry.ts
+++ b/packages/backend/src/boot/entry.ts
@@ -75,7 +75,7 @@ async function main() {
 			ev.mount();
 		}
 	}
-	if (cluster.isWorker || envOption.disableClustering) {
+	if (cluster.isWorker) {
 		await workerMain();
 	}
 
diff --git a/packages/backend/src/config.ts b/packages/backend/src/config.ts
index c99bc7ae03..f6ce9b3cdf 100644
--- a/packages/backend/src/config.ts
+++ b/packages/backend/src/config.ts
@@ -212,6 +212,8 @@ export function loadConfig(): Config {
 			{} as Source,
 		) as Source;
 
+	applyEnvOverrides(config);
+
 	const url = tryCreateUrl(config.url);
 	const version = meta.version;
 	const host = url.host;
@@ -304,3 +306,123 @@ function convertRedisOptions(options: RedisOptionsSource, host: string): RedisOp
 		db: options.db ?? 0,
 	};
 }
+
+/*
+	this function allows overriding any string-valued config option with
+	a sensible-named environment variable
+
+	e.g. `MK_CONFIG_MEILISEARCH_APIKEY` sets `config.meilisearch.apikey`
+
+	you can also override a single `dbSlave` value,
+	e.g. `MK_CONFIG_DBSLAVES_1_PASS` sets the password for the 2nd
+	database replica (the first one would be
+	`MK_CONFIG_DBSLAVES_0_PASS`); in this case, `config.dbSlaves` must
+	be set to an array of the right size already in the file
+
+	values can be read from files, too: setting `MK_DB_PASS_FILE` to
+	`/some/file` would set the main database password to the contents of
+	`/some/file` (trimmed of whitespaces)
+ */
+function applyEnvOverrides(config: Source) {
+	// these inner functions recurse through the config structure, using
+	// the given steps, building the env variable name
+
+	function _apply_top(steps: (string | number)[]) {
+		_walk('', [], steps);
+	}
+
+	function _walk(name: string, path: (string | number)[], steps: (string | number)[]) {
+		// are there more steps after this one? recurse
+		if (steps.length > 1) {
+			const thisStep = steps.shift();
+			if (thisStep === null || thisStep === undefined) return;
+
+			// if a step is not a simple value, iterate through it
+			if (typeof thisStep === 'object') {
+				for (const thisOneStep of thisStep) {
+					_descend(name, path, thisOneStep, steps);
+				}
+			} else {
+				_descend(name, path, thisStep, steps);
+			}
+
+			// the actual override has happened at the bottom of the
+			// recursion, we're done
+			return;
+		}
+
+		// this is the last step, same thing as above
+		const lastStep = steps[0];
+
+		if (typeof lastStep === 'object') {
+			for (const lastOneStep of lastStep) {
+				_lastBit(name, path, lastOneStep);
+			}
+		} else {
+			_lastBit(name, path, lastStep);
+		}
+	}
+
+	function _step2name(step: string|number): string {
+		return step.toString().replaceAll(/[^a-z0-9]+/gi,'').toUpperCase();
+	}
+
+	// this recurses down, bailing out if there's no config to override
+	function _descend(name: string, path: (string | number)[], thisStep: string | number, steps: (string | number)[]) {
+		name = `${name}${_step2name(thisStep)}_`;
+		path = [ ...path, thisStep ];
+		_walk(name, path, steps);
+	}
+
+	// this is the bottom of the recursion: look at the environment and
+	// set the value
+	function _lastBit(name: string, path: (string | number)[], lastStep: string | number) {
+		name = `MK_CONFIG_${name}${_step2name(lastStep)}`;
+
+		const val = process.env[name];
+		if (val != null && val != undefined) {
+			_assign(path, lastStep, val);
+		}
+
+		const file = process.env[`${name}_FILE`];
+		if (file) {
+			_assign(path, lastStep, fs.readFileSync(file, 'utf-8').trim());
+		}
+	}
+
+	const alwaysStrings = { 'chmodSocket': 1 };
+
+	function _assign(path: (string | number)[], lastStep: string | number, value: string) {
+		let thisConfig = config;
+		for (const step of path) {
+			if (!thisConfig[step]) {
+				thisConfig[step] = {};
+			}
+			thisConfig = thisConfig[step];
+		}
+
+		if (!alwaysStrings[lastStep]) {
+			if (value.match(/^[0-9]+$/)) {
+				value = parseInt(value);
+			} else if (value.match(/^(true|false)$/i)) {
+				value = !!value.match(/^true$/i);
+			}
+		}
+
+		thisConfig[lastStep] = value;
+	}
+
+	// these are all the settings that can be overridden
+
+	_apply_top([['url', 'port', 'socket', 'chmodSocket', 'disableHsts']]);
+	_apply_top(['db', ['host', 'port', 'db', 'user', 'pass']]);
+	_apply_top(['dbSlaves', config.dbSlaves?.keys(), ['host', 'port', 'db', 'user', 'pass']]);
+	_apply_top([
+		['redis', 'redisForPubsub', 'redisForJobQueue', 'redisForTimelines'],
+		['host','port','username','pass','db','prefix'],
+	]);
+	_apply_top(['meilisearch', ['host', 'port', 'apikey', 'ssl', 'index', 'scope']]);
+	_apply_top([['clusterLimit', 'deliverJobConcurrency', 'inboxJobConcurrency', 'relashionshipJobConcurrency', 'deliverJobPerSec', 'inboxJobPerSec', 'relashionshipJobPerSec', 'deliverJobMaxAttempts', 'inboxJobMaxAttempts']]);
+	_apply_top([['outgoingAddress', 'outgoingAddressFamily', 'proxy', 'proxySmtp', 'mediaProxy', 'videoThumbnailGenerator']]);
+	_apply_top([['maxFileSize', 'maxNoteLength', 'pidFile']]);
+}
diff --git a/packages/backend/src/core/AntennaService.ts b/packages/backend/src/core/AntennaService.ts
index 793d8974b3..89e475b5f1 100644
--- a/packages/backend/src/core/AntennaService.ts
+++ b/packages/backend/src/core/AntennaService.ts
@@ -133,13 +133,17 @@ export class AntennaService implements OnApplicationShutdown {
 				const { username, host } = Acct.parse(x);
 				return this.utilityService.getFullApAccount(username, host).toLowerCase();
 			});
-			if (!accts.includes(this.utilityService.getFullApAccount(noteUser.username, noteUser.host).toLowerCase())) return false;
+			const matchUser = this.utilityService.getFullApAccount(noteUser.username, noteUser.host).toLowerCase();
+			const matchWildcard = this.utilityService.getFullApAccount('*', noteUser.host).toLowerCase();
+			if (!accts.includes(matchUser) && !accts.includes(matchWildcard)) return false;
 		} else if (antenna.src === 'users_blacklist') {
 			const accts = antenna.users.map(x => {
 				const { username, host } = Acct.parse(x);
 				return this.utilityService.getFullApAccount(username, host).toLowerCase();
 			});
-			if (accts.includes(this.utilityService.getFullApAccount(noteUser.username, noteUser.host).toLowerCase())) return false;
+			const matchUser = this.utilityService.getFullApAccount(noteUser.username, noteUser.host).toLowerCase();
+			const matchWildcard = this.utilityService.getFullApAccount('*', noteUser.host).toLowerCase();
+			if (accts.includes(matchUser) || accts.includes(matchWildcard)) return false;
 		}
 
 		const keywords = antenna.keywords
diff --git a/packages/backend/src/core/DriveService.ts b/packages/backend/src/core/DriveService.ts
index f64568ee9a..4203b03c74 100644
--- a/packages/backend/src/core/DriveService.ts
+++ b/packages/backend/src/core/DriveService.ts
@@ -632,7 +632,8 @@ export class DriveService {
 
 	@bindThis
 	public async updateFile(file: MiDriveFile, values: Partial<MiDriveFile>, updater: MiUser) {
-		const alwaysMarkNsfw = (await this.roleService.getUserPolicies(file.userId)).alwaysMarkNsfw;
+		const profile = await this.userProfilesRepository.findOneBy({ userId: file.userId });
+		const alwaysMarkNsfw = (await this.roleService.getUserPolicies(file.userId)).alwaysMarkNsfw || (profile !== null && profile!.alwaysMarkNsfw);
 
 		if (values.name != null && !this.driveFileEntityService.validateFileName(values.name)) {
 			throw new DriveService.InvalidFileNameError();
diff --git a/packages/backend/src/core/NoteEditService.ts b/packages/backend/src/core/NoteEditService.ts
index 34017f015a..244f7e78d4 100644
--- a/packages/backend/src/core/NoteEditService.ts
+++ b/packages/backend/src/core/NoteEditService.ts
@@ -699,6 +699,24 @@ export class NoteEditService implements OnApplicationShutdown {
 						dm.addFollowersRecipe();
 					}
 
+					if (['public', 'home'].includes(note.visibility)) {
+						// Send edit event to all users who replied to,
+						// renoted a post or reacted to a note.
+						const noteId = note.id;
+						const users = await this.usersRepository.createQueryBuilder()
+							.where(
+								'id IN (SELECT "userId" FROM note WHERE "replyId" = :noteId OR "renoteId" = :noteId UNION SELECT "userId" FROM note_reaction WHERE "noteId" = :noteId)',
+								{ noteId },
+							)
+							.andWhere('host IS NOT NULL')
+							.getMany();
+						for (const u of users) {
+							// User was verified to be remote by checking
+							// whether host IS NOT NULL in SQL query.
+							dm.addDirectRecipe(u as MiRemoteUser);
+						}
+					}
+
 					if (['public'].includes(note.visibility)) {
 						this.relayService.deliverToRelays(user, noteActivity);
 					}
diff --git a/packages/backend/src/server/api/endpoints/channels/timeline.ts b/packages/backend/src/server/api/endpoints/channels/timeline.ts
index 8c55673590..295fc5686c 100644
--- a/packages/backend/src/server/api/endpoints/channels/timeline.ts
+++ b/packages/backend/src/server/api/endpoints/channels/timeline.ts
@@ -51,6 +51,12 @@ export const paramDef = {
 		sinceDate: { type: 'integer' },
 		untilDate: { type: 'integer' },
 		allowPartial: { type: 'boolean', default: false }, // true is recommended but for compatibility false by default
+		withRenotes: { type: 'boolean', default: true },
+		withFiles: {
+			type: 'boolean',
+			default: false,
+			description: 'Only show notes that have attached files.',
+		},
 	},
 	required: ['channelId'],
 } as const;
@@ -89,7 +95,7 @@ export default class extends Endpoint<typeof meta, typeof paramDef> { // eslint-
 			if (me) this.activeUsersChart.read(me);
 
 			if (!serverSettings.enableFanoutTimeline) {
-				return await this.noteEntityService.packMany(await this.getFromDb({ untilId, sinceId, limit: ps.limit, channelId: channel.id }, me), me);
+				return await this.noteEntityService.packMany(await this.getFromDb({ untilId, sinceId, limit: ps.limit, channelId: channel.id, withFiles: ps.withFiles, withRenotes: ps.withRenotes }, me), me);
 			}
 
 			return await this.fanoutTimelineEndpointService.timeline({
@@ -100,9 +106,10 @@ export default class extends Endpoint<typeof meta, typeof paramDef> { // eslint-
 				me,
 				useDbFallback: true,
 				redisTimelines: [`channelTimeline:${channel.id}`],
-				excludePureRenotes: false,
+				excludePureRenotes: !ps.withRenotes,
+				excludeNoFiles: ps.withFiles,
 				dbFallback: async (untilId, sinceId, limit) => {
-					return await this.getFromDb({ untilId, sinceId, limit, channelId: channel.id }, me);
+					return await this.getFromDb({ untilId, sinceId, limit, channelId: channel.id, withFiles: ps.withFiles, withRenotes: ps.withRenotes }, me);
 				},
 			});
 		});
@@ -112,7 +119,9 @@ export default class extends Endpoint<typeof meta, typeof paramDef> { // eslint-
 		untilId: string | null,
 		sinceId: string | null,
 		limit: number,
-		channelId: string
+		channelId: string,
+		withFiles: boolean,
+		withRenotes: boolean,
 	}, me: MiLocalUser | null) {
 		//#region fallback to database
 		const query = this.queryService.makePaginationQuery(this.notesRepository.createQueryBuilder('note'), ps.sinceId, ps.untilId)
@@ -128,6 +137,20 @@ export default class extends Endpoint<typeof meta, typeof paramDef> { // eslint-
 			this.queryService.generateMutedUserQuery(query, me);
 			this.queryService.generateBlockedUserQuery(query, me);
 		}
+
+		if (ps.withRenotes === false) {
+			query.andWhere(new Brackets(qb => {
+				qb.orWhere('note.renoteId IS NULL');
+				qb.orWhere(new Brackets(qb => {
+					qb.orWhere('note.text IS NOT NULL');
+					qb.orWhere('note.fileIds != \'{}\'');
+				}));
+			}));
+		}
+
+		if (ps.withFiles) {
+			query.andWhere('note.fileIds != \'{}\'');
+		}
 		//#endregion
 
 		return await query.limit(ps.limit).getMany();
diff --git a/packages/backend/src/server/api/mastodon/converters.ts b/packages/backend/src/server/api/mastodon/converters.ts
index 326d3a1d5c..ea219b933d 100644
--- a/packages/backend/src/server/api/mastodon/converters.ts
+++ b/packages/backend/src/server/api/mastodon/converters.ts
@@ -146,8 +146,8 @@ export class MastoConverters {
 			display_name: user.name ?? user.username,
 			locked: user.isLocked,
 			created_at: this.idService.parse(user.id).date.toISOString(),
-			followers_count: user.followersCount,
-			following_count: user.followingCount,
+			followers_count: profile?.followersVisibility === 'public' ? user.followersCount : 0,
+			following_count: profile?.followingVisibility === 'public' ? user.followingCount : 0,
 			statuses_count: user.notesCount,
 			note: profile?.description ?? '',
 			url: user.uri ?? acctUrl,
diff --git a/packages/backend/src/server/api/stream/channels/channel.ts b/packages/backend/src/server/api/stream/channels/channel.ts
index 140dd3dd9b..865e4fed19 100644
--- a/packages/backend/src/server/api/stream/channels/channel.ts
+++ b/packages/backend/src/server/api/stream/channels/channel.ts
@@ -15,6 +15,8 @@ class ChannelChannel extends Channel {
 	public static shouldShare = false;
 	public static requireCredential = false as const;
 	private channelId: string;
+	private withFiles: boolean;
+	private withRenotes: boolean;
 
 	constructor(
 		private noteEntityService: NoteEntityService,
@@ -29,6 +31,8 @@ class ChannelChannel extends Channel {
 	@bindThis
 	public async init(params: any) {
 		this.channelId = params.channelId as string;
+		this.withFiles = params.withFiles ?? false;
+		this.withRenotes = params.withRenotes ?? true;
 
 		// Subscribe stream
 		this.subscriber.on('notesStream', this.onNote);
@@ -38,6 +42,10 @@ class ChannelChannel extends Channel {
 	private async onNote(note: Packed<'Note'>) {
 		if (note.channelId !== this.channelId) return;
 
+		if (this.withFiles && (note.fileIds == null || note.fileIds.length === 0)) return;
+
+		if (note.renote && note.text == null && (note.fileIds == null || note.fileIds.length === 0) && !this.withRenotes) return;
+
 		if (this.isNoteMutedOrBlocked(note)) return;
 
 		if (this.user && isRenotePacked(note) && !isQuotePacked(note)) {