Merge branch 'develop' into merge/2024-02-03

# Conflicts:
#	packages/backend/src/server/ActivityPubServerService.ts
#	pnpm-lock.yaml

5 files changed, 34 insertions, 291 deletions

diff --git a/packages/backend/src/server/api/ApiCallService.ts b/packages/backend/src/server/api/ApiCallService.ts
index 03f25a51fe..5ce358d68f 100644
--- a/packages/backend/src/server/api/ApiCallService.ts
+++ b/packages/backend/src/server/api/ApiCallService.ts
@@ -19,7 +19,7 @@ import { bindThis } from '@/decorators.js';
 import { RoleService } from '@/core/RoleService.js';
 import type { Config } from '@/config.js';
 import { sendRateLimitHeaders } from '@/misc/rate-limit-utils.js';
-import { SkRateLimiterService } from '@/server/api/SkRateLimiterService.js';
+import { SkRateLimiterService } from '@/server/SkRateLimiterService.js';
 import { ApiError } from './error.js';
 import { ApiLoggerService } from './ApiLoggerService.js';
 import { AuthenticateService, AuthenticationError } from './AuthenticateService.js';
@@ -313,35 +313,30 @@ export class ApiCallService implements OnApplicationShutdown {
 		// eslint-disable-next-line @typescript-eslint/no-unnecessary-condition
 		if (endpointLimit) {
 			// koa will automatically load the `X-Forwarded-For` header if `proxy: true` is configured in the app.
-			let limitActor: string;
+			let limitActor: string | MiLocalUser;
 			if (user) {
-				limitActor = user.id;
+				limitActor = user;
 			} else {
 				limitActor = getIpHash(request.ip);
 			}
 
-			// TODO: 毎リクエスト計算するのもあれだしキャッシュしたい
-			const factor = user ? (await this.roleService.getUserPolicies(user.id)).rateLimitFactor : 1;
+			const limit = {
+				key: ep.name,
+				...endpointLimit,
+			};
 
-			if (factor > 0) {
-				const limit = {
-					key: ep.name,
-					...endpointLimit,
-				};
+			// Rate limit
+			const info = await this.rateLimiterService.limit(limit, limitActor);
 
-				// Rate limit
-				const info = await this.rateLimiterService.limit(limit, limitActor, factor);
+			sendRateLimitHeaders(reply, info);
 
-				sendRateLimitHeaders(reply, info);
-
-				if (info.blocked) {
-					throw new ApiError({
-						message: 'Rate limit exceeded. Please try again later.',
-						code: 'RATE_LIMIT_EXCEEDED',
-						id: 'd5826d14-3982-4d2e-8011-b9e9f02499ef',
-						httpStatusCode: 429,
-					}, info);
-				}
+			if (info.blocked) {
+				throw new ApiError({
+					message: 'Rate limit exceeded. Please try again later.',
+					code: 'RATE_LIMIT_EXCEEDED',
+					id: 'd5826d14-3982-4d2e-8011-b9e9f02499ef',
+					httpStatusCode: 429,
+				}, info);
 			}
 		}
 
diff --git a/packages/backend/src/server/api/SigninApiService.ts b/packages/backend/src/server/api/SigninApiService.ts
index fa9155d82d..72712bce60 100644
--- a/packages/backend/src/server/api/SigninApiService.ts
+++ b/packages/backend/src/server/api/SigninApiService.ts
@@ -26,12 +26,19 @@ import { UserAuthService } from '@/core/UserAuthService.js';
 import { CaptchaService } from '@/core/CaptchaService.js';
 import { FastifyReplyError } from '@/misc/fastify-reply-error.js';
 import { isSystemAccount } from '@/misc/is-system-account.js';
-import { SkRateLimiterService } from '@/server/api/SkRateLimiterService.js';
-import { sendRateLimitHeaders } from '@/misc/rate-limit-utils.js';
+import { SkRateLimiterService } from '@/server/SkRateLimiterService.js';
+import { Keyed, RateLimit, sendRateLimitHeaders } from '@/misc/rate-limit-utils.js';
 import { SigninService } from './SigninService.js';
 import type { AuthenticationResponseJSON } from '@simplewebauthn/types';
 import type { FastifyReply, FastifyRequest } from 'fastify';
 
+// Up to 10 attempts, then 1 per minute
+const signinRateLimit: Keyed<RateLimit> = {
+	key: 'signin',
+	max: 10,
+	dripRate: 1000 * 60,
+};
+
 @Injectable()
 export class SigninApiService {
 	constructor(
@@ -94,7 +101,7 @@ export class SigninApiService {
 		}
 
 		// not more than 1 attempt per second and not more than 10 attempts per hour
-		const rateLimit = await this.rateLimiterService.limit({ key: 'signin', duration: 60 * 60 * 1000, max: 10, minInterval: 1000 }, getIpHash(request.ip));
+		const rateLimit = await this.rateLimiterService.limit(signinRateLimit, getIpHash(request.ip));
 
 		sendRateLimitHeaders(reply, rateLimit);
 
diff --git a/packages/backend/src/server/api/SigninWithPasskeyApiService.ts b/packages/backend/src/server/api/SigninWithPasskeyApiService.ts
index e94d2b6b68..f84f50523b 100644
--- a/packages/backend/src/server/api/SigninWithPasskeyApiService.ts
+++ b/packages/backend/src/server/api/SigninWithPasskeyApiService.ts
@@ -21,7 +21,7 @@ import { WebAuthnService } from '@/core/WebAuthnService.js';
 import Logger from '@/logger.js';
 import { LoggerService } from '@/core/LoggerService.js';
 import type { IdentifiableError } from '@/misc/identifiable-error.js';
-import { SkRateLimiterService } from '@/server/api/SkRateLimiterService.js';
+import { SkRateLimiterService } from '@/server/SkRateLimiterService.js';
 import { sendRateLimitHeaders } from '@/misc/rate-limit-utils.js';
 import { SigninService } from './SigninService.js';
 import type { AuthenticationResponseJSON } from '@simplewebauthn/types';
diff --git a/packages/backend/src/server/api/SkRateLimiterService.ts b/packages/backend/src/server/api/SkRateLimiterService.ts
deleted file mode 100644
index 38c97b63df..0000000000
--- a/packages/backend/src/server/api/SkRateLimiterService.ts
+++ /dev/null
@@ -1,253 +0,0 @@
-/*
- * SPDX-FileCopyrightText: hazelnoot and other Sharkey contributors
- * SPDX-License-Identifier: AGPL-3.0-only
- */
-
-import { Inject, Injectable } from '@nestjs/common';
-import Redis from 'ioredis';
-import { TimeService } from '@/core/TimeService.js';
-import { EnvService } from '@/core/EnvService.js';
-import { BucketRateLimit, LegacyRateLimit, LimitInfo, RateLimit, hasMinLimit, isLegacyRateLimit, Keyed, hasMaxLimit, disabledLimitInfo, MaxLegacyLimit, MinLegacyLimit } from '@/misc/rate-limit-utils.js';
-import { DI } from '@/di-symbols.js';
-
-@Injectable()
-export class SkRateLimiterService {
-	private readonly disabled: boolean;
-
-	constructor(
-		@Inject(TimeService)
-		private readonly timeService: TimeService,
-
-		@Inject(DI.redis)
-		private readonly redisClient: Redis.Redis,
-
-		@Inject(EnvService)
-		envService: EnvService,
-	) {
-		this.disabled = envService.env.NODE_ENV === 'test';
-	}
-
-	/**
-	 * Check & increment a rate limit
-	 * @param limit The limit definition
-	 * @param actor Client who is calling this limit
-	 * @param factor Scaling factor - smaller = larger limit (less restrictive)
-	 */
-	public async limit(limit: Keyed<RateLimit>, actor: string, factor = 1): Promise<LimitInfo> {
-		if (this.disabled || factor === 0) {
-			return disabledLimitInfo;
-		}
-
-		if (factor < 0) {
-			throw new Error(`Rate limit factor is zero or negative: ${factor}`);
-		}
-
-		return await this.tryLimit(limit, actor, factor);
-	}
-
-	private async tryLimit(limit: Keyed<RateLimit>, actor: string, factor: number): Promise<LimitInfo> {
-		if (isLegacyRateLimit(limit)) {
-			return await this.limitLegacy(limit, actor, factor);
-		} else {
-			return await this.limitBucket(limit, actor, factor);
-		}
-	}
-
-	private async limitLegacy(limit: Keyed<LegacyRateLimit>, actor: string, factor: number): Promise<LimitInfo> {
-		if (hasMaxLimit(limit)) {
-			return await this.limitLegacyMinMax(limit, actor, factor);
-		} else if (hasMinLimit(limit)) {
-			return await this.limitLegacyMinOnly(limit, actor, factor);
-		} else {
-			return disabledLimitInfo;
-		}
-	}
-
-	private async limitLegacyMinMax(limit: Keyed<MaxLegacyLimit>, actor: string, factor: number): Promise<LimitInfo> {
-		if (limit.duration === 0) return disabledLimitInfo;
-		if (limit.duration < 0) throw new Error(`Invalid rate limit ${limit.key}: duration is negative (${limit.duration})`);
-		if (limit.max < 1) throw new Error(`Invalid rate limit ${limit.key}: max is less than 1 (${limit.max})`);
-
-		// Derive initial dripRate from minInterval OR duration/max.
-		const initialDripRate = Math.max(limit.minInterval ?? Math.round(limit.duration / limit.max), 1);
-
-		// Calculate dripSize to reach max at exactly duration
-		const dripSize = Math.max(Math.round(limit.max / (limit.duration / initialDripRate)), 1);
-
-		// Calculate final dripRate from dripSize and duration/max
-		const dripRate = Math.max(Math.round(limit.duration / (limit.max / dripSize)), 1);
-
-		const bucketLimit: Keyed<BucketRateLimit> = {
-			type: 'bucket',
-			key: limit.key,
-			size: limit.max,
-			dripRate,
-			dripSize,
-		};
-		return await this.limitBucket(bucketLimit, actor, factor);
-	}
-
-	private async limitLegacyMinOnly(limit: Keyed<MinLegacyLimit>, actor: string, factor: number): Promise<LimitInfo> {
-		if (limit.minInterval === 0) return disabledLimitInfo;
-		if (limit.minInterval < 0) throw new Error(`Invalid rate limit ${limit.key}: minInterval is negative (${limit.minInterval})`);
-
-		const dripRate = Math.max(Math.round(limit.minInterval), 1);
-		const bucketLimit: Keyed<BucketRateLimit> = {
-			type: 'bucket',
-			key: limit.key,
-			size: 1,
-			dripRate,
-			dripSize: 1,
-		};
-		return await this.limitBucket(bucketLimit, actor, factor);
-	}
-
-	/**
-	 * Implementation of Leaky Bucket rate limiting - see SkRateLimiterService.md for details.
-	 */
-	private async limitBucket(limit: Keyed<BucketRateLimit>, actor: string, factor: number): Promise<LimitInfo> {
-		if (limit.size < 1) throw new Error(`Invalid rate limit ${limit.key}: size is less than 1 (${limit.size})`);
-		if (limit.dripRate != null && limit.dripRate < 1) throw new Error(`Invalid rate limit ${limit.key}: dripRate is less than 1 (${limit.dripRate})`);
-		if (limit.dripSize != null && limit.dripSize < 1) throw new Error(`Invalid rate limit ${limit.key}: dripSize is less than 1 (${limit.dripSize})`);
-
-		// 0 - Calculate
-		const now = this.timeService.now;
-		const bucketSize = Math.max(Math.ceil(limit.size / factor), 1);
-		const dripRate = Math.ceil(limit.dripRate ?? 1000);
-		const dripSize = Math.ceil(limit.dripSize ?? 1);
-		const expirationSec = Math.max(Math.ceil((dripRate * Math.ceil(bucketSize / dripSize)) / 1000), 1);
-
-		// 1 - Read
-		const counterKey = createLimitKey(limit, actor, 'c');
-		const timestampKey = createLimitKey(limit, actor, 't');
-		const counter = await this.getLimitCounter(counterKey, timestampKey);
-
-		// 2 - Drip
-		const dripsSinceLastTick = Math.floor((now - counter.timestamp) / dripRate) * dripSize;
-		const deltaCounter = Math.min(dripsSinceLastTick, counter.counter);
-		const deltaTimestamp = dripsSinceLastTick * dripRate;
-		if (deltaCounter > 0) {
-			// Execute the next drip(s)
-			const results = await this.executeRedisMulti(
-				['get', timestampKey],
-				['incrby', timestampKey, deltaTimestamp],
-				['expire', timestampKey, expirationSec],
-				['get', timestampKey],
-				['decrby', counterKey, deltaCounter],
-				['expire', counterKey, expirationSec],
-				['get', counterKey],
-			);
-			const expectedTimestamp = counter.timestamp;
-			const canaryTimestamp = results[0] ? parseInt(results[0]) : 0;
-			counter.timestamp = results[3] ? parseInt(results[3]) : 0;
-			counter.counter = results[6] ? parseInt(results[6]) : 0;
-
-			// Check for a data collision and rollback
-			if (canaryTimestamp !== expectedTimestamp) {
-				const rollbackResults = await this.executeRedisMulti(
-					['decrby', timestampKey, deltaTimestamp],
-					['get', timestampKey],
-					['incrby', counterKey, deltaCounter],
-					['get', counterKey],
-				);
-				counter.timestamp = rollbackResults[1] ? parseInt(rollbackResults[1]) : 0;
-				counter.counter = rollbackResults[3] ? parseInt(rollbackResults[3]) : 0;
-			}
-		}
-
-		// 3 - Check
-		const blocked = counter.counter >= bucketSize;
-		if (!blocked) {
-			if (counter.timestamp === 0) {
-				const results = await this.executeRedisMulti(
-					['set', timestampKey, now],
-					['expire', timestampKey, expirationSec],
-					['incr', counterKey],
-					['expire', counterKey, expirationSec],
-					['get', counterKey],
-				);
-				counter.timestamp = now;
-				counter.counter = results[4] ? parseInt(results[4]) : 0;
-			} else {
-				const results = await this.executeRedisMulti(
-					['incr', counterKey],
-					['expire', counterKey, expirationSec],
-					['get', counterKey],
-				);
-				counter.counter = results[2] ? parseInt(results[2]) : 0;
-			}
-		}
-
-		// Calculate how much time is needed to free up a bucket slot
-		const overflow = Math.max((counter.counter + 1) - bucketSize, 0);
-		const dripsNeeded = Math.ceil(overflow / dripSize);
-		const timeNeeded = Math.max((dripRate * dripsNeeded) - (this.timeService.now - counter.timestamp), 0);
-
-		// Calculate limit status
-		const remaining = Math.max(bucketSize - counter.counter, 0);
-		const resetMs = timeNeeded;
-		const resetSec = Math.ceil(resetMs / 1000);
-		const fullResetMs = Math.ceil(counter.counter / dripSize) * dripRate;
-		const fullResetSec = Math.ceil(fullResetMs / 1000);
-		return { blocked, remaining, resetSec, resetMs, fullResetSec, fullResetMs };
-	}
-
-	private async getLimitCounter(counterKey: string, timestampKey: string): Promise<LimitCounter> {
-		const [counter, timestamp] = await this.executeRedisMulti(
-			['get', counterKey],
-			['get', timestampKey],
-		);
-
-		return {
-			counter: counter ? parseInt(counter) : 0,
-			timestamp: timestamp ? parseInt(timestamp) : 0,
-		};
-	}
-
-	private async executeRedisMulti(...batch: RedisCommand[]): Promise<RedisResult[]> {
-		const results = await this.redisClient.multi(batch).exec();
-
-		// Transaction conflict (retryable)
-		if (!results) {
-			throw new ConflictError('Redis error: transaction conflict');
-		}
-
-		// Transaction failed (fatal)
-		if (results.length !== batch.length) {
-			throw new Error('Redis error: failed to execute batch');
-		}
-
-		// Map responses
-		const errors: Error[] = [];
-		const responses: RedisResult[] = [];
-		for (const [error, response] of results) {
-			if (error) errors.push(error);
-			responses.push(response as RedisResult);
-		}
-
-		// Command failed (fatal)
-		if (errors.length > 0) {
-			const errorMessages = errors
-				.map((e, i) => `Error in command ${i}: ${e}`)
-				.join('\', \'');
-			throw new AggregateError(errors, `Redis error: failed to execute command(s): '${errorMessages}'`);
-		}
-
-		return responses;
-	}
-}
-
-// Not correct, but good enough for the basic commands we use.
-type RedisResult = string | null;
-type RedisCommand = [command: string, ...args: unknown[]];
-
-function createLimitKey(limit: Keyed<RateLimit>, actor: string, value: string): string {
-	return `rl_${actor}_${limit.key}_${value}`;
-}
-
-class ConflictError extends Error {}
-
-interface LimitCounter {
-	timestamp: number;
-	counter: number;
-}
diff --git a/packages/backend/src/server/api/StreamingApiServerService.ts b/packages/backend/src/server/api/StreamingApiServerService.ts
index e3fd1312ae..6e7abcfae6 100644
--- a/packages/backend/src/server/api/StreamingApiServerService.ts
+++ b/packages/backend/src/server/api/StreamingApiServerService.ts
@@ -18,10 +18,9 @@ import { CacheService } from '@/core/CacheService.js';
 import { MiLocalUser } from '@/models/User.js';
 import { UserService } from '@/core/UserService.js';
 import { ChannelFollowingService } from '@/core/ChannelFollowingService.js';
-import { RoleService } from '@/core/RoleService.js';
 import { getIpHash } from '@/misc/get-ip-hash.js';
 import { LoggerService } from '@/core/LoggerService.js';
-import { SkRateLimiterService } from '@/server/api/SkRateLimiterService.js';
+import { SkRateLimiterService } from '@/server/SkRateLimiterService.js';
 import { AuthenticateService, AuthenticationError } from './AuthenticateService.js';
 import MainStreamConnection from './stream/Connection.js';
 import { ChannelsService } from './stream/ChannelsService.js';
@@ -49,7 +48,6 @@ export class StreamingApiServerService {
 		private usersService: UserService,
 		private channelFollowingService: ChannelFollowingService,
 		private rateLimiterService: SkRateLimiterService,
-		private roleService: RoleService,
 		private loggerService: LoggerService,
 	) {
 	}
@@ -57,22 +55,18 @@ export class StreamingApiServerService {
 	@bindThis
 	private async rateLimitThis(
 		user: MiLocalUser | null | undefined,
-		requestIp: string | undefined,
+		requestIp: string,
 		limit: IEndpointMeta['limit'] & { key: NonNullable<string> },
 	) : Promise<boolean> {
-		let limitActor: string;
+		let limitActor: string | MiLocalUser;
 		if (user) {
-			limitActor = user.id;
+			limitActor = user;
 		} else {
-			limitActor = getIpHash(requestIp || 'wtf');
+			limitActor = getIpHash(requestIp);
 		}
 
-		const factor = user ? (await this.roleService.getUserPolicies(user.id)).rateLimitFactor : 1;
-
-		if (factor <= 0) return false;
-
 		// Rate limit
-		const rateLimit = await this.rateLimiterService.limit(limit, limitActor, factor);
+		const rateLimit = await this.rateLimiterService.limit(limit, limitActor);
 		return rateLimit.blocked;
 	}