From 09669d72e7e2474141a2712a12c6dafe290ccf88 Mon Sep 17 00:00:00 2001 From: Hazelnoot Date: Sun, 2 Feb 2025 22:02:08 -0500 Subject: lookup and cache rate limit factors directly within SkRateLimiterService --- packages/backend/src/server/FileServerService.ts | 21 +++++++++------------ 1 file changed, 9 insertions(+), 12 deletions(-) (limited to 'packages/backend/src/server/FileServerService.ts') diff --git a/packages/backend/src/server/FileServerService.ts b/packages/backend/src/server/FileServerService.ts index 5293d529ad..18cdda5430 100644 --- a/packages/backend/src/server/FileServerService.ts +++ b/packages/backend/src/server/FileServerService.ts @@ -11,7 +11,7 @@ import rename from 'rename'; import sharp from 'sharp'; import { sharpBmp } from '@misskey-dev/sharp-read-bmp'; import type { Config } from '@/config.js'; -import type { MiDriveFile, DriveFilesRepository } from '@/models/_.js'; +import type { MiDriveFile, DriveFilesRepository, MiUser } from '@/models/_.js'; import { DI } from '@/di-symbols.js'; import { createTemp } from '@/misc/create-temp.js'; import { FILE_TYPE_BROWSERSAFE } from '@/const.js'; @@ -30,7 +30,6 @@ import { correctFilename } from '@/misc/correct-filename.js'; import { handleRequestRedirectToOmitSearch } from '@/misc/fastify-hook-handlers.js'; import { getIpHash } from '@/misc/get-ip-hash.js'; import { AuthenticateService } from '@/server/api/AuthenticateService.js'; -import { RoleService } from '@/core/RoleService.js'; import { SkRateLimiterService } from '@/server/api/SkRateLimiterService.js'; import { Keyed, RateLimit, sendRateLimitHeaders } from '@/misc/rate-limit-utils.js'; import type { FastifyInstance, FastifyRequest, FastifyReply, FastifyPluginOptions } from 'fastify'; @@ -59,7 +58,6 @@ export class FileServerService { private loggerService: LoggerService, private authenticateService: AuthenticateService, private rateLimiterService: SkRateLimiterService, - private roleService: RoleService, ) { this.logger = this.loggerService.getLogger('server', 'gray'); @@ -625,14 +623,13 @@ export class FileServerService { // koa will automatically load the `X-Forwarded-For` header if `proxy: true` is configured in the app. const [user] = await this.authenticateService.authenticate(token); - const actor = user?.id ?? getIpHash(request.ip); - const factor = user ? (await this.roleService.getUserPolicies(user.id)).rateLimitFactor : 1; + const actor = user ?? getIpHash(request.ip); // Call both limits: the per-resource limit and the shared cross-resource limit - return await this.checkResourceLimit(reply, actor, group, resource, factor) && await this.checkSharedLimit(reply, actor, group, factor); + return await this.checkResourceLimit(reply, actor, group, resource) && await this.checkSharedLimit(reply, actor, group); } - private async checkResourceLimit(reply: FastifyReply, actor: string, group: string, resource: string, factor = 1): Promise { + private async checkResourceLimit(reply: FastifyReply, actor: string | MiUser, group: string, resource: string): Promise { const limit: Keyed = { // Group by resource key: `${group}${resource}`, @@ -643,10 +640,10 @@ export class FileServerService { dripRate: 1000 * 60, }; - return await this.checkLimit(reply, actor, limit, factor); + return await this.checkLimit(reply, actor, limit); } - private async checkSharedLimit(reply: FastifyReply, actor: string, group: string, factor = 1): Promise { + private async checkSharedLimit(reply: FastifyReply, actor: string | MiUser, group: string): Promise { const limit: Keyed = { key: group, type: 'bucket', @@ -655,11 +652,11 @@ export class FileServerService { size: 3600, }; - return await this.checkLimit(reply, actor, limit, factor); + return await this.checkLimit(reply, actor, limit); } - private async checkLimit(reply: FastifyReply, actor: string, limit: Keyed, factor = 1): Promise { - const info = await this.rateLimiterService.limit(limit, actor, factor); + private async checkLimit(reply: FastifyReply, actor: string | MiUser, limit: Keyed): Promise { + const info = await this.rateLimiterService.limit(limit, actor); sendRateLimitHeaders(reply, info); -- cgit v1.2.3-freya From f92fb3bb8c70b5ca35f4f61a437b101bd5bae7e8 Mon Sep 17 00:00:00 2001 From: Hazelnoot Date: Wed, 5 Feb 2025 11:16:39 -0500 Subject: move SkRateLimiterService to correct directory --- packages/backend/src/server/FileServerService.ts | 2 +- packages/backend/src/server/ServerModule.ts | 2 +- .../backend/src/server/SkRateLimiterService.ts | 280 +++++++++++++++++++++ packages/backend/src/server/api/ApiCallService.ts | 2 +- .../backend/src/server/api/SigninApiService.ts | 2 +- .../src/server/api/SigninWithPasskeyApiService.ts | 2 +- .../backend/src/server/api/SkRateLimiterService.ts | 280 --------------------- .../src/server/api/StreamingApiServerService.ts | 2 +- .../test/unit/SigninWithPasskeyApiService.ts | 2 +- .../unit/server/api/SkRateLimiterServiceTests.ts | 2 +- 10 files changed, 288 insertions(+), 288 deletions(-) create mode 100644 packages/backend/src/server/SkRateLimiterService.ts delete mode 100644 packages/backend/src/server/api/SkRateLimiterService.ts (limited to 'packages/backend/src/server/FileServerService.ts') diff --git a/packages/backend/src/server/FileServerService.ts b/packages/backend/src/server/FileServerService.ts index 18cdda5430..a7e13a1b78 100644 --- a/packages/backend/src/server/FileServerService.ts +++ b/packages/backend/src/server/FileServerService.ts @@ -30,7 +30,7 @@ import { correctFilename } from '@/misc/correct-filename.js'; import { handleRequestRedirectToOmitSearch } from '@/misc/fastify-hook-handlers.js'; import { getIpHash } from '@/misc/get-ip-hash.js'; import { AuthenticateService } from '@/server/api/AuthenticateService.js'; -import { SkRateLimiterService } from '@/server/api/SkRateLimiterService.js'; +import { SkRateLimiterService } from '@/server/SkRateLimiterService.js'; import { Keyed, RateLimit, sendRateLimitHeaders } from '@/misc/rate-limit-utils.js'; import type { FastifyInstance, FastifyRequest, FastifyReply, FastifyPluginOptions } from 'fastify'; diff --git a/packages/backend/src/server/ServerModule.ts b/packages/backend/src/server/ServerModule.ts index c1d7c088f1..6b5156106a 100644 --- a/packages/backend/src/server/ServerModule.ts +++ b/packages/backend/src/server/ServerModule.ts @@ -6,7 +6,7 @@ import { Module } from '@nestjs/common'; import { EndpointsModule } from '@/server/api/EndpointsModule.js'; import { CoreModule } from '@/core/CoreModule.js'; -import { SkRateLimiterService } from '@/server/api/SkRateLimiterService.js'; +import { SkRateLimiterService } from '@/server/SkRateLimiterService.js'; import { ApiCallService } from './api/ApiCallService.js'; import { FileServerService } from './FileServerService.js'; import { HealthServerService } from './HealthServerService.js'; diff --git a/packages/backend/src/server/SkRateLimiterService.ts b/packages/backend/src/server/SkRateLimiterService.ts new file mode 100644 index 0000000000..038f12cb25 --- /dev/null +++ b/packages/backend/src/server/SkRateLimiterService.ts @@ -0,0 +1,280 @@ +/* + * SPDX-FileCopyrightText: hazelnoot and other Sharkey contributors + * SPDX-License-Identifier: AGPL-3.0-only + */ + +import { Inject, Injectable } from '@nestjs/common'; +import Redis from 'ioredis'; +import type { TimeService } from '@/core/TimeService.js'; +import type { EnvService } from '@/core/EnvService.js'; +import { BucketRateLimit, LegacyRateLimit, LimitInfo, RateLimit, hasMinLimit, isLegacyRateLimit, Keyed, hasMaxLimit, disabledLimitInfo, MaxLegacyLimit, MinLegacyLimit } from '@/misc/rate-limit-utils.js'; +import { DI } from '@/di-symbols.js'; +import { MemoryKVCache } from '@/misc/cache.js'; +import type { MiUser } from '@/models/_.js'; +import type { RoleService } from '@/core/RoleService.js'; + +// Sentinel value used for caching the default role template. +// Required because MemoryKVCache doesn't support null keys. +const defaultUserKey = ''; + +@Injectable() +export class SkRateLimiterService { + // 1-minute cache interval + private readonly factorCache = new MemoryKVCache(1000 * 60); + private readonly disabled: boolean; + + constructor( + @Inject('TimeService') + private readonly timeService: TimeService, + + @Inject(DI.redis) + private readonly redisClient: Redis.Redis, + + @Inject('RoleService') + private readonly roleService: RoleService, + + @Inject('EnvService') + envService: EnvService, + ) { + this.disabled = envService.env.NODE_ENV === 'test'; + } + + /** + * Check & increment a rate limit for a client. + * + * If the client (actorOrUser) is passed as a string, then it uses the default rate limit factor from the role template. + * If the client (actorOrUser) is passed as an MiUser, then it queries the user's actual rate limit factor from their assigned roles. + * + * A factor of zero (0) will disable the limit, while any negative number will produce an error. + * A factor between zero (0) and one (1) will increase the limit from its default values (allowing more actions per time interval). + * A factor greater than one (1) will decrease the limit from its default values (allowing fewer actions per time interval). + * + * @param limit The limit definition + * @param actorOrUser authenticated client user or IP hash + */ + public async limit(limit: Keyed, actorOrUser: string | MiUser): Promise { + if (this.disabled) { + return disabledLimitInfo; + } + + const actor = typeof(actorOrUser) === 'object' ? actorOrUser.id : actorOrUser; + const userCacheKey = typeof(actorOrUser) === 'object' ? actorOrUser.id : defaultUserKey; + const userRoleKey = typeof(actorOrUser) === 'object' ? actorOrUser.id : null; + const factor = this.factorCache.get(userCacheKey) ?? await this.factorCache.fetch(userCacheKey, async () => { + const role = await this.roleService.getUserPolicies(userRoleKey); + return role.rateLimitFactor; + }); + + if (factor === 0) { + return disabledLimitInfo; + } + + if (factor < 0) { + throw new Error(`Rate limit factor is zero or negative: ${factor}`); + } + + if (isLegacyRateLimit(limit)) { + return await this.limitLegacy(limit, actor, factor); + } else { + return await this.limitBucket(limit, actor, factor); + } + } + + private async limitLegacy(limit: Keyed, actor: string, factor: number): Promise { + if (hasMaxLimit(limit)) { + return await this.limitLegacyMinMax(limit, actor, factor); + } else if (hasMinLimit(limit)) { + return await this.limitLegacyMinOnly(limit, actor, factor); + } else { + return disabledLimitInfo; + } + } + + private async limitLegacyMinMax(limit: Keyed, actor: string, factor: number): Promise { + if (limit.duration === 0) return disabledLimitInfo; + if (limit.duration < 0) throw new Error(`Invalid rate limit ${limit.key}: duration is negative (${limit.duration})`); + if (limit.max < 1) throw new Error(`Invalid rate limit ${limit.key}: max is less than 1 (${limit.max})`); + + // Derive initial dripRate from minInterval OR duration/max. + const initialDripRate = Math.max(limit.minInterval ?? Math.round(limit.duration / limit.max), 1); + + // Calculate dripSize to reach max at exactly duration + const dripSize = Math.max(Math.round(limit.max / (limit.duration / initialDripRate)), 1); + + // Calculate final dripRate from dripSize and duration/max + const dripRate = Math.max(Math.round(limit.duration / (limit.max / dripSize)), 1); + + const bucketLimit: Keyed = { + type: 'bucket', + key: limit.key, + size: limit.max, + dripRate, + dripSize, + }; + return await this.limitBucket(bucketLimit, actor, factor); + } + + private async limitLegacyMinOnly(limit: Keyed, actor: string, factor: number): Promise { + if (limit.minInterval === 0) return disabledLimitInfo; + if (limit.minInterval < 0) throw new Error(`Invalid rate limit ${limit.key}: minInterval is negative (${limit.minInterval})`); + + const dripRate = Math.max(Math.round(limit.minInterval), 1); + const bucketLimit: Keyed = { + type: 'bucket', + key: limit.key, + size: 1, + dripRate, + dripSize: 1, + }; + return await this.limitBucket(bucketLimit, actor, factor); + } + + /** + * Implementation of Leaky Bucket rate limiting - see SkRateLimiterService.md for details. + */ + private async limitBucket(limit: Keyed, actor: string, factor: number): Promise { + if (limit.size < 1) throw new Error(`Invalid rate limit ${limit.key}: size is less than 1 (${limit.size})`); + if (limit.dripRate != null && limit.dripRate < 1) throw new Error(`Invalid rate limit ${limit.key}: dripRate is less than 1 (${limit.dripRate})`); + if (limit.dripSize != null && limit.dripSize < 1) throw new Error(`Invalid rate limit ${limit.key}: dripSize is less than 1 (${limit.dripSize})`); + + // 0 - Calculate + const now = this.timeService.now; + const bucketSize = Math.max(Math.ceil(limit.size / factor), 1); + const dripRate = Math.ceil(limit.dripRate ?? 1000); + const dripSize = Math.ceil(limit.dripSize ?? 1); + const expirationSec = Math.max(Math.ceil((dripRate * Math.ceil(bucketSize / dripSize)) / 1000), 1); + + // 1 - Read + const counterKey = createLimitKey(limit, actor, 'c'); + const timestampKey = createLimitKey(limit, actor, 't'); + const counter = await this.getLimitCounter(counterKey, timestampKey); + + // 2 - Drip + const dripsSinceLastTick = Math.floor((now - counter.timestamp) / dripRate) * dripSize; + const deltaCounter = Math.min(dripsSinceLastTick, counter.counter); + const deltaTimestamp = dripsSinceLastTick * dripRate; + if (deltaCounter > 0) { + // Execute the next drip(s) + const results = await this.executeRedisMulti( + ['get', timestampKey], + ['incrby', timestampKey, deltaTimestamp], + ['expire', timestampKey, expirationSec], + ['get', timestampKey], + ['decrby', counterKey, deltaCounter], + ['expire', counterKey, expirationSec], + ['get', counterKey], + ); + const expectedTimestamp = counter.timestamp; + const canaryTimestamp = results[0] ? parseInt(results[0]) : 0; + counter.timestamp = results[3] ? parseInt(results[3]) : 0; + counter.counter = results[6] ? parseInt(results[6]) : 0; + + // Check for a data collision and rollback + if (canaryTimestamp !== expectedTimestamp) { + const rollbackResults = await this.executeRedisMulti( + ['decrby', timestampKey, deltaTimestamp], + ['get', timestampKey], + ['incrby', counterKey, deltaCounter], + ['get', counterKey], + ); + counter.timestamp = rollbackResults[1] ? parseInt(rollbackResults[1]) : 0; + counter.counter = rollbackResults[3] ? parseInt(rollbackResults[3]) : 0; + } + } + + // 3 - Check + const blocked = counter.counter >= bucketSize; + if (!blocked) { + if (counter.timestamp === 0) { + const results = await this.executeRedisMulti( + ['set', timestampKey, now], + ['expire', timestampKey, expirationSec], + ['incr', counterKey], + ['expire', counterKey, expirationSec], + ['get', counterKey], + ); + counter.timestamp = now; + counter.counter = results[4] ? parseInt(results[4]) : 0; + } else { + const results = await this.executeRedisMulti( + ['incr', counterKey], + ['expire', counterKey, expirationSec], + ['get', counterKey], + ); + counter.counter = results[2] ? parseInt(results[2]) : 0; + } + } + + // Calculate how much time is needed to free up a bucket slot + const overflow = Math.max((counter.counter + 1) - bucketSize, 0); + const dripsNeeded = Math.ceil(overflow / dripSize); + const timeNeeded = Math.max((dripRate * dripsNeeded) - (this.timeService.now - counter.timestamp), 0); + + // Calculate limit status + const remaining = Math.max(bucketSize - counter.counter, 0); + const resetMs = timeNeeded; + const resetSec = Math.ceil(resetMs / 1000); + const fullResetMs = Math.ceil(counter.counter / dripSize) * dripRate; + const fullResetSec = Math.ceil(fullResetMs / 1000); + return { blocked, remaining, resetSec, resetMs, fullResetSec, fullResetMs }; + } + + private async getLimitCounter(counterKey: string, timestampKey: string): Promise { + const [counter, timestamp] = await this.executeRedisMulti( + ['get', counterKey], + ['get', timestampKey], + ); + + return { + counter: counter ? parseInt(counter) : 0, + timestamp: timestamp ? parseInt(timestamp) : 0, + }; + } + + private async executeRedisMulti(...batch: RedisCommand[]): Promise { + const results = await this.redisClient.multi(batch).exec(); + + // Transaction conflict (retryable) + if (!results) { + throw new ConflictError('Redis error: transaction conflict'); + } + + // Transaction failed (fatal) + if (results.length !== batch.length) { + throw new Error('Redis error: failed to execute batch'); + } + + // Map responses + const errors: Error[] = []; + const responses: RedisResult[] = []; + for (const [error, response] of results) { + if (error) errors.push(error); + responses.push(response as RedisResult); + } + + // Command failed (fatal) + if (errors.length > 0) { + const errorMessages = errors + .map((e, i) => `Error in command ${i}: ${e}`) + .join('\', \''); + throw new AggregateError(errors, `Redis error: failed to execute command(s): '${errorMessages}'`); + } + + return responses; + } +} + +// Not correct, but good enough for the basic commands we use. +type RedisResult = string | null; +type RedisCommand = [command: string, ...args: unknown[]]; + +function createLimitKey(limit: Keyed, actor: string, value: string): string { + return `rl_${actor}_${limit.key}_${value}`; +} + +class ConflictError extends Error {} + +interface LimitCounter { + timestamp: number; + counter: number; +} diff --git a/packages/backend/src/server/api/ApiCallService.ts b/packages/backend/src/server/api/ApiCallService.ts index 9c3952d541..5ce358d68f 100644 --- a/packages/backend/src/server/api/ApiCallService.ts +++ b/packages/backend/src/server/api/ApiCallService.ts @@ -19,7 +19,7 @@ import { bindThis } from '@/decorators.js'; import { RoleService } from '@/core/RoleService.js'; import type { Config } from '@/config.js'; import { sendRateLimitHeaders } from '@/misc/rate-limit-utils.js'; -import { SkRateLimiterService } from '@/server/api/SkRateLimiterService.js'; +import { SkRateLimiterService } from '@/server/SkRateLimiterService.js'; import { ApiError } from './error.js'; import { ApiLoggerService } from './ApiLoggerService.js'; import { AuthenticateService, AuthenticationError } from './AuthenticateService.js'; diff --git a/packages/backend/src/server/api/SigninApiService.ts b/packages/backend/src/server/api/SigninApiService.ts index 25ea03a356..72712bce60 100644 --- a/packages/backend/src/server/api/SigninApiService.ts +++ b/packages/backend/src/server/api/SigninApiService.ts @@ -26,7 +26,7 @@ import { UserAuthService } from '@/core/UserAuthService.js'; import { CaptchaService } from '@/core/CaptchaService.js'; import { FastifyReplyError } from '@/misc/fastify-reply-error.js'; import { isSystemAccount } from '@/misc/is-system-account.js'; -import { SkRateLimiterService } from '@/server/api/SkRateLimiterService.js'; +import { SkRateLimiterService } from '@/server/SkRateLimiterService.js'; import { Keyed, RateLimit, sendRateLimitHeaders } from '@/misc/rate-limit-utils.js'; import { SigninService } from './SigninService.js'; import type { AuthenticationResponseJSON } from '@simplewebauthn/types'; diff --git a/packages/backend/src/server/api/SigninWithPasskeyApiService.ts b/packages/backend/src/server/api/SigninWithPasskeyApiService.ts index e94d2b6b68..f84f50523b 100644 --- a/packages/backend/src/server/api/SigninWithPasskeyApiService.ts +++ b/packages/backend/src/server/api/SigninWithPasskeyApiService.ts @@ -21,7 +21,7 @@ import { WebAuthnService } from '@/core/WebAuthnService.js'; import Logger from '@/logger.js'; import { LoggerService } from '@/core/LoggerService.js'; import type { IdentifiableError } from '@/misc/identifiable-error.js'; -import { SkRateLimiterService } from '@/server/api/SkRateLimiterService.js'; +import { SkRateLimiterService } from '@/server/SkRateLimiterService.js'; import { sendRateLimitHeaders } from '@/misc/rate-limit-utils.js'; import { SigninService } from './SigninService.js'; import type { AuthenticationResponseJSON } from '@simplewebauthn/types'; diff --git a/packages/backend/src/server/api/SkRateLimiterService.ts b/packages/backend/src/server/api/SkRateLimiterService.ts deleted file mode 100644 index 038f12cb25..0000000000 --- a/packages/backend/src/server/api/SkRateLimiterService.ts +++ /dev/null @@ -1,280 +0,0 @@ -/* - * SPDX-FileCopyrightText: hazelnoot and other Sharkey contributors - * SPDX-License-Identifier: AGPL-3.0-only - */ - -import { Inject, Injectable } from '@nestjs/common'; -import Redis from 'ioredis'; -import type { TimeService } from '@/core/TimeService.js'; -import type { EnvService } from '@/core/EnvService.js'; -import { BucketRateLimit, LegacyRateLimit, LimitInfo, RateLimit, hasMinLimit, isLegacyRateLimit, Keyed, hasMaxLimit, disabledLimitInfo, MaxLegacyLimit, MinLegacyLimit } from '@/misc/rate-limit-utils.js'; -import { DI } from '@/di-symbols.js'; -import { MemoryKVCache } from '@/misc/cache.js'; -import type { MiUser } from '@/models/_.js'; -import type { RoleService } from '@/core/RoleService.js'; - -// Sentinel value used for caching the default role template. -// Required because MemoryKVCache doesn't support null keys. -const defaultUserKey = ''; - -@Injectable() -export class SkRateLimiterService { - // 1-minute cache interval - private readonly factorCache = new MemoryKVCache(1000 * 60); - private readonly disabled: boolean; - - constructor( - @Inject('TimeService') - private readonly timeService: TimeService, - - @Inject(DI.redis) - private readonly redisClient: Redis.Redis, - - @Inject('RoleService') - private readonly roleService: RoleService, - - @Inject('EnvService') - envService: EnvService, - ) { - this.disabled = envService.env.NODE_ENV === 'test'; - } - - /** - * Check & increment a rate limit for a client. - * - * If the client (actorOrUser) is passed as a string, then it uses the default rate limit factor from the role template. - * If the client (actorOrUser) is passed as an MiUser, then it queries the user's actual rate limit factor from their assigned roles. - * - * A factor of zero (0) will disable the limit, while any negative number will produce an error. - * A factor between zero (0) and one (1) will increase the limit from its default values (allowing more actions per time interval). - * A factor greater than one (1) will decrease the limit from its default values (allowing fewer actions per time interval). - * - * @param limit The limit definition - * @param actorOrUser authenticated client user or IP hash - */ - public async limit(limit: Keyed, actorOrUser: string | MiUser): Promise { - if (this.disabled) { - return disabledLimitInfo; - } - - const actor = typeof(actorOrUser) === 'object' ? actorOrUser.id : actorOrUser; - const userCacheKey = typeof(actorOrUser) === 'object' ? actorOrUser.id : defaultUserKey; - const userRoleKey = typeof(actorOrUser) === 'object' ? actorOrUser.id : null; - const factor = this.factorCache.get(userCacheKey) ?? await this.factorCache.fetch(userCacheKey, async () => { - const role = await this.roleService.getUserPolicies(userRoleKey); - return role.rateLimitFactor; - }); - - if (factor === 0) { - return disabledLimitInfo; - } - - if (factor < 0) { - throw new Error(`Rate limit factor is zero or negative: ${factor}`); - } - - if (isLegacyRateLimit(limit)) { - return await this.limitLegacy(limit, actor, factor); - } else { - return await this.limitBucket(limit, actor, factor); - } - } - - private async limitLegacy(limit: Keyed, actor: string, factor: number): Promise { - if (hasMaxLimit(limit)) { - return await this.limitLegacyMinMax(limit, actor, factor); - } else if (hasMinLimit(limit)) { - return await this.limitLegacyMinOnly(limit, actor, factor); - } else { - return disabledLimitInfo; - } - } - - private async limitLegacyMinMax(limit: Keyed, actor: string, factor: number): Promise { - if (limit.duration === 0) return disabledLimitInfo; - if (limit.duration < 0) throw new Error(`Invalid rate limit ${limit.key}: duration is negative (${limit.duration})`); - if (limit.max < 1) throw new Error(`Invalid rate limit ${limit.key}: max is less than 1 (${limit.max})`); - - // Derive initial dripRate from minInterval OR duration/max. - const initialDripRate = Math.max(limit.minInterval ?? Math.round(limit.duration / limit.max), 1); - - // Calculate dripSize to reach max at exactly duration - const dripSize = Math.max(Math.round(limit.max / (limit.duration / initialDripRate)), 1); - - // Calculate final dripRate from dripSize and duration/max - const dripRate = Math.max(Math.round(limit.duration / (limit.max / dripSize)), 1); - - const bucketLimit: Keyed = { - type: 'bucket', - key: limit.key, - size: limit.max, - dripRate, - dripSize, - }; - return await this.limitBucket(bucketLimit, actor, factor); - } - - private async limitLegacyMinOnly(limit: Keyed, actor: string, factor: number): Promise { - if (limit.minInterval === 0) return disabledLimitInfo; - if (limit.minInterval < 0) throw new Error(`Invalid rate limit ${limit.key}: minInterval is negative (${limit.minInterval})`); - - const dripRate = Math.max(Math.round(limit.minInterval), 1); - const bucketLimit: Keyed = { - type: 'bucket', - key: limit.key, - size: 1, - dripRate, - dripSize: 1, - }; - return await this.limitBucket(bucketLimit, actor, factor); - } - - /** - * Implementation of Leaky Bucket rate limiting - see SkRateLimiterService.md for details. - */ - private async limitBucket(limit: Keyed, actor: string, factor: number): Promise { - if (limit.size < 1) throw new Error(`Invalid rate limit ${limit.key}: size is less than 1 (${limit.size})`); - if (limit.dripRate != null && limit.dripRate < 1) throw new Error(`Invalid rate limit ${limit.key}: dripRate is less than 1 (${limit.dripRate})`); - if (limit.dripSize != null && limit.dripSize < 1) throw new Error(`Invalid rate limit ${limit.key}: dripSize is less than 1 (${limit.dripSize})`); - - // 0 - Calculate - const now = this.timeService.now; - const bucketSize = Math.max(Math.ceil(limit.size / factor), 1); - const dripRate = Math.ceil(limit.dripRate ?? 1000); - const dripSize = Math.ceil(limit.dripSize ?? 1); - const expirationSec = Math.max(Math.ceil((dripRate * Math.ceil(bucketSize / dripSize)) / 1000), 1); - - // 1 - Read - const counterKey = createLimitKey(limit, actor, 'c'); - const timestampKey = createLimitKey(limit, actor, 't'); - const counter = await this.getLimitCounter(counterKey, timestampKey); - - // 2 - Drip - const dripsSinceLastTick = Math.floor((now - counter.timestamp) / dripRate) * dripSize; - const deltaCounter = Math.min(dripsSinceLastTick, counter.counter); - const deltaTimestamp = dripsSinceLastTick * dripRate; - if (deltaCounter > 0) { - // Execute the next drip(s) - const results = await this.executeRedisMulti( - ['get', timestampKey], - ['incrby', timestampKey, deltaTimestamp], - ['expire', timestampKey, expirationSec], - ['get', timestampKey], - ['decrby', counterKey, deltaCounter], - ['expire', counterKey, expirationSec], - ['get', counterKey], - ); - const expectedTimestamp = counter.timestamp; - const canaryTimestamp = results[0] ? parseInt(results[0]) : 0; - counter.timestamp = results[3] ? parseInt(results[3]) : 0; - counter.counter = results[6] ? parseInt(results[6]) : 0; - - // Check for a data collision and rollback - if (canaryTimestamp !== expectedTimestamp) { - const rollbackResults = await this.executeRedisMulti( - ['decrby', timestampKey, deltaTimestamp], - ['get', timestampKey], - ['incrby', counterKey, deltaCounter], - ['get', counterKey], - ); - counter.timestamp = rollbackResults[1] ? parseInt(rollbackResults[1]) : 0; - counter.counter = rollbackResults[3] ? parseInt(rollbackResults[3]) : 0; - } - } - - // 3 - Check - const blocked = counter.counter >= bucketSize; - if (!blocked) { - if (counter.timestamp === 0) { - const results = await this.executeRedisMulti( - ['set', timestampKey, now], - ['expire', timestampKey, expirationSec], - ['incr', counterKey], - ['expire', counterKey, expirationSec], - ['get', counterKey], - ); - counter.timestamp = now; - counter.counter = results[4] ? parseInt(results[4]) : 0; - } else { - const results = await this.executeRedisMulti( - ['incr', counterKey], - ['expire', counterKey, expirationSec], - ['get', counterKey], - ); - counter.counter = results[2] ? parseInt(results[2]) : 0; - } - } - - // Calculate how much time is needed to free up a bucket slot - const overflow = Math.max((counter.counter + 1) - bucketSize, 0); - const dripsNeeded = Math.ceil(overflow / dripSize); - const timeNeeded = Math.max((dripRate * dripsNeeded) - (this.timeService.now - counter.timestamp), 0); - - // Calculate limit status - const remaining = Math.max(bucketSize - counter.counter, 0); - const resetMs = timeNeeded; - const resetSec = Math.ceil(resetMs / 1000); - const fullResetMs = Math.ceil(counter.counter / dripSize) * dripRate; - const fullResetSec = Math.ceil(fullResetMs / 1000); - return { blocked, remaining, resetSec, resetMs, fullResetSec, fullResetMs }; - } - - private async getLimitCounter(counterKey: string, timestampKey: string): Promise { - const [counter, timestamp] = await this.executeRedisMulti( - ['get', counterKey], - ['get', timestampKey], - ); - - return { - counter: counter ? parseInt(counter) : 0, - timestamp: timestamp ? parseInt(timestamp) : 0, - }; - } - - private async executeRedisMulti(...batch: RedisCommand[]): Promise { - const results = await this.redisClient.multi(batch).exec(); - - // Transaction conflict (retryable) - if (!results) { - throw new ConflictError('Redis error: transaction conflict'); - } - - // Transaction failed (fatal) - if (results.length !== batch.length) { - throw new Error('Redis error: failed to execute batch'); - } - - // Map responses - const errors: Error[] = []; - const responses: RedisResult[] = []; - for (const [error, response] of results) { - if (error) errors.push(error); - responses.push(response as RedisResult); - } - - // Command failed (fatal) - if (errors.length > 0) { - const errorMessages = errors - .map((e, i) => `Error in command ${i}: ${e}`) - .join('\', \''); - throw new AggregateError(errors, `Redis error: failed to execute command(s): '${errorMessages}'`); - } - - return responses; - } -} - -// Not correct, but good enough for the basic commands we use. -type RedisResult = string | null; -type RedisCommand = [command: string, ...args: unknown[]]; - -function createLimitKey(limit: Keyed, actor: string, value: string): string { - return `rl_${actor}_${limit.key}_${value}`; -} - -class ConflictError extends Error {} - -interface LimitCounter { - timestamp: number; - counter: number; -} diff --git a/packages/backend/src/server/api/StreamingApiServerService.ts b/packages/backend/src/server/api/StreamingApiServerService.ts index f30bbb928b..6e7abcfae6 100644 --- a/packages/backend/src/server/api/StreamingApiServerService.ts +++ b/packages/backend/src/server/api/StreamingApiServerService.ts @@ -20,7 +20,7 @@ import { UserService } from '@/core/UserService.js'; import { ChannelFollowingService } from '@/core/ChannelFollowingService.js'; import { getIpHash } from '@/misc/get-ip-hash.js'; import { LoggerService } from '@/core/LoggerService.js'; -import { SkRateLimiterService } from '@/server/api/SkRateLimiterService.js'; +import { SkRateLimiterService } from '@/server/SkRateLimiterService.js'; import { AuthenticateService, AuthenticationError } from './AuthenticateService.js'; import MainStreamConnection from './stream/Connection.js'; import { ChannelsService } from './stream/ChannelsService.js'; diff --git a/packages/backend/test/unit/SigninWithPasskeyApiService.ts b/packages/backend/test/unit/SigninWithPasskeyApiService.ts index 7df991c15c..efed905e02 100644 --- a/packages/backend/test/unit/SigninWithPasskeyApiService.ts +++ b/packages/backend/test/unit/SigninWithPasskeyApiService.ts @@ -20,7 +20,7 @@ import { SigninWithPasskeyApiService } from '@/server/api/SigninWithPasskeyApiSe import { WebAuthnService } from '@/core/WebAuthnService.js'; import { SigninService } from '@/server/api/SigninService.js'; import { IdentifiableError } from '@/misc/identifiable-error.js'; -import { SkRateLimiterService } from '@/server/api/SkRateLimiterService.js'; +import { SkRateLimiterService } from '@/server/SkRateLimiterService.js'; import { LimitInfo } from '@/misc/rate-limit-utils.js'; const moduleMocker = new ModuleMocker(global); diff --git a/packages/backend/test/unit/server/api/SkRateLimiterServiceTests.ts b/packages/backend/test/unit/server/api/SkRateLimiterServiceTests.ts index 6d4f117c87..b1f100698b 100644 --- a/packages/backend/test/unit/server/api/SkRateLimiterServiceTests.ts +++ b/packages/backend/test/unit/server/api/SkRateLimiterServiceTests.ts @@ -6,7 +6,7 @@ import type Redis from 'ioredis'; import type { MiUser } from '@/models/User.js'; import type { RolePolicies, RoleService } from '@/core/RoleService.js'; -import { SkRateLimiterService } from '@/server/api/SkRateLimiterService.js'; +import { SkRateLimiterService } from '@/server/SkRateLimiterService.js'; import { BucketRateLimit, Keyed, LegacyRateLimit } from '@/misc/rate-limit-utils.js'; /* eslint-disable @typescript-eslint/no-non-null-assertion */ -- cgit v1.2.3-freya