From 402933004a8094038e2fc9949626a7c2aa9ff9b1 Mon Sep 17 00:00:00 2001 From: Hazelnoot Date: Sun, 2 Feb 2025 17:23:36 -0500 Subject: increase sign-in rate limit --- packages/backend/src/server/api/SigninApiService.ts | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) (limited to 'packages/backend/src/server') diff --git a/packages/backend/src/server/api/SigninApiService.ts b/packages/backend/src/server/api/SigninApiService.ts index fa9155d82d..25ea03a356 100644 --- a/packages/backend/src/server/api/SigninApiService.ts +++ b/packages/backend/src/server/api/SigninApiService.ts @@ -27,11 +27,18 @@ import { CaptchaService } from '@/core/CaptchaService.js'; import { FastifyReplyError } from '@/misc/fastify-reply-error.js'; import { isSystemAccount } from '@/misc/is-system-account.js'; import { SkRateLimiterService } from '@/server/api/SkRateLimiterService.js'; -import { sendRateLimitHeaders } from '@/misc/rate-limit-utils.js'; +import { Keyed, RateLimit, sendRateLimitHeaders } from '@/misc/rate-limit-utils.js'; import { SigninService } from './SigninService.js'; import type { AuthenticationResponseJSON } from '@simplewebauthn/types'; import type { FastifyReply, FastifyRequest } from 'fastify'; +// Up to 10 attempts, then 1 per minute +const signinRateLimit: Keyed = { + key: 'signin', + max: 10, + dripRate: 1000 * 60, +}; + @Injectable() export class SigninApiService { constructor( @@ -94,7 +101,7 @@ export class SigninApiService { } // not more than 1 attempt per second and not more than 10 attempts per hour - const rateLimit = await this.rateLimiterService.limit({ key: 'signin', duration: 60 * 60 * 1000, max: 10, minInterval: 1000 }, getIpHash(request.ip)); + const rateLimit = await this.rateLimiterService.limit(signinRateLimit, getIpHash(request.ip)); sendRateLimitHeaders(reply, rateLimit); -- cgit v1.2.3-freya From 09669d72e7e2474141a2712a12c6dafe290ccf88 Mon Sep 17 00:00:00 2001 From: Hazelnoot Date: Sun, 2 Feb 2025 22:02:08 -0500 Subject: lookup and cache rate limit factors directly within SkRateLimiterService --- packages/backend/src/core/CoreModule.ts | 6 ++ .../backend/src/server/ActivityPubServerService.ts | 4 +- packages/backend/src/server/FileServerService.ts | 21 +++---- packages/backend/src/server/api/ApiCallService.ts | 37 ++++++------ .../backend/src/server/api/SkRateLimiterService.ts | 45 ++++++++++----- .../src/server/api/StreamingApiServerService.ts | 16 ++---- .../unit/server/api/SkRateLimiterServiceTests.ts | 66 ++++++++++++++++++---- 7 files changed, 124 insertions(+), 71 deletions(-) (limited to 'packages/backend/src/server') diff --git a/packages/backend/src/core/CoreModule.ts b/packages/backend/src/core/CoreModule.ts index 141f905d7f..8c9f419c44 100644 --- a/packages/backend/src/core/CoreModule.ts +++ b/packages/backend/src/core/CoreModule.ts @@ -232,6 +232,8 @@ const $FanoutTimelineEndpointService: Provider = { provide: 'FanoutTimelineEndpo const $ChannelFollowingService: Provider = { provide: 'ChannelFollowingService', useExisting: ChannelFollowingService }; const $RegistryApiService: Provider = { provide: 'RegistryApiService', useExisting: RegistryApiService }; const $ReversiService: Provider = { provide: 'ReversiService', useExisting: ReversiService }; +const $TimeService: Provider = { provide: 'TimeService', useExisting: TimeService }; +const $EnvService: Provider = { provide: 'EnvService', useExisting: EnvService }; const $ChartLoggerService: Provider = { provide: 'ChartLoggerService', useExisting: ChartLoggerService }; const $FederationChart: Provider = { provide: 'FederationChart', useExisting: FederationChart }; @@ -538,6 +540,8 @@ const $SponsorsService: Provider = { provide: 'SponsorsService', useExisting: Sp $ChannelFollowingService, $RegistryApiService, $ReversiService, + $TimeService, + $EnvService, $ChartLoggerService, $FederationChart, @@ -839,6 +843,8 @@ const $SponsorsService: Provider = { provide: 'SponsorsService', useExisting: Sp $ChannelFollowingService, $RegistryApiService, $ReversiService, + $TimeService, + $EnvService, $FederationChart, $NotesChart, diff --git a/packages/backend/src/server/ActivityPubServerService.ts b/packages/backend/src/server/ActivityPubServerService.ts index 815bf278c7..19049e528c 100644 --- a/packages/backend/src/server/ActivityPubServerService.ts +++ b/packages/backend/src/server/ActivityPubServerService.ts @@ -34,10 +34,10 @@ import { bindThis } from '@/decorators.js'; import { IActivity } from '@/core/activitypub/type.js'; import { isQuote, isRenote } from '@/misc/is-renote.js'; import * as Acct from '@/misc/acct.js'; -import type { FastifyInstance, FastifyRequest, FastifyReply, FastifyPluginOptions, FastifyBodyParser } from 'fastify'; -import type { FindOptionsWhere } from 'typeorm'; import type Logger from '@/logger.js'; import { LoggerService } from '@/core/LoggerService.js'; +import type { FastifyInstance, FastifyRequest, FastifyReply, FastifyPluginOptions, FastifyBodyParser } from 'fastify'; +import type { FindOptionsWhere } from 'typeorm'; const ACTIVITY_JSON = 'application/activity+json; charset=utf-8'; const LD_JSON = 'application/ld+json; profile="https://www.w3.org/ns/activitystreams"; charset=utf-8'; diff --git a/packages/backend/src/server/FileServerService.ts b/packages/backend/src/server/FileServerService.ts index 5293d529ad..18cdda5430 100644 --- a/packages/backend/src/server/FileServerService.ts +++ b/packages/backend/src/server/FileServerService.ts @@ -11,7 +11,7 @@ import rename from 'rename'; import sharp from 'sharp'; import { sharpBmp } from '@misskey-dev/sharp-read-bmp'; import type { Config } from '@/config.js'; -import type { MiDriveFile, DriveFilesRepository } from '@/models/_.js'; +import type { MiDriveFile, DriveFilesRepository, MiUser } from '@/models/_.js'; import { DI } from '@/di-symbols.js'; import { createTemp } from '@/misc/create-temp.js'; import { FILE_TYPE_BROWSERSAFE } from '@/const.js'; @@ -30,7 +30,6 @@ import { correctFilename } from '@/misc/correct-filename.js'; import { handleRequestRedirectToOmitSearch } from '@/misc/fastify-hook-handlers.js'; import { getIpHash } from '@/misc/get-ip-hash.js'; import { AuthenticateService } from '@/server/api/AuthenticateService.js'; -import { RoleService } from '@/core/RoleService.js'; import { SkRateLimiterService } from '@/server/api/SkRateLimiterService.js'; import { Keyed, RateLimit, sendRateLimitHeaders } from '@/misc/rate-limit-utils.js'; import type { FastifyInstance, FastifyRequest, FastifyReply, FastifyPluginOptions } from 'fastify'; @@ -59,7 +58,6 @@ export class FileServerService { private loggerService: LoggerService, private authenticateService: AuthenticateService, private rateLimiterService: SkRateLimiterService, - private roleService: RoleService, ) { this.logger = this.loggerService.getLogger('server', 'gray'); @@ -625,14 +623,13 @@ export class FileServerService { // koa will automatically load the `X-Forwarded-For` header if `proxy: true` is configured in the app. const [user] = await this.authenticateService.authenticate(token); - const actor = user?.id ?? getIpHash(request.ip); - const factor = user ? (await this.roleService.getUserPolicies(user.id)).rateLimitFactor : 1; + const actor = user ?? getIpHash(request.ip); // Call both limits: the per-resource limit and the shared cross-resource limit - return await this.checkResourceLimit(reply, actor, group, resource, factor) && await this.checkSharedLimit(reply, actor, group, factor); + return await this.checkResourceLimit(reply, actor, group, resource) && await this.checkSharedLimit(reply, actor, group); } - private async checkResourceLimit(reply: FastifyReply, actor: string, group: string, resource: string, factor = 1): Promise { + private async checkResourceLimit(reply: FastifyReply, actor: string | MiUser, group: string, resource: string): Promise { const limit: Keyed = { // Group by resource key: `${group}${resource}`, @@ -643,10 +640,10 @@ export class FileServerService { dripRate: 1000 * 60, }; - return await this.checkLimit(reply, actor, limit, factor); + return await this.checkLimit(reply, actor, limit); } - private async checkSharedLimit(reply: FastifyReply, actor: string, group: string, factor = 1): Promise { + private async checkSharedLimit(reply: FastifyReply, actor: string | MiUser, group: string): Promise { const limit: Keyed = { key: group, type: 'bucket', @@ -655,11 +652,11 @@ export class FileServerService { size: 3600, }; - return await this.checkLimit(reply, actor, limit, factor); + return await this.checkLimit(reply, actor, limit); } - private async checkLimit(reply: FastifyReply, actor: string, limit: Keyed, factor = 1): Promise { - const info = await this.rateLimiterService.limit(limit, actor, factor); + private async checkLimit(reply: FastifyReply, actor: string | MiUser, limit: Keyed): Promise { + const info = await this.rateLimiterService.limit(limit, actor); sendRateLimitHeaders(reply, info); diff --git a/packages/backend/src/server/api/ApiCallService.ts b/packages/backend/src/server/api/ApiCallService.ts index 03f25a51fe..9c3952d541 100644 --- a/packages/backend/src/server/api/ApiCallService.ts +++ b/packages/backend/src/server/api/ApiCallService.ts @@ -313,35 +313,30 @@ export class ApiCallService implements OnApplicationShutdown { // eslint-disable-next-line @typescript-eslint/no-unnecessary-condition if (endpointLimit) { // koa will automatically load the `X-Forwarded-For` header if `proxy: true` is configured in the app. - let limitActor: string; + let limitActor: string | MiLocalUser; if (user) { - limitActor = user.id; + limitActor = user; } else { limitActor = getIpHash(request.ip); } - // TODO: 毎リクエスト計算するのもあれだしキャッシュしたい - const factor = user ? (await this.roleService.getUserPolicies(user.id)).rateLimitFactor : 1; + const limit = { + key: ep.name, + ...endpointLimit, + }; - if (factor > 0) { - const limit = { - key: ep.name, - ...endpointLimit, - }; + // Rate limit + const info = await this.rateLimiterService.limit(limit, limitActor); - // Rate limit - const info = await this.rateLimiterService.limit(limit, limitActor, factor); + sendRateLimitHeaders(reply, info); - sendRateLimitHeaders(reply, info); - - if (info.blocked) { - throw new ApiError({ - message: 'Rate limit exceeded. Please try again later.', - code: 'RATE_LIMIT_EXCEEDED', - id: 'd5826d14-3982-4d2e-8011-b9e9f02499ef', - httpStatusCode: 429, - }, info); - } + if (info.blocked) { + throw new ApiError({ + message: 'Rate limit exceeded. Please try again later.', + code: 'RATE_LIMIT_EXCEEDED', + id: 'd5826d14-3982-4d2e-8011-b9e9f02499ef', + httpStatusCode: 429, + }, info); } } diff --git a/packages/backend/src/server/api/SkRateLimiterService.ts b/packages/backend/src/server/api/SkRateLimiterService.ts index 38c97b63df..70103222f3 100644 --- a/packages/backend/src/server/api/SkRateLimiterService.ts +++ b/packages/backend/src/server/api/SkRateLimiterService.ts @@ -5,36 +5,59 @@ import { Inject, Injectable } from '@nestjs/common'; import Redis from 'ioredis'; -import { TimeService } from '@/core/TimeService.js'; -import { EnvService } from '@/core/EnvService.js'; +import type { TimeService } from '@/core/TimeService.js'; +import type { EnvService } from '@/core/EnvService.js'; import { BucketRateLimit, LegacyRateLimit, LimitInfo, RateLimit, hasMinLimit, isLegacyRateLimit, Keyed, hasMaxLimit, disabledLimitInfo, MaxLegacyLimit, MinLegacyLimit } from '@/misc/rate-limit-utils.js'; import { DI } from '@/di-symbols.js'; +import { MemoryKVCache } from '@/misc/cache.js'; +import type { MiUser } from '@/models/_.js'; +import type { RoleService } from '@/core/RoleService.js'; + +// Sentinel value used for caching the default role template. +// Required because MemoryKVCache doesn't support null keys. +const defaultUserKey = ''; @Injectable() export class SkRateLimiterService { + // 1-minute cache interval + private readonly factorCache = new MemoryKVCache(1000 * 60); private readonly disabled: boolean; constructor( - @Inject(TimeService) + @Inject('TimeService') private readonly timeService: TimeService, @Inject(DI.redis) private readonly redisClient: Redis.Redis, - @Inject(EnvService) + @Inject('RoleService') + private readonly roleService: RoleService, + + @Inject('EnvService') envService: EnvService, ) { this.disabled = envService.env.NODE_ENV === 'test'; } /** - * Check & increment a rate limit + * Check & increment a rate limit for a client * @param limit The limit definition - * @param actor Client who is calling this limit - * @param factor Scaling factor - smaller = larger limit (less restrictive) + * @param actorOrUser authenticated client user or IP hash */ - public async limit(limit: Keyed, actor: string, factor = 1): Promise { - if (this.disabled || factor === 0) { + public async limit(limit: Keyed, actorOrUser: string | MiUser): Promise { + if (this.disabled) { + return disabledLimitInfo; + } + + const actor = typeof(actorOrUser) === 'object' ? actorOrUser.id : actorOrUser; + const userCacheKey = typeof(actorOrUser) === 'object' ? actorOrUser.id : defaultUserKey; + const userRoleKey = typeof(actorOrUser) === 'object' ? actorOrUser.id : null; + const factor = this.factorCache.get(userCacheKey) ?? await this.factorCache.fetch(userCacheKey, async () => { + const role = await this.roleService.getUserPolicies(userRoleKey); + return role.rateLimitFactor; + }); + + if (factor === 0) { return disabledLimitInfo; } @@ -42,10 +65,6 @@ export class SkRateLimiterService { throw new Error(`Rate limit factor is zero or negative: ${factor}`); } - return await this.tryLimit(limit, actor, factor); - } - - private async tryLimit(limit: Keyed, actor: string, factor: number): Promise { if (isLegacyRateLimit(limit)) { return await this.limitLegacy(limit, actor, factor); } else { diff --git a/packages/backend/src/server/api/StreamingApiServerService.ts b/packages/backend/src/server/api/StreamingApiServerService.ts index e3fd1312ae..f30bbb928b 100644 --- a/packages/backend/src/server/api/StreamingApiServerService.ts +++ b/packages/backend/src/server/api/StreamingApiServerService.ts @@ -18,7 +18,6 @@ import { CacheService } from '@/core/CacheService.js'; import { MiLocalUser } from '@/models/User.js'; import { UserService } from '@/core/UserService.js'; import { ChannelFollowingService } from '@/core/ChannelFollowingService.js'; -import { RoleService } from '@/core/RoleService.js'; import { getIpHash } from '@/misc/get-ip-hash.js'; import { LoggerService } from '@/core/LoggerService.js'; import { SkRateLimiterService } from '@/server/api/SkRateLimiterService.js'; @@ -49,7 +48,6 @@ export class StreamingApiServerService { private usersService: UserService, private channelFollowingService: ChannelFollowingService, private rateLimiterService: SkRateLimiterService, - private roleService: RoleService, private loggerService: LoggerService, ) { } @@ -57,22 +55,18 @@ export class StreamingApiServerService { @bindThis private async rateLimitThis( user: MiLocalUser | null | undefined, - requestIp: string | undefined, + requestIp: string, limit: IEndpointMeta['limit'] & { key: NonNullable }, ) : Promise { - let limitActor: string; + let limitActor: string | MiLocalUser; if (user) { - limitActor = user.id; + limitActor = user; } else { - limitActor = getIpHash(requestIp || 'wtf'); + limitActor = getIpHash(requestIp); } - const factor = user ? (await this.roleService.getUserPolicies(user.id)).rateLimitFactor : 1; - - if (factor <= 0) return false; - // Rate limit - const rateLimit = await this.rateLimiterService.limit(limit, limitActor, factor); + const rateLimit = await this.rateLimiterService.limit(limit, limitActor); return rateLimit.blocked; } diff --git a/packages/backend/test/unit/server/api/SkRateLimiterServiceTests.ts b/packages/backend/test/unit/server/api/SkRateLimiterServiceTests.ts index d13dbd2a71..6d4f117c87 100644 --- a/packages/backend/test/unit/server/api/SkRateLimiterServiceTests.ts +++ b/packages/backend/test/unit/server/api/SkRateLimiterServiceTests.ts @@ -4,6 +4,8 @@ */ import type Redis from 'ioredis'; +import type { MiUser } from '@/models/User.js'; +import type { RolePolicies, RoleService } from '@/core/RoleService.js'; import { SkRateLimiterService } from '@/server/api/SkRateLimiterService.js'; import { BucketRateLimit, Keyed, LegacyRateLimit } from '@/misc/rate-limit-utils.js'; @@ -15,6 +17,8 @@ describe(SkRateLimiterService, () => { let mockRedisExec: (batch: [string, ...unknown[]][]) => Promise<[Error | null, unknown][] | null> = null!; let mockEnvironment: Record = null!; let serviceUnderTest: () => SkRateLimiterService = null!; + let mockDefaultUserPolicies: Partial = null!; + let mockUserPolicies: Record> = null!; beforeEach(() => { mockTimeService = { @@ -69,9 +73,18 @@ describe(SkRateLimiterService, () => { env: mockEnvironment, }; + mockDefaultUserPolicies = { rateLimitFactor: 1 }; + mockUserPolicies = {}; + const mockRoleService = { + getUserPolicies(key: string | null) { + const policies = key != null ? mockUserPolicies[key] : null; + return Promise.resolve(policies ?? mockDefaultUserPolicies); + }, + } as unknown as RoleService; + let service: SkRateLimiterService | undefined = undefined; serviceUnderTest = () => { - return service ??= new SkRateLimiterService(mockTimeService, mockRedisClient, mockEnvService); + return service ??= new SkRateLimiterService(mockTimeService, mockRedisClient, mockRoleService, mockEnvService); }; }); @@ -284,11 +297,12 @@ describe(SkRateLimiterService, () => { }); it('should scale limit by factor', async () => { + mockDefaultUserPolicies.rateLimitFactor = 0.5; limitCounter = 1; limitTimestamp = 0; - const i1 = await serviceUnderTest().limit(limit, actor, 0.5); // 1 + 1 = 2 - const i2 = await serviceUnderTest().limit(limit, actor, 0.5); // 2 + 1 = 3 + const i1 = await serviceUnderTest().limit(limit, actor); // 1 + 1 = 2 + const i2 = await serviceUnderTest().limit(limit, actor); // 2 + 1 = 3 expect(i1.blocked).toBeFalsy(); expect(i2.blocked).toBeTruthy(); @@ -330,14 +344,18 @@ describe(SkRateLimiterService, () => { }); it('should skip if factor is zero', async () => { - const info = await serviceUnderTest().limit(limit, actor, 0); + mockDefaultUserPolicies.rateLimitFactor = 0; + + const info = await serviceUnderTest().limit(limit, actor); expect(info.blocked).toBeFalsy(); expect(info.remaining).toBe(Number.MAX_SAFE_INTEGER); }); it('should throw if factor is negative', async () => { - const promise = serviceUnderTest().limit(limit, actor, -1); + mockDefaultUserPolicies.rateLimitFactor = -1; + + const promise = serviceUnderTest().limit(limit, actor); await expect(promise).rejects.toThrow(/factor is zero or negative/); }); @@ -426,6 +444,19 @@ describe(SkRateLimiterService, () => { expect(info.fullResetMs).toBe(2000); expect(info.fullResetSec).toBe(2); }); + + it('should look up factor by user ID', async () => { + const userActor = { id: actor } as unknown as MiUser; + mockUserPolicies[actor] = { rateLimitFactor: 0.5 }; + limitCounter = 1; + limitTimestamp = 0; + + const i1 = await serviceUnderTest().limit(limit, userActor); // 1 + 1 = 2 + const i2 = await serviceUnderTest().limit(limit, userActor); // 2 + 1 = 3 + + expect(i1.blocked).toBeFalsy(); + expect(i2.blocked).toBeTruthy(); + }); }); describe('with min interval', () => { @@ -529,11 +560,12 @@ describe(SkRateLimiterService, () => { }); it('should scale interval by factor', async () => { + mockDefaultUserPolicies.rateLimitFactor = 0.5; limitCounter = 1; limitTimestamp = 0; mockTimeService.now += 500; - const info = await serviceUnderTest().limit(limit, actor, 0.5); + const info = await serviceUnderTest().limit(limit, actor); expect(info.blocked).toBeFalsy(); }); @@ -574,14 +606,18 @@ describe(SkRateLimiterService, () => { }); it('should skip if factor is zero', async () => { - const info = await serviceUnderTest().limit(limit, actor, 0); + mockDefaultUserPolicies.rateLimitFactor = 0; + + const info = await serviceUnderTest().limit(limit, actor); expect(info.blocked).toBeFalsy(); expect(info.remaining).toBe(Number.MAX_SAFE_INTEGER); }); it('should throw if factor is negative', async () => { - const promise = serviceUnderTest().limit(limit, actor, -1); + mockDefaultUserPolicies.rateLimitFactor = -1; + + const promise = serviceUnderTest().limit(limit, actor); await expect(promise).rejects.toThrow(/factor is zero or negative/); }); @@ -701,10 +737,11 @@ describe(SkRateLimiterService, () => { }); it('should scale limit by factor', async () => { + mockDefaultUserPolicies.rateLimitFactor = 0.5; limitCounter = 10; limitTimestamp = 0; - const info = await serviceUnderTest().limit(limit, actor, 0.5); // 10 + 1 = 11 + const info = await serviceUnderTest().limit(limit, actor); // 10 + 1 = 11 expect(info.blocked).toBeTruthy(); }); @@ -760,14 +797,18 @@ describe(SkRateLimiterService, () => { }); it('should skip if factor is zero', async () => { - const info = await serviceUnderTest().limit(limit, actor, 0); + mockDefaultUserPolicies.rateLimitFactor = 0; + + const info = await serviceUnderTest().limit(limit, actor); expect(info.blocked).toBeFalsy(); expect(info.remaining).toBe(Number.MAX_SAFE_INTEGER); }); it('should throw if factor is negative', async () => { - const promise = serviceUnderTest().limit(limit, actor, -1); + mockDefaultUserPolicies.rateLimitFactor = -1; + + const promise = serviceUnderTest().limit(limit, actor); await expect(promise).rejects.toThrow(/factor is zero or negative/); }); @@ -890,11 +931,12 @@ describe(SkRateLimiterService, () => { }); it('should scale limit and interval by factor', async () => { + mockDefaultUserPolicies.rateLimitFactor = 0.5; limitCounter = 5; limitTimestamp = 0; mockTimeService.now += 500; - const info = await serviceUnderTest().limit(limit, actor, 0.5); + const info = await serviceUnderTest().limit(limit, actor); expect(info.blocked).toBeFalsy(); }); -- cgit v1.2.3-freya From d4311ea041eb746f3fdc5b4d8dbba2adb2470380 Mon Sep 17 00:00:00 2001 From: Hazelnoot Date: Wed, 5 Feb 2025 10:04:42 -0500 Subject: document new rate limit factor calculations --- packages/backend/src/server/SkRateLimiterService.md | 5 +++++ packages/backend/src/server/api/SkRateLimiterService.ts | 10 +++++++++- 2 files changed, 14 insertions(+), 1 deletion(-) (limited to 'packages/backend/src/server') diff --git a/packages/backend/src/server/SkRateLimiterService.md b/packages/backend/src/server/SkRateLimiterService.md index 762f8dfe14..fb007538fa 100644 --- a/packages/backend/src/server/SkRateLimiterService.md +++ b/packages/backend/src/server/SkRateLimiterService.md @@ -12,6 +12,11 @@ SkRateLimiterService is not quite plug-and-play compatible with existing call si Instead, the returned LimitInfo object will have `blocked` set to true. Callers are responsible for checking this property and taking any desired action, such as rejecting a request or returning limit details. +Rate limit factors are also handled differently. +Instead of providing an optional parameter for callers, SkRateLimiterServer accepts an `MiUser` parameter that is used to compute the factor directly. +If a user is not available (such as for unauthenticated callers), then the Role Template factor is used instead. +To avoid confusion, the `factor` parameter has been removed entirely and is now an implementation detail. + ## Headers LimitInfo objects (returned by `SkRateLimitService.limit()`) can be passed to `rate-limit-utils.sendRateLimitHeaders()` to send standard rate limit headers with an HTTP response. diff --git a/packages/backend/src/server/api/SkRateLimiterService.ts b/packages/backend/src/server/api/SkRateLimiterService.ts index 70103222f3..038f12cb25 100644 --- a/packages/backend/src/server/api/SkRateLimiterService.ts +++ b/packages/backend/src/server/api/SkRateLimiterService.ts @@ -40,7 +40,15 @@ export class SkRateLimiterService { } /** - * Check & increment a rate limit for a client + * Check & increment a rate limit for a client. + * + * If the client (actorOrUser) is passed as a string, then it uses the default rate limit factor from the role template. + * If the client (actorOrUser) is passed as an MiUser, then it queries the user's actual rate limit factor from their assigned roles. + * + * A factor of zero (0) will disable the limit, while any negative number will produce an error. + * A factor between zero (0) and one (1) will increase the limit from its default values (allowing more actions per time interval). + * A factor greater than one (1) will decrease the limit from its default values (allowing fewer actions per time interval). + * * @param limit The limit definition * @param actorOrUser authenticated client user or IP hash */ -- cgit v1.2.3-freya From f92fb3bb8c70b5ca35f4f61a437b101bd5bae7e8 Mon Sep 17 00:00:00 2001 From: Hazelnoot Date: Wed, 5 Feb 2025 11:16:39 -0500 Subject: move SkRateLimiterService to correct directory --- packages/backend/src/server/FileServerService.ts | 2 +- packages/backend/src/server/ServerModule.ts | 2 +- .../backend/src/server/SkRateLimiterService.ts | 280 +++++++++++++++++++++ packages/backend/src/server/api/ApiCallService.ts | 2 +- .../backend/src/server/api/SigninApiService.ts | 2 +- .../src/server/api/SigninWithPasskeyApiService.ts | 2 +- .../backend/src/server/api/SkRateLimiterService.ts | 280 --------------------- .../src/server/api/StreamingApiServerService.ts | 2 +- .../test/unit/SigninWithPasskeyApiService.ts | 2 +- .../unit/server/api/SkRateLimiterServiceTests.ts | 2 +- 10 files changed, 288 insertions(+), 288 deletions(-) create mode 100644 packages/backend/src/server/SkRateLimiterService.ts delete mode 100644 packages/backend/src/server/api/SkRateLimiterService.ts (limited to 'packages/backend/src/server') diff --git a/packages/backend/src/server/FileServerService.ts b/packages/backend/src/server/FileServerService.ts index 18cdda5430..a7e13a1b78 100644 --- a/packages/backend/src/server/FileServerService.ts +++ b/packages/backend/src/server/FileServerService.ts @@ -30,7 +30,7 @@ import { correctFilename } from '@/misc/correct-filename.js'; import { handleRequestRedirectToOmitSearch } from '@/misc/fastify-hook-handlers.js'; import { getIpHash } from '@/misc/get-ip-hash.js'; import { AuthenticateService } from '@/server/api/AuthenticateService.js'; -import { SkRateLimiterService } from '@/server/api/SkRateLimiterService.js'; +import { SkRateLimiterService } from '@/server/SkRateLimiterService.js'; import { Keyed, RateLimit, sendRateLimitHeaders } from '@/misc/rate-limit-utils.js'; import type { FastifyInstance, FastifyRequest, FastifyReply, FastifyPluginOptions } from 'fastify'; diff --git a/packages/backend/src/server/ServerModule.ts b/packages/backend/src/server/ServerModule.ts index c1d7c088f1..6b5156106a 100644 --- a/packages/backend/src/server/ServerModule.ts +++ b/packages/backend/src/server/ServerModule.ts @@ -6,7 +6,7 @@ import { Module } from '@nestjs/common'; import { EndpointsModule } from '@/server/api/EndpointsModule.js'; import { CoreModule } from '@/core/CoreModule.js'; -import { SkRateLimiterService } from '@/server/api/SkRateLimiterService.js'; +import { SkRateLimiterService } from '@/server/SkRateLimiterService.js'; import { ApiCallService } from './api/ApiCallService.js'; import { FileServerService } from './FileServerService.js'; import { HealthServerService } from './HealthServerService.js'; diff --git a/packages/backend/src/server/SkRateLimiterService.ts b/packages/backend/src/server/SkRateLimiterService.ts new file mode 100644 index 0000000000..038f12cb25 --- /dev/null +++ b/packages/backend/src/server/SkRateLimiterService.ts @@ -0,0 +1,280 @@ +/* + * SPDX-FileCopyrightText: hazelnoot and other Sharkey contributors + * SPDX-License-Identifier: AGPL-3.0-only + */ + +import { Inject, Injectable } from '@nestjs/common'; +import Redis from 'ioredis'; +import type { TimeService } from '@/core/TimeService.js'; +import type { EnvService } from '@/core/EnvService.js'; +import { BucketRateLimit, LegacyRateLimit, LimitInfo, RateLimit, hasMinLimit, isLegacyRateLimit, Keyed, hasMaxLimit, disabledLimitInfo, MaxLegacyLimit, MinLegacyLimit } from '@/misc/rate-limit-utils.js'; +import { DI } from '@/di-symbols.js'; +import { MemoryKVCache } from '@/misc/cache.js'; +import type { MiUser } from '@/models/_.js'; +import type { RoleService } from '@/core/RoleService.js'; + +// Sentinel value used for caching the default role template. +// Required because MemoryKVCache doesn't support null keys. +const defaultUserKey = ''; + +@Injectable() +export class SkRateLimiterService { + // 1-minute cache interval + private readonly factorCache = new MemoryKVCache(1000 * 60); + private readonly disabled: boolean; + + constructor( + @Inject('TimeService') + private readonly timeService: TimeService, + + @Inject(DI.redis) + private readonly redisClient: Redis.Redis, + + @Inject('RoleService') + private readonly roleService: RoleService, + + @Inject('EnvService') + envService: EnvService, + ) { + this.disabled = envService.env.NODE_ENV === 'test'; + } + + /** + * Check & increment a rate limit for a client. + * + * If the client (actorOrUser) is passed as a string, then it uses the default rate limit factor from the role template. + * If the client (actorOrUser) is passed as an MiUser, then it queries the user's actual rate limit factor from their assigned roles. + * + * A factor of zero (0) will disable the limit, while any negative number will produce an error. + * A factor between zero (0) and one (1) will increase the limit from its default values (allowing more actions per time interval). + * A factor greater than one (1) will decrease the limit from its default values (allowing fewer actions per time interval). + * + * @param limit The limit definition + * @param actorOrUser authenticated client user or IP hash + */ + public async limit(limit: Keyed, actorOrUser: string | MiUser): Promise { + if (this.disabled) { + return disabledLimitInfo; + } + + const actor = typeof(actorOrUser) === 'object' ? actorOrUser.id : actorOrUser; + const userCacheKey = typeof(actorOrUser) === 'object' ? actorOrUser.id : defaultUserKey; + const userRoleKey = typeof(actorOrUser) === 'object' ? actorOrUser.id : null; + const factor = this.factorCache.get(userCacheKey) ?? await this.factorCache.fetch(userCacheKey, async () => { + const role = await this.roleService.getUserPolicies(userRoleKey); + return role.rateLimitFactor; + }); + + if (factor === 0) { + return disabledLimitInfo; + } + + if (factor < 0) { + throw new Error(`Rate limit factor is zero or negative: ${factor}`); + } + + if (isLegacyRateLimit(limit)) { + return await this.limitLegacy(limit, actor, factor); + } else { + return await this.limitBucket(limit, actor, factor); + } + } + + private async limitLegacy(limit: Keyed, actor: string, factor: number): Promise { + if (hasMaxLimit(limit)) { + return await this.limitLegacyMinMax(limit, actor, factor); + } else if (hasMinLimit(limit)) { + return await this.limitLegacyMinOnly(limit, actor, factor); + } else { + return disabledLimitInfo; + } + } + + private async limitLegacyMinMax(limit: Keyed, actor: string, factor: number): Promise { + if (limit.duration === 0) return disabledLimitInfo; + if (limit.duration < 0) throw new Error(`Invalid rate limit ${limit.key}: duration is negative (${limit.duration})`); + if (limit.max < 1) throw new Error(`Invalid rate limit ${limit.key}: max is less than 1 (${limit.max})`); + + // Derive initial dripRate from minInterval OR duration/max. + const initialDripRate = Math.max(limit.minInterval ?? Math.round(limit.duration / limit.max), 1); + + // Calculate dripSize to reach max at exactly duration + const dripSize = Math.max(Math.round(limit.max / (limit.duration / initialDripRate)), 1); + + // Calculate final dripRate from dripSize and duration/max + const dripRate = Math.max(Math.round(limit.duration / (limit.max / dripSize)), 1); + + const bucketLimit: Keyed = { + type: 'bucket', + key: limit.key, + size: limit.max, + dripRate, + dripSize, + }; + return await this.limitBucket(bucketLimit, actor, factor); + } + + private async limitLegacyMinOnly(limit: Keyed, actor: string, factor: number): Promise { + if (limit.minInterval === 0) return disabledLimitInfo; + if (limit.minInterval < 0) throw new Error(`Invalid rate limit ${limit.key}: minInterval is negative (${limit.minInterval})`); + + const dripRate = Math.max(Math.round(limit.minInterval), 1); + const bucketLimit: Keyed = { + type: 'bucket', + key: limit.key, + size: 1, + dripRate, + dripSize: 1, + }; + return await this.limitBucket(bucketLimit, actor, factor); + } + + /** + * Implementation of Leaky Bucket rate limiting - see SkRateLimiterService.md for details. + */ + private async limitBucket(limit: Keyed, actor: string, factor: number): Promise { + if (limit.size < 1) throw new Error(`Invalid rate limit ${limit.key}: size is less than 1 (${limit.size})`); + if (limit.dripRate != null && limit.dripRate < 1) throw new Error(`Invalid rate limit ${limit.key}: dripRate is less than 1 (${limit.dripRate})`); + if (limit.dripSize != null && limit.dripSize < 1) throw new Error(`Invalid rate limit ${limit.key}: dripSize is less than 1 (${limit.dripSize})`); + + // 0 - Calculate + const now = this.timeService.now; + const bucketSize = Math.max(Math.ceil(limit.size / factor), 1); + const dripRate = Math.ceil(limit.dripRate ?? 1000); + const dripSize = Math.ceil(limit.dripSize ?? 1); + const expirationSec = Math.max(Math.ceil((dripRate * Math.ceil(bucketSize / dripSize)) / 1000), 1); + + // 1 - Read + const counterKey = createLimitKey(limit, actor, 'c'); + const timestampKey = createLimitKey(limit, actor, 't'); + const counter = await this.getLimitCounter(counterKey, timestampKey); + + // 2 - Drip + const dripsSinceLastTick = Math.floor((now - counter.timestamp) / dripRate) * dripSize; + const deltaCounter = Math.min(dripsSinceLastTick, counter.counter); + const deltaTimestamp = dripsSinceLastTick * dripRate; + if (deltaCounter > 0) { + // Execute the next drip(s) + const results = await this.executeRedisMulti( + ['get', timestampKey], + ['incrby', timestampKey, deltaTimestamp], + ['expire', timestampKey, expirationSec], + ['get', timestampKey], + ['decrby', counterKey, deltaCounter], + ['expire', counterKey, expirationSec], + ['get', counterKey], + ); + const expectedTimestamp = counter.timestamp; + const canaryTimestamp = results[0] ? parseInt(results[0]) : 0; + counter.timestamp = results[3] ? parseInt(results[3]) : 0; + counter.counter = results[6] ? parseInt(results[6]) : 0; + + // Check for a data collision and rollback + if (canaryTimestamp !== expectedTimestamp) { + const rollbackResults = await this.executeRedisMulti( + ['decrby', timestampKey, deltaTimestamp], + ['get', timestampKey], + ['incrby', counterKey, deltaCounter], + ['get', counterKey], + ); + counter.timestamp = rollbackResults[1] ? parseInt(rollbackResults[1]) : 0; + counter.counter = rollbackResults[3] ? parseInt(rollbackResults[3]) : 0; + } + } + + // 3 - Check + const blocked = counter.counter >= bucketSize; + if (!blocked) { + if (counter.timestamp === 0) { + const results = await this.executeRedisMulti( + ['set', timestampKey, now], + ['expire', timestampKey, expirationSec], + ['incr', counterKey], + ['expire', counterKey, expirationSec], + ['get', counterKey], + ); + counter.timestamp = now; + counter.counter = results[4] ? parseInt(results[4]) : 0; + } else { + const results = await this.executeRedisMulti( + ['incr', counterKey], + ['expire', counterKey, expirationSec], + ['get', counterKey], + ); + counter.counter = results[2] ? parseInt(results[2]) : 0; + } + } + + // Calculate how much time is needed to free up a bucket slot + const overflow = Math.max((counter.counter + 1) - bucketSize, 0); + const dripsNeeded = Math.ceil(overflow / dripSize); + const timeNeeded = Math.max((dripRate * dripsNeeded) - (this.timeService.now - counter.timestamp), 0); + + // Calculate limit status + const remaining = Math.max(bucketSize - counter.counter, 0); + const resetMs = timeNeeded; + const resetSec = Math.ceil(resetMs / 1000); + const fullResetMs = Math.ceil(counter.counter / dripSize) * dripRate; + const fullResetSec = Math.ceil(fullResetMs / 1000); + return { blocked, remaining, resetSec, resetMs, fullResetSec, fullResetMs }; + } + + private async getLimitCounter(counterKey: string, timestampKey: string): Promise { + const [counter, timestamp] = await this.executeRedisMulti( + ['get', counterKey], + ['get', timestampKey], + ); + + return { + counter: counter ? parseInt(counter) : 0, + timestamp: timestamp ? parseInt(timestamp) : 0, + }; + } + + private async executeRedisMulti(...batch: RedisCommand[]): Promise { + const results = await this.redisClient.multi(batch).exec(); + + // Transaction conflict (retryable) + if (!results) { + throw new ConflictError('Redis error: transaction conflict'); + } + + // Transaction failed (fatal) + if (results.length !== batch.length) { + throw new Error('Redis error: failed to execute batch'); + } + + // Map responses + const errors: Error[] = []; + const responses: RedisResult[] = []; + for (const [error, response] of results) { + if (error) errors.push(error); + responses.push(response as RedisResult); + } + + // Command failed (fatal) + if (errors.length > 0) { + const errorMessages = errors + .map((e, i) => `Error in command ${i}: ${e}`) + .join('\', \''); + throw new AggregateError(errors, `Redis error: failed to execute command(s): '${errorMessages}'`); + } + + return responses; + } +} + +// Not correct, but good enough for the basic commands we use. +type RedisResult = string | null; +type RedisCommand = [command: string, ...args: unknown[]]; + +function createLimitKey(limit: Keyed, actor: string, value: string): string { + return `rl_${actor}_${limit.key}_${value}`; +} + +class ConflictError extends Error {} + +interface LimitCounter { + timestamp: number; + counter: number; +} diff --git a/packages/backend/src/server/api/ApiCallService.ts b/packages/backend/src/server/api/ApiCallService.ts index 9c3952d541..5ce358d68f 100644 --- a/packages/backend/src/server/api/ApiCallService.ts +++ b/packages/backend/src/server/api/ApiCallService.ts @@ -19,7 +19,7 @@ import { bindThis } from '@/decorators.js'; import { RoleService } from '@/core/RoleService.js'; import type { Config } from '@/config.js'; import { sendRateLimitHeaders } from '@/misc/rate-limit-utils.js'; -import { SkRateLimiterService } from '@/server/api/SkRateLimiterService.js'; +import { SkRateLimiterService } from '@/server/SkRateLimiterService.js'; import { ApiError } from './error.js'; import { ApiLoggerService } from './ApiLoggerService.js'; import { AuthenticateService, AuthenticationError } from './AuthenticateService.js'; diff --git a/packages/backend/src/server/api/SigninApiService.ts b/packages/backend/src/server/api/SigninApiService.ts index 25ea03a356..72712bce60 100644 --- a/packages/backend/src/server/api/SigninApiService.ts +++ b/packages/backend/src/server/api/SigninApiService.ts @@ -26,7 +26,7 @@ import { UserAuthService } from '@/core/UserAuthService.js'; import { CaptchaService } from '@/core/CaptchaService.js'; import { FastifyReplyError } from '@/misc/fastify-reply-error.js'; import { isSystemAccount } from '@/misc/is-system-account.js'; -import { SkRateLimiterService } from '@/server/api/SkRateLimiterService.js'; +import { SkRateLimiterService } from '@/server/SkRateLimiterService.js'; import { Keyed, RateLimit, sendRateLimitHeaders } from '@/misc/rate-limit-utils.js'; import { SigninService } from './SigninService.js'; import type { AuthenticationResponseJSON } from '@simplewebauthn/types'; diff --git a/packages/backend/src/server/api/SigninWithPasskeyApiService.ts b/packages/backend/src/server/api/SigninWithPasskeyApiService.ts index e94d2b6b68..f84f50523b 100644 --- a/packages/backend/src/server/api/SigninWithPasskeyApiService.ts +++ b/packages/backend/src/server/api/SigninWithPasskeyApiService.ts @@ -21,7 +21,7 @@ import { WebAuthnService } from '@/core/WebAuthnService.js'; import Logger from '@/logger.js'; import { LoggerService } from '@/core/LoggerService.js'; import type { IdentifiableError } from '@/misc/identifiable-error.js'; -import { SkRateLimiterService } from '@/server/api/SkRateLimiterService.js'; +import { SkRateLimiterService } from '@/server/SkRateLimiterService.js'; import { sendRateLimitHeaders } from '@/misc/rate-limit-utils.js'; import { SigninService } from './SigninService.js'; import type { AuthenticationResponseJSON } from '@simplewebauthn/types'; diff --git a/packages/backend/src/server/api/SkRateLimiterService.ts b/packages/backend/src/server/api/SkRateLimiterService.ts deleted file mode 100644 index 038f12cb25..0000000000 --- a/packages/backend/src/server/api/SkRateLimiterService.ts +++ /dev/null @@ -1,280 +0,0 @@ -/* - * SPDX-FileCopyrightText: hazelnoot and other Sharkey contributors - * SPDX-License-Identifier: AGPL-3.0-only - */ - -import { Inject, Injectable } from '@nestjs/common'; -import Redis from 'ioredis'; -import type { TimeService } from '@/core/TimeService.js'; -import type { EnvService } from '@/core/EnvService.js'; -import { BucketRateLimit, LegacyRateLimit, LimitInfo, RateLimit, hasMinLimit, isLegacyRateLimit, Keyed, hasMaxLimit, disabledLimitInfo, MaxLegacyLimit, MinLegacyLimit } from '@/misc/rate-limit-utils.js'; -import { DI } from '@/di-symbols.js'; -import { MemoryKVCache } from '@/misc/cache.js'; -import type { MiUser } from '@/models/_.js'; -import type { RoleService } from '@/core/RoleService.js'; - -// Sentinel value used for caching the default role template. -// Required because MemoryKVCache doesn't support null keys. -const defaultUserKey = ''; - -@Injectable() -export class SkRateLimiterService { - // 1-minute cache interval - private readonly factorCache = new MemoryKVCache(1000 * 60); - private readonly disabled: boolean; - - constructor( - @Inject('TimeService') - private readonly timeService: TimeService, - - @Inject(DI.redis) - private readonly redisClient: Redis.Redis, - - @Inject('RoleService') - private readonly roleService: RoleService, - - @Inject('EnvService') - envService: EnvService, - ) { - this.disabled = envService.env.NODE_ENV === 'test'; - } - - /** - * Check & increment a rate limit for a client. - * - * If the client (actorOrUser) is passed as a string, then it uses the default rate limit factor from the role template. - * If the client (actorOrUser) is passed as an MiUser, then it queries the user's actual rate limit factor from their assigned roles. - * - * A factor of zero (0) will disable the limit, while any negative number will produce an error. - * A factor between zero (0) and one (1) will increase the limit from its default values (allowing more actions per time interval). - * A factor greater than one (1) will decrease the limit from its default values (allowing fewer actions per time interval). - * - * @param limit The limit definition - * @param actorOrUser authenticated client user or IP hash - */ - public async limit(limit: Keyed, actorOrUser: string | MiUser): Promise { - if (this.disabled) { - return disabledLimitInfo; - } - - const actor = typeof(actorOrUser) === 'object' ? actorOrUser.id : actorOrUser; - const userCacheKey = typeof(actorOrUser) === 'object' ? actorOrUser.id : defaultUserKey; - const userRoleKey = typeof(actorOrUser) === 'object' ? actorOrUser.id : null; - const factor = this.factorCache.get(userCacheKey) ?? await this.factorCache.fetch(userCacheKey, async () => { - const role = await this.roleService.getUserPolicies(userRoleKey); - return role.rateLimitFactor; - }); - - if (factor === 0) { - return disabledLimitInfo; - } - - if (factor < 0) { - throw new Error(`Rate limit factor is zero or negative: ${factor}`); - } - - if (isLegacyRateLimit(limit)) { - return await this.limitLegacy(limit, actor, factor); - } else { - return await this.limitBucket(limit, actor, factor); - } - } - - private async limitLegacy(limit: Keyed, actor: string, factor: number): Promise { - if (hasMaxLimit(limit)) { - return await this.limitLegacyMinMax(limit, actor, factor); - } else if (hasMinLimit(limit)) { - return await this.limitLegacyMinOnly(limit, actor, factor); - } else { - return disabledLimitInfo; - } - } - - private async limitLegacyMinMax(limit: Keyed, actor: string, factor: number): Promise { - if (limit.duration === 0) return disabledLimitInfo; - if (limit.duration < 0) throw new Error(`Invalid rate limit ${limit.key}: duration is negative (${limit.duration})`); - if (limit.max < 1) throw new Error(`Invalid rate limit ${limit.key}: max is less than 1 (${limit.max})`); - - // Derive initial dripRate from minInterval OR duration/max. - const initialDripRate = Math.max(limit.minInterval ?? Math.round(limit.duration / limit.max), 1); - - // Calculate dripSize to reach max at exactly duration - const dripSize = Math.max(Math.round(limit.max / (limit.duration / initialDripRate)), 1); - - // Calculate final dripRate from dripSize and duration/max - const dripRate = Math.max(Math.round(limit.duration / (limit.max / dripSize)), 1); - - const bucketLimit: Keyed = { - type: 'bucket', - key: limit.key, - size: limit.max, - dripRate, - dripSize, - }; - return await this.limitBucket(bucketLimit, actor, factor); - } - - private async limitLegacyMinOnly(limit: Keyed, actor: string, factor: number): Promise { - if (limit.minInterval === 0) return disabledLimitInfo; - if (limit.minInterval < 0) throw new Error(`Invalid rate limit ${limit.key}: minInterval is negative (${limit.minInterval})`); - - const dripRate = Math.max(Math.round(limit.minInterval), 1); - const bucketLimit: Keyed = { - type: 'bucket', - key: limit.key, - size: 1, - dripRate, - dripSize: 1, - }; - return await this.limitBucket(bucketLimit, actor, factor); - } - - /** - * Implementation of Leaky Bucket rate limiting - see SkRateLimiterService.md for details. - */ - private async limitBucket(limit: Keyed, actor: string, factor: number): Promise { - if (limit.size < 1) throw new Error(`Invalid rate limit ${limit.key}: size is less than 1 (${limit.size})`); - if (limit.dripRate != null && limit.dripRate < 1) throw new Error(`Invalid rate limit ${limit.key}: dripRate is less than 1 (${limit.dripRate})`); - if (limit.dripSize != null && limit.dripSize < 1) throw new Error(`Invalid rate limit ${limit.key}: dripSize is less than 1 (${limit.dripSize})`); - - // 0 - Calculate - const now = this.timeService.now; - const bucketSize = Math.max(Math.ceil(limit.size / factor), 1); - const dripRate = Math.ceil(limit.dripRate ?? 1000); - const dripSize = Math.ceil(limit.dripSize ?? 1); - const expirationSec = Math.max(Math.ceil((dripRate * Math.ceil(bucketSize / dripSize)) / 1000), 1); - - // 1 - Read - const counterKey = createLimitKey(limit, actor, 'c'); - const timestampKey = createLimitKey(limit, actor, 't'); - const counter = await this.getLimitCounter(counterKey, timestampKey); - - // 2 - Drip - const dripsSinceLastTick = Math.floor((now - counter.timestamp) / dripRate) * dripSize; - const deltaCounter = Math.min(dripsSinceLastTick, counter.counter); - const deltaTimestamp = dripsSinceLastTick * dripRate; - if (deltaCounter > 0) { - // Execute the next drip(s) - const results = await this.executeRedisMulti( - ['get', timestampKey], - ['incrby', timestampKey, deltaTimestamp], - ['expire', timestampKey, expirationSec], - ['get', timestampKey], - ['decrby', counterKey, deltaCounter], - ['expire', counterKey, expirationSec], - ['get', counterKey], - ); - const expectedTimestamp = counter.timestamp; - const canaryTimestamp = results[0] ? parseInt(results[0]) : 0; - counter.timestamp = results[3] ? parseInt(results[3]) : 0; - counter.counter = results[6] ? parseInt(results[6]) : 0; - - // Check for a data collision and rollback - if (canaryTimestamp !== expectedTimestamp) { - const rollbackResults = await this.executeRedisMulti( - ['decrby', timestampKey, deltaTimestamp], - ['get', timestampKey], - ['incrby', counterKey, deltaCounter], - ['get', counterKey], - ); - counter.timestamp = rollbackResults[1] ? parseInt(rollbackResults[1]) : 0; - counter.counter = rollbackResults[3] ? parseInt(rollbackResults[3]) : 0; - } - } - - // 3 - Check - const blocked = counter.counter >= bucketSize; - if (!blocked) { - if (counter.timestamp === 0) { - const results = await this.executeRedisMulti( - ['set', timestampKey, now], - ['expire', timestampKey, expirationSec], - ['incr', counterKey], - ['expire', counterKey, expirationSec], - ['get', counterKey], - ); - counter.timestamp = now; - counter.counter = results[4] ? parseInt(results[4]) : 0; - } else { - const results = await this.executeRedisMulti( - ['incr', counterKey], - ['expire', counterKey, expirationSec], - ['get', counterKey], - ); - counter.counter = results[2] ? parseInt(results[2]) : 0; - } - } - - // Calculate how much time is needed to free up a bucket slot - const overflow = Math.max((counter.counter + 1) - bucketSize, 0); - const dripsNeeded = Math.ceil(overflow / dripSize); - const timeNeeded = Math.max((dripRate * dripsNeeded) - (this.timeService.now - counter.timestamp), 0); - - // Calculate limit status - const remaining = Math.max(bucketSize - counter.counter, 0); - const resetMs = timeNeeded; - const resetSec = Math.ceil(resetMs / 1000); - const fullResetMs = Math.ceil(counter.counter / dripSize) * dripRate; - const fullResetSec = Math.ceil(fullResetMs / 1000); - return { blocked, remaining, resetSec, resetMs, fullResetSec, fullResetMs }; - } - - private async getLimitCounter(counterKey: string, timestampKey: string): Promise { - const [counter, timestamp] = await this.executeRedisMulti( - ['get', counterKey], - ['get', timestampKey], - ); - - return { - counter: counter ? parseInt(counter) : 0, - timestamp: timestamp ? parseInt(timestamp) : 0, - }; - } - - private async executeRedisMulti(...batch: RedisCommand[]): Promise { - const results = await this.redisClient.multi(batch).exec(); - - // Transaction conflict (retryable) - if (!results) { - throw new ConflictError('Redis error: transaction conflict'); - } - - // Transaction failed (fatal) - if (results.length !== batch.length) { - throw new Error('Redis error: failed to execute batch'); - } - - // Map responses - const errors: Error[] = []; - const responses: RedisResult[] = []; - for (const [error, response] of results) { - if (error) errors.push(error); - responses.push(response as RedisResult); - } - - // Command failed (fatal) - if (errors.length > 0) { - const errorMessages = errors - .map((e, i) => `Error in command ${i}: ${e}`) - .join('\', \''); - throw new AggregateError(errors, `Redis error: failed to execute command(s): '${errorMessages}'`); - } - - return responses; - } -} - -// Not correct, but good enough for the basic commands we use. -type RedisResult = string | null; -type RedisCommand = [command: string, ...args: unknown[]]; - -function createLimitKey(limit: Keyed, actor: string, value: string): string { - return `rl_${actor}_${limit.key}_${value}`; -} - -class ConflictError extends Error {} - -interface LimitCounter { - timestamp: number; - counter: number; -} diff --git a/packages/backend/src/server/api/StreamingApiServerService.ts b/packages/backend/src/server/api/StreamingApiServerService.ts index f30bbb928b..6e7abcfae6 100644 --- a/packages/backend/src/server/api/StreamingApiServerService.ts +++ b/packages/backend/src/server/api/StreamingApiServerService.ts @@ -20,7 +20,7 @@ import { UserService } from '@/core/UserService.js'; import { ChannelFollowingService } from '@/core/ChannelFollowingService.js'; import { getIpHash } from '@/misc/get-ip-hash.js'; import { LoggerService } from '@/core/LoggerService.js'; -import { SkRateLimiterService } from '@/server/api/SkRateLimiterService.js'; +import { SkRateLimiterService } from '@/server/SkRateLimiterService.js'; import { AuthenticateService, AuthenticationError } from './AuthenticateService.js'; import MainStreamConnection from './stream/Connection.js'; import { ChannelsService } from './stream/ChannelsService.js'; diff --git a/packages/backend/test/unit/SigninWithPasskeyApiService.ts b/packages/backend/test/unit/SigninWithPasskeyApiService.ts index 7df991c15c..efed905e02 100644 --- a/packages/backend/test/unit/SigninWithPasskeyApiService.ts +++ b/packages/backend/test/unit/SigninWithPasskeyApiService.ts @@ -20,7 +20,7 @@ import { SigninWithPasskeyApiService } from '@/server/api/SigninWithPasskeyApiSe import { WebAuthnService } from '@/core/WebAuthnService.js'; import { SigninService } from '@/server/api/SigninService.js'; import { IdentifiableError } from '@/misc/identifiable-error.js'; -import { SkRateLimiterService } from '@/server/api/SkRateLimiterService.js'; +import { SkRateLimiterService } from '@/server/SkRateLimiterService.js'; import { LimitInfo } from '@/misc/rate-limit-utils.js'; const moduleMocker = new ModuleMocker(global); diff --git a/packages/backend/test/unit/server/api/SkRateLimiterServiceTests.ts b/packages/backend/test/unit/server/api/SkRateLimiterServiceTests.ts index 6d4f117c87..b1f100698b 100644 --- a/packages/backend/test/unit/server/api/SkRateLimiterServiceTests.ts +++ b/packages/backend/test/unit/server/api/SkRateLimiterServiceTests.ts @@ -6,7 +6,7 @@ import type Redis from 'ioredis'; import type { MiUser } from '@/models/User.js'; import type { RolePolicies, RoleService } from '@/core/RoleService.js'; -import { SkRateLimiterService } from '@/server/api/SkRateLimiterService.js'; +import { SkRateLimiterService } from '@/server/SkRateLimiterService.js'; import { BucketRateLimit, Keyed, LegacyRateLimit } from '@/misc/rate-limit-utils.js'; /* eslint-disable @typescript-eslint/no-non-null-assertion */ -- cgit v1.2.3-freya From b171c1c4408c1438976f3473c8104b68305a53f9 Mon Sep 17 00:00:00 2001 From: Hazelnoot Date: Wed, 5 Feb 2025 11:18:53 -0500 Subject: move imports to fix git diff in ActivityPubServerService.ts --- packages/backend/src/server/ActivityPubServerService.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'packages/backend/src/server') diff --git a/packages/backend/src/server/ActivityPubServerService.ts b/packages/backend/src/server/ActivityPubServerService.ts index 19049e528c..6e8c249471 100644 --- a/packages/backend/src/server/ActivityPubServerService.ts +++ b/packages/backend/src/server/ActivityPubServerService.ts @@ -30,12 +30,12 @@ import type { MiNote } from '@/models/Note.js'; import { QueryService } from '@/core/QueryService.js'; import { UtilityService } from '@/core/UtilityService.js'; import { UserEntityService } from '@/core/entities/UserEntityService.js'; +import type Logger from '@/logger.js'; +import { LoggerService } from '@/core/LoggerService.js'; import { bindThis } from '@/decorators.js'; import { IActivity } from '@/core/activitypub/type.js'; import { isQuote, isRenote } from '@/misc/is-renote.js'; import * as Acct from '@/misc/acct.js'; -import type Logger from '@/logger.js'; -import { LoggerService } from '@/core/LoggerService.js'; import type { FastifyInstance, FastifyRequest, FastifyReply, FastifyPluginOptions, FastifyBodyParser } from 'fastify'; import type { FindOptionsWhere } from 'typeorm'; -- cgit v1.2.3-freya