From 71cecdbcf2d46d00ff58634eafec9227f4264b08 Mon Sep 17 00:00:00 2001 From: おさむのひと <46447427+samunohito@users.noreply.github.com> Date: Tue, 14 Jan 2025 21:01:01 +0900 Subject: feat(backend): pgroongaに対応（configの構成変更あり） (#14978) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat(backend): pgroongaに対応（configの構成変更あり） * fix CHANGELOG.md * fix CHANGELOG.md * add using provider logging * fix CHANGELOG.md * Update CHANGELOG.md --------- Co-authored-by: syuilo <4439005+syuilo@users.noreply.github.com> --- packages/backend/src/GlobalModule.ts | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) (limited to 'packages/backend/src/GlobalModule.ts') diff --git a/packages/backend/src/GlobalModule.ts b/packages/backend/src/GlobalModule.ts index 6ae8ccfbb3..ace7f7841c 100644 --- a/packages/backend/src/GlobalModule.ts +++ b/packages/backend/src/GlobalModule.ts @@ -7,14 +7,14 @@ import { Global, Inject, Module } from '@nestjs/common'; import * as Redis from 'ioredis'; import { DataSource } from 'typeorm'; import { MeiliSearch } from 'meilisearch'; +import { MiMeta } from '@/models/Meta.js'; import { DI } from './di-symbols.js'; import { Config, loadConfig } from './config.js'; import { createPostgresDataSource } from './postgres.js'; import { RepositoryModule } from './models/RepositoryModule.js'; import { allSettled } from './misc/promise-tracker.js'; -import type { Provider, OnApplicationShutdown } from '@nestjs/common'; -import { MiMeta } from '@/models/Meta.js'; import { GlobalEvents } from './core/GlobalEventService.js'; +import type { Provider, OnApplicationShutdown } from '@nestjs/common'; const $config: Provider = { provide: DI.config, @@ -33,7 +33,11 @@ const $db: Provider = { const $meilisearch: Provider = { provide: DI.meilisearch, useFactory: (config: Config) => { - if (config.meilisearch) { + if (config.fulltextSearch?.provider === 'meilisearch') { + if (!config.meilisearch) { + throw new Error('MeiliSearch is enabled but no configuration is provided'); + } + return new MeiliSearch({ host: `${config.meilisearch.ssl ? 'https' : 'http'}://${config.meilisearch.host}:${config.meilisearch.port}`, apiKey: config.meilisearch.apiKey, -- cgit v1.2.3-freya From 788751d24d91afd1f97b92b633c2177ee38398f4 Mon Sep 17 00:00:00 2001 From: Hazelnoot Date: Tue, 18 Feb 2025 10:36:29 -0500 Subject: implement redisForRateLimit --- .config/ci.yml | 10 ++++++++++ .config/cypress-devcontainer.yml | 10 ++++++++++ .config/docker_example.yml | 10 ++++++++++ .config/example.yml | 10 ++++++++++ packages/backend/scripts/check_connect.js | 1 + packages/backend/src/GlobalModule.ts | 14 ++++++++++++-- packages/backend/src/config.ts | 5 ++++- packages/backend/src/di-symbols.ts | 1 + packages/backend/src/server/SkRateLimiterService.md | 1 + packages/backend/src/server/SkRateLimiterService.ts | 2 +- 10 files changed, 60 insertions(+), 4 deletions(-) (limited to 'packages/backend/src/GlobalModule.ts') diff --git a/.config/ci.yml b/.config/ci.yml index 311a98d8fb..d8c365a47e 100644 --- a/.config/ci.yml +++ b/.config/ci.yml @@ -103,6 +103,16 @@ redis: # #prefix: example-prefix # #db: 1 +#redisForRateLimit: +# host: localhost +# port: 6379 +# #family: 0 # 0=Both, 4=IPv4, 6=IPv6 +# #pass: example-pass +# #prefix: example-prefix +# #db: 1 +# # You can specify more ioredis options... +# #username: example-username + # ┌───────────────────────────────┐ #───┘ Fulltext search configuration └───────────────────────────── diff --git a/.config/cypress-devcontainer.yml b/.config/cypress-devcontainer.yml index 391bc9998c..02442360fe 100644 --- a/.config/cypress-devcontainer.yml +++ b/.config/cypress-devcontainer.yml @@ -124,6 +124,16 @@ redis: # #prefix: example-prefix # #db: 1 +#redisForRateLimit: +# host: localhost +# port: 6379 +# #family: 0 # 0=Both, 4=IPv4, 6=IPv6 +# #pass: example-pass +# #prefix: example-prefix +# #db: 1 +# # You can specify more ioredis options... +# #username: example-username + # ┌───────────────────────────────┐ #───┘ Fulltext search configuration └───────────────────────────── diff --git a/.config/docker_example.yml b/.config/docker_example.yml index 1e03e902bf..fee5e35b72 100644 --- a/.config/docker_example.yml +++ b/.config/docker_example.yml @@ -171,6 +171,16 @@ redis: # #prefix: example-prefix # #db: 1 +#redisForRateLimit: +# host: localhost +# port: 6379 +# #family: 0 # 0=Both, 4=IPv4, 6=IPv6 +# #pass: example-pass +# #prefix: example-prefix +# #db: 1 +# # You can specify more ioredis options... +# #username: example-username + # ┌───────────────────────────────┐ #───┘ Fulltext search configuration └───────────────────────────── diff --git a/.config/example.yml b/.config/example.yml index 7d4cd0c659..2a1afd8481 100644 --- a/.config/example.yml +++ b/.config/example.yml @@ -198,6 +198,16 @@ redis: # # You can specify more ioredis options... # #username: example-username +#redisForRateLimit: +# host: localhost +# port: 6379 +# #family: 0 # 0=Both, 4=IPv4, 6=IPv6 +# #pass: example-pass +# #prefix: example-prefix +# #db: 1 +# # You can specify more ioredis options... +# #username: example-username + # ┌───────────────────────────────┐ #───┘ Fulltext search configuration └───────────────────────────── diff --git a/packages/backend/scripts/check_connect.js b/packages/backend/scripts/check_connect.js index 17b198ef62..b15ce51ec8 100644 --- a/packages/backend/scripts/check_connect.js +++ b/packages/backend/scripts/check_connect.js @@ -51,6 +51,7 @@ const promises = Array config.redisForJobQueue, config.redisForTimelines, config.redisForReactions, + config.redisForRateLimit, ])) .map(connectToRedis) .concat([ diff --git a/packages/backend/src/GlobalModule.ts b/packages/backend/src/GlobalModule.ts index ace7f7841c..7ca566477d 100644 --- a/packages/backend/src/GlobalModule.ts +++ b/packages/backend/src/GlobalModule.ts @@ -92,6 +92,14 @@ const $redisForReactions: Provider = { inject: [DI.config], }; +const $redisForRateLimit: Provider = { + provide: DI.redisForRateLimit, + useFactory: (config: Config) => { + return new Redis.Redis(config.redisForRateLimit); + }, + inject: [DI.config], +}; + const $meta: Provider = { provide: DI.meta, useFactory: async (db: DataSource, redisForSub: Redis.Redis) => { @@ -152,8 +160,8 @@ const $meta: Provider = { @Global() @Module({ imports: [RepositoryModule], - providers: [$config, $db, $meta, $meilisearch, $redis, $redisForPub, $redisForSub, $redisForTimelines, $redisForReactions], - exports: [$config, $db, $meta, $meilisearch, $redis, $redisForPub, $redisForSub, $redisForTimelines, $redisForReactions, RepositoryModule], + providers: [$config, $db, $meta, $meilisearch, $redis, $redisForPub, $redisForSub, $redisForTimelines, $redisForReactions, $redisForRateLimit], + exports: [$config, $db, $meta, $meilisearch, $redis, $redisForPub, $redisForSub, $redisForTimelines, $redisForReactions, $redisForRateLimit, RepositoryModule], }) export class GlobalModule implements OnApplicationShutdown { constructor( @@ -163,6 +171,7 @@ export class GlobalModule implements OnApplicationShutdown { @Inject(DI.redisForSub) private redisForSub: Redis.Redis, @Inject(DI.redisForTimelines) private redisForTimelines: Redis.Redis, @Inject(DI.redisForReactions) private redisForReactions: Redis.Redis, + @Inject(DI.redisForRateLimit) private redisForRateLimit: Redis.Redis, ) { } public async dispose(): Promise { @@ -176,6 +185,7 @@ export class GlobalModule implements OnApplicationShutdown { this.redisForSub.disconnect(), this.redisForTimelines.disconnect(), this.redisForReactions.disconnect(), + this.redisForRateLimit.disconnect(), ]); } diff --git a/packages/backend/src/config.ts b/packages/backend/src/config.ts index d35befdc2b..45aa1c1e22 100644 --- a/packages/backend/src/config.ts +++ b/packages/backend/src/config.ts @@ -53,6 +53,7 @@ type Source = { redisForJobQueue?: RedisOptionsSource; redisForTimelines?: RedisOptionsSource; redisForReactions?: RedisOptionsSource; + redisForRateLimit?: RedisOptionsSource; fulltextSearch?: { provider?: FulltextSearchProvider; }; @@ -225,6 +226,7 @@ export type Config = { redisForJobQueue: RedisOptions & RedisOptionsSource; redisForTimelines: RedisOptions & RedisOptionsSource; redisForReactions: RedisOptions & RedisOptionsSource; + redisForRateLimit: RedisOptions & RedisOptionsSource; sentryForBackend: { options: Partial; enableNodeProfiling: boolean; } | undefined; sentryForFrontend: { options: Partial } | undefined; perChannelMaxNoteCacheCount: number; @@ -333,6 +335,7 @@ export function loadConfig(): Config { redisForJobQueue: config.redisForJobQueue ? convertRedisOptions(config.redisForJobQueue, host) : redis, redisForTimelines: config.redisForTimelines ? convertRedisOptions(config.redisForTimelines, host) : redis, redisForReactions: config.redisForReactions ? convertRedisOptions(config.redisForReactions, host) : redis, + redisForRateLimit: config.redisForRateLimit ? convertRedisOptions(config.redisForRateLimit, host) : redis, sentryForBackend: config.sentryForBackend, sentryForFrontend: config.sentryForFrontend, id: config.id, @@ -518,7 +521,7 @@ function applyEnvOverrides(config: Source) { _apply_top(['db', ['host', 'port', 'db', 'user', 'pass', 'disableCache']]); _apply_top(['dbSlaves', Array.from((config.dbSlaves ?? []).keys()), ['host', 'port', 'db', 'user', 'pass']]); _apply_top([ - ['redis', 'redisForPubsub', 'redisForJobQueue', 'redisForTimelines', 'redisForReactions'], + ['redis', 'redisForPubsub', 'redisForJobQueue', 'redisForTimelines', 'redisForReactions', 'redisForRateLimit'], ['host', 'port', 'username', 'pass', 'db', 'prefix'], ]); _apply_top(['fulltextSearch', 'provider']); diff --git a/packages/backend/src/di-symbols.ts b/packages/backend/src/di-symbols.ts index 296cc4815b..71f627dc4e 100644 --- a/packages/backend/src/di-symbols.ts +++ b/packages/backend/src/di-symbols.ts @@ -13,6 +13,7 @@ export const DI = { redisForSub: Symbol('redisForSub'), redisForTimelines: Symbol('redisForTimelines'), redisForReactions: Symbol('redisForReactions'), + redisForRateLimit: Symbol('redisForRateLimit'), //#region Repositories usersRepository: Symbol('usersRepository'), diff --git a/packages/backend/src/server/SkRateLimiterService.md b/packages/backend/src/server/SkRateLimiterService.md index fb007538fa..c8a2b4e85c 100644 --- a/packages/backend/src/server/SkRateLimiterService.md +++ b/packages/backend/src/server/SkRateLimiterService.md @@ -39,6 +39,7 @@ The first call is read-only, while the others perform at least one write operati Two integer keys are stored per client/subject, and both expire together after the maximum duration of the limit. While performance has not been formally tested, it's expected that SkRateLimiterService has an impact roughly on par with the legacy RateLimiterService. Redis memory usage should be notably lower due to the reduced number of keys and avoidance of set / array constructions. +If redis load does become a concern, then a dedicated node can be assigned via the `redisForRateLimit` config setting. ## Concurrency and Multi-Node Correctness diff --git a/packages/backend/src/server/SkRateLimiterService.ts b/packages/backend/src/server/SkRateLimiterService.ts index 038f12cb25..30bf092e4f 100644 --- a/packages/backend/src/server/SkRateLimiterService.ts +++ b/packages/backend/src/server/SkRateLimiterService.ts @@ -27,7 +27,7 @@ export class SkRateLimiterService { @Inject('TimeService') private readonly timeService: TimeService, - @Inject(DI.redis) + @Inject(DI.redisForRateLimit) private readonly redisClient: Redis.Redis, @Inject('RoleService') -- cgit v1.2.3-freya