From e2261b63e9053fb5116dd0ef393f464bf16da190 Mon Sep 17 00:00:00 2001 From: Balazs Nadasdi Date: Thu, 22 Jun 2023 08:56:40 +0200 Subject: fix: clear queue endpoint error with redis script (#11037) Error message: ``` ReplyError: ERR value is not an integer or out of range script: 720d973b3877f92b4fb3285ced83c97cdd204979, on @user_script:209. ``` The whole error can be tracked back to one of the arguments, which is `Infinity` in the codebase, but it has to be a number. The documentation in bullmq says `0` is unlimited[^1], and bullmq tries to parse the argument with `tonumber` which returns with `-9223372036854775808` if the argument is `"Infinity"` which is out of bound. ``` 127.0.0.1:6379> eval 'return tonumber(ARGV[3])' '2' 'slippy.xyz:queue:inbox:inbox:delayed' 'slippy.xyz:queue:inbox:inbox:events' 'slippy.xyz:queue:inbox:inbox:' '1687183763944' Infinity 'delayed' (integer) -9223372036854775808 127.0.0.1:6379> ``` [^1]: https://github.com/taskforcesh/bullmq/blob/master/src/commands/cleanJobsInSet-2.lua#L10 Signed-off-by: Efertone --- packages/backend/src/core/QueueService.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'packages/backend/src/core/QueueService.ts') diff --git a/packages/backend/src/core/QueueService.ts b/packages/backend/src/core/QueueService.ts index 2ae8a2b754..5b7359074e 100644 --- a/packages/backend/src/core/QueueService.ts +++ b/packages/backend/src/core/QueueService.ts @@ -400,11 +400,11 @@ export class QueueService { this.deliverQueue.once('cleaned', (jobs, status) => { //deliverLogger.succ(`Cleaned ${jobs.length} ${status} jobs`); }); - this.deliverQueue.clean(0, Infinity, 'delayed'); + this.deliverQueue.clean(0, 0, 'delayed'); this.inboxQueue.once('cleaned', (jobs, status) => { //inboxLogger.succ(`Cleaned ${jobs.length} ${status} jobs`); }); - this.inboxQueue.clean(0, Infinity, 'delayed'); + this.inboxQueue.clean(0, 0, 'delayed'); } } -- cgit v1.2.3-freya From 22227fa641f80240ab5f134007067d1b1048095a Mon Sep 17 00:00:00 2001 From: tamaina Date: Wed, 5 Jul 2023 12:15:48 +0900 Subject: perf(backend): Use addBulk to add deliver queues (#11114) --- packages/backend/src/core/QueueService.ts | 36 ++++++++++++++++++++-- .../core/activitypub/ApDeliverManagerService.ts | 19 +++++++----- 2 files changed, 45 insertions(+), 10 deletions(-) (limited to 'packages/backend/src/core/QueueService.ts') diff --git a/packages/backend/src/core/QueueService.ts b/packages/backend/src/core/QueueService.ts index 5b7359074e..48ff00c8ce 100644 --- a/packages/backend/src/core/QueueService.ts +++ b/packages/backend/src/core/QueueService.ts @@ -8,7 +8,7 @@ import { DI } from '@/di-symbols.js'; import { bindThis } from '@/decorators.js'; import type { Antenna } from '@/server/api/endpoints/i/import-antennas.js'; import type { DbQueue, DeliverQueue, EndedPollNotificationQueue, InboxQueue, ObjectStorageQueue, RelationshipQueue, SystemQueue, WebhookDeliverQueue } from './QueueModule.js'; -import type { DbJobData, RelationshipJobData, ThinUser } from '../queue/types.js'; +import type { DbJobData, DeliverJobData, RelationshipJobData, ThinUser } from '../queue/types.js'; import type httpSignature from '@peertube/http-signature'; import type * as Bull from 'bullmq'; @@ -69,7 +69,7 @@ export class QueueService { if (content == null) return null; if (to == null) return null; - const data = { + const data: DeliverJobData = { user: { id: user.id, }, @@ -88,6 +88,38 @@ export class QueueService { }); } + /** + * ApDeliverManager-DeliverManager.execute()からinboxesを突っ込んでaddBulkしたい + * @param user `{ id: string; }` この関数ではThinUserに変換しないので前もって変換してください + * @param content IActivity | null + * @param inboxes `Map` / key: to (inbox url), value: isSharedInbox (whether it is sharedInbox) + * @returns void + */ + @bindThis + public async deliverMany(user: ThinUser, content: IActivity | null, inboxes: Map) { + const opts = { + attempts: this.config.deliverJobMaxAttempts ?? 12, + backoff: { + type: 'custom', + }, + removeOnComplete: true, + removeOnFail: true, + }; + + await this.deliverQueue.addBulk(Array.from(inboxes.entries()).map(d => ({ + name: d[0], + data: { + user, + content, + to: d[0], + isSharedInbox: d[1], + } as DeliverJobData, + opts, + }))); + + return; + } + @bindThis public inbox(activity: IActivity, signature: httpSignature.IParsedSignature) { const data = { diff --git a/packages/backend/src/core/activitypub/ApDeliverManagerService.ts b/packages/backend/src/core/activitypub/ApDeliverManagerService.ts index 62a2a33a19..66e7761187 100644 --- a/packages/backend/src/core/activitypub/ApDeliverManagerService.ts +++ b/packages/backend/src/core/activitypub/ApDeliverManagerService.ts @@ -7,6 +7,7 @@ import type { LocalUser, RemoteUser, User } from '@/models/entities/User.js'; import { QueueService } from '@/core/QueueService.js'; import { UserEntityService } from '@/core/entities/UserEntityService.js'; import { bindThis } from '@/decorators.js'; +import { ThinUser } from '@/queue/types.js'; interface IRecipe { type: string; @@ -94,7 +95,7 @@ export class ApDeliverManagerService { } class DeliverManager { - private actor: { id: User['id']; host: null; }; + private actor: ThinUser; private activity: any; private recipes: IRecipe[] = []; @@ -111,7 +112,13 @@ class DeliverManager { actor: { id: User['id']; host: null; }, activity: any, ) { - this.actor = actor; + // 型で弾いてはいるが一応ローカルユーザーかチェック + if (actor.host != null) throw new Error('actor.host must be null'); + + // パフォーマンス向上のためキューに突っ込むのはidのみに絞る + this.actor = { + id: actor.id, + }; this.activity = activity; } @@ -155,9 +162,8 @@ class DeliverManager { */ @bindThis public async execute() { - if (!this.userEntityService.isLocalUser(this.actor)) return; - // The value flags whether it is shared or not. + // key: inbox URL, value: whether it is sharedInbox const inboxes = new Map(); /* @@ -201,9 +207,6 @@ class DeliverManager { .forEach(recipe => inboxes.set(recipe.to.inbox!, false)); // deliver - for (const inbox of inboxes) { - // inbox[0]: inbox, inbox[1]: whether it is sharedInbox - this.queueService.deliver(this.actor, this.activity, inbox[0], inbox[1]); - } + this.queueService.deliverMany(this.actor, this.activity, inboxes); } } -- cgit v1.2.3-freya From b318789354a8100fb1a2025ae70e958dc770101e Mon Sep 17 00:00:00 2001 From: tamaina Date: Fri, 7 Jul 2023 23:15:04 +0000 Subject: fix(backend): deliverManyにcontentのnullチェックを追加 https://github.com/MisskeyIO/misskey/pull/99 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- packages/backend/src/core/QueueService.ts | 2 ++ 1 file changed, 2 insertions(+) (limited to 'packages/backend/src/core/QueueService.ts') diff --git a/packages/backend/src/core/QueueService.ts b/packages/backend/src/core/QueueService.ts index 48ff00c8ce..e1da0516d1 100644 --- a/packages/backend/src/core/QueueService.ts +++ b/packages/backend/src/core/QueueService.ts @@ -97,6 +97,8 @@ export class QueueService { */ @bindThis public async deliverMany(user: ThinUser, content: IActivity | null, inboxes: Map) { + if (content == null) return null; + const opts = { attempts: this.config.deliverJobMaxAttempts ?? 12, backoff: { -- cgit v1.2.3-freya From b392f44b81548ddbb9b7c483468ce894bdd025ea Mon Sep 17 00:00:00 2001 From: woxtu Date: Sat, 15 Jul 2023 18:39:38 +0900 Subject: refactor(backend): Improve UUID generation (#11286) * Replace with `crypto.randomUUID()` * Remove uuid --- packages/backend/package.json | 2 -- .../backend/src/core/CreateSystemUserService.ts | 4 ++-- packages/backend/src/core/DriveService.ts | 26 +++++++++++----------- packages/backend/src/core/QueueService.ts | 4 ++-- .../src/core/activitypub/ApRendererService.ts | 5 ++--- packages/backend/src/server/api/ApiCallService.ts | 4 ++-- .../server/api/endpoints/auth/session/generate.ts | 4 ++-- .../backend/src/server/web/ClientServerService.ts | 4 ++-- pnpm-lock.yaml | 6 ----- 9 files changed, 25 insertions(+), 34 deletions(-) (limited to 'packages/backend/src/core/QueueService.ts') diff --git a/packages/backend/package.json b/packages/backend/package.json index 777971527a..185651f394 100644 --- a/packages/backend/package.json +++ b/packages/backend/package.json @@ -158,7 +158,6 @@ "typescript": "5.1.6", "ulid": "2.3.0", "unzipper": "0.10.14", - "uuid": "9.0.0", "vary": "1.1.2", "web-push": "3.6.3", "ws": "8.13.0", @@ -201,7 +200,6 @@ "@types/tinycolor2": "1.4.3", "@types/tmp": "0.2.3", "@types/unzipper": "0.10.6", - "@types/uuid": "9.0.2", "@types/vary": "1.1.0", "@types/web-push": "3.3.2", "@types/websocket": "1.0.5", diff --git a/packages/backend/src/core/CreateSystemUserService.ts b/packages/backend/src/core/CreateSystemUserService.ts index 0bfbe2b173..2741cb41ee 100644 --- a/packages/backend/src/core/CreateSystemUserService.ts +++ b/packages/backend/src/core/CreateSystemUserService.ts @@ -1,6 +1,6 @@ +import { randomUUID } from 'node:crypto'; import { Inject, Injectable } from '@nestjs/common'; import bcrypt from 'bcryptjs'; -import { v4 as uuid } from 'uuid'; import { IsNull, DataSource } from 'typeorm'; import { genRsaKeyPair } from '@/misc/gen-key-pair.js'; import { User } from '@/models/entities/User.js'; @@ -24,7 +24,7 @@ export class CreateSystemUserService { @bindThis public async createSystemUser(username: string): Promise { - const password = uuid(); + const password = randomUUID(); // Generate hash of password const salt = await bcrypt.genSalt(8); diff --git a/packages/backend/src/core/DriveService.ts b/packages/backend/src/core/DriveService.ts index 1483b55469..355e5e8c0d 100644 --- a/packages/backend/src/core/DriveService.ts +++ b/packages/backend/src/core/DriveService.ts @@ -1,6 +1,6 @@ +import { randomUUID } from 'node:crypto'; import * as fs from 'node:fs'; import { Inject, Injectable } from '@nestjs/common'; -import { v4 as uuid } from 'uuid'; import sharp from 'sharp'; import { sharpBmp } from 'sharp-read-bmp'; import { IsNull } from 'typeorm'; @@ -162,7 +162,7 @@ export class DriveService { ?? `${ meta.objectStorageUseSSL ? 'https' : 'http' }://${ meta.objectStorageEndpoint }${ meta.objectStoragePort ? `:${meta.objectStoragePort}` : '' }/${ meta.objectStorageBucket }`; // for original - const key = `${meta.objectStoragePrefix}/${uuid()}${ext}`; + const key = `${meta.objectStoragePrefix}/${randomUUID()}${ext}`; const url = `${ baseUrl }/${ key }`; // for alts @@ -179,7 +179,7 @@ export class DriveService { ]; if (alts.webpublic) { - webpublicKey = `${meta.objectStoragePrefix}/webpublic-${uuid()}.${alts.webpublic.ext}`; + webpublicKey = `${meta.objectStoragePrefix}/webpublic-${randomUUID()}.${alts.webpublic.ext}`; webpublicUrl = `${ baseUrl }/${ webpublicKey }`; this.registerLogger.info(`uploading webpublic: ${webpublicKey}`); @@ -187,7 +187,7 @@ export class DriveService { } if (alts.thumbnail) { - thumbnailKey = `${meta.objectStoragePrefix}/thumbnail-${uuid()}.${alts.thumbnail.ext}`; + thumbnailKey = `${meta.objectStoragePrefix}/thumbnail-${randomUUID()}.${alts.thumbnail.ext}`; thumbnailUrl = `${ baseUrl }/${ thumbnailKey }`; this.registerLogger.info(`uploading thumbnail: ${thumbnailKey}`); @@ -212,9 +212,9 @@ export class DriveService { return await this.driveFilesRepository.insert(file).then(x => this.driveFilesRepository.findOneByOrFail(x.identifiers[0])); } else { // use internal storage - const accessKey = uuid(); - const thumbnailAccessKey = 'thumbnail-' + uuid(); - const webpublicAccessKey = 'webpublic-' + uuid(); + const accessKey = randomUUID(); + const thumbnailAccessKey = 'thumbnail-' + randomUUID(); + const webpublicAccessKey = 'webpublic-' + randomUUID(); const url = this.internalStorageService.saveFromPath(accessKey, path); @@ -584,9 +584,9 @@ export class DriveService { if (isLink) { file.url = url; // ローカルプロキシ用 - file.accessKey = uuid(); - file.thumbnailAccessKey = 'thumbnail-' + uuid(); - file.webpublicAccessKey = 'webpublic-' + uuid(); + file.accessKey = randomUUID(); + file.thumbnailAccessKey = 'thumbnail-' + randomUUID(); + file.webpublicAccessKey = 'webpublic-' + randomUUID(); } } @@ -713,9 +713,9 @@ export class DriveService { webpublicUrl: null, storedInternal: false, // ローカルプロキシ用 - accessKey: uuid(), - thumbnailAccessKey: 'thumbnail-' + uuid(), - webpublicAccessKey: 'webpublic-' + uuid(), + accessKey: randomUUID(), + thumbnailAccessKey: 'thumbnail-' + randomUUID(), + webpublicAccessKey: 'webpublic-' + randomUUID(), }); } else { this.driveFilesRepository.delete(file.id); diff --git a/packages/backend/src/core/QueueService.ts b/packages/backend/src/core/QueueService.ts index e1da0516d1..d0d4f802ee 100644 --- a/packages/backend/src/core/QueueService.ts +++ b/packages/backend/src/core/QueueService.ts @@ -1,5 +1,5 @@ +import { randomUUID } from 'node:crypto'; import { Inject, Injectable } from '@nestjs/common'; -import { v4 as uuid } from 'uuid'; import type { IActivity } from '@/core/activitypub/type.js'; import type { DriveFile } from '@/models/entities/DriveFile.js'; import type { Webhook, webhookEventTypes } from '@/models/entities/Webhook.js'; @@ -416,7 +416,7 @@ export class QueueService { to: webhook.url, secret: webhook.secret, createdAt: Date.now(), - eventId: uuid(), + eventId: randomUUID(), }; return this.webhookDeliverQueue.add(webhook.id, data, { diff --git a/packages/backend/src/core/activitypub/ApRendererService.ts b/packages/backend/src/core/activitypub/ApRendererService.ts index 95e3a936d2..797c6267b1 100644 --- a/packages/backend/src/core/activitypub/ApRendererService.ts +++ b/packages/backend/src/core/activitypub/ApRendererService.ts @@ -1,7 +1,6 @@ -import { createPublicKey } from 'node:crypto'; +import { createPublicKey, randomUUID } from 'node:crypto'; import { Inject, Injectable } from '@nestjs/common'; import { In } from 'typeorm'; -import { v4 as uuid } from 'uuid'; import * as mfm from 'mfm-js'; import { DI } from '@/di-symbols.js'; import type { Config } from '@/config.js'; @@ -613,7 +612,7 @@ export class ApRendererService { @bindThis public addContext(x: T): T & { '@context': any; id: string; } { if (typeof x === 'object' && x.id == null) { - x.id = `${this.config.url}/${uuid()}`; + x.id = `${this.config.url}/${randomUUID()}`; } return Object.assign({ diff --git a/packages/backend/src/server/api/ApiCallService.ts b/packages/backend/src/server/api/ApiCallService.ts index 09e3724394..c4c02e7afe 100644 --- a/packages/backend/src/server/api/ApiCallService.ts +++ b/packages/backend/src/server/api/ApiCallService.ts @@ -1,8 +1,8 @@ +import { randomUUID } from 'node:crypto'; import { pipeline } from 'node:stream'; import * as fs from 'node:fs'; import { promisify } from 'node:util'; import { Inject, Injectable } from '@nestjs/common'; -import { v4 as uuid } from 'uuid'; import { DI } from '@/di-symbols.js'; import { getIpHash } from '@/misc/get-ip-hash.js'; import type { LocalUser, User } from '@/models/entities/User.js'; @@ -362,7 +362,7 @@ export class ApiCallService implements OnApplicationShutdown { if (err instanceof ApiError || err instanceof AuthenticationError) { throw err; } else { - const errId = uuid(); + const errId = randomUUID(); this.logger.error(`Internal error occurred in ${ep.name}: ${err.message}`, { ep: ep.name, ps: data, diff --git a/packages/backend/src/server/api/endpoints/auth/session/generate.ts b/packages/backend/src/server/api/endpoints/auth/session/generate.ts index 6108d8202d..631fb4f024 100644 --- a/packages/backend/src/server/api/endpoints/auth/session/generate.ts +++ b/packages/backend/src/server/api/endpoints/auth/session/generate.ts @@ -1,4 +1,4 @@ -import { v4 as uuid } from 'uuid'; +import { randomUUID } from 'node:crypto'; import { Inject, Injectable } from '@nestjs/common'; import { Endpoint } from '@/server/api/endpoint-base.js'; import type { AppsRepository, AuthSessionsRepository } from '@/models/index.js'; @@ -71,7 +71,7 @@ export default class extends Endpoint { } // Generate token - const token = uuid(); + const token = randomUUID(); // Create session token document const doc = await this.authSessionsRepository.insert({ diff --git a/packages/backend/src/server/web/ClientServerService.ts b/packages/backend/src/server/web/ClientServerService.ts index 07ba2731c3..b5eea07775 100644 --- a/packages/backend/src/server/web/ClientServerService.ts +++ b/packages/backend/src/server/web/ClientServerService.ts @@ -1,7 +1,7 @@ +import { randomUUID } from 'node:crypto'; import { dirname } from 'node:path'; import { fileURLToPath } from 'node:url'; import { Inject, Injectable } from '@nestjs/common'; -import { v4 as uuid } from 'uuid'; import { createBullBoard } from '@bull-board/api'; import { BullAdapter } from '@bull-board/api/bullAdapter.js'; import { FastifyAdapter } from '@bull-board/fastify'; @@ -676,7 +676,7 @@ export class ClientServerService { }); fastify.setErrorHandler(async (error, request, reply) => { - const errId = uuid(); + const errId = randomUUID(); this.clientLoggerService.logger.error(`Internal error occured in ${request.routerPath}: ${error.message}`, { path: request.routerPath, params: request.params, diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 73a7cb1674..d97a8b7980 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -380,9 +380,6 @@ importers: unzipper: specifier: 0.10.14 version: 0.10.14 - uuid: - specifier: 9.0.0 - version: 9.0.0 vary: specifier: 1.1.2 version: 1.1.2 @@ -586,9 +583,6 @@ importers: '@types/unzipper': specifier: 0.10.6 version: 0.10.6 - '@types/uuid': - specifier: 9.0.2 - version: 9.0.2 '@types/vary': specifier: 1.1.0 version: 1.1.0 -- cgit v1.2.3-freya From c9b9db13c7203f8065e232a10a4635619d8de72a Mon Sep 17 00:00:00 2001 From: "Acid Chicken (硫酸鶏)" Date: Fri, 21 Jul 2023 10:07:03 +0000 Subject: refactor: avoid redundant method chain --- packages/backend/src/core/QueueService.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'packages/backend/src/core/QueueService.ts') diff --git a/packages/backend/src/core/QueueService.ts b/packages/backend/src/core/QueueService.ts index d0d4f802ee..546b4cee1b 100644 --- a/packages/backend/src/core/QueueService.ts +++ b/packages/backend/src/core/QueueService.ts @@ -108,7 +108,7 @@ export class QueueService { removeOnFail: true, }; - await this.deliverQueue.addBulk(Array.from(inboxes.entries()).map(d => ({ + await this.deliverQueue.addBulk(Array.from(inboxes.entries(), d => ({ name: d[0], data: { user, -- cgit v1.2.3-freya