diff options
| author | syuilo <Syuilotan@yahoo.co.jp> | 2023-03-25 08:36:41 +0900 |
|---|---|---|
| committer | GitHub <noreply@github.com> | 2023-03-25 08:36:41 +0900 |
| commit | f54a9542bb2458e5f83bc7bcf25803a1bb76cef8 (patch) | |
| tree | 03a62e0b9508b75e55b71300b93a1bf540f84812 /packages/backend | |
| parent | Merge pull request #10388 from misskey-dev/develop (diff) | |
| parent | New Crowdin updates (#10405) (diff) | |
| download | misskey-f54a9542bb2458e5f83bc7bcf25803a1bb76cef8.tar.gz misskey-f54a9542bb2458e5f83bc7bcf25803a1bb76cef8.tar.bz2 misskey-f54a9542bb2458e5f83bc7bcf25803a1bb76cef8.zip | |
Merge pull request #10402 from misskey-dev/develop
Release: 13.10.3
Diffstat (limited to 'packages/backend')
34 files changed, 487 insertions, 185 deletions
diff --git a/packages/backend/migration/1679639483253-enableChartsForRemoteUser.js b/packages/backend/migration/1679639483253-enableChartsForRemoteUser.js new file mode 100644 index 0000000000..42faab7466 --- /dev/null +++ b/packages/backend/migration/1679639483253-enableChartsForRemoteUser.js @@ -0,0 +1,11 @@ +export class enableChartsForRemoteUser1679639483253 { + name = 'enableChartsForRemoteUser1679639483253' + + async up(queryRunner) { + await queryRunner.query(`ALTER TABLE "meta" ADD "enableChartsForRemoteUser" boolean NOT NULL DEFAULT true`); + } + + async down(queryRunner) { + await queryRunner.query(`ALTER TABLE "meta" DROP COLUMN "enableChartsForRemoteUser"`); + } +} diff --git a/packages/backend/migration/1679651580149-cleanup.js b/packages/backend/migration/1679651580149-cleanup.js new file mode 100644 index 0000000000..1f00f3cc1f --- /dev/null +++ b/packages/backend/migration/1679651580149-cleanup.js @@ -0,0 +1,11 @@ +export class cleanup1679651580149 { + name = 'cleanup1679651580149' + + async up(queryRunner) { + await queryRunner.query(`ALTER TABLE "meta" DROP COLUMN "useStarForReactionFallback"`); + } + + async down(queryRunner) { + await queryRunner.query(`ALTER TABLE "meta" ADD "useStarForReactionFallback" boolean NOT NULL DEFAULT false`); + } +} diff --git a/packages/backend/migration/1679652081809-enableChartsForFederatedInstances.js b/packages/backend/migration/1679652081809-enableChartsForFederatedInstances.js new file mode 100644 index 0000000000..0733339841 --- /dev/null +++ b/packages/backend/migration/1679652081809-enableChartsForFederatedInstances.js @@ -0,0 +1,11 @@ +export class enableChartsForFederatedInstances1679652081809 { + name = 'enableChartsForFederatedInstances1679652081809' + + async up(queryRunner) { + await queryRunner.query(`ALTER TABLE "meta" ADD "enableChartsForFederatedInstances" boolean NOT NULL DEFAULT true`); + } + + async down(queryRunner) { + await queryRunner.query(`ALTER TABLE "meta" DROP COLUMN "enableChartsForFederatedInstances"`); + } +} diff --git a/packages/backend/package.json b/packages/backend/package.json index 5a3dcfb5e7..3f640c4a63 100644 --- a/packages/backend/package.json +++ b/packages/backend/package.json @@ -37,6 +37,9 @@ "@tensorflow/tfjs-node": "4.2.0" }, "dependencies": { + "@aws-sdk/client-s3": "^3.294.0", + "@aws-sdk/lib-storage": "^3.294.0", + "@aws-sdk/node-http-handler": "^3.292.0", "@bull-board/api": "5.0.0", "@bull-board/fastify": "5.0.0", "@bull-board/ui": "5.0.0", @@ -59,7 +62,6 @@ "ajv": "8.12.0", "archiver": "5.3.1", "autwh": "0.1.0", - "aws-sdk": "2.1318.0", "bcryptjs": "2.4.3", "blurhash": "2.0.5", "bull": "4.10.4", @@ -190,6 +192,7 @@ "@types/ws": "8.5.4", "@typescript-eslint/eslint-plugin": "5.54.1", "@typescript-eslint/parser": "5.54.1", + "aws-sdk-client-mock": "^2.1.1", "cross-env": "7.0.3", "eslint": "8.35.0", "eslint-plugin-import": "2.27.5", diff --git a/packages/backend/src/core/CustomEmojiService.ts b/packages/backend/src/core/CustomEmojiService.ts index b404848d7d..a62854c61c 100644 --- a/packages/backend/src/core/CustomEmojiService.ts +++ b/packages/backend/src/core/CustomEmojiService.ts @@ -8,7 +8,7 @@ import type { DriveFile } from '@/models/entities/DriveFile.js'; import type { Emoji } from '@/models/entities/Emoji.js'; import type { EmojisRepository, Note } from '@/models/index.js'; import { bindThis } from '@/decorators.js'; -import { Cache } from '@/misc/cache.js'; +import { KVCache } from '@/misc/cache.js'; import { UtilityService } from '@/core/UtilityService.js'; import type { Config } from '@/config.js'; import { ReactionService } from '@/core/ReactionService.js'; @@ -16,7 +16,7 @@ import { query } from '@/misc/prelude/url.js'; @Injectable() export class CustomEmojiService { - private cache: Cache<Emoji | null>; + private cache: KVCache<Emoji | null>; constructor( @Inject(DI.config) @@ -34,7 +34,7 @@ export class CustomEmojiService { private globalEventService: GlobalEventService, private reactionService: ReactionService, ) { - this.cache = new Cache<Emoji | null>(1000 * 60 * 60 * 12); + this.cache = new KVCache<Emoji | null>(1000 * 60 * 60 * 12); } @bindThis diff --git a/packages/backend/src/core/DriveService.ts b/packages/backend/src/core/DriveService.ts index f1e93d6dd9..c6258474ec 100644 --- a/packages/backend/src/core/DriveService.ts +++ b/packages/backend/src/core/DriveService.ts @@ -4,6 +4,7 @@ import { v4 as uuid } from 'uuid'; import sharp from 'sharp'; import { sharpBmp } from 'sharp-read-bmp'; import { IsNull } from 'typeorm'; +import { DeleteObjectCommandInput, PutObjectCommandInput, NoSuchKey } from '@aws-sdk/client-s3'; import { DI } from '@/di-symbols.js'; import type { DriveFilesRepository, UsersRepository, DriveFoldersRepository, UserProfilesRepository } from '@/models/index.js'; import type { Config } from '@/config.js'; @@ -36,7 +37,6 @@ import { bindThis } from '@/decorators.js'; import { RoleService } from '@/core/RoleService.js'; import { correctFilename } from '@/misc/correct-filename.js'; import { isMimeImage } from '@/misc/is-mime-image.js'; -import type S3 from 'aws-sdk/clients/s3.js'; type AddFileArgs = { /** User who wish to add file */ @@ -81,6 +81,7 @@ type UploadFromUrlArgs = { export class DriveService { private registerLogger: Logger; private downloaderLogger: Logger; + private deleteLogger: Logger; constructor( @Inject(DI.config) @@ -118,6 +119,7 @@ export class DriveService { const logger = new Logger('drive', 'blue'); this.registerLogger = logger.createSubLogger('register', 'yellow'); this.downloaderLogger = logger.createSubLogger('downloader'); + this.deleteLogger = logger.createSubLogger('delete'); } /*** @@ -368,7 +370,7 @@ export class DriveService { Body: stream, ContentType: type, CacheControl: 'max-age=31536000, immutable', - } as S3.PutObjectRequest; + } as PutObjectCommandInput; if (filename) params.ContentDisposition = contentDisposition( 'inline', @@ -378,21 +380,16 @@ export class DriveService { ); if (meta.objectStorageSetPublicRead) params.ACL = 'public-read'; - const s3 = this.s3Service.getS3(meta); - - const upload = s3.upload(params, { - partSize: s3.endpoint.hostname === 'storage.googleapis.com' ? 500 * 1024 * 1024 : 8 * 1024 * 1024, - }); - - await upload.promise() + await this.s3Service.upload(meta, params) .then( result => { - if (result) { + if ('Bucket' in result) { // CompleteMultipartUploadCommandOutput this.registerLogger.debug(`Uploaded: ${result.Bucket}/${result.Key} => ${result.Location}`); - } else { - this.registerLogger.error(`Upload Result Empty: key = ${key}, filename = ${filename}`); + } else { // AbortMultipartUploadCommandOutput + this.registerLogger.error(`Upload Result Aborted: key = ${key}, filename = ${filename}`); } - }, + }) + .catch( err => { this.registerLogger.error(`Upload Failed: key = ${key}, filename = ${filename}`, err); }, @@ -528,10 +525,10 @@ export class DriveService { }; const properties: { - width?: number; - height?: number; - orientation?: number; - } = {}; + width?: number; + height?: number; + orientation?: number; + } = {}; if (info.width) { properties['width'] = info.width; @@ -616,17 +613,20 @@ export class DriveService { if (user) { this.driveFileEntityService.pack(file, { self: true }).then(packedFile => { - // Publish driveFileCreated event + // Publish driveFileCreated event this.globalEventService.publishMainStream(user.id, 'driveFileCreated', packedFile); this.globalEventService.publishDriveStream(user.id, 'fileCreated', packedFile); }); } - // 統計を更新 this.driveChart.update(file, true); - this.perUserDriveChart.update(file, true); - if (file.userHost !== null) { - this.instanceChart.updateDrive(file, true); + if (file.userHost == null) { + // ローカルユーザーのみ + this.perUserDriveChart.update(file, true); + } else { + if ((await this.metaService.fetch()).enableChartsForFederatedInstances) { + this.instanceChart.updateDrive(file, true); + } } return file; @@ -692,7 +692,7 @@ export class DriveService { @bindThis private async deletePostProcess(file: DriveFile, isExpired = false) { - // リモートファイル期限切れ削除後は直リンクにする + // リモートファイル期限切れ削除後は直リンクにする if (isExpired && file.userHost !== null && file.uri != null) { this.driveFilesRepository.update(file.id, { isLink: true, @@ -709,33 +709,36 @@ export class DriveService { this.driveFilesRepository.delete(file.id); } - // 統計を更新 this.driveChart.update(file, false); - this.perUserDriveChart.update(file, false); - if (file.userHost !== null) { - this.instanceChart.updateDrive(file, false); + if (file.userHost == null) { + // ローカルユーザーのみ + this.perUserDriveChart.update(file, false); + } else { + if ((await this.metaService.fetch()).enableChartsForFederatedInstances) { + this.instanceChart.updateDrive(file, false); + } } } @bindThis public async deleteObjectStorageFile(key: string) { const meta = await this.metaService.fetch(); - - const s3 = this.s3Service.getS3(meta); - try { - await s3.deleteObject({ - Bucket: meta.objectStorageBucket!, + const param = { + Bucket: meta.objectStorageBucket, Key: key, - }).promise(); + } as DeleteObjectCommandInput; + + await this.s3Service.delete(meta, param); } catch (err: any) { - if (err.code === 'NoSuchKey') { - console.warn(`The object storage had no such key to delete: ${key}. Skipping this.`, err); + if (err.name === 'NoSuchKey') { + this.deleteLogger.warn(`The object storage had no such key to delete: ${key}. Skipping this.`, err as Error); return; + } else { + throw new Error(`Failed to delete the file from the object storage with the given key: ${key}`, { + cause: err, + }); } - throw new Error(`Failed to delete the file from the object storage with the given key: ${key}`, { - cause: err, - }); } } diff --git a/packages/backend/src/core/FederatedInstanceService.ts b/packages/backend/src/core/FederatedInstanceService.ts index e83b037dd7..b85791e43f 100644 --- a/packages/backend/src/core/FederatedInstanceService.ts +++ b/packages/backend/src/core/FederatedInstanceService.ts @@ -1,7 +1,7 @@ import { Inject, Injectable } from '@nestjs/common'; import type { InstancesRepository } from '@/models/index.js'; import type { Instance } from '@/models/entities/Instance.js'; -import { Cache } from '@/misc/cache.js'; +import { KVCache } from '@/misc/cache.js'; import { IdService } from '@/core/IdService.js'; import { DI } from '@/di-symbols.js'; import { UtilityService } from '@/core/UtilityService.js'; @@ -9,7 +9,7 @@ import { bindThis } from '@/decorators.js'; @Injectable() export class FederatedInstanceService { - private cache: Cache<Instance>; + private cache: KVCache<Instance>; constructor( @Inject(DI.instancesRepository) @@ -18,7 +18,7 @@ export class FederatedInstanceService { private utilityService: UtilityService, private idService: IdService, ) { - this.cache = new Cache<Instance>(1000 * 60 * 60); + this.cache = new KVCache<Instance>(1000 * 60 * 60); } @bindThis diff --git a/packages/backend/src/core/InstanceActorService.ts b/packages/backend/src/core/InstanceActorService.ts index ee9ae0733f..ef87051a74 100644 --- a/packages/backend/src/core/InstanceActorService.ts +++ b/packages/backend/src/core/InstanceActorService.ts @@ -2,7 +2,7 @@ import { Inject, Injectable } from '@nestjs/common'; import { IsNull } from 'typeorm'; import type { LocalUser } from '@/models/entities/User.js'; import type { UsersRepository } from '@/models/index.js'; -import { Cache } from '@/misc/cache.js'; +import { KVCache } from '@/misc/cache.js'; import { DI } from '@/di-symbols.js'; import { CreateSystemUserService } from '@/core/CreateSystemUserService.js'; import { bindThis } from '@/decorators.js'; @@ -11,7 +11,7 @@ const ACTOR_USERNAME = 'instance.actor' as const; @Injectable() export class InstanceActorService { - private cache: Cache<LocalUser>; + private cache: KVCache<LocalUser>; constructor( @Inject(DI.usersRepository) @@ -19,7 +19,7 @@ export class InstanceActorService { private createSystemUserService: CreateSystemUserService, ) { - this.cache = new Cache<LocalUser>(Infinity); + this.cache = new KVCache<LocalUser>(Infinity); } @bindThis diff --git a/packages/backend/src/core/NoteCreateService.ts b/packages/backend/src/core/NoteCreateService.ts index 2fc2a3d54f..7d08053761 100644 --- a/packages/backend/src/core/NoteCreateService.ts +++ b/packages/backend/src/core/NoteCreateService.ts @@ -19,7 +19,7 @@ import { isDuplicateKeyValueError } from '@/misc/is-duplicate-key-value-error.js import { checkWordMute } from '@/misc/check-word-mute.js'; import type { Channel } from '@/models/entities/Channel.js'; import { normalizeForSearch } from '@/misc/normalize-for-search.js'; -import { Cache } from '@/misc/cache.js'; +import { KVCache } from '@/misc/cache.js'; import type { UserProfile } from '@/models/entities/UserProfile.js'; import { RelayService } from '@/core/RelayService.js'; import { FederatedInstanceService } from '@/core/FederatedInstanceService.js'; @@ -46,7 +46,7 @@ import { DB_MAX_NOTE_TEXT_LENGTH } from '@/const.js'; import { RoleService } from '@/core/RoleService.js'; import { MetaService } from '@/core/MetaService.js'; -const mutedWordsCache = new Cache<{ userId: UserProfile['userId']; mutedWords: UserProfile['mutedWords']; }[]>(1000 * 60 * 5); +const mutedWordsCache = new KVCache<{ userId: UserProfile['userId']; mutedWords: UserProfile['mutedWords']; }[]>(1000 * 60 * 5); type NotificationType = 'reply' | 'renote' | 'quote' | 'mention'; @@ -435,15 +435,20 @@ export class NoteCreateService implements OnApplicationShutdown { createdAt: User['createdAt']; isBot: User['isBot']; }, data: Option, silent: boolean, tags: string[], mentionedUsers: MinimumUser[]) { - // 統計を更新 + const meta = await this.metaService.fetch(); + this.notesChart.update(note, true); - this.perUserNotesChart.update(user, note, true); + if (meta.enableChartsForRemoteUser || (user.host == null)) { + this.perUserNotesChart.update(user, note, true); + } // Register host if (this.userEntityService.isRemoteUser(user)) { - this.federatedInstanceService.fetch(user.host).then(i => { + this.federatedInstanceService.fetch(user.host).then(async i => { this.instancesRepository.increment({ id: i.id }, 'notesCount', 1); - this.instanceChart.updateNote(i.host, note, true); + if ((await this.metaService.fetch()).enableChartsForFederatedInstances) { + this.instanceChart.updateNote(i.host, note, true); + } }); } diff --git a/packages/backend/src/core/NoteDeleteService.ts b/packages/backend/src/core/NoteDeleteService.ts index 571b625523..dd878f7bba 100644 --- a/packages/backend/src/core/NoteDeleteService.ts +++ b/packages/backend/src/core/NoteDeleteService.ts @@ -16,6 +16,7 @@ import { ApDeliverManagerService } from '@/core/activitypub/ApDeliverManagerServ import { UserEntityService } from '@/core/entities/UserEntityService.js'; import { NoteEntityService } from '@/core/entities/NoteEntityService.js'; import { bindThis } from '@/decorators.js'; +import { MetaService } from '@/core/MetaService.js'; @Injectable() export class NoteDeleteService { @@ -39,6 +40,7 @@ export class NoteDeleteService { private federatedInstanceService: FederatedInstanceService, private apRendererService: ApRendererService, private apDeliverManagerService: ApDeliverManagerService, + private metaService: MetaService, private notesChart: NotesChart, private perUserNotesChart: PerUserNotesChart, private instanceChart: InstanceChart, @@ -95,14 +97,19 @@ export class NoteDeleteService { } //#endregion - // 統計を更新 + const meta = await this.metaService.fetch(); + this.notesChart.update(note, false); - this.perUserNotesChart.update(user, note, false); + if (meta.enableChartsForRemoteUser || (user.host == null)) { + this.perUserNotesChart.update(user, note, false); + } if (this.userEntityService.isRemoteUser(user)) { - this.federatedInstanceService.fetch(user.host).then(i => { + this.federatedInstanceService.fetch(user.host).then(async i => { this.instancesRepository.decrement({ id: i.id }, 'notesCount', 1); - this.instanceChart.updateNote(i.host, note, false); + if ((await this.metaService.fetch()).enableChartsForFederatedInstances) { + this.instanceChart.updateNote(i.host, note, false); + } }); } } diff --git a/packages/backend/src/core/ReactionService.ts b/packages/backend/src/core/ReactionService.ts index 271ba79176..b3aea878d6 100644 --- a/packages/backend/src/core/ReactionService.ts +++ b/packages/backend/src/core/ReactionService.ts @@ -21,6 +21,8 @@ import { bindThis } from '@/decorators.js'; import { UtilityService } from '@/core/UtilityService.js'; import { UserBlockingService } from '@/core/UserBlockingService.js'; +const FALLBACK = '❤'; + const legacies: Record<string, string> = { 'like': '👍', 'love': '❤', // ここに記述する場合は異体字セレクタを入れない @@ -147,7 +149,11 @@ export class ReactionService { .where('id = :id', { id: note.id }) .execute(); - this.perUserReactionsChart.update(user, note); + const meta = await this.metaService.fetch(); + + if (meta.enableChartsForRemoteUser || (user.host == null)) { + this.perUserReactionsChart.update(user, note); + } // カスタム絵文字リアクションだったら絵文字情報も送る const decodedReaction = this.decodeReaction(reaction); @@ -252,12 +258,6 @@ export class ReactionService { } @bindThis - public async getFallbackReaction(): Promise<string> { - const meta = await this.metaService.fetch(); - return meta.useStarForReactionFallback ? '⭐' : '👍'; - } - - @bindThis public convertLegacyReactions(reactions: Record<string, number>) { const _reactions = {} as Record<string, number>; @@ -290,7 +290,7 @@ export class ReactionService { @bindThis public async toDbReaction(reaction?: string | null, reacterHost?: string | null): Promise<string> { - if (reaction == null) return await this.getFallbackReaction(); + if (reaction == null) return FALLBACK; reacterHost = this.utilityService.toPunyNullable(reacterHost); @@ -318,7 +318,7 @@ export class ReactionService { if (emoji) return reacterHost ? `:${name}@${reacterHost}:` : `:${name}:`; } - return await this.getFallbackReaction(); + return FALLBACK; } @bindThis diff --git a/packages/backend/src/core/RelayService.ts b/packages/backend/src/core/RelayService.ts index 86f983cc78..4537f1b81a 100644 --- a/packages/backend/src/core/RelayService.ts +++ b/packages/backend/src/core/RelayService.ts @@ -3,7 +3,7 @@ import { IsNull } from 'typeorm'; import type { LocalUser, User } from '@/models/entities/User.js'; import type { RelaysRepository, UsersRepository } from '@/models/index.js'; import { IdService } from '@/core/IdService.js'; -import { Cache } from '@/misc/cache.js'; +import { KVCache } from '@/misc/cache.js'; import type { Relay } from '@/models/entities/Relay.js'; import { QueueService } from '@/core/QueueService.js'; import { CreateSystemUserService } from '@/core/CreateSystemUserService.js'; @@ -16,7 +16,7 @@ const ACTOR_USERNAME = 'relay.actor' as const; @Injectable() export class RelayService { - private relaysCache: Cache<Relay[]>; + private relaysCache: KVCache<Relay[]>; constructor( @Inject(DI.usersRepository) @@ -30,7 +30,7 @@ export class RelayService { private createSystemUserService: CreateSystemUserService, private apRendererService: ApRendererService, ) { - this.relaysCache = new Cache<Relay[]>(1000 * 60 * 10); + this.relaysCache = new KVCache<Relay[]>(1000 * 60 * 10); } @bindThis diff --git a/packages/backend/src/core/RoleService.ts b/packages/backend/src/core/RoleService.ts index 4775196c6f..7b63e43cb1 100644 --- a/packages/backend/src/core/RoleService.ts +++ b/packages/backend/src/core/RoleService.ts @@ -2,7 +2,7 @@ import { Inject, Injectable } from '@nestjs/common'; import Redis from 'ioredis'; import { In } from 'typeorm'; import type { Role, RoleAssignment, RoleAssignmentsRepository, RolesRepository, UsersRepository } from '@/models/index.js'; -import { Cache } from '@/misc/cache.js'; +import { KVCache } from '@/misc/cache.js'; import type { User } from '@/models/entities/User.js'; import { DI } from '@/di-symbols.js'; import { bindThis } from '@/decorators.js'; @@ -57,8 +57,8 @@ export const DEFAULT_POLICIES: RolePolicies = { @Injectable() export class RoleService implements OnApplicationShutdown { - private rolesCache: Cache<Role[]>; - private roleAssignmentByUserIdCache: Cache<RoleAssignment[]>; + private rolesCache: KVCache<Role[]>; + private roleAssignmentByUserIdCache: KVCache<RoleAssignment[]>; public static AlreadyAssignedError = class extends Error {}; public static NotAssignedError = class extends Error {}; @@ -84,8 +84,8 @@ export class RoleService implements OnApplicationShutdown { ) { //this.onMessage = this.onMessage.bind(this); - this.rolesCache = new Cache<Role[]>(Infinity); - this.roleAssignmentByUserIdCache = new Cache<RoleAssignment[]>(Infinity); + this.rolesCache = new KVCache<Role[]>(Infinity); + this.roleAssignmentByUserIdCache = new KVCache<RoleAssignment[]>(Infinity); this.redisSubscriber.on('message', this.onMessage); } @@ -192,6 +192,12 @@ export class RoleService implements OnApplicationShutdown { case 'followingMoreThanOrEq': { return user.followingCount >= value.value; } + case 'notesLessThanOrEq': { + return user.notesCount <= value.value; + } + case 'notesMoreThanOrEq': { + return user.notesCount >= value.value; + } default: return false; } diff --git a/packages/backend/src/core/S3Service.ts b/packages/backend/src/core/S3Service.ts index cc8f950813..629278d915 100644 --- a/packages/backend/src/core/S3Service.ts +++ b/packages/backend/src/core/S3Service.ts @@ -1,11 +1,16 @@ import { URL } from 'node:url'; +import * as http from 'node:http'; +import * as https from 'node:https'; import { Inject, Injectable } from '@nestjs/common'; -import S3 from 'aws-sdk/clients/s3.js'; +import { DeleteObjectCommand, S3Client } from '@aws-sdk/client-s3'; +import { Upload } from '@aws-sdk/lib-storage'; +import { NodeHttpHandler, NodeHttpHandlerOptions } from '@aws-sdk/node-http-handler'; import { DI } from '@/di-symbols.js'; import type { Config } from '@/config.js'; import type { Meta } from '@/models/entities/Meta.js'; import { HttpRequestService } from '@/core/HttpRequestService.js'; import { bindThis } from '@/decorators.js'; +import type { DeleteObjectCommandInput, PutObjectCommandInput } from '@aws-sdk/client-s3'; @Injectable() export class S3Service { @@ -18,25 +23,47 @@ export class S3Service { } @bindThis - public getS3(meta: Meta) { + public getS3Client(meta: Meta): S3Client { const u = meta.objectStorageEndpoint - ? `${meta.objectStorageUseSSL ? 'https://' : 'http://'}${meta.objectStorageEndpoint}` - : `${meta.objectStorageUseSSL ? 'https://' : 'http://'}example.net`; + ? `${meta.objectStorageUseSSL ? 'https' : 'http'}://${meta.objectStorageEndpoint}` + : `${meta.objectStorageUseSSL ? 'https' : 'http'}://example.net`; // dummy url to select http(s) agent - return new S3({ - endpoint: meta.objectStorageEndpoint && meta.objectStorageEndpoint.length > 0 - ? meta.objectStorageEndpoint - : undefined, - accessKeyId: meta.objectStorageAccessKey!, - secretAccessKey: meta.objectStorageSecretKey!, + const agent = this.httpRequestService.getAgentByUrl(new URL(u), !meta.objectStorageUseProxy); + const handlerOption: NodeHttpHandlerOptions = {}; + if (meta.objectStorageUseSSL) { + handlerOption.httpsAgent = agent as https.Agent; + } else { + handlerOption.httpAgent = agent as http.Agent; + } + + return new S3Client({ + endpoint: meta.objectStorageEndpoint ? u : undefined, + credentials: (meta.objectStorageAccessKey !== null && meta.objectStorageSecretKey !== null) ? { + accessKeyId: meta.objectStorageAccessKey, + secretAccessKey: meta.objectStorageSecretKey, + } : undefined, region: meta.objectStorageRegion ?? undefined, - sslEnabled: meta.objectStorageUseSSL, - s3ForcePathStyle: !meta.objectStorageEndpoint // AWS with endPoint omitted - ? false - : meta.objectStorageS3ForcePathStyle, - httpOptions: { - agent: this.httpRequestService.getAgentByUrl(new URL(u), !meta.objectStorageUseProxy), - }, + tls: meta.objectStorageUseSSL, + forcePathStyle: meta.objectStorageEndpoint ? meta.objectStorageS3ForcePathStyle : false, // AWS with endPoint omitted + requestHandler: new NodeHttpHandler(handlerOption), }); } + + @bindThis + public async upload(meta: Meta, input: PutObjectCommandInput) { + const client = this.getS3Client(meta); + return new Upload({ + client, + params: input, + partSize: (client.config.endpoint && (await client.config.endpoint()).hostname === 'storage.googleapis.com') + ? 500 * 1024 * 1024 + : 8 * 1024 * 1024, + }).done(); + } + + @bindThis + public delete(meta: Meta, input: DeleteObjectCommandInput) { + const client = this.getS3Client(meta); + return client.send(new DeleteObjectCommand(input)); + } } diff --git a/packages/backend/src/core/UserBlockingService.ts b/packages/backend/src/core/UserBlockingService.ts index 92408da342..33b51537a6 100644 --- a/packages/backend/src/core/UserBlockingService.ts +++ b/packages/backend/src/core/UserBlockingService.ts @@ -15,7 +15,7 @@ import { ApRendererService } from '@/core/activitypub/ApRendererService.js'; import { LoggerService } from '@/core/LoggerService.js'; import { WebhookService } from '@/core/WebhookService.js'; import { bindThis } from '@/decorators.js'; -import { Cache } from '@/misc/cache.js'; +import { KVCache } from '@/misc/cache.js'; import { StreamMessages } from '@/server/api/stream/types.js'; @Injectable() @@ -23,7 +23,7 @@ export class UserBlockingService implements OnApplicationShutdown { private logger: Logger; // キーがユーザーIDで、値がそのユーザーがブロックしているユーザーのIDのリストなキャッシュ - private blockingsByUserIdCache: Cache<User['id'][]>; + private blockingsByUserIdCache: KVCache<User['id'][]>; constructor( @Inject(DI.redisSubscriber) @@ -58,7 +58,7 @@ export class UserBlockingService implements OnApplicationShutdown { ) { this.logger = this.loggerService.getLogger('user-block'); - this.blockingsByUserIdCache = new Cache<User['id'][]>(Infinity); + this.blockingsByUserIdCache = new KVCache<User['id'][]>(Infinity); this.redisSubscriber.on('message', this.onMessage); } diff --git a/packages/backend/src/core/UserCacheService.ts b/packages/backend/src/core/UserCacheService.ts index fc383d1c08..631eb44062 100644 --- a/packages/backend/src/core/UserCacheService.ts +++ b/packages/backend/src/core/UserCacheService.ts @@ -1,7 +1,7 @@ import { Inject, Injectable } from '@nestjs/common'; import Redis from 'ioredis'; import type { UsersRepository } from '@/models/index.js'; -import { Cache } from '@/misc/cache.js'; +import { KVCache } from '@/misc/cache.js'; import type { LocalUser, User } from '@/models/entities/User.js'; import { DI } from '@/di-symbols.js'; import { UserEntityService } from '@/core/entities/UserEntityService.js'; @@ -11,10 +11,10 @@ import type { OnApplicationShutdown } from '@nestjs/common'; @Injectable() export class UserCacheService implements OnApplicationShutdown { - public userByIdCache: Cache<User>; - public localUserByNativeTokenCache: Cache<LocalUser | null>; - public localUserByIdCache: Cache<LocalUser>; - public uriPersonCache: Cache<User | null>; + public userByIdCache: KVCache<User>; + public localUserByNativeTokenCache: KVCache<LocalUser | null>; + public localUserByIdCache: KVCache<LocalUser>; + public uriPersonCache: KVCache<User | null>; constructor( @Inject(DI.redisSubscriber) @@ -27,10 +27,10 @@ export class UserCacheService implements OnApplicationShutdown { ) { //this.onMessage = this.onMessage.bind(this); - this.userByIdCache = new Cache<User>(Infinity); - this.localUserByNativeTokenCache = new Cache<LocalUser | null>(Infinity); - this.localUserByIdCache = new Cache<LocalUser>(Infinity); - this.uriPersonCache = new Cache<User | null>(Infinity); + this.userByIdCache = new KVCache<User>(Infinity); + this.localUserByNativeTokenCache = new KVCache<LocalUser | null>(Infinity); + this.localUserByIdCache = new KVCache<LocalUser>(Infinity); + this.uriPersonCache = new KVCache<User | null>(Infinity); this.redisSubscriber.on('message', this.onMessage); } diff --git a/packages/backend/src/core/UserFollowingService.ts b/packages/backend/src/core/UserFollowingService.ts index 1c85504353..b51b553c70 100644 --- a/packages/backend/src/core/UserFollowingService.ts +++ b/packages/backend/src/core/UserFollowingService.ts @@ -17,6 +17,7 @@ import { UserEntityService } from '@/core/entities/UserEntityService.js'; import { ApRendererService } from '@/core/activitypub/ApRendererService.js'; import { bindThis } from '@/decorators.js'; import { UserBlockingService } from '@/core/UserBlockingService.js'; +import { MetaService } from '@/core/MetaService.js'; import Logger from '../logger.js'; const logger = new Logger('following/create'); @@ -57,6 +58,7 @@ export class UserFollowingService { private idService: IdService, private queueService: QueueService, private globalEventService: GlobalEventService, + private metaService: MetaService, private notificationService: NotificationService, private federatedInstanceService: FederatedInstanceService, private webhookService: WebhookService, @@ -200,14 +202,18 @@ export class UserFollowingService { //#region Update instance stats if (this.userEntityService.isRemoteUser(follower) && this.userEntityService.isLocalUser(followee)) { - this.federatedInstanceService.fetch(follower.host).then(i => { + this.federatedInstanceService.fetch(follower.host).then(async i => { this.instancesRepository.increment({ id: i.id }, 'followingCount', 1); - this.instanceChart.updateFollowing(i.host, true); + if ((await this.metaService.fetch()).enableChartsForFederatedInstances) { + this.instanceChart.updateFollowing(i.host, true); + } }); } else if (this.userEntityService.isLocalUser(follower) && this.userEntityService.isRemoteUser(followee)) { - this.federatedInstanceService.fetch(followee.host).then(i => { + this.federatedInstanceService.fetch(followee.host).then(async i => { this.instancesRepository.increment({ id: i.id }, 'followersCount', 1); - this.instanceChart.updateFollowers(i.host, true); + if ((await this.metaService.fetch()).enableChartsForFederatedInstances) { + this.instanceChart.updateFollowers(i.host, true); + } }); } //#endregion @@ -320,14 +326,18 @@ export class UserFollowingService { //#region Update instance stats if (this.userEntityService.isRemoteUser(follower) && this.userEntityService.isLocalUser(followee)) { - this.federatedInstanceService.fetch(follower.host).then(i => { + this.federatedInstanceService.fetch(follower.host).then(async i => { this.instancesRepository.decrement({ id: i.id }, 'followingCount', 1); - this.instanceChart.updateFollowing(i.host, false); + if ((await this.metaService.fetch()).enableChartsForFederatedInstances) { + this.instanceChart.updateFollowing(i.host, false); + } }); } else if (this.userEntityService.isLocalUser(follower) && this.userEntityService.isRemoteUser(followee)) { - this.federatedInstanceService.fetch(followee.host).then(i => { + this.federatedInstanceService.fetch(followee.host).then(async i => { this.instancesRepository.decrement({ id: i.id }, 'followersCount', 1); - this.instanceChart.updateFollowers(i.host, false); + if ((await this.metaService.fetch()).enableChartsForFederatedInstances) { + this.instanceChart.updateFollowers(i.host, false); + } }); } //#endregion diff --git a/packages/backend/src/core/UserKeypairStoreService.ts b/packages/backend/src/core/UserKeypairStoreService.ts index 1d3cc87c8d..61c9293f86 100644 --- a/packages/backend/src/core/UserKeypairStoreService.ts +++ b/packages/backend/src/core/UserKeypairStoreService.ts @@ -1,20 +1,20 @@ import { Inject, Injectable } from '@nestjs/common'; import type { User } from '@/models/entities/User.js'; import type { UserKeypairsRepository } from '@/models/index.js'; -import { Cache } from '@/misc/cache.js'; +import { KVCache } from '@/misc/cache.js'; import type { UserKeypair } from '@/models/entities/UserKeypair.js'; import { DI } from '@/di-symbols.js'; import { bindThis } from '@/decorators.js'; @Injectable() export class UserKeypairStoreService { - private cache: Cache<UserKeypair>; + private cache: KVCache<UserKeypair>; constructor( @Inject(DI.userKeypairsRepository) private userKeypairsRepository: UserKeypairsRepository, ) { - this.cache = new Cache<UserKeypair>(Infinity); + this.cache = new KVCache<UserKeypair>(Infinity); } @bindThis diff --git a/packages/backend/src/core/activitypub/ApDbResolverService.ts b/packages/backend/src/core/activitypub/ApDbResolverService.ts index d0a4ad7a75..c3b3875613 100644 --- a/packages/backend/src/core/activitypub/ApDbResolverService.ts +++ b/packages/backend/src/core/activitypub/ApDbResolverService.ts @@ -3,7 +3,7 @@ import escapeRegexp from 'escape-regexp'; import { DI } from '@/di-symbols.js'; import type { NotesRepository, UserPublickeysRepository, UsersRepository } from '@/models/index.js'; import type { Config } from '@/config.js'; -import { Cache } from '@/misc/cache.js'; +import { KVCache } from '@/misc/cache.js'; import type { UserPublickey } from '@/models/entities/UserPublickey.js'; import { UserCacheService } from '@/core/UserCacheService.js'; import type { Note } from '@/models/entities/Note.js'; @@ -31,8 +31,8 @@ export type UriParseResult = { @Injectable() export class ApDbResolverService { - private publicKeyCache: Cache<UserPublickey | null>; - private publicKeyByUserIdCache: Cache<UserPublickey | null>; + private publicKeyCache: KVCache<UserPublickey | null>; + private publicKeyByUserIdCache: KVCache<UserPublickey | null>; constructor( @Inject(DI.config) @@ -50,8 +50,8 @@ export class ApDbResolverService { private userCacheService: UserCacheService, private apPersonService: ApPersonService, ) { - this.publicKeyCache = new Cache<UserPublickey | null>(Infinity); - this.publicKeyByUserIdCache = new Cache<UserPublickey | null>(Infinity); + this.publicKeyCache = new KVCache<UserPublickey | null>(Infinity); + this.publicKeyByUserIdCache = new KVCache<UserPublickey | null>(Infinity); } @bindThis diff --git a/packages/backend/src/core/activitypub/models/ApPersonService.ts b/packages/backend/src/core/activitypub/models/ApPersonService.ts index d06958da0c..41f7eafa41 100644 --- a/packages/backend/src/core/activitypub/models/ApPersonService.ts +++ b/packages/backend/src/core/activitypub/models/ApPersonService.ts @@ -30,6 +30,7 @@ import { StatusError } from '@/misc/status-error.js'; import type { UtilityService } from '@/core/UtilityService.js'; import type { UserEntityService } from '@/core/entities/UserEntityService.js'; import { bindThis } from '@/decorators.js'; +import { MetaService } from '@/core/MetaService.js'; import { getApId, getApType, getOneApHrefNullable, isActor, isCollection, isCollectionOrOrderedCollection, isPropertyValue } from '../type.js'; import { extractApHashtags } from './tag.js'; import type { OnModuleInit } from '@nestjs/common'; @@ -50,6 +51,7 @@ export class ApPersonService implements OnModuleInit { private userEntityService: UserEntityService; private idService: IdService; private globalEventService: GlobalEventService; + private metaService: MetaService; private federatedInstanceService: FederatedInstanceService; private fetchInstanceMetadataService: FetchInstanceMetadataService; private userCacheService: UserCacheService; @@ -92,6 +94,7 @@ export class ApPersonService implements OnModuleInit { //private userEntityService: UserEntityService, //private idService: IdService, //private globalEventService: GlobalEventService, + //private metaService: MetaService, //private federatedInstanceService: FederatedInstanceService, //private fetchInstanceMetadataService: FetchInstanceMetadataService, //private userCacheService: UserCacheService, @@ -112,6 +115,7 @@ export class ApPersonService implements OnModuleInit { this.userEntityService = this.moduleRef.get('UserEntityService'); this.idService = this.moduleRef.get('IdService'); this.globalEventService = this.moduleRef.get('GlobalEventService'); + this.metaService = this.moduleRef.get('MetaService'); this.federatedInstanceService = this.moduleRef.get('FederatedInstanceService'); this.fetchInstanceMetadataService = this.moduleRef.get('FetchInstanceMetadataService'); this.userCacheService = this.moduleRef.get('UserCacheService'); @@ -327,10 +331,12 @@ export class ApPersonService implements OnModuleInit { } // Register host - this.federatedInstanceService.fetch(host).then(i => { + this.federatedInstanceService.fetch(host).then(async i => { this.instancesRepository.increment({ id: i.id }, 'usersCount', 1); - this.instanceChart.newUser(i.host); this.fetchInstanceMetadataService.fetchInstanceMetadata(i); + if ((await this.metaService.fetch()).enableChartsForFederatedInstances) { + this.instanceChart.newUser(i.host); + } }); this.usersChart.update(user!, true); diff --git a/packages/backend/src/core/entities/UserEntityService.ts b/packages/backend/src/core/entities/UserEntityService.ts index 068ffad09d..b693883e06 100644 --- a/packages/backend/src/core/entities/UserEntityService.ts +++ b/packages/backend/src/core/entities/UserEntityService.ts @@ -8,7 +8,7 @@ import type { Packed } from '@/misc/json-schema.js'; import type { Promiseable } from '@/misc/prelude/await-all.js'; import { awaitAll } from '@/misc/prelude/await-all.js'; import { USER_ACTIVE_THRESHOLD, USER_ONLINE_THRESHOLD } from '@/const.js'; -import { Cache } from '@/misc/cache.js'; +import { KVCache } from '@/misc/cache.js'; import type { Instance } from '@/models/entities/Instance.js'; import type { LocalUser, RemoteUser, User } from '@/models/entities/User.js'; import { birthdaySchema, descriptionSchema, localUsernameSchema, locationSchema, nameSchema, passwordSchema } from '@/models/entities/User.js'; @@ -52,7 +52,7 @@ export class UserEntityService implements OnModuleInit { private customEmojiService: CustomEmojiService; private antennaService: AntennaService; private roleService: RoleService; - private userInstanceCache: Cache<Instance | null>; + private userInstanceCache: KVCache<Instance | null>; constructor( private moduleRef: ModuleRef, @@ -121,7 +121,7 @@ export class UserEntityService implements OnModuleInit { //private antennaService: AntennaService, //private roleService: RoleService, ) { - this.userInstanceCache = new Cache<Instance | null>(1000 * 60 * 60 * 3); + this.userInstanceCache = new KVCache<Instance | null>(1000 * 60 * 60 * 3); } onModuleInit() { diff --git a/packages/backend/src/misc/cache.ts b/packages/backend/src/misc/cache.ts index 43a71a2b57..b249cf4480 100644 --- a/packages/backend/src/misc/cache.ts +++ b/packages/backend/src/misc/cache.ts @@ -2,11 +2,11 @@ import { bindThis } from '@/decorators.js'; // TODO: メモリ節約のためあまり参照されないキーを定期的に削除できるようにする? -export class Cache<T> { +export class KVCache<T> { public cache: Map<string | null, { date: number; value: T; }>; private lifetime: number; - constructor(lifetime: Cache<never>['lifetime']) { + constructor(lifetime: KVCache<never>['lifetime']) { this.cache = new Map(); this.lifetime = lifetime; } @@ -87,3 +87,88 @@ export class Cache<T> { return value; } } + +export class Cache<T> { + private cachedAt: number | null = null; + private value: T | undefined; + private lifetime: number; + + constructor(lifetime: Cache<never>['lifetime']) { + this.lifetime = lifetime; + } + + @bindThis + public set(value: T): void { + this.cachedAt = Date.now(); + this.value = value; + } + + @bindThis + public get(): T | undefined { + if (this.cachedAt == null) return undefined; + if ((Date.now() - this.cachedAt) > this.lifetime) { + this.value = undefined; + this.cachedAt = null; + return undefined; + } + return this.value; + } + + @bindThis + public delete() { + this.value = undefined; + this.cachedAt = null; + } + + /** + * キャッシュがあればそれを返し、無ければfetcherを呼び出して結果をキャッシュ&返します + * optional: キャッシュが存在してもvalidatorでfalseを返すとキャッシュ無効扱いにします + */ + @bindThis + public async fetch(fetcher: () => Promise<T>, validator?: (cachedValue: T) => boolean): Promise<T> { + const cachedValue = this.get(); + if (cachedValue !== undefined) { + if (validator) { + if (validator(cachedValue)) { + // Cache HIT + return cachedValue; + } + } else { + // Cache HIT + return cachedValue; + } + } + + // Cache MISS + const value = await fetcher(); + this.set(value); + return value; + } + + /** + * キャッシュがあればそれを返し、無ければfetcherを呼び出して結果をキャッシュ&返します + * optional: キャッシュが存在してもvalidatorでfalseを返すとキャッシュ無効扱いにします + */ + @bindThis + public async fetchMaybe(fetcher: () => Promise<T | undefined>, validator?: (cachedValue: T) => boolean): Promise<T | undefined> { + const cachedValue = this.get(); + if (cachedValue !== undefined) { + if (validator) { + if (validator(cachedValue)) { + // Cache HIT + return cachedValue; + } + } else { + // Cache HIT + return cachedValue; + } + } + + // Cache MISS + const value = await fetcher(); + if (value !== undefined) { + this.set(value); + } + return value; + } +} diff --git a/packages/backend/src/models/entities/Meta.ts b/packages/backend/src/models/entities/Meta.ts index 57338ecbd2..2e4f90b57f 100644 --- a/packages/backend/src/models/entities/Meta.ts +++ b/packages/backend/src/models/entities/Meta.ts @@ -42,11 +42,6 @@ export class Meta { }) public disableRegistration: boolean; - @Column('boolean', { - default: false, - }) - public useStarForReactionFallback: boolean; - @Column('varchar', { length: 1024, array: true, default: '{}', }) @@ -396,6 +391,16 @@ export class Meta { }) public enableActiveEmailValidation: boolean; + @Column('boolean', { + default: true, + }) + public enableChartsForRemoteUser: boolean; + + @Column('boolean', { + default: true, + }) + public enableChartsForFederatedInstances: boolean; + @Column('jsonb', { default: { }, }) diff --git a/packages/backend/src/models/entities/Role.ts b/packages/backend/src/models/entities/Role.ts index 85ff266740..eca9bcf270 100644 --- a/packages/backend/src/models/entities/Role.ts +++ b/packages/backend/src/models/entities/Role.ts @@ -54,6 +54,16 @@ type CondFormulaValueFollowingMoreThanOrEq = { value: number; }; +type CondFormulaValueNotesLessThanOrEq = { + type: 'notesLessThanOrEq'; + value: number; +}; + +type CondFormulaValueNotesMoreThanOrEq = { + type: 'notesMoreThanOrEq'; + value: number; +}; + export type RoleCondFormulaValue = CondFormulaValueAnd | CondFormulaValueOr | @@ -65,7 +75,9 @@ export type RoleCondFormulaValue = CondFormulaValueFollowersLessThanOrEq | CondFormulaValueFollowersMoreThanOrEq | CondFormulaValueFollowingLessThanOrEq | - CondFormulaValueFollowingMoreThanOrEq; + CondFormulaValueFollowingMoreThanOrEq | + CondFormulaValueNotesLessThanOrEq | + CondFormulaValueNotesMoreThanOrEq; @Entity() export class Role { diff --git a/packages/backend/src/queue/processors/DeliverProcessorService.ts b/packages/backend/src/queue/processors/DeliverProcessorService.ts index 43a92bb267..f637bf8818 100644 --- a/packages/backend/src/queue/processors/DeliverProcessorService.ts +++ b/packages/backend/src/queue/processors/DeliverProcessorService.ts @@ -7,7 +7,7 @@ import { MetaService } from '@/core/MetaService.js'; import { ApRequestService } from '@/core/activitypub/ApRequestService.js'; import { FederatedInstanceService } from '@/core/FederatedInstanceService.js'; import { FetchInstanceMetadataService } from '@/core/FetchInstanceMetadataService.js'; -import { Cache } from '@/misc/cache.js'; +import { KVCache } from '@/misc/cache.js'; import type { Instance } from '@/models/entities/Instance.js'; import InstanceChart from '@/core/chart/charts/instance.js'; import ApRequestChart from '@/core/chart/charts/ap-request.js'; @@ -22,7 +22,7 @@ import type { DeliverJobData } from '../types.js'; @Injectable() export class DeliverProcessorService { private logger: Logger; - private suspendedHostsCache: Cache<Instance[]>; + private suspendedHostsCache: KVCache<Instance[]>; private latest: string | null; constructor( @@ -46,7 +46,7 @@ export class DeliverProcessorService { private queueLoggerService: QueueLoggerService, ) { this.logger = this.queueLoggerService.logger.createSubLogger('deliver'); - this.suspendedHostsCache = new Cache<Instance[]>(1000 * 60 * 60); + this.suspendedHostsCache = new KVCache<Instance[]>(1000 * 60 * 60); } @bindThis @@ -88,10 +88,12 @@ export class DeliverProcessorService { } this.fetchInstanceMetadataService.fetchInstanceMetadata(i); - - this.instanceChart.requestSent(i.host, true); this.apRequestChart.deliverSucc(); this.federationChart.deliverd(i.host, true); + + if (meta.enableChartsForFederatedInstances) { + this.instanceChart.requestSent(i.host, true); + } }); return 'Success'; @@ -107,9 +109,12 @@ export class DeliverProcessorService { }); } - this.instanceChart.requestSent(i.host, false); this.apRequestChart.deliverFail(); this.federationChart.deliverd(i.host, false); + + if (meta.enableChartsForFederatedInstances) { + this.instanceChart.requestSent(i.host, false); + } }); if (res instanceof StatusError) { diff --git a/packages/backend/src/queue/processors/InboxProcessorService.ts b/packages/backend/src/queue/processors/InboxProcessorService.ts index 41fe06b7c3..ed7f38d013 100644 --- a/packages/backend/src/queue/processors/InboxProcessorService.ts +++ b/packages/backend/src/queue/processors/InboxProcessorService.ts @@ -184,9 +184,12 @@ export class InboxProcessorService { this.fetchInstanceMetadataService.fetchInstanceMetadata(i); - this.instanceChart.requestReceived(i.host); this.apRequestChart.inbox(); this.federationChart.inbox(i.host); + + if (meta.enableChartsForFederatedInstances) { + this.instanceChart.requestReceived(i.host); + } }); // アクティビティを処理 diff --git a/packages/backend/src/server/NodeinfoServerService.ts b/packages/backend/src/server/NodeinfoServerService.ts index 364b46696d..86019d4166 100644 --- a/packages/backend/src/server/NodeinfoServerService.ts +++ b/packages/backend/src/server/NodeinfoServerService.ts @@ -4,7 +4,7 @@ import type { NotesRepository, UsersRepository } from '@/models/index.js'; import type { Config } from '@/config.js'; import { MetaService } from '@/core/MetaService.js'; import { MAX_NOTE_TEXT_LENGTH } from '@/const.js'; -import { Cache } from '@/misc/cache.js'; +import { KVCache } from '@/misc/cache.js'; import { UserEntityService } from '@/core/entities/UserEntityService.js'; import { bindThis } from '@/decorators.js'; import NotesChart from '@/core/chart/charts/notes.js'; @@ -118,7 +118,7 @@ export class NodeinfoServerService { }; }; - const cache = new Cache<Awaited<ReturnType<typeof nodeinfo2>>>(1000 * 60 * 10); + const cache = new KVCache<Awaited<ReturnType<typeof nodeinfo2>>>(1000 * 60 * 10); fastify.get(nodeinfo2_1path, async (request, reply) => { const base = await cache.fetch(null, () => nodeinfo2()); diff --git a/packages/backend/src/server/api/AuthenticateService.ts b/packages/backend/src/server/api/AuthenticateService.ts index 87438c348d..a1895e3705 100644 --- a/packages/backend/src/server/api/AuthenticateService.ts +++ b/packages/backend/src/server/api/AuthenticateService.ts @@ -3,7 +3,7 @@ import { DI } from '@/di-symbols.js'; import type { AccessTokensRepository, AppsRepository, UsersRepository } from '@/models/index.js'; import type { LocalUser } from '@/models/entities/User.js'; import type { AccessToken } from '@/models/entities/AccessToken.js'; -import { Cache } from '@/misc/cache.js'; +import { KVCache } from '@/misc/cache.js'; import type { App } from '@/models/entities/App.js'; import { UserCacheService } from '@/core/UserCacheService.js'; import isNativeToken from '@/misc/is-native-token.js'; @@ -18,7 +18,7 @@ export class AuthenticationError extends Error { @Injectable() export class AuthenticateService { - private appCache: Cache<App>; + private appCache: KVCache<App>; constructor( @Inject(DI.usersRepository) @@ -32,7 +32,7 @@ export class AuthenticateService { private userCacheService: UserCacheService, ) { - this.appCache = new Cache<App>(Infinity); + this.appCache = new KVCache<App>(Infinity); } @bindThis diff --git a/packages/backend/src/server/api/endpoints/admin/meta.ts b/packages/backend/src/server/api/endpoints/admin/meta.ts index ce7e0d569d..fc318a621a 100644 --- a/packages/backend/src/server/api/endpoints/admin/meta.ts +++ b/packages/backend/src/server/api/endpoints/admin/meta.ts @@ -239,6 +239,14 @@ export const meta = { type: 'boolean', optional: true, nullable: false, }, + enableChartsForRemoteUser: { + type: 'boolean', + optional: false, nullable: false, + }, + enableChartsForFederatedInstances: { + type: 'boolean', + optional: false, nullable: false, + }, policies: { type: 'object', optional: false, nullable: false, @@ -299,7 +307,6 @@ export default class extends Endpoint<typeof meta, typeof paramDef> { enableServiceWorker: instance.enableServiceWorker, translatorAvailable: instance.deeplAuthKey != null, cacheRemoteFiles: instance.cacheRemoteFiles, - useStarForReactionFallback: instance.useStarForReactionFallback, pinnedUsers: instance.pinnedUsers, hiddenTags: instance.hiddenTags, blockedHosts: instance.blockedHosts, @@ -337,6 +344,8 @@ export default class extends Endpoint<typeof meta, typeof paramDef> { deeplIsPro: instance.deeplIsPro, enableIpLogging: instance.enableIpLogging, enableActiveEmailValidation: instance.enableActiveEmailValidation, + enableChartsForRemoteUser: instance.enableChartsForRemoteUser, + enableChartsForFederatedInstances: instance.enableChartsForFederatedInstances, policies: { ...DEFAULT_POLICIES, ...instance.policies }, }; }); diff --git a/packages/backend/src/server/api/endpoints/admin/update-meta.ts b/packages/backend/src/server/api/endpoints/admin/update-meta.ts index 2f23aca243..11de29bf83 100644 --- a/packages/backend/src/server/api/endpoints/admin/update-meta.ts +++ b/packages/backend/src/server/api/endpoints/admin/update-meta.ts @@ -17,7 +17,6 @@ export const paramDef = { type: 'object', properties: { disableRegistration: { type: 'boolean', nullable: true }, - useStarForReactionFallback: { type: 'boolean', nullable: true }, pinnedUsers: { type: 'array', nullable: true, items: { type: 'string', } }, @@ -93,6 +92,8 @@ export const paramDef = { objectStorageS3ForcePathStyle: { type: 'boolean' }, enableIpLogging: { type: 'boolean' }, enableActiveEmailValidation: { type: 'boolean' }, + enableChartsForRemoteUser: { type: 'boolean' }, + enableChartsForFederatedInstances: { type: 'boolean' }, }, required: [], } as const; @@ -114,10 +115,6 @@ export default class extends Endpoint<typeof meta, typeof paramDef> { set.disableRegistration = ps.disableRegistration; } - if (typeof ps.useStarForReactionFallback === 'boolean') { - set.useStarForReactionFallback = ps.useStarForReactionFallback; - } - if (Array.isArray(ps.pinnedUsers)) { set.pinnedUsers = ps.pinnedUsers.filter(Boolean); } @@ -382,6 +379,14 @@ export default class extends Endpoint<typeof meta, typeof paramDef> { set.enableActiveEmailValidation = ps.enableActiveEmailValidation; } + if (ps.enableChartsForRemoteUser !== undefined) { + set.enableChartsForRemoteUser = ps.enableChartsForRemoteUser; + } + + if (ps.enableChartsForFederatedInstances !== undefined) { + set.enableChartsForFederatedInstances = ps.enableChartsForFederatedInstances; + } + await this.metaService.update(set); this.moderationLogService.insertModerationLog(me, 'updateMeta'); }); diff --git a/packages/backend/src/server/api/endpoints/antennas/create.ts b/packages/backend/src/server/api/endpoints/antennas/create.ts index d147ddb7f1..b7ce3363a9 100644 --- a/packages/backend/src/server/api/endpoints/antennas/create.ts +++ b/packages/backend/src/server/api/endpoints/antennas/create.ts @@ -79,7 +79,7 @@ export default class extends Endpoint<typeof meta, typeof paramDef> { private globalEventService: GlobalEventService, ) { super(meta, paramDef, async (ps, me) => { - if (ps.keywords.length === 0) { + if ((ps.keywords.length === 0) || ps.keywords[0].every(x => x === '')) { throw new Error('invalid param'); } diff --git a/packages/backend/test/unit/DriveService.ts b/packages/backend/test/unit/DriveService.ts index 0549800a68..4065665579 100644 --- a/packages/backend/test/unit/DriveService.ts +++ b/packages/backend/test/unit/DriveService.ts @@ -1,55 +1,56 @@ process.env.NODE_ENV = 'test'; -import { jest } from '@jest/globals'; import { Test } from '@nestjs/testing'; +import { DeleteObjectCommandOutput, DeleteObjectCommand, NoSuchKey, InvalidObjectState, S3Client } from '@aws-sdk/client-s3'; +import { mockClient } from 'aws-sdk-client-mock'; import { GlobalModule } from '@/GlobalModule.js'; import { DriveService } from '@/core/DriveService.js'; import { CoreModule } from '@/core/CoreModule.js'; -import { S3Service } from '@/core/S3Service'; -import type { Meta } from '@/models'; -import type { DeleteObjectOutput } from 'aws-sdk/clients/s3'; -import type { AWSError } from 'aws-sdk/lib/error'; -import type { PromiseResult, Request } from 'aws-sdk/lib/request'; import type { TestingModule } from '@nestjs/testing'; describe('DriveService', () => { let app: TestingModule; let driveService: DriveService; + const s3Mock = mockClient(S3Client); - beforeEach(async () => { + beforeAll(async () => { app = await Test.createTestingModule({ imports: [GlobalModule, CoreModule], - providers: [DriveService, S3Service], + providers: [DriveService], }).compile(); app.enableShutdownHooks(); driveService = app.get<DriveService>(DriveService); + }); - const s3Service = app.get<S3Service>(S3Service); - const s3 = s3Service.getS3({} as Meta); + beforeEach(async () => { + s3Mock.reset(); + }); - // new S3() surprisingly does not return an instance of class S3. - // Let's use getPrototypeOf here to get a real prototype, since spying on S3.prototype doesn't work. - // TODO: Use `aws-sdk-client-mock` package when upgrading to AWS SDK v3. - jest.spyOn(Object.getPrototypeOf(s3), 'deleteObject').mockImplementation(() => { - // Roughly mock AWS request object - return { - async promise(): Promise<PromiseResult<DeleteObjectOutput, AWSError>> { - const err = new Error('mock') as AWSError; - err.code = 'NoSuchKey'; - throw err; - }, - } as Request<DeleteObjectOutput, AWSError>; - }); + afterAll(async () => { + await app.close(); }); describe('Object storage', () => { + test('delete a file', async () => { + s3Mock.on(DeleteObjectCommand) + .resolves({} as DeleteObjectCommandOutput); + + await driveService.deleteObjectStorageFile('peace of the world'); + }); + + test('delete a file then unexpected error', async () => { + s3Mock.on(DeleteObjectCommand) + .rejects(new InvalidObjectState({ $metadata: {}, message: '' })); + + await expect(driveService.deleteObjectStorageFile('unexpected')).rejects.toThrowError(Error); + }); + test('delete a file with no valid key', async () => { - try { - await driveService.deleteObjectStorageFile('lol no way'); - } catch (err: any) { - console.log(err.cause); - throw err; - } + // Some S3 implementations returns 404 Not Found on deleting with a non-existent key + s3Mock.on(DeleteObjectCommand) + .rejects(new NoSuchKey({ $metadata: {}, message: 'allowed error.' })); + + await driveService.deleteObjectStorageFile('lol no way'); }); }); }); diff --git a/packages/backend/test/unit/ReactionService.ts b/packages/backend/test/unit/ReactionService.ts index 6a20a1e08e..38db081ac0 100644 --- a/packages/backend/test/unit/ReactionService.ts +++ b/packages/backend/test/unit/ReactionService.ts @@ -74,19 +74,19 @@ describe('ReactionService', () => { }); test('fallback - undefined', async () => { - assert.strictEqual(await reactionService.toDbReaction(undefined), '👍'); + assert.strictEqual(await reactionService.toDbReaction(undefined), '❤'); }); test('fallback - null', async () => { - assert.strictEqual(await reactionService.toDbReaction(null), '👍'); + assert.strictEqual(await reactionService.toDbReaction(null), '❤'); }); test('fallback - empty', async () => { - assert.strictEqual(await reactionService.toDbReaction(''), '👍'); + assert.strictEqual(await reactionService.toDbReaction(''), '❤'); }); test('fallback - unknown', async () => { - assert.strictEqual(await reactionService.toDbReaction('unknown'), '👍'); + assert.strictEqual(await reactionService.toDbReaction('unknown'), '❤'); }); }); }); diff --git a/packages/backend/test/unit/S3Service.ts b/packages/backend/test/unit/S3Service.ts new file mode 100644 index 0000000000..1dfa22afd2 --- /dev/null +++ b/packages/backend/test/unit/S3Service.ts @@ -0,0 +1,77 @@ +process.env.NODE_ENV = 'test'; + +import { Test } from '@nestjs/testing'; +import { UploadPartCommand, CompleteMultipartUploadCommand, CreateMultipartUploadCommand, S3Client, PutObjectCommand } from '@aws-sdk/client-s3'; +import { mockClient } from 'aws-sdk-client-mock'; +import { GlobalModule } from '@/GlobalModule.js'; +import { CoreModule } from '@/core/CoreModule.js'; +import { S3Service } from '@/core/S3Service'; +import { Meta } from '@/models'; +import type { TestingModule } from '@nestjs/testing'; + +describe('S3Service', () => { + let app: TestingModule; + let s3Service: S3Service; + const s3Mock = mockClient(S3Client); + + beforeAll(async () => { + app = await Test.createTestingModule({ + imports: [GlobalModule, CoreModule], + providers: [S3Service], + }).compile(); + app.enableShutdownHooks(); + s3Service = app.get<S3Service>(S3Service); + }); + + beforeEach(async () => { + s3Mock.reset(); + }); + + afterAll(async () => { + await app.close(); + }); + + describe('upload', () => { + test('upload a file', async () => { + s3Mock.on(PutObjectCommand).resolves({}); + + await s3Service.upload({ objectStorageRegion: 'us-east-1' } as Meta, { + Bucket: 'fake', + Key: 'fake', + Body: 'x', + }); + }); + + test('upload a large file', async () => { + s3Mock.on(CreateMultipartUploadCommand).resolves({ UploadId: '1' }); + s3Mock.on(UploadPartCommand).resolves({ ETag: '1' }); + s3Mock.on(CompleteMultipartUploadCommand).resolves({ Bucket: 'fake', Key: 'fake' }); + + await s3Service.upload({} as Meta, { + Bucket: 'fake', + Key: 'fake', + Body: 'x'.repeat(8 * 1024 * 1024 + 1), // デフォルトpartSizeにしている 8 * 1024 * 1024 を越えるサイズ + }); + }); + + test('upload a file error', async () => { + s3Mock.on(PutObjectCommand).rejects({ name: 'Fake Error' }); + + await expect(s3Service.upload({ objectStorageRegion: 'us-east-1' } as Meta, { + Bucket: 'fake', + Key: 'fake', + Body: 'x', + })).rejects.toThrowError(Error); + }); + + test('upload a large file error', async () => { + s3Mock.on(UploadPartCommand).rejects(); + + await expect(s3Service.upload({} as Meta, { + Bucket: 'fake', + Key: 'fake', + Body: 'x'.repeat(8 * 1024 * 1024 + 1), // デフォルトpartSizeにしている 8 * 1024 * 1024 を越えるサイズ + })).rejects.toThrowError(Error); + }); + }); +}); |