From 32872181ddf14cfbf006dc93d04eeacac5eaf7a0 Mon Sep 17 00:00:00 2001 From: Hazel K Date: Thu, 25 Jul 2024 10:37:23 -0400 Subject: feat: implement `attachLdSignatureForRelays` to control signing of Relayed activities --- .../src/core/activitypub/ApRendererService.ts | 97 +++++++++++++++------- 1 file changed, 68 insertions(+), 29 deletions(-) (limited to 'packages/backend/src/core') diff --git a/packages/backend/src/core/activitypub/ApRendererService.ts b/packages/backend/src/core/activitypub/ApRendererService.ts index 90784fdc1d..28c5dcf150 100644 --- a/packages/backend/src/core/activitypub/ApRendererService.ts +++ b/packages/backend/src/core/activitypub/ApRendererService.ts @@ -3,36 +3,69 @@ * SPDX-License-Identifier: AGPL-3.0-only */ -import { createPublicKey, randomUUID } from 'node:crypto'; -import { Inject, Injectable } from '@nestjs/common'; -import { In } from 'typeorm'; +import {createPublicKey, randomUUID} from 'node:crypto'; +import {Inject, Injectable} from '@nestjs/common'; +import {In} from 'typeorm'; import * as mfm from '@transfem-org/sfm-js'; -import { DI } from '@/di-symbols.js'; -import type { Config } from '@/config.js'; -import type { MiPartialLocalUser, MiLocalUser, MiPartialRemoteUser, MiRemoteUser, MiUser } from '@/models/User.js'; -import type { IMentionedRemoteUsers, MiNote } from '@/models/Note.js'; -import type { MiBlocking } from '@/models/Blocking.js'; -import type { MiRelay } from '@/models/Relay.js'; -import type { MiDriveFile } from '@/models/DriveFile.js'; -import type { MiNoteReaction } from '@/models/NoteReaction.js'; -import type { MiEmoji } from '@/models/Emoji.js'; -import type { MiPoll } from '@/models/Poll.js'; -import type { MiPollVote } from '@/models/PollVote.js'; -import { UserKeypairService } from '@/core/UserKeypairService.js'; -import { MfmService } from '@/core/MfmService.js'; -import { UserEntityService } from '@/core/entities/UserEntityService.js'; -import { DriveFileEntityService } from '@/core/entities/DriveFileEntityService.js'; -import type { MiUserKeypair } from '@/models/UserKeypair.js'; -import type { UsersRepository, UserProfilesRepository, NotesRepository, DriveFilesRepository, PollsRepository, InstancesRepository } from '@/models/_.js'; -import { bindThis } from '@/decorators.js'; -import { CustomEmojiService } from '@/core/CustomEmojiService.js'; -import { isNotNull } from '@/misc/is-not-null.js'; -import { IdService } from '@/core/IdService.js'; -import { MetaService } from '../MetaService.js'; -import { JsonLdService } from './JsonLdService.js'; -import { ApMfmService } from './ApMfmService.js'; -import { CONTEXT } from './misc/contexts.js'; -import type { IAccept, IActivity, IAdd, IAnnounce, IApDocument, IApEmoji, IApHashtag, IApImage, IApMention, IBlock, ICreate, IDelete, IFlag, IFollow, IKey, ILike, IMove, IObject, IPost, IQuestion, IReject, IRemove, ITombstone, IUndo, IUpdate } from './type.js'; +import {DI} from '@/di-symbols.js'; +import type {Config} from '@/config.js'; +import type {MiLocalUser, MiPartialLocalUser, MiPartialRemoteUser, MiRemoteUser, MiUser} from '@/models/User.js'; +import type {IMentionedRemoteUsers, MiNote} from '@/models/Note.js'; +import type {MiBlocking} from '@/models/Blocking.js'; +import type {MiRelay} from '@/models/Relay.js'; +import type {MiDriveFile} from '@/models/DriveFile.js'; +import type {MiNoteReaction} from '@/models/NoteReaction.js'; +import type {MiEmoji} from '@/models/Emoji.js'; +import type {MiPoll} from '@/models/Poll.js'; +import type {MiPollVote} from '@/models/PollVote.js'; +import {UserKeypairService} from '@/core/UserKeypairService.js'; +import {MfmService} from '@/core/MfmService.js'; +import {UserEntityService} from '@/core/entities/UserEntityService.js'; +import {DriveFileEntityService} from '@/core/entities/DriveFileEntityService.js'; +import type {MiUserKeypair} from '@/models/UserKeypair.js'; +import type { + DriveFilesRepository, + InstancesRepository, + NotesRepository, + PollsRepository, + UserProfilesRepository, + UsersRepository +} from '@/models/_.js'; +import {bindThis} from '@/decorators.js'; +import {CustomEmojiService} from '@/core/CustomEmojiService.js'; +import {isNotNull} from '@/misc/is-not-null.js'; +import {IdService} from '@/core/IdService.js'; +import {MetaService} from '../MetaService.js'; +import {JsonLdService} from './JsonLdService.js'; +import {ApMfmService} from './ApMfmService.js'; +import {CONTEXT} from './misc/contexts.js'; +import type { + IAccept, + IActivity, + IAdd, + IAnnounce, + IApDocument, + IApEmoji, + IApHashtag, + IApImage, + IApMention, + IBlock, + ICreate, + IDelete, + IFlag, + IFollow, + IKey, + ILike, + IMove, + IObject, + IPost, + IQuestion, + IReject, + IRemove, + ITombstone, + IUndo, + IUpdate +} from './type.js'; @Injectable() export class ApRendererService { @@ -793,6 +826,12 @@ export class ApRendererService { @bindThis public async attachLdSignature(activity: any, user: { id: MiUser['id']; host: null; }): Promise { + // When using authorized fetch, Linked Data signatures are often undesired (as it can allow blocked instances to bypass the check). + // We allow admins to disable LD signatures for increased privacy, at the expense of increased incoming fetch (GET) requests. + if (!this.config.attachLdSignatureForRelays) { + return activity; + } + const keypair = await this.userKeypairService.getUserKeypair(user.id); const jsonLd = this.jsonLdService.use(); -- cgit v1.2.3-freya From fecdff7fa092ca9d8ca0cd50b5694eb3f8279a91 Mon Sep 17 00:00:00 2001 From: Hazel K Date: Fri, 26 Jul 2024 09:42:49 -0400 Subject: revert import changes --- .../src/core/activitypub/ApRendererService.ts | 91 +++++++--------------- 1 file changed, 29 insertions(+), 62 deletions(-) (limited to 'packages/backend/src/core') diff --git a/packages/backend/src/core/activitypub/ApRendererService.ts b/packages/backend/src/core/activitypub/ApRendererService.ts index 28c5dcf150..8db9199e5d 100644 --- a/packages/backend/src/core/activitypub/ApRendererService.ts +++ b/packages/backend/src/core/activitypub/ApRendererService.ts @@ -3,69 +3,36 @@ * SPDX-License-Identifier: AGPL-3.0-only */ -import {createPublicKey, randomUUID} from 'node:crypto'; -import {Inject, Injectable} from '@nestjs/common'; -import {In} from 'typeorm'; +import { createPublicKey, randomUUID } from 'node:crypto'; +import { Inject, Injectable } from '@nestjs/common'; +import { In } from 'typeorm'; import * as mfm from '@transfem-org/sfm-js'; -import {DI} from '@/di-symbols.js'; -import type {Config} from '@/config.js'; -import type {MiLocalUser, MiPartialLocalUser, MiPartialRemoteUser, MiRemoteUser, MiUser} from '@/models/User.js'; -import type {IMentionedRemoteUsers, MiNote} from '@/models/Note.js'; -import type {MiBlocking} from '@/models/Blocking.js'; -import type {MiRelay} from '@/models/Relay.js'; -import type {MiDriveFile} from '@/models/DriveFile.js'; -import type {MiNoteReaction} from '@/models/NoteReaction.js'; -import type {MiEmoji} from '@/models/Emoji.js'; -import type {MiPoll} from '@/models/Poll.js'; -import type {MiPollVote} from '@/models/PollVote.js'; -import {UserKeypairService} from '@/core/UserKeypairService.js'; -import {MfmService} from '@/core/MfmService.js'; -import {UserEntityService} from '@/core/entities/UserEntityService.js'; -import {DriveFileEntityService} from '@/core/entities/DriveFileEntityService.js'; -import type {MiUserKeypair} from '@/models/UserKeypair.js'; -import type { - DriveFilesRepository, - InstancesRepository, - NotesRepository, - PollsRepository, - UserProfilesRepository, - UsersRepository -} from '@/models/_.js'; -import {bindThis} from '@/decorators.js'; -import {CustomEmojiService} from '@/core/CustomEmojiService.js'; -import {isNotNull} from '@/misc/is-not-null.js'; -import {IdService} from '@/core/IdService.js'; -import {MetaService} from '../MetaService.js'; -import {JsonLdService} from './JsonLdService.js'; -import {ApMfmService} from './ApMfmService.js'; -import {CONTEXT} from './misc/contexts.js'; -import type { - IAccept, - IActivity, - IAdd, - IAnnounce, - IApDocument, - IApEmoji, - IApHashtag, - IApImage, - IApMention, - IBlock, - ICreate, - IDelete, - IFlag, - IFollow, - IKey, - ILike, - IMove, - IObject, - IPost, - IQuestion, - IReject, - IRemove, - ITombstone, - IUndo, - IUpdate -} from './type.js'; +import { DI } from '@/di-symbols.js'; +import type { Config } from '@/config.js'; +import type { MiPartialLocalUser, MiLocalUser, MiPartialRemoteUser, MiRemoteUser, MiUser } from '@/models/User.js'; +import type { IMentionedRemoteUsers, MiNote } from '@/models/Note.js'; +import type { MiBlocking } from '@/models/Blocking.js'; +import type { MiRelay } from '@/models/Relay.js'; +import type { MiDriveFile } from '@/models/DriveFile.js'; +import type { MiNoteReaction } from '@/models/NoteReaction.js'; +import type { MiEmoji } from '@/models/Emoji.js'; +import type { MiPoll } from '@/models/Poll.js'; +import type { MiPollVote } from '@/models/PollVote.js'; +import { UserKeypairService } from '@/core/UserKeypairService.js'; +import { MfmService } from '@/core/MfmService.js'; +import { UserEntityService } from '@/core/entities/UserEntityService.js'; +import { DriveFileEntityService } from '@/core/entities/DriveFileEntityService.js'; +import type { MiUserKeypair } from '@/models/UserKeypair.js'; +import type { UsersRepository, UserProfilesRepository, NotesRepository, DriveFilesRepository, PollsRepository, InstancesRepository } from '@/models/_.js'; +import { bindThis } from '@/decorators.js'; +import { CustomEmojiService } from '@/core/CustomEmojiService.js'; +import { isNotNull } from '@/misc/is-not-null.js'; +import { IdService } from '@/core/IdService.js'; +import { MetaService } from '../MetaService.js'; +import { JsonLdService } from './JsonLdService.js'; +import { ApMfmService } from './ApMfmService.js'; +import { CONTEXT } from './misc/contexts.js'; +import type { IAccept, IActivity, IAdd, IAnnounce, IApDocument, IApEmoji, IApHashtag, IApImage, IApMention, IBlock, ICreate, IDelete, IFlag, IFollow, IKey, ILike, IMove, IObject, IPost, IQuestion, IReject, IRemove, ITombstone, IUndo, IUpdate } from './type.js'; @Injectable() export class ApRendererService { -- cgit v1.2.3-freya From 378408226b5e8968313058de4862b9916d08d6e0 Mon Sep 17 00:00:00 2001 From: Hazel K Date: Fri, 26 Jul 2024 22:45:07 -0400 Subject: tweak wording --- .config/ci.yml | 3 +++ .config/docker_example.yml | 3 +++ .config/example.yml | 3 +++ chart/files/default.yml | 3 +++ packages/backend/src/core/activitypub/ApRendererService.ts | 5 +++-- 5 files changed, 15 insertions(+), 2 deletions(-) (limited to 'packages/backend/src/core') diff --git a/.config/ci.yml b/.config/ci.yml index 02081e5971..44092d3662 100644 --- a/.config/ci.yml +++ b/.config/ci.yml @@ -201,6 +201,9 @@ proxyRemoteFiles: true # Sign outgoing ActivityPub GET request (default: true) signToActivityPubGet: true # Sign outgoing ActivityPub Activities (default: true) +# Linked Data signatures are cryptographic signatures attached to each activity to provide proof of authenticity. +# When using authorized fetch, this is often undesired as any signed activity can be forwarded to a blocked instance by relays and other instances. +# This setting allows admins to disable LD signatures for increased privacy, at the expense of fewer relayed activities and additional inbound fetch (GET) requests. attachLdSignatureForRelays: true # check that inbound ActivityPub GET requests are signed ("authorized fetch") checkActivityPubGetSignature: false diff --git a/.config/docker_example.yml b/.config/docker_example.yml index 375753e79f..f4645d672d 100644 --- a/.config/docker_example.yml +++ b/.config/docker_example.yml @@ -273,6 +273,9 @@ proxyRemoteFiles: true # Sign outgoing ActivityPub GET request (default: true) signToActivityPubGet: true # Sign outgoing ActivityPub Activities (default: true) +# Linked Data signatures are cryptographic signatures attached to each activity to provide proof of authenticity. +# When using authorized fetch, this is often undesired as any signed activity can be forwarded to a blocked instance by relays and other instances. +# This setting allows admins to disable LD signatures for increased privacy, at the expense of fewer relayed activities and additional inbound fetch (GET) requests. attachLdSignatureForRelays: true # check that inbound ActivityPub GET requests are signed ("authorized fetch") checkActivityPubGetSignature: false diff --git a/.config/example.yml b/.config/example.yml index 4b6aaae63b..21e85b7b89 100644 --- a/.config/example.yml +++ b/.config/example.yml @@ -288,6 +288,9 @@ proxyRemoteFiles: true # Sign outgoing ActivityPub GET request (default: true) signToActivityPubGet: true # Sign outgoing ActivityPub Activities (default: true) +# Linked Data signatures are cryptographic signatures attached to each activity to provide proof of authenticity. +# When using authorized fetch, this is often undesired as any signed activity can be forwarded to a blocked instance by relays and other instances. +# This setting allows admins to disable LD signatures for increased privacy, at the expense of fewer relayed activities and additional inbound fetch (GET) requests. attachLdSignatureForRelays: true # check that inbound ActivityPub GET requests are signed ("authorized fetch") checkActivityPubGetSignature: false diff --git a/chart/files/default.yml b/chart/files/default.yml index 7c94bcbea3..aab7ed6ce1 100644 --- a/chart/files/default.yml +++ b/chart/files/default.yml @@ -211,6 +211,9 @@ id: "aidx" # Sign outgoing ActivityPub GET request (default: true) signToActivityPubGet: true # Sign outgoing ActivityPub Activities (default: true) +# Linked Data signatures are cryptographic signatures attached to each activity to provide proof of authenticity. +# When using authorized fetch, this is often undesired as any signed activity can be forwarded to a blocked instance by relays and other instances. +# This setting allows admins to disable LD signatures for increased privacy, at the expense of fewer relayed activities and additional inbound fetch (GET) requests. attachLdSignatureForRelays: true # check that inbound ActivityPub GET requests are signed ("authorized fetch") checkActivityPubGetSignature: false diff --git a/packages/backend/src/core/activitypub/ApRendererService.ts b/packages/backend/src/core/activitypub/ApRendererService.ts index 8db9199e5d..98fc647a83 100644 --- a/packages/backend/src/core/activitypub/ApRendererService.ts +++ b/packages/backend/src/core/activitypub/ApRendererService.ts @@ -793,8 +793,9 @@ export class ApRendererService { @bindThis public async attachLdSignature(activity: any, user: { id: MiUser['id']; host: null; }): Promise { - // When using authorized fetch, Linked Data signatures are often undesired (as it can allow blocked instances to bypass the check). - // We allow admins to disable LD signatures for increased privacy, at the expense of increased incoming fetch (GET) requests. + // Linked Data signatures are cryptographic signatures attached to each activity to provide proof of authenticity. + // When using authorized fetch, this is often undesired as any signed activity can be forwarded to a blocked instance by relays and other instances. + // This setting allows admins to disable LD signatures for increased privacy, at the expense of fewer relayed activities and additional inbound fetch (GET) requests. if (!this.config.attachLdSignatureForRelays) { return activity; } -- cgit v1.2.3-freya From 114b6980346470fcf8cf1a11e50038c8fb15c48e Mon Sep 17 00:00:00 2001 From: Hazel K Date: Sat, 3 Aug 2024 09:18:44 -0400 Subject: encapsulate `MemoryKVCache` --- packages/backend/src/core/CacheService.ts | 4 ++-- packages/backend/src/misc/cache.ts | 24 +++++++++--------------- 2 files changed, 11 insertions(+), 17 deletions(-) (limited to 'packages/backend/src/core') diff --git a/packages/backend/src/core/CacheService.ts b/packages/backend/src/core/CacheService.ts index d008e7ec52..4afcef02be 100644 --- a/packages/backend/src/core/CacheService.ts +++ b/packages/backend/src/core/CacheService.ts @@ -135,14 +135,14 @@ export class CacheService implements OnApplicationShutdown { if (user == null) { this.userByIdCache.delete(body.id); this.localUserByIdCache.delete(body.id); - for (const [k, v] of this.uriPersonCache.cache.entries()) { + for (const [k, v] of this.uriPersonCache.entries) { if (v.value?.id === body.id) { this.uriPersonCache.delete(k); } } } else { this.userByIdCache.set(user.id, user); - for (const [k, v] of this.uriPersonCache.cache.entries()) { + for (const [k, v] of this.uriPersonCache.entries) { if (v.value?.id === user.id) { this.uriPersonCache.set(k, user); } diff --git a/packages/backend/src/misc/cache.ts b/packages/backend/src/misc/cache.ts index bba64a06ef..fe27d44692 100644 --- a/packages/backend/src/misc/cache.ts +++ b/packages/backend/src/misc/cache.ts @@ -187,22 +187,12 @@ export class RedisSingleCache { // TODO: メモリ節約のためあまり参照されないキーを定期的に削除できるようにする？ export class MemoryKVCache { - /** - * データを持つマップ - * @deprecated これを直接操作するべきではない - */ - public cache: Map; - private lifetime: number; - private gcIntervalHandle: NodeJS.Timeout; - - constructor(lifetime: MemoryKVCache['lifetime']) { - this.cache = new Map(); - this.lifetime = lifetime; + private readonly cache = new Map(); + private readonly gcIntervalHandle = setInterval(() => this.gc(), 1000 * 60 * 3); - this.gcIntervalHandle = setInterval(() => { - this.gc(); - }, 1000 * 60 * 3); - } + constructor( + private readonly lifetime: number, + ) {} @bindThis /** @@ -298,6 +288,10 @@ export class MemoryKVCache { public dispose(): void { clearInterval(this.gcIntervalHandle); } + + public get entries() { + return this.cache.entries(); + } } export class MemorySingleCache { -- cgit v1.2.3-freya From bc236a4bd250fc700bdd2d5549bf9647c02cb946 Mon Sep 17 00:00:00 2001 From: Hazel K Date: Sat, 3 Aug 2024 13:42:23 -0400 Subject: remove infinity caches --- packages/backend/src/core/CacheService.ts | 8 ++++---- packages/backend/src/core/UserKeypairService.ts | 2 +- packages/backend/src/core/activitypub/ApDbResolverService.ts | 4 ++-- packages/backend/src/server/api/AuthenticateService.ts | 2 +- 4 files changed, 8 insertions(+), 8 deletions(-) (limited to 'packages/backend/src/core') diff --git a/packages/backend/src/core/CacheService.ts b/packages/backend/src/core/CacheService.ts index 4afcef02be..6725ebe75b 100644 --- a/packages/backend/src/core/CacheService.ts +++ b/packages/backend/src/core/CacheService.ts @@ -56,10 +56,10 @@ export class CacheService implements OnApplicationShutdown { ) { //this.onMessage = this.onMessage.bind(this); - this.userByIdCache = new MemoryKVCache(Infinity); - this.localUserByNativeTokenCache = new MemoryKVCache(Infinity); - this.localUserByIdCache = new MemoryKVCache(Infinity); - this.uriPersonCache = new MemoryKVCache(Infinity); + this.userByIdCache = new MemoryKVCache(1000 * 60 * 5); // 5m + this.localUserByNativeTokenCache = new MemoryKVCache(1000 * 60 * 5); // 5m + this.localUserByIdCache = new MemoryKVCache(1000 * 60 * 5); // 5m + this.uriPersonCache = new MemoryKVCache(1000 * 60 * 5); // 5m this.userProfileCache = new RedisKVCache(this.redisClient, 'userProfile', { lifetime: 1000 * 60 * 30, // 30m diff --git a/packages/backend/src/core/UserKeypairService.ts b/packages/backend/src/core/UserKeypairService.ts index 51ac99179a..eb7a95da3e 100644 --- a/packages/backend/src/core/UserKeypairService.ts +++ b/packages/backend/src/core/UserKeypairService.ts @@ -25,7 +25,7 @@ export class UserKeypairService implements OnApplicationShutdown { ) { this.cache = new RedisKVCache(this.redisClient, 'userKeypair', { lifetime: 1000 * 60 * 60 * 24, // 24h - memoryCacheLifetime: Infinity, + memoryCacheLifetime: 1000 * 60 * 60 * 12, // 12h fetcher: (key) => this.userKeypairsRepository.findOneByOrFail({ userId: key }), toRedisConverter: (value) => JSON.stringify(value), fromRedisConverter: (value) => JSON.parse(value), diff --git a/packages/backend/src/core/activitypub/ApDbResolverService.ts b/packages/backend/src/core/activitypub/ApDbResolverService.ts index 44680a2ed5..062af39732 100644 --- a/packages/backend/src/core/activitypub/ApDbResolverService.ts +++ b/packages/backend/src/core/activitypub/ApDbResolverService.ts @@ -54,8 +54,8 @@ export class ApDbResolverService implements OnApplicationShutdown { private cacheService: CacheService, private apPersonService: ApPersonService, ) { - this.publicKeyCache = new MemoryKVCache(Infinity); - this.publicKeyByUserIdCache = new MemoryKVCache(Infinity); + this.publicKeyCache = new MemoryKVCache(1000 * 60 * 60 * 12); // 12h + this.publicKeyByUserIdCache = new MemoryKVCache(1000 * 60 * 60 * 12); // 12h } @bindThis diff --git a/packages/backend/src/server/api/AuthenticateService.ts b/packages/backend/src/server/api/AuthenticateService.ts index ddef8db987..690ff2e022 100644 --- a/packages/backend/src/server/api/AuthenticateService.ts +++ b/packages/backend/src/server/api/AuthenticateService.ts @@ -37,7 +37,7 @@ export class AuthenticateService implements OnApplicationShutdown { private cacheService: CacheService, ) { - this.appCache = new MemoryKVCache(Infinity); + this.appCache = new MemoryKVCache(1000 * 60 * 60 * 24 * 7); // 1w } @bindThis -- cgit v1.2.3-freya From 672f1ea68476f5d325c2c996eb3020fd69e99aad Mon Sep 17 00:00:00 2001 From: Hazel K Date: Sat, 3 Aug 2024 14:49:06 -0400 Subject: tune cache lifetimes --- packages/backend/src/core/AvatarDecorationService.ts | 2 +- packages/backend/src/core/CustomEmojiService.ts | 14 +++++++------- packages/backend/src/core/RelayService.ts | 2 +- packages/backend/src/core/RoleService.ts | 6 ++---- packages/backend/src/core/UserKeypairService.ts | 2 +- .../src/queue/processors/DeliverProcessorService.ts | 2 +- packages/backend/src/server/NodeinfoServerService.ts | 2 +- packages/backend/src/server/web/UrlPreviewService.ts | 4 ++-- 8 files changed, 16 insertions(+), 18 deletions(-) (limited to 'packages/backend/src/core') diff --git a/packages/backend/src/core/AvatarDecorationService.ts b/packages/backend/src/core/AvatarDecorationService.ts index 21e31d79a4..fa3f63677e 100644 --- a/packages/backend/src/core/AvatarDecorationService.ts +++ b/packages/backend/src/core/AvatarDecorationService.ts @@ -29,7 +29,7 @@ export class AvatarDecorationService implements OnApplicationShutdown { private moderationLogService: ModerationLogService, private globalEventService: GlobalEventService, ) { - this.cache = new MemorySingleCache(1000 * 60 * 30); + this.cache = new MemorySingleCache(1000 * 60 * 30); // 30s this.redisForSub.on('message', this.onMessage); } diff --git a/packages/backend/src/core/CustomEmojiService.ts b/packages/backend/src/core/CustomEmojiService.ts index bfbc2b172d..098e94991c 100644 --- a/packages/backend/src/core/CustomEmojiService.ts +++ b/packages/backend/src/core/CustomEmojiService.ts @@ -26,7 +26,7 @@ const parseEmojiStrRegexp = /^([-\w]+)(?:@([\w.-]+))?$/; @Injectable() export class CustomEmojiService implements OnApplicationShutdown { - private cache: MemoryKVCache; + private emojisCache: MemoryKVCache; public localEmojisCache: RedisSingleCache>; constructor( @@ -49,7 +49,7 @@ export class CustomEmojiService implements OnApplicationShutdown { private globalEventService: GlobalEventService, private driveService: DriveService, ) { - this.cache = new MemoryKVCache(1000 * 60 * 60 * 12); + this.emojisCache = new MemoryKVCache(1000 * 60 * 60 * 12); // 12h this.localEmojisCache = new RedisSingleCache>(this.redisClient, 'localEmojis', { lifetime: 1000 * 60 * 30, // 30m @@ -350,14 +350,14 @@ export class CustomEmojiService implements OnApplicationShutdown { if (name == null) return null; if (host == null) return null; - const newHost = host === this.config.host ? null : host; + const newHost = host === this.config.host ? null : host; const queryOrNull = async () => (await this.emojisRepository.findOneBy({ name, host: newHost ?? IsNull(), })) ?? null; - const emoji = await this.cache.fetch(`${name} ${host}`, queryOrNull); + const emoji = await this.emojisCache.fetch(`${name} ${host}`, queryOrNull); if (emoji == null) return null; return emoji.publicUrl || emoji.originalUrl; // || emoji.originalUrl してるのは後方互換性のため（publicUrlはstringなので??はだめ） @@ -384,7 +384,7 @@ export class CustomEmojiService implements OnApplicationShutdown { */ @bindThis public async prefetchEmojis(emojis: { name: string; host: string | null; }[]): Promise { - const notCachedEmojis = emojis.filter(emoji => this.cache.get(`${emoji.name} ${emoji.host}`) == null); + const notCachedEmojis = emojis.filter(emoji => this.emojisCache.get(`${emoji.name} ${emoji.host}`) == null); const emojisQuery: any[] = []; const hosts = new Set(notCachedEmojis.map(e => e.host)); for (const host of hosts) { @@ -399,7 +399,7 @@ export class CustomEmojiService implements OnApplicationShutdown { select: ['name', 'host', 'originalUrl', 'publicUrl'], }) : []; for (const emoji of _emojis) { - this.cache.set(`${emoji.name} ${emoji.host}`, emoji); + this.emojisCache.set(`${emoji.name} ${emoji.host}`, emoji); } } @@ -424,7 +424,7 @@ export class CustomEmojiService implements OnApplicationShutdown { @bindThis public dispose(): void { - this.cache.dispose(); + this.emojisCache.dispose(); } @bindThis diff --git a/packages/backend/src/core/RelayService.ts b/packages/backend/src/core/RelayService.ts index e9dc9b57af..9caeaf1714 100644 --- a/packages/backend/src/core/RelayService.ts +++ b/packages/backend/src/core/RelayService.ts @@ -35,7 +35,7 @@ export class RelayService { private createSystemUserService: CreateSystemUserService, private apRendererService: ApRendererService, ) { - this.relaysCache = new MemorySingleCache(1000 * 60 * 10); + this.relaysCache = new MemorySingleCache(1000 * 60 * 10); // 10s } @bindThis diff --git a/packages/backend/src/core/RoleService.ts b/packages/backend/src/core/RoleService.ts index f5a753afc7..f46aacaef4 100644 --- a/packages/backend/src/core/RoleService.ts +++ b/packages/backend/src/core/RoleService.ts @@ -129,10 +129,8 @@ export class RoleService implements OnApplicationShutdown, OnModuleInit { private moderationLogService: ModerationLogService, private fanoutTimelineService: FanoutTimelineService, ) { - //this.onMessage = this.onMessage.bind(this); - - this.rolesCache = new MemorySingleCache(1000 * 60 * 60 * 1); - this.roleAssignmentByUserIdCache = new MemoryKVCache(1000 * 60 * 60 * 1); + this.rolesCache = new MemorySingleCache(1000 * 60 * 60); // 1h + this.roleAssignmentByUserIdCache = new MemoryKVCache(1000 * 60 * 5); // 1h this.redisForSub.on('message', this.onMessage); } diff --git a/packages/backend/src/core/UserKeypairService.ts b/packages/backend/src/core/UserKeypairService.ts index eb7a95da3e..92d61cd103 100644 --- a/packages/backend/src/core/UserKeypairService.ts +++ b/packages/backend/src/core/UserKeypairService.ts @@ -25,7 +25,7 @@ export class UserKeypairService implements OnApplicationShutdown { ) { this.cache = new RedisKVCache(this.redisClient, 'userKeypair', { lifetime: 1000 * 60 * 60 * 24, // 24h - memoryCacheLifetime: 1000 * 60 * 60 * 12, // 12h + memoryCacheLifetime: 1000 * 60 * 60, // 1h fetcher: (key) => this.userKeypairsRepository.findOneByOrFail({ userId: key }), toRedisConverter: (value) => JSON.stringify(value), fromRedisConverter: (value) => JSON.parse(value), diff --git a/packages/backend/src/queue/processors/DeliverProcessorService.ts b/packages/backend/src/queue/processors/DeliverProcessorService.ts index d665945861..95477aa2cd 100644 --- a/packages/backend/src/queue/processors/DeliverProcessorService.ts +++ b/packages/backend/src/queue/processors/DeliverProcessorService.ts @@ -45,7 +45,7 @@ export class DeliverProcessorService { private queueLoggerService: QueueLoggerService, ) { this.logger = this.queueLoggerService.logger.createSubLogger('deliver'); - this.suspendedHostsCache = new MemorySingleCache(1000 * 60 * 60); + this.suspendedHostsCache = new MemorySingleCache(1000 * 60 * 60); // 1m } @bindThis diff --git a/packages/backend/src/server/NodeinfoServerService.ts b/packages/backend/src/server/NodeinfoServerService.ts index 716bb0944b..bf12822964 100644 --- a/packages/backend/src/server/NodeinfoServerService.ts +++ b/packages/backend/src/server/NodeinfoServerService.ts @@ -135,7 +135,7 @@ export class NodeinfoServerService { return document; }; - const cache = new MemorySingleCache>>(1000 * 60 * 10); + const cache = new MemorySingleCache>>(1000 * 60 * 10); // 10s fastify.get(nodeinfo2_1path, async (request, reply) => { const base = await cache.fetch(() => nodeinfo2(21)); diff --git a/packages/backend/src/server/web/UrlPreviewService.ts b/packages/backend/src/server/web/UrlPreviewService.ts index 96038d9c1e..ef804b5bfd 100644 --- a/packages/backend/src/server/web/UrlPreviewService.ts +++ b/packages/backend/src/server/web/UrlPreviewService.ts @@ -38,8 +38,8 @@ export class UrlPreviewService { ) { this.logger = this.loggerService.getLogger('url-preview'); this.previewCache = new RedisKVCache(this.redisClient, 'summaly', { - lifetime: 1000 * 86400, - memoryCacheLifetime: 1000 * 10 * 60, + lifetime: 1000 * 60 * 60 * 24, // 1d + memoryCacheLifetime: 1000 * 60 * 10, // 10m fetcher: (key: string) => { throw new Error('the UrlPreview cache should never fetch'); }, toRedisConverter: (value) => JSON.stringify(value), fromRedisConverter: (value) => JSON.parse(value), -- cgit v1.2.3-freya From 1e86cba7dc1cf81d03b805529fc0556c96ff286e Mon Sep 17 00:00:00 2001 From: dakkar Date: Mon, 5 Aug 2024 09:27:06 +0100 Subject: delete old emoji file when replaced - fixes #608 it's the same code that 5f7fc54ee9359d7dae82ad70e89f930d6a2b2e61 added to `delete` and `deleteBulk`, with the extra check that we're not deleting the same file we're setting --- packages/backend/src/core/CustomEmojiService.ts | 7 +++++++ 1 file changed, 7 insertions(+) (limited to 'packages/backend/src/core') diff --git a/packages/backend/src/core/CustomEmojiService.ts b/packages/backend/src/core/CustomEmojiService.ts index bfbc2b172d..eea0f9228d 100644 --- a/packages/backend/src/core/CustomEmojiService.ts +++ b/packages/backend/src/core/CustomEmojiService.ts @@ -142,6 +142,13 @@ export class CustomEmojiService implements OnApplicationShutdown { this.localEmojisCache.refresh(); + if (data.driveFile != null) { + const file = await this.driveFilesRepository.findOneBy({ url: emoji.originalUrl, userHost: emoji.host ? emoji.host : IsNull() }); + if (file && file.id != data.driveFile.id) { + await this.driveService.deleteFile(file, false, moderator ? moderator : undefined); + } + } + const packed = await this.emojiEntityService.packDetailed(emoji.id); if (emoji.name === data.name) { -- cgit v1.2.3-freya From 61c13241babbd9424e9d8ac21c7fe84ecc6c5018 Mon Sep 17 00:00:00 2001 From: dakkar Date: Tue, 6 Aug 2024 10:13:53 +0100 Subject: use `XMLSerializer` for `toMastoApiHtml` - fixes #556 the `inline` bit is not pretty, but does the job --- packages/backend/src/core/MfmService.ts | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) (limited to 'packages/backend/src/core') diff --git a/packages/backend/src/core/MfmService.ts b/packages/backend/src/core/MfmService.ts index 625df1feaa..76d0eb2339 100644 --- a/packages/backend/src/core/MfmService.ts +++ b/packages/backend/src/core/MfmService.ts @@ -6,7 +6,7 @@ import { URL } from 'node:url'; import { Inject, Injectable } from '@nestjs/common'; import * as parse5 from 'parse5'; -import { Window, XMLSerializer } from 'happy-dom'; +import { Window, DocumentFragment, XMLSerializer } from 'happy-dom'; import { DI } from '@/di-symbols.js'; import type { Config } from '@/config.js'; import { intersperse } from '@/misc/prelude/array.js'; @@ -483,6 +483,8 @@ export class MfmService { const doc = window.document; + const body = doc.createElement('p'); + async function appendChildren(children: mfm.MfmNode[], targetElement: any): Promise { if (children) { for (const child of await Promise.all(children.map(async (x) => await (handlers as any)[x.type](x)))) targetElement.appendChild(child); @@ -661,7 +663,7 @@ export class MfmService { }, }; - await appendChildren(nodes, doc.body); + await appendChildren(nodes, body); if (quoteUri !== null) { const a = doc.createElement('a'); @@ -675,9 +677,15 @@ export class MfmService { quote.innerHTML += 'RE: '; quote.appendChild(a); - doc.body.appendChild(quote); + body.appendChild(quote); + } + + let result = new XMLSerializer().serializeToString(body); + + if (inline) { + result = result.replace(/^

/,'').replace(/<\/p>$/,''); } - return inline ? doc.body.innerHTML : `

${doc.body.innerHTML}

`; + return result; } } -- cgit v1.2.3-freya From 9930c64f2d4a198551cfcff1a7c84b5ab10b54f4 Mon Sep 17 00:00:00 2001 From: Hazel K Date: Mon, 5 Aug 2024 21:19:29 -0400 Subject: Fix timeout comments MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: かっこかり <67428053+kakkokari-gtyih@users.noreply.github.com> --- packages/backend/src/core/RelayService.ts | 2 +- packages/backend/src/core/RoleService.ts | 2 +- packages/backend/src/queue/processors/DeliverProcessorService.ts | 2 +- packages/backend/src/server/NodeinfoServerService.ts | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) (limited to 'packages/backend/src/core') diff --git a/packages/backend/src/core/RelayService.ts b/packages/backend/src/core/RelayService.ts index 9caeaf1714..91857dc683 100644 --- a/packages/backend/src/core/RelayService.ts +++ b/packages/backend/src/core/RelayService.ts @@ -35,7 +35,7 @@ export class RelayService { private createSystemUserService: CreateSystemUserService, private apRendererService: ApRendererService, ) { - this.relaysCache = new MemorySingleCache(1000 * 60 * 10); // 10s + this.relaysCache = new MemorySingleCache(1000 * 60 * 10); // 10m } @bindThis diff --git a/packages/backend/src/core/RoleService.ts b/packages/backend/src/core/RoleService.ts index f46aacaef4..2b6089fd3a 100644 --- a/packages/backend/src/core/RoleService.ts +++ b/packages/backend/src/core/RoleService.ts @@ -130,7 +130,7 @@ export class RoleService implements OnApplicationShutdown, OnModuleInit { private fanoutTimelineService: FanoutTimelineService, ) { this.rolesCache = new MemorySingleCache(1000 * 60 * 60); // 1h - this.roleAssignmentByUserIdCache = new MemoryKVCache(1000 * 60 * 5); // 1h + this.roleAssignmentByUserIdCache = new MemoryKVCache(1000 * 60 * 5); // 5m this.redisForSub.on('message', this.onMessage); } diff --git a/packages/backend/src/queue/processors/DeliverProcessorService.ts b/packages/backend/src/queue/processors/DeliverProcessorService.ts index 95477aa2cd..4076e9da90 100644 --- a/packages/backend/src/queue/processors/DeliverProcessorService.ts +++ b/packages/backend/src/queue/processors/DeliverProcessorService.ts @@ -45,7 +45,7 @@ export class DeliverProcessorService { private queueLoggerService: QueueLoggerService, ) { this.logger = this.queueLoggerService.logger.createSubLogger('deliver'); - this.suspendedHostsCache = new MemorySingleCache(1000 * 60 * 60); // 1m + this.suspendedHostsCache = new MemorySingleCache(1000 * 60 * 60); // 1h } @bindThis diff --git a/packages/backend/src/server/NodeinfoServerService.ts b/packages/backend/src/server/NodeinfoServerService.ts index bf12822964..bc8d3c0411 100644 --- a/packages/backend/src/server/NodeinfoServerService.ts +++ b/packages/backend/src/server/NodeinfoServerService.ts @@ -135,7 +135,7 @@ export class NodeinfoServerService { return document; }; - const cache = new MemorySingleCache>>(1000 * 60 * 10); // 10s + const cache = new MemorySingleCache>>(1000 * 60 * 10); // 10m fastify.get(nodeinfo2_1path, async (request, reply) => { const base = await cache.fetch(() => nodeinfo2(21)); -- cgit v1.2.3-freya