From 32872181ddf14cfbf006dc93d04eeacac5eaf7a0 Mon Sep 17 00:00:00 2001 From: Hazel K Date: Thu, 25 Jul 2024 10:37:23 -0400 Subject: feat: implement `attachLdSignatureForRelays` to control signing of Relayed activities --- .config/ci.yml | 8 +- .config/docker_example.yml | 4 +- .config/example.yml | 4 +- chart/files/default.yml | 4 +- packages/backend/src/config.ts | 11 ++- .../src/core/activitypub/ApRendererService.ts | 97 +++++++++++++++------- 6 files changed, 89 insertions(+), 39 deletions(-) diff --git a/.config/ci.yml b/.config/ci.yml index c381d21d92..02081e5971 100644 --- a/.config/ci.yml +++ b/.config/ci.yml @@ -106,7 +106,7 @@ redis: # ┌───────────────────────────┐ #───┘ MeiliSearch configuration └───────────────────────────── -# You can set scope to local (default value) or global +# You can set scope to local (default value) or global # (include notes from remote). #meilisearch: @@ -198,13 +198,15 @@ proxyRemoteFiles: true # https://example.com/thumbnail.webp?thumbnail=1&url=https%3A%2F%2Fstorage.example.com%2Fpath%2Fto%2Fvideo.mp4 #videoThumbnailGenerator: https://example.com -# Sign to ActivityPub GET request (default: true) +# Sign outgoing ActivityPub GET request (default: true) signToActivityPubGet: true +# Sign outgoing ActivityPub Activities (default: true) +attachLdSignatureForRelays: true # check that inbound ActivityPub GET requests are signed ("authorized fetch") checkActivityPubGetSignature: false # For security reasons, uploading attachments from the intranet is prohibited, -# but exceptions can be made from the following settings. Default value is "undefined". +# but exceptions can be made from the following settings. Default value is "undefined". # Read changelog to learn more (Improvements of 12.90.0 (2021/09/04)). #allowedPrivateNetworks: [ # '127.0.0.1/32' diff --git a/.config/docker_example.yml b/.config/docker_example.yml index c22bd83c2e..375753e79f 100644 --- a/.config/docker_example.yml +++ b/.config/docker_example.yml @@ -270,8 +270,10 @@ proxyRemoteFiles: true # https://example.com/thumbnail.webp?thumbnail=1&url=https%3A%2F%2Fstorage.example.com%2Fpath%2Fto%2Fvideo.mp4 #videoThumbnailGenerator: https://example.com -# Sign to ActivityPub GET request (default: true) +# Sign outgoing ActivityPub GET request (default: true) signToActivityPubGet: true +# Sign outgoing ActivityPub Activities (default: true) +attachLdSignatureForRelays: true # check that inbound ActivityPub GET requests are signed ("authorized fetch") checkActivityPubGetSignature: false diff --git a/.config/example.yml b/.config/example.yml index ae55b983bb..4b6aaae63b 100644 --- a/.config/example.yml +++ b/.config/example.yml @@ -285,8 +285,10 @@ proxyRemoteFiles: true # https://example.com/thumbnail.webp?thumbnail=1&url=https%3A%2F%2Fstorage.example.com%2Fpath%2Fto%2Fvideo.mp4 #videoThumbnailGenerator: https://example.com -# Sign to ActivityPub GET request (default: true) +# Sign outgoing ActivityPub GET request (default: true) signToActivityPubGet: true +# Sign outgoing ActivityPub Activities (default: true) +attachLdSignatureForRelays: true # check that inbound ActivityPub GET requests are signed ("authorized fetch") checkActivityPubGetSignature: false diff --git a/chart/files/default.yml b/chart/files/default.yml index 2e1381ec57..7c94bcbea3 100644 --- a/chart/files/default.yml +++ b/chart/files/default.yml @@ -208,8 +208,10 @@ id: "aidx" # Media Proxy #mediaProxy: https://example.com/proxy -# Sign to ActivityPub GET request (default: true) +# Sign outgoing ActivityPub GET request (default: true) signToActivityPubGet: true +# Sign outgoing ActivityPub Activities (default: true) +attachLdSignatureForRelays: true # check that inbound ActivityPub GET requests are signed ("authorized fetch") checkActivityPubGetSignature: false diff --git a/packages/backend/src/config.ts b/packages/backend/src/config.ts index 58c4d028aa..10a63f8ae2 100644 --- a/packages/backend/src/config.ts +++ b/packages/backend/src/config.ts @@ -4,12 +4,12 @@ */ import * as fs from 'node:fs'; -import { fileURLToPath } from 'node:url'; -import { dirname, resolve } from 'node:path'; +import {fileURLToPath} from 'node:url'; +import {dirname, resolve} from 'node:path'; import * as yaml from 'js-yaml'; -import { globSync } from 'glob'; +import {globSync} from 'glob'; import * as Sentry from '@sentry/node'; -import type { RedisOptions } from 'ioredis'; +import type {RedisOptions} from 'ioredis'; type RedisOptionsSource = Partial & { host: string; @@ -95,6 +95,7 @@ type Source = { customMOTD?: string[]; signToActivityPubGet?: boolean; + attachLdSignatureForRelays?: boolean; checkActivityPubGetSignature?: boolean; perChannelMaxNoteCacheCount?: number; @@ -161,6 +162,7 @@ export type Config = { proxyRemoteFiles: boolean | undefined; customMOTD: string[] | undefined; signToActivityPubGet: boolean; + attachLdSignatureForRelays: boolean; checkActivityPubGetSignature: boolean | undefined; version: string; @@ -291,6 +293,7 @@ export function loadConfig(): Config { proxyRemoteFiles: config.proxyRemoteFiles, customMOTD: config.customMOTD, signToActivityPubGet: config.signToActivityPubGet ?? true, + attachLdSignatureForRelays: config.attachLdSignatureForRelays ?? true, checkActivityPubGetSignature: config.checkActivityPubGetSignature, mediaProxy: externalMediaProxy ?? internalMediaProxy, externalMediaProxyEnabled: externalMediaProxy !== null && externalMediaProxy !== internalMediaProxy, diff --git a/packages/backend/src/core/activitypub/ApRendererService.ts b/packages/backend/src/core/activitypub/ApRendererService.ts index 90784fdc1d..28c5dcf150 100644 --- a/packages/backend/src/core/activitypub/ApRendererService.ts +++ b/packages/backend/src/core/activitypub/ApRendererService.ts @@ -3,36 +3,69 @@ * SPDX-License-Identifier: AGPL-3.0-only */ -import { createPublicKey, randomUUID } from 'node:crypto'; -import { Inject, Injectable } from '@nestjs/common'; -import { In } from 'typeorm'; +import {createPublicKey, randomUUID} from 'node:crypto'; +import {Inject, Injectable} from '@nestjs/common'; +import {In} from 'typeorm'; import * as mfm from '@transfem-org/sfm-js'; -import { DI } from '@/di-symbols.js'; -import type { Config } from '@/config.js'; -import type { MiPartialLocalUser, MiLocalUser, MiPartialRemoteUser, MiRemoteUser, MiUser } from '@/models/User.js'; -import type { IMentionedRemoteUsers, MiNote } from '@/models/Note.js'; -import type { MiBlocking } from '@/models/Blocking.js'; -import type { MiRelay } from '@/models/Relay.js'; -import type { MiDriveFile } from '@/models/DriveFile.js'; -import type { MiNoteReaction } from '@/models/NoteReaction.js'; -import type { MiEmoji } from '@/models/Emoji.js'; -import type { MiPoll } from '@/models/Poll.js'; -import type { MiPollVote } from '@/models/PollVote.js'; -import { UserKeypairService } from '@/core/UserKeypairService.js'; -import { MfmService } from '@/core/MfmService.js'; -import { UserEntityService } from '@/core/entities/UserEntityService.js'; -import { DriveFileEntityService } from '@/core/entities/DriveFileEntityService.js'; -import type { MiUserKeypair } from '@/models/UserKeypair.js'; -import type { UsersRepository, UserProfilesRepository, NotesRepository, DriveFilesRepository, PollsRepository, InstancesRepository } from '@/models/_.js'; -import { bindThis } from '@/decorators.js'; -import { CustomEmojiService } from '@/core/CustomEmojiService.js'; -import { isNotNull } from '@/misc/is-not-null.js'; -import { IdService } from '@/core/IdService.js'; -import { MetaService } from '../MetaService.js'; -import { JsonLdService } from './JsonLdService.js'; -import { ApMfmService } from './ApMfmService.js'; -import { CONTEXT } from './misc/contexts.js'; -import type { IAccept, IActivity, IAdd, IAnnounce, IApDocument, IApEmoji, IApHashtag, IApImage, IApMention, IBlock, ICreate, IDelete, IFlag, IFollow, IKey, ILike, IMove, IObject, IPost, IQuestion, IReject, IRemove, ITombstone, IUndo, IUpdate } from './type.js'; +import {DI} from '@/di-symbols.js'; +import type {Config} from '@/config.js'; +import type {MiLocalUser, MiPartialLocalUser, MiPartialRemoteUser, MiRemoteUser, MiUser} from '@/models/User.js'; +import type {IMentionedRemoteUsers, MiNote} from '@/models/Note.js'; +import type {MiBlocking} from '@/models/Blocking.js'; +import type {MiRelay} from '@/models/Relay.js'; +import type {MiDriveFile} from '@/models/DriveFile.js'; +import type {MiNoteReaction} from '@/models/NoteReaction.js'; +import type {MiEmoji} from '@/models/Emoji.js'; +import type {MiPoll} from '@/models/Poll.js'; +import type {MiPollVote} from '@/models/PollVote.js'; +import {UserKeypairService} from '@/core/UserKeypairService.js'; +import {MfmService} from '@/core/MfmService.js'; +import {UserEntityService} from '@/core/entities/UserEntityService.js'; +import {DriveFileEntityService} from '@/core/entities/DriveFileEntityService.js'; +import type {MiUserKeypair} from '@/models/UserKeypair.js'; +import type { + DriveFilesRepository, + InstancesRepository, + NotesRepository, + PollsRepository, + UserProfilesRepository, + UsersRepository +} from '@/models/_.js'; +import {bindThis} from '@/decorators.js'; +import {CustomEmojiService} from '@/core/CustomEmojiService.js'; +import {isNotNull} from '@/misc/is-not-null.js'; +import {IdService} from '@/core/IdService.js'; +import {MetaService} from '../MetaService.js'; +import {JsonLdService} from './JsonLdService.js'; +import {ApMfmService} from './ApMfmService.js'; +import {CONTEXT} from './misc/contexts.js'; +import type { + IAccept, + IActivity, + IAdd, + IAnnounce, + IApDocument, + IApEmoji, + IApHashtag, + IApImage, + IApMention, + IBlock, + ICreate, + IDelete, + IFlag, + IFollow, + IKey, + ILike, + IMove, + IObject, + IPost, + IQuestion, + IReject, + IRemove, + ITombstone, + IUndo, + IUpdate +} from './type.js'; @Injectable() export class ApRendererService { @@ -793,6 +826,12 @@ export class ApRendererService { @bindThis public async attachLdSignature(activity: any, user: { id: MiUser['id']; host: null; }): Promise { + // When using authorized fetch, Linked Data signatures are often undesired (as it can allow blocked instances to bypass the check). + // We allow admins to disable LD signatures for increased privacy, at the expense of increased incoming fetch (GET) requests. + if (!this.config.attachLdSignatureForRelays) { + return activity; + } + const keypair = await this.userKeypairService.getUserKeypair(user.id); const jsonLd = this.jsonLdService.use(); -- cgit v1.2.3-freya From fecdff7fa092ca9d8ca0cd50b5694eb3f8279a91 Mon Sep 17 00:00:00 2001 From: Hazel K Date: Fri, 26 Jul 2024 09:42:49 -0400 Subject: revert import changes --- .../src/core/activitypub/ApRendererService.ts | 91 +++++++--------------- 1 file changed, 29 insertions(+), 62 deletions(-) diff --git a/packages/backend/src/core/activitypub/ApRendererService.ts b/packages/backend/src/core/activitypub/ApRendererService.ts index 28c5dcf150..8db9199e5d 100644 --- a/packages/backend/src/core/activitypub/ApRendererService.ts +++ b/packages/backend/src/core/activitypub/ApRendererService.ts @@ -3,69 +3,36 @@ * SPDX-License-Identifier: AGPL-3.0-only */ -import {createPublicKey, randomUUID} from 'node:crypto'; -import {Inject, Injectable} from '@nestjs/common'; -import {In} from 'typeorm'; +import { createPublicKey, randomUUID } from 'node:crypto'; +import { Inject, Injectable } from '@nestjs/common'; +import { In } from 'typeorm'; import * as mfm from '@transfem-org/sfm-js'; -import {DI} from '@/di-symbols.js'; -import type {Config} from '@/config.js'; -import type {MiLocalUser, MiPartialLocalUser, MiPartialRemoteUser, MiRemoteUser, MiUser} from '@/models/User.js'; -import type {IMentionedRemoteUsers, MiNote} from '@/models/Note.js'; -import type {MiBlocking} from '@/models/Blocking.js'; -import type {MiRelay} from '@/models/Relay.js'; -import type {MiDriveFile} from '@/models/DriveFile.js'; -import type {MiNoteReaction} from '@/models/NoteReaction.js'; -import type {MiEmoji} from '@/models/Emoji.js'; -import type {MiPoll} from '@/models/Poll.js'; -import type {MiPollVote} from '@/models/PollVote.js'; -import {UserKeypairService} from '@/core/UserKeypairService.js'; -import {MfmService} from '@/core/MfmService.js'; -import {UserEntityService} from '@/core/entities/UserEntityService.js'; -import {DriveFileEntityService} from '@/core/entities/DriveFileEntityService.js'; -import type {MiUserKeypair} from '@/models/UserKeypair.js'; -import type { - DriveFilesRepository, - InstancesRepository, - NotesRepository, - PollsRepository, - UserProfilesRepository, - UsersRepository -} from '@/models/_.js'; -import {bindThis} from '@/decorators.js'; -import {CustomEmojiService} from '@/core/CustomEmojiService.js'; -import {isNotNull} from '@/misc/is-not-null.js'; -import {IdService} from '@/core/IdService.js'; -import {MetaService} from '../MetaService.js'; -import {JsonLdService} from './JsonLdService.js'; -import {ApMfmService} from './ApMfmService.js'; -import {CONTEXT} from './misc/contexts.js'; -import type { - IAccept, - IActivity, - IAdd, - IAnnounce, - IApDocument, - IApEmoji, - IApHashtag, - IApImage, - IApMention, - IBlock, - ICreate, - IDelete, - IFlag, - IFollow, - IKey, - ILike, - IMove, - IObject, - IPost, - IQuestion, - IReject, - IRemove, - ITombstone, - IUndo, - IUpdate -} from './type.js'; +import { DI } from '@/di-symbols.js'; +import type { Config } from '@/config.js'; +import type { MiPartialLocalUser, MiLocalUser, MiPartialRemoteUser, MiRemoteUser, MiUser } from '@/models/User.js'; +import type { IMentionedRemoteUsers, MiNote } from '@/models/Note.js'; +import type { MiBlocking } from '@/models/Blocking.js'; +import type { MiRelay } from '@/models/Relay.js'; +import type { MiDriveFile } from '@/models/DriveFile.js'; +import type { MiNoteReaction } from '@/models/NoteReaction.js'; +import type { MiEmoji } from '@/models/Emoji.js'; +import type { MiPoll } from '@/models/Poll.js'; +import type { MiPollVote } from '@/models/PollVote.js'; +import { UserKeypairService } from '@/core/UserKeypairService.js'; +import { MfmService } from '@/core/MfmService.js'; +import { UserEntityService } from '@/core/entities/UserEntityService.js'; +import { DriveFileEntityService } from '@/core/entities/DriveFileEntityService.js'; +import type { MiUserKeypair } from '@/models/UserKeypair.js'; +import type { UsersRepository, UserProfilesRepository, NotesRepository, DriveFilesRepository, PollsRepository, InstancesRepository } from '@/models/_.js'; +import { bindThis } from '@/decorators.js'; +import { CustomEmojiService } from '@/core/CustomEmojiService.js'; +import { isNotNull } from '@/misc/is-not-null.js'; +import { IdService } from '@/core/IdService.js'; +import { MetaService } from '../MetaService.js'; +import { JsonLdService } from './JsonLdService.js'; +import { ApMfmService } from './ApMfmService.js'; +import { CONTEXT } from './misc/contexts.js'; +import type { IAccept, IActivity, IAdd, IAnnounce, IApDocument, IApEmoji, IApHashtag, IApImage, IApMention, IBlock, ICreate, IDelete, IFlag, IFollow, IKey, ILike, IMove, IObject, IPost, IQuestion, IReject, IRemove, ITombstone, IUndo, IUpdate } from './type.js'; @Injectable() export class ApRendererService { -- cgit v1.2.3-freya From 916509dd6a6216277b2439a7e622890c563c4a78 Mon Sep 17 00:00:00 2001 From: Hazel K Date: Fri, 26 Jul 2024 10:17:02 -0400 Subject: revert more import changes --- packages/backend/src/config.ts | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/packages/backend/src/config.ts b/packages/backend/src/config.ts index 10a63f8ae2..c8170a6a50 100644 --- a/packages/backend/src/config.ts +++ b/packages/backend/src/config.ts @@ -4,12 +4,12 @@ */ import * as fs from 'node:fs'; -import {fileURLToPath} from 'node:url'; -import {dirname, resolve} from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { dirname, resolve } from 'node:path'; import * as yaml from 'js-yaml'; -import {globSync} from 'glob'; +import { globSync } from 'glob'; import * as Sentry from '@sentry/node'; -import type {RedisOptions} from 'ioredis'; +import type { RedisOptions } from 'ioredis'; type RedisOptionsSource = Partial & { host: string; -- cgit v1.2.3-freya From 378408226b5e8968313058de4862b9916d08d6e0 Mon Sep 17 00:00:00 2001 From: Hazel K Date: Fri, 26 Jul 2024 22:45:07 -0400 Subject: tweak wording --- .config/ci.yml | 3 +++ .config/docker_example.yml | 3 +++ .config/example.yml | 3 +++ chart/files/default.yml | 3 +++ packages/backend/src/core/activitypub/ApRendererService.ts | 5 +++-- 5 files changed, 15 insertions(+), 2 deletions(-) diff --git a/.config/ci.yml b/.config/ci.yml index 02081e5971..44092d3662 100644 --- a/.config/ci.yml +++ b/.config/ci.yml @@ -201,6 +201,9 @@ proxyRemoteFiles: true # Sign outgoing ActivityPub GET request (default: true) signToActivityPubGet: true # Sign outgoing ActivityPub Activities (default: true) +# Linked Data signatures are cryptographic signatures attached to each activity to provide proof of authenticity. +# When using authorized fetch, this is often undesired as any signed activity can be forwarded to a blocked instance by relays and other instances. +# This setting allows admins to disable LD signatures for increased privacy, at the expense of fewer relayed activities and additional inbound fetch (GET) requests. attachLdSignatureForRelays: true # check that inbound ActivityPub GET requests are signed ("authorized fetch") checkActivityPubGetSignature: false diff --git a/.config/docker_example.yml b/.config/docker_example.yml index 375753e79f..f4645d672d 100644 --- a/.config/docker_example.yml +++ b/.config/docker_example.yml @@ -273,6 +273,9 @@ proxyRemoteFiles: true # Sign outgoing ActivityPub GET request (default: true) signToActivityPubGet: true # Sign outgoing ActivityPub Activities (default: true) +# Linked Data signatures are cryptographic signatures attached to each activity to provide proof of authenticity. +# When using authorized fetch, this is often undesired as any signed activity can be forwarded to a blocked instance by relays and other instances. +# This setting allows admins to disable LD signatures for increased privacy, at the expense of fewer relayed activities and additional inbound fetch (GET) requests. attachLdSignatureForRelays: true # check that inbound ActivityPub GET requests are signed ("authorized fetch") checkActivityPubGetSignature: false diff --git a/.config/example.yml b/.config/example.yml index 4b6aaae63b..21e85b7b89 100644 --- a/.config/example.yml +++ b/.config/example.yml @@ -288,6 +288,9 @@ proxyRemoteFiles: true # Sign outgoing ActivityPub GET request (default: true) signToActivityPubGet: true # Sign outgoing ActivityPub Activities (default: true) +# Linked Data signatures are cryptographic signatures attached to each activity to provide proof of authenticity. +# When using authorized fetch, this is often undesired as any signed activity can be forwarded to a blocked instance by relays and other instances. +# This setting allows admins to disable LD signatures for increased privacy, at the expense of fewer relayed activities and additional inbound fetch (GET) requests. attachLdSignatureForRelays: true # check that inbound ActivityPub GET requests are signed ("authorized fetch") checkActivityPubGetSignature: false diff --git a/chart/files/default.yml b/chart/files/default.yml index 7c94bcbea3..aab7ed6ce1 100644 --- a/chart/files/default.yml +++ b/chart/files/default.yml @@ -211,6 +211,9 @@ id: "aidx" # Sign outgoing ActivityPub GET request (default: true) signToActivityPubGet: true # Sign outgoing ActivityPub Activities (default: true) +# Linked Data signatures are cryptographic signatures attached to each activity to provide proof of authenticity. +# When using authorized fetch, this is often undesired as any signed activity can be forwarded to a blocked instance by relays and other instances. +# This setting allows admins to disable LD signatures for increased privacy, at the expense of fewer relayed activities and additional inbound fetch (GET) requests. attachLdSignatureForRelays: true # check that inbound ActivityPub GET requests are signed ("authorized fetch") checkActivityPubGetSignature: false diff --git a/packages/backend/src/core/activitypub/ApRendererService.ts b/packages/backend/src/core/activitypub/ApRendererService.ts index 8db9199e5d..98fc647a83 100644 --- a/packages/backend/src/core/activitypub/ApRendererService.ts +++ b/packages/backend/src/core/activitypub/ApRendererService.ts @@ -793,8 +793,9 @@ export class ApRendererService { @bindThis public async attachLdSignature(activity: any, user: { id: MiUser['id']; host: null; }): Promise { - // When using authorized fetch, Linked Data signatures are often undesired (as it can allow blocked instances to bypass the check). - // We allow admins to disable LD signatures for increased privacy, at the expense of increased incoming fetch (GET) requests. + // Linked Data signatures are cryptographic signatures attached to each activity to provide proof of authenticity. + // When using authorized fetch, this is often undesired as any signed activity can be forwarded to a blocked instance by relays and other instances. + // This setting allows admins to disable LD signatures for increased privacy, at the expense of fewer relayed activities and additional inbound fetch (GET) requests. if (!this.config.attachLdSignatureForRelays) { return activity; } -- cgit v1.2.3-freya From 114b6980346470fcf8cf1a11e50038c8fb15c48e Mon Sep 17 00:00:00 2001 From: Hazel K Date: Sat, 3 Aug 2024 09:18:44 -0400 Subject: encapsulate `MemoryKVCache` --- packages/backend/src/core/CacheService.ts | 4 ++-- packages/backend/src/misc/cache.ts | 24 +++++++++--------------- 2 files changed, 11 insertions(+), 17 deletions(-) diff --git a/packages/backend/src/core/CacheService.ts b/packages/backend/src/core/CacheService.ts index d008e7ec52..4afcef02be 100644 --- a/packages/backend/src/core/CacheService.ts +++ b/packages/backend/src/core/CacheService.ts @@ -135,14 +135,14 @@ export class CacheService implements OnApplicationShutdown { if (user == null) { this.userByIdCache.delete(body.id); this.localUserByIdCache.delete(body.id); - for (const [k, v] of this.uriPersonCache.cache.entries()) { + for (const [k, v] of this.uriPersonCache.entries) { if (v.value?.id === body.id) { this.uriPersonCache.delete(k); } } } else { this.userByIdCache.set(user.id, user); - for (const [k, v] of this.uriPersonCache.cache.entries()) { + for (const [k, v] of this.uriPersonCache.entries) { if (v.value?.id === user.id) { this.uriPersonCache.set(k, user); } diff --git a/packages/backend/src/misc/cache.ts b/packages/backend/src/misc/cache.ts index bba64a06ef..fe27d44692 100644 --- a/packages/backend/src/misc/cache.ts +++ b/packages/backend/src/misc/cache.ts @@ -187,22 +187,12 @@ export class RedisSingleCache { // TODO: メモリ節約のためあまり参照されないキーを定期的に削除できるようにする？ export class MemoryKVCache { - /** - * データを持つマップ - * @deprecated これを直接操作するべきではない - */ - public cache: Map; - private lifetime: number; - private gcIntervalHandle: NodeJS.Timeout; - - constructor(lifetime: MemoryKVCache['lifetime']) { - this.cache = new Map(); - this.lifetime = lifetime; + private readonly cache = new Map(); + private readonly gcIntervalHandle = setInterval(() => this.gc(), 1000 * 60 * 3); - this.gcIntervalHandle = setInterval(() => { - this.gc(); - }, 1000 * 60 * 3); - } + constructor( + private readonly lifetime: number, + ) {} @bindThis /** @@ -298,6 +288,10 @@ export class MemoryKVCache { public dispose(): void { clearInterval(this.gcIntervalHandle); } + + public get entries() { + return this.cache.entries(); + } } export class MemorySingleCache { -- cgit v1.2.3-freya From bc236a4bd250fc700bdd2d5549bf9647c02cb946 Mon Sep 17 00:00:00 2001 From: Hazel K Date: Sat, 3 Aug 2024 13:42:23 -0400 Subject: remove infinity caches --- packages/backend/src/core/CacheService.ts | 8 ++++---- packages/backend/src/core/UserKeypairService.ts | 2 +- packages/backend/src/core/activitypub/ApDbResolverService.ts | 4 ++-- packages/backend/src/server/api/AuthenticateService.ts | 2 +- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/packages/backend/src/core/CacheService.ts b/packages/backend/src/core/CacheService.ts index 4afcef02be..6725ebe75b 100644 --- a/packages/backend/src/core/CacheService.ts +++ b/packages/backend/src/core/CacheService.ts @@ -56,10 +56,10 @@ export class CacheService implements OnApplicationShutdown { ) { //this.onMessage = this.onMessage.bind(this); - this.userByIdCache = new MemoryKVCache(Infinity); - this.localUserByNativeTokenCache = new MemoryKVCache(Infinity); - this.localUserByIdCache = new MemoryKVCache(Infinity); - this.uriPersonCache = new MemoryKVCache(Infinity); + this.userByIdCache = new MemoryKVCache(1000 * 60 * 5); // 5m + this.localUserByNativeTokenCache = new MemoryKVCache(1000 * 60 * 5); // 5m + this.localUserByIdCache = new MemoryKVCache(1000 * 60 * 5); // 5m + this.uriPersonCache = new MemoryKVCache(1000 * 60 * 5); // 5m this.userProfileCache = new RedisKVCache(this.redisClient, 'userProfile', { lifetime: 1000 * 60 * 30, // 30m diff --git a/packages/backend/src/core/UserKeypairService.ts b/packages/backend/src/core/UserKeypairService.ts index 51ac99179a..eb7a95da3e 100644 --- a/packages/backend/src/core/UserKeypairService.ts +++ b/packages/backend/src/core/UserKeypairService.ts @@ -25,7 +25,7 @@ export class UserKeypairService implements OnApplicationShutdown { ) { this.cache = new RedisKVCache(this.redisClient, 'userKeypair', { lifetime: 1000 * 60 * 60 * 24, // 24h - memoryCacheLifetime: Infinity, + memoryCacheLifetime: 1000 * 60 * 60 * 12, // 12h fetcher: (key) => this.userKeypairsRepository.findOneByOrFail({ userId: key }), toRedisConverter: (value) => JSON.stringify(value), fromRedisConverter: (value) => JSON.parse(value), diff --git a/packages/backend/src/core/activitypub/ApDbResolverService.ts b/packages/backend/src/core/activitypub/ApDbResolverService.ts index 44680a2ed5..062af39732 100644 --- a/packages/backend/src/core/activitypub/ApDbResolverService.ts +++ b/packages/backend/src/core/activitypub/ApDbResolverService.ts @@ -54,8 +54,8 @@ export class ApDbResolverService implements OnApplicationShutdown { private cacheService: CacheService, private apPersonService: ApPersonService, ) { - this.publicKeyCache = new MemoryKVCache(Infinity); - this.publicKeyByUserIdCache = new MemoryKVCache(Infinity); + this.publicKeyCache = new MemoryKVCache(1000 * 60 * 60 * 12); // 12h + this.publicKeyByUserIdCache = new MemoryKVCache(1000 * 60 * 60 * 12); // 12h } @bindThis diff --git a/packages/backend/src/server/api/AuthenticateService.ts b/packages/backend/src/server/api/AuthenticateService.ts index ddef8db987..690ff2e022 100644 --- a/packages/backend/src/server/api/AuthenticateService.ts +++ b/packages/backend/src/server/api/AuthenticateService.ts @@ -37,7 +37,7 @@ export class AuthenticateService implements OnApplicationShutdown { private cacheService: CacheService, ) { - this.appCache = new MemoryKVCache(Infinity); + this.appCache = new MemoryKVCache(1000 * 60 * 60 * 24 * 7); // 1w } @bindThis -- cgit v1.2.3-freya From b1f1e3eb0e9a809839d2ee2ccf71d64c47dc7efa Mon Sep 17 00:00:00 2001 From: Hazel K Date: Sat, 3 Aug 2024 13:54:59 -0400 Subject: encapsulate other caches --- packages/backend/src/misc/cache.ts | 75 +++++++++++++++++++------------------- 1 file changed, 37 insertions(+), 38 deletions(-) diff --git a/packages/backend/src/misc/cache.ts b/packages/backend/src/misc/cache.ts index fe27d44692..b6eca73b03 100644 --- a/packages/backend/src/misc/cache.ts +++ b/packages/backend/src/misc/cache.ts @@ -7,23 +7,23 @@ import * as Redis from 'ioredis'; import { bindThis } from '@/decorators.js'; export class RedisKVCache { - private redisClient: Redis.Redis; - private name: string; - private lifetime: number; - private memoryCache: MemoryKVCache; - private fetcher: (key: string) => Promise; - private toRedisConverter: (value: T) => string; - private fromRedisConverter: (value: string) => T | undefined; - - constructor(redisClient: RedisKVCache['redisClient'], name: RedisKVCache['name'], opts: { - lifetime: RedisKVCache['lifetime']; - memoryCacheLifetime: number; - fetcher: RedisKVCache['fetcher']; - toRedisConverter: RedisKVCache['toRedisConverter']; - fromRedisConverter: RedisKVCache['fromRedisConverter']; - }) { - this.redisClient = redisClient; - this.name = name; + private readonly lifetime: number; + private readonly memoryCache: MemoryKVCache; + private readonly fetcher: (key: string) => Promise; + private readonly toRedisConverter: (value: T) => string; + private readonly fromRedisConverter: (value: string) => T | undefined; + + constructor( + private redisClient: Redis.Redis, + private name: string, + opts: { + lifetime: RedisKVCache['lifetime']; + memoryCacheLifetime: number; + fetcher: RedisKVCache['fetcher']; + toRedisConverter: RedisKVCache['toRedisConverter']; + fromRedisConverter: RedisKVCache['fromRedisConverter']; + }, + ) { this.lifetime = opts.lifetime; this.memoryCache = new MemoryKVCache(opts.memoryCacheLifetime); this.fetcher = opts.fetcher; @@ -101,23 +101,23 @@ export class RedisKVCache { } export class RedisSingleCache { - private redisClient: Redis.Redis; - private name: string; - private lifetime: number; - private memoryCache: MemorySingleCache; - private fetcher: () => Promise; - private toRedisConverter: (value: T) => string; - private fromRedisConverter: (value: string) => T | undefined; - - constructor(redisClient: RedisSingleCache['redisClient'], name: RedisSingleCache['name'], opts: { - lifetime: RedisSingleCache['lifetime']; - memoryCacheLifetime: number; - fetcher: RedisSingleCache['fetcher']; - toRedisConverter: RedisSingleCache['toRedisConverter']; - fromRedisConverter: RedisSingleCache['fromRedisConverter']; - }) { - this.redisClient = redisClient; - this.name = name; + private readonly lifetime: number; + private readonly memoryCache: MemorySingleCache; + private readonly fetcher: () => Promise; + private readonly toRedisConverter: (value: T) => string; + private readonly fromRedisConverter: (value: string) => T | undefined; + + constructor( + private redisClient: Redis.Redis, + private name: string, + opts: { + lifetime: number; + memoryCacheLifetime: number; + fetcher: RedisSingleCache['fetcher']; + toRedisConverter: RedisSingleCache['toRedisConverter']; + fromRedisConverter: RedisSingleCache['fromRedisConverter']; + }, + ) { this.lifetime = opts.lifetime; this.memoryCache = new MemorySingleCache(opts.memoryCacheLifetime); this.fetcher = opts.fetcher; @@ -297,11 +297,10 @@ export class MemoryKVCache { export class MemorySingleCache { private cachedAt: number | null = null; private value: T | undefined; - private lifetime: number; - constructor(lifetime: MemorySingleCache['lifetime']) { - this.lifetime = lifetime; - } + constructor( + private lifetime: number, + ) {} @bindThis public set(value: T): void { -- cgit v1.2.3-freya From 613706f6b87951bc10e36ebf915e81b0efb2bcac Mon Sep 17 00:00:00 2001 From: Hazel K Date: Sat, 3 Aug 2024 14:02:18 -0400 Subject: add missing awaits to internally synchronize caches --- packages/backend/src/misc/cache.ts | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/packages/backend/src/misc/cache.ts b/packages/backend/src/misc/cache.ts index b6eca73b03..68397e1563 100644 --- a/packages/backend/src/misc/cache.ts +++ b/packages/backend/src/misc/cache.ts @@ -77,14 +77,14 @@ export class RedisKVCache { // Cache MISS const value = await this.fetcher(key); - this.set(key, value); + await this.set(key, value); return value; } @bindThis public async refresh(key: string) { const value = await this.fetcher(key); - this.set(key, value); + await this.set(key, value); // TODO: イベント発行して他プロセスのメモリキャッシュも更新できるようにする } @@ -171,14 +171,14 @@ export class RedisSingleCache { // Cache MISS const value = await this.fetcher(); - this.set(value); + await this.set(value); return value; } @bindThis public async refresh() { const value = await this.fetcher(); - this.set(value); + await this.set(value); // TODO: イベント発行して他プロセスのメモリキャッシュも更新できるようにする } -- cgit v1.2.3-freya From 3688f1dadf8e5f5601700a167ca4aa9514d71469 Mon Sep 17 00:00:00 2001 From: Hazel K Date: Sat, 3 Aug 2024 14:09:08 -0400 Subject: implement pull-through caching --- packages/backend/src/misc/cache.ts | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/packages/backend/src/misc/cache.ts b/packages/backend/src/misc/cache.ts index 68397e1563..fc98ce8132 100644 --- a/packages/backend/src/misc/cache.ts +++ b/packages/backend/src/misc/cache.ts @@ -55,7 +55,13 @@ export class RedisKVCache { const cached = await this.redisClient.get(`kvcache:${this.name}:${key}`); if (cached == null) return undefined; - return this.fromRedisConverter(cached); + + const value = this.fromRedisConverter(cached); + if (value !== undefined) { + this.memoryCache.set(key, value); + } + + return value; } @bindThis @@ -149,7 +155,13 @@ export class RedisSingleCache { const cached = await this.redisClient.get(`singlecache:${this.name}`); if (cached == null) return undefined; - return this.fromRedisConverter(cached); + + const value = this.fromRedisConverter(cached); + if (value !== undefined) { + this.memoryCache.set(value); + } + + return value; } @bindThis -- cgit v1.2.3-freya From 672f1ea68476f5d325c2c996eb3020fd69e99aad Mon Sep 17 00:00:00 2001 From: Hazel K Date: Sat, 3 Aug 2024 14:49:06 -0400 Subject: tune cache lifetimes --- packages/backend/src/core/AvatarDecorationService.ts | 2 +- packages/backend/src/core/CustomEmojiService.ts | 14 +++++++------- packages/backend/src/core/RelayService.ts | 2 +- packages/backend/src/core/RoleService.ts | 6 ++---- packages/backend/src/core/UserKeypairService.ts | 2 +- .../src/queue/processors/DeliverProcessorService.ts | 2 +- packages/backend/src/server/NodeinfoServerService.ts | 2 +- packages/backend/src/server/web/UrlPreviewService.ts | 4 ++-- 8 files changed, 16 insertions(+), 18 deletions(-) diff --git a/packages/backend/src/core/AvatarDecorationService.ts b/packages/backend/src/core/AvatarDecorationService.ts index 21e31d79a4..fa3f63677e 100644 --- a/packages/backend/src/core/AvatarDecorationService.ts +++ b/packages/backend/src/core/AvatarDecorationService.ts @@ -29,7 +29,7 @@ export class AvatarDecorationService implements OnApplicationShutdown { private moderationLogService: ModerationLogService, private globalEventService: GlobalEventService, ) { - this.cache = new MemorySingleCache(1000 * 60 * 30); + this.cache = new MemorySingleCache(1000 * 60 * 30); // 30s this.redisForSub.on('message', this.onMessage); } diff --git a/packages/backend/src/core/CustomEmojiService.ts b/packages/backend/src/core/CustomEmojiService.ts index bfbc2b172d..098e94991c 100644 --- a/packages/backend/src/core/CustomEmojiService.ts +++ b/packages/backend/src/core/CustomEmojiService.ts @@ -26,7 +26,7 @@ const parseEmojiStrRegexp = /^([-\w]+)(?:@([\w.-]+))?$/; @Injectable() export class CustomEmojiService implements OnApplicationShutdown { - private cache: MemoryKVCache; + private emojisCache: MemoryKVCache; public localEmojisCache: RedisSingleCache>; constructor( @@ -49,7 +49,7 @@ export class CustomEmojiService implements OnApplicationShutdown { private globalEventService: GlobalEventService, private driveService: DriveService, ) { - this.cache = new MemoryKVCache(1000 * 60 * 60 * 12); + this.emojisCache = new MemoryKVCache(1000 * 60 * 60 * 12); // 12h this.localEmojisCache = new RedisSingleCache>(this.redisClient, 'localEmojis', { lifetime: 1000 * 60 * 30, // 30m @@ -350,14 +350,14 @@ export class CustomEmojiService implements OnApplicationShutdown { if (name == null) return null; if (host == null) return null; - const newHost = host === this.config.host ? null : host; + const newHost = host === this.config.host ? null : host; const queryOrNull = async () => (await this.emojisRepository.findOneBy({ name, host: newHost ?? IsNull(), })) ?? null; - const emoji = await this.cache.fetch(`${name} ${host}`, queryOrNull); + const emoji = await this.emojisCache.fetch(`${name} ${host}`, queryOrNull); if (emoji == null) return null; return emoji.publicUrl || emoji.originalUrl; // || emoji.originalUrl してるのは後方互換性のため（publicUrlはstringなので??はだめ） @@ -384,7 +384,7 @@ export class CustomEmojiService implements OnApplicationShutdown { */ @bindThis public async prefetchEmojis(emojis: { name: string; host: string | null; }[]): Promise { - const notCachedEmojis = emojis.filter(emoji => this.cache.get(`${emoji.name} ${emoji.host}`) == null); + const notCachedEmojis = emojis.filter(emoji => this.emojisCache.get(`${emoji.name} ${emoji.host}`) == null); const emojisQuery: any[] = []; const hosts = new Set(notCachedEmojis.map(e => e.host)); for (const host of hosts) { @@ -399,7 +399,7 @@ export class CustomEmojiService implements OnApplicationShutdown { select: ['name', 'host', 'originalUrl', 'publicUrl'], }) : []; for (const emoji of _emojis) { - this.cache.set(`${emoji.name} ${emoji.host}`, emoji); + this.emojisCache.set(`${emoji.name} ${emoji.host}`, emoji); } } @@ -424,7 +424,7 @@ export class CustomEmojiService implements OnApplicationShutdown { @bindThis public dispose(): void { - this.cache.dispose(); + this.emojisCache.dispose(); } @bindThis diff --git a/packages/backend/src/core/RelayService.ts b/packages/backend/src/core/RelayService.ts index e9dc9b57af..9caeaf1714 100644 --- a/packages/backend/src/core/RelayService.ts +++ b/packages/backend/src/core/RelayService.ts @@ -35,7 +35,7 @@ export class RelayService { private createSystemUserService: CreateSystemUserService, private apRendererService: ApRendererService, ) { - this.relaysCache = new MemorySingleCache(1000 * 60 * 10); + this.relaysCache = new MemorySingleCache(1000 * 60 * 10); // 10s } @bindThis diff --git a/packages/backend/src/core/RoleService.ts b/packages/backend/src/core/RoleService.ts index f5a753afc7..f46aacaef4 100644 --- a/packages/backend/src/core/RoleService.ts +++ b/packages/backend/src/core/RoleService.ts @@ -129,10 +129,8 @@ export class RoleService implements OnApplicationShutdown, OnModuleInit { private moderationLogService: ModerationLogService, private fanoutTimelineService: FanoutTimelineService, ) { - //this.onMessage = this.onMessage.bind(this); - - this.rolesCache = new MemorySingleCache(1000 * 60 * 60 * 1); - this.roleAssignmentByUserIdCache = new MemoryKVCache(1000 * 60 * 60 * 1); + this.rolesCache = new MemorySingleCache(1000 * 60 * 60); // 1h + this.roleAssignmentByUserIdCache = new MemoryKVCache(1000 * 60 * 5); // 1h this.redisForSub.on('message', this.onMessage); } diff --git a/packages/backend/src/core/UserKeypairService.ts b/packages/backend/src/core/UserKeypairService.ts index eb7a95da3e..92d61cd103 100644 --- a/packages/backend/src/core/UserKeypairService.ts +++ b/packages/backend/src/core/UserKeypairService.ts @@ -25,7 +25,7 @@ export class UserKeypairService implements OnApplicationShutdown { ) { this.cache = new RedisKVCache(this.redisClient, 'userKeypair', { lifetime: 1000 * 60 * 60 * 24, // 24h - memoryCacheLifetime: 1000 * 60 * 60 * 12, // 12h + memoryCacheLifetime: 1000 * 60 * 60, // 1h fetcher: (key) => this.userKeypairsRepository.findOneByOrFail({ userId: key }), toRedisConverter: (value) => JSON.stringify(value), fromRedisConverter: (value) => JSON.parse(value), diff --git a/packages/backend/src/queue/processors/DeliverProcessorService.ts b/packages/backend/src/queue/processors/DeliverProcessorService.ts index d665945861..95477aa2cd 100644 --- a/packages/backend/src/queue/processors/DeliverProcessorService.ts +++ b/packages/backend/src/queue/processors/DeliverProcessorService.ts @@ -45,7 +45,7 @@ export class DeliverProcessorService { private queueLoggerService: QueueLoggerService, ) { this.logger = this.queueLoggerService.logger.createSubLogger('deliver'); - this.suspendedHostsCache = new MemorySingleCache(1000 * 60 * 60); + this.suspendedHostsCache = new MemorySingleCache(1000 * 60 * 60); // 1m } @bindThis diff --git a/packages/backend/src/server/NodeinfoServerService.ts b/packages/backend/src/server/NodeinfoServerService.ts index 716bb0944b..bf12822964 100644 --- a/packages/backend/src/server/NodeinfoServerService.ts +++ b/packages/backend/src/server/NodeinfoServerService.ts @@ -135,7 +135,7 @@ export class NodeinfoServerService { return document; }; - const cache = new MemorySingleCache>>(1000 * 60 * 10); + const cache = new MemorySingleCache>>(1000 * 60 * 10); // 10s fastify.get(nodeinfo2_1path, async (request, reply) => { const base = await cache.fetch(() => nodeinfo2(21)); diff --git a/packages/backend/src/server/web/UrlPreviewService.ts b/packages/backend/src/server/web/UrlPreviewService.ts index 96038d9c1e..ef804b5bfd 100644 --- a/packages/backend/src/server/web/UrlPreviewService.ts +++ b/packages/backend/src/server/web/UrlPreviewService.ts @@ -38,8 +38,8 @@ export class UrlPreviewService { ) { this.logger = this.loggerService.getLogger('url-preview'); this.previewCache = new RedisKVCache(this.redisClient, 'summaly', { - lifetime: 1000 * 86400, - memoryCacheLifetime: 1000 * 10 * 60, + lifetime: 1000 * 60 * 60 * 24, // 1d + memoryCacheLifetime: 1000 * 60 * 10, // 10m fetcher: (key: string) => { throw new Error('the UrlPreview cache should never fetch'); }, toRedisConverter: (value) => JSON.stringify(value), fromRedisConverter: (value) => JSON.parse(value), -- cgit v1.2.3-freya From 4ed4547f4a14aa912054edcb8ee669c3eceae9fc Mon Sep 17 00:00:00 2001 From: Marie Date: Sat, 3 Aug 2024 22:45:49 +0000 Subject: upd: add icon for moving files/folders --- .../fonts/sharkey-icons/custom-sharkey-icons.svg | 3 ++- .../fonts/sharkey-icons/custom-sharkey-icons.ttf | Bin 1896 -> 2180 bytes .../fonts/sharkey-icons/custom-sharkey-icons.woff | Bin 1292 -> 1520 bytes packages/backend/assets/fonts/sharkey-icons/style.css | 4 ++++ 4 files changed, 6 insertions(+), 1 deletion(-) diff --git a/packages/backend/assets/fonts/sharkey-icons/custom-sharkey-icons.svg b/packages/backend/assets/fonts/sharkey-icons/custom-sharkey-icons.svg index 9d21137072..777fe5ab05 100644 --- a/packages/backend/assets/fonts/sharkey-icons/custom-sharkey-icons.svg +++ b/packages/backend/assets/fonts/sharkey-icons/custom-sharkey-icons.svg @@ -7,5 +7,6 @@ - + + diff --git a/packages/backend/assets/fonts/sharkey-icons/custom-sharkey-icons.ttf b/packages/backend/assets/fonts/sharkey-icons/custom-sharkey-icons.ttf index a2601e0f1b..e48ffbdf73 100644 Binary files a/packages/backend/assets/fonts/sharkey-icons/custom-sharkey-icons.ttf and b/packages/backend/assets/fonts/sharkey-icons/custom-sharkey-icons.ttf differ diff --git a/packages/backend/assets/fonts/sharkey-icons/custom-sharkey-icons.woff b/packages/backend/assets/fonts/sharkey-icons/custom-sharkey-icons.woff index d9f471fa35..2ac8730e3e 100644 Binary files a/packages/backend/assets/fonts/sharkey-icons/custom-sharkey-icons.woff and b/packages/backend/assets/fonts/sharkey-icons/custom-sharkey-icons.woff differ diff --git a/packages/backend/assets/fonts/sharkey-icons/style.css b/packages/backend/assets/fonts/sharkey-icons/style.css index 7fb0f94504..4cae7787d4 100644 --- a/packages/backend/assets/fonts/sharkey-icons/style.css +++ b/packages/backend/assets/fonts/sharkey-icons/style.css @@ -29,3 +29,7 @@ .sk-icons.sk-misskey:before { content: "\62"; } + +.sk-icons.sk-foldermove:before { + content: "\63"; +} -- cgit v1.2.3-freya From ba093382682380ddf1bb756e9d83b31b776fa7af Mon Sep 17 00:00:00 2001 From: Hazel K Date: Sun, 4 Aug 2024 09:58:01 -0400 Subject: optimize cache GC by stopping early --- packages/backend/src/misc/cache.ts | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/packages/backend/src/misc/cache.ts b/packages/backend/src/misc/cache.ts index fc98ce8132..d968069ca3 100644 --- a/packages/backend/src/misc/cache.ts +++ b/packages/backend/src/misc/cache.ts @@ -200,7 +200,7 @@ export class RedisSingleCache { export class MemoryKVCache { private readonly cache = new Map(); - private readonly gcIntervalHandle = setInterval(() => this.gc(), 1000 * 60 * 3); + private readonly gcIntervalHandle = setInterval(() => this.gc(), 1000 * 60 * 3); // 3m constructor( private readonly lifetime: number, @@ -289,10 +289,14 @@ export class MemoryKVCache { @bindThis public gc(): void { const now = Date.now(); + for (const [key, { date }] of this.cache.entries()) { - if ((now - date) > this.lifetime) { - this.cache.delete(key); - } + // The map is ordered from oldest to youngest. + // We can stop once we find an entry that's still active, because all following entries must *also* be active. + const age = now - date; + if (age < this.lifetime) break; + + this.cache.delete(key); } } -- cgit v1.2.3-freya From 1e86cba7dc1cf81d03b805529fc0556c96ff286e Mon Sep 17 00:00:00 2001 From: dakkar Date: Mon, 5 Aug 2024 09:27:06 +0100 Subject: delete old emoji file when replaced - fixes #608 it's the same code that 5f7fc54ee9359d7dae82ad70e89f930d6a2b2e61 added to `delete` and `deleteBulk`, with the extra check that we're not deleting the same file we're setting --- packages/backend/src/core/CustomEmojiService.ts | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/packages/backend/src/core/CustomEmojiService.ts b/packages/backend/src/core/CustomEmojiService.ts index bfbc2b172d..eea0f9228d 100644 --- a/packages/backend/src/core/CustomEmojiService.ts +++ b/packages/backend/src/core/CustomEmojiService.ts @@ -142,6 +142,13 @@ export class CustomEmojiService implements OnApplicationShutdown { this.localEmojisCache.refresh(); + if (data.driveFile != null) { + const file = await this.driveFilesRepository.findOneBy({ url: emoji.originalUrl, userHost: emoji.host ? emoji.host : IsNull() }); + if (file && file.id != data.driveFile.id) { + await this.driveService.deleteFile(file, false, moderator ? moderator : undefined); + } + } + const packed = await this.emojiEntityService.packDetailed(emoji.id); if (emoji.name === data.name) { -- cgit v1.2.3-freya From 0386e52d6f8231a879ccf72e166b04f71b9d11db Mon Sep 17 00:00:00 2001 From: 4censord Date: Sun, 4 Aug 2024 18:09:48 +0200 Subject: Impove the check_connect script --- packages/backend/scripts/check_connect.js | 32 ++++++++++++++++++++++++++----- 1 file changed, 27 insertions(+), 5 deletions(-) diff --git a/packages/backend/scripts/check_connect.js b/packages/backend/scripts/check_connect.js index ba25fd416c..d4bf4baf43 100644 --- a/packages/backend/scripts/check_connect.js +++ b/packages/backend/scripts/check_connect.js @@ -5,11 +5,33 @@ import Redis from 'ioredis'; import { loadConfig } from '../built/config.js'; +import { createPostgresDataSource } from '../built/postgres.js'; const config = loadConfig(); -const redis = new Redis(config.redis); -redis.on('connect', () => redis.disconnect()); -redis.on('error', (e) => { - throw e; -}); +// createPostgresDataSource handels primaries and replicas automatically. +// usually, it only opens connections first use, so we force it using +// .initialize() +createPostgresDataSource(config) + .initialize() + .then(c => { c.destroy() }) + .catch(e => { throw e }); + + +// Connect to all redis servers +function connectToRedis(redisOptions) { + const redis = new Redis(redisOptions); + redis.on('connect', () => redis.disconnect()); + redis.on('error', (e) => { + throw e; + }); +} + +// If not all of these are defined, the default one gets reused. +// so we use a Set to only try connecting once to each **uniq** redis. +(new Set([ + config.redis, + config.redisForPubsub, + config.redisForJobQueue, + config.redisForTimelines, +])).forEach(connectToRedis); -- cgit v1.2.3-freya From 61c13241babbd9424e9d8ac21c7fe84ecc6c5018 Mon Sep 17 00:00:00 2001 From: dakkar Date: Tue, 6 Aug 2024 10:13:53 +0100 Subject: use `XMLSerializer` for `toMastoApiHtml` - fixes #556 the `inline` bit is not pretty, but does the job --- packages/backend/src/core/MfmService.ts | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/packages/backend/src/core/MfmService.ts b/packages/backend/src/core/MfmService.ts index 625df1feaa..76d0eb2339 100644 --- a/packages/backend/src/core/MfmService.ts +++ b/packages/backend/src/core/MfmService.ts @@ -6,7 +6,7 @@ import { URL } from 'node:url'; import { Inject, Injectable } from '@nestjs/common'; import * as parse5 from 'parse5'; -import { Window, XMLSerializer } from 'happy-dom'; +import { Window, DocumentFragment, XMLSerializer } from 'happy-dom'; import { DI } from '@/di-symbols.js'; import type { Config } from '@/config.js'; import { intersperse } from '@/misc/prelude/array.js'; @@ -483,6 +483,8 @@ export class MfmService { const doc = window.document; + const body = doc.createElement('p'); + async function appendChildren(children: mfm.MfmNode[], targetElement: any): Promise { if (children) { for (const child of await Promise.all(children.map(async (x) => await (handlers as any)[x.type](x)))) targetElement.appendChild(child); @@ -661,7 +663,7 @@ export class MfmService { }, }; - await appendChildren(nodes, doc.body); + await appendChildren(nodes, body); if (quoteUri !== null) { const a = doc.createElement('a'); @@ -675,9 +677,15 @@ export class MfmService { quote.innerHTML += 'RE: '; quote.appendChild(a); - doc.body.appendChild(quote); + body.appendChild(quote); + } + + let result = new XMLSerializer().serializeToString(body); + + if (inline) { + result = result.replace(/^

/,'').replace(/<\/p>$/,''); } - return inline ? doc.body.innerHTML : `

${doc.body.innerHTML}

`; + return result; } } -- cgit v1.2.3-freya From 9d4d2a1fad27abe853b90cb4bab5ba9ea331fe23 Mon Sep 17 00:00:00 2001 From: Marie Date: Tue, 6 Aug 2024 15:35:52 +0000 Subject: upd: align font with new repo --- .../fonts/sharkey-icons/custom-sharkey-icons.svg | 12 --- .../fonts/sharkey-icons/custom-sharkey-icons.ttf | Bin 2180 -> 0 bytes .../fonts/sharkey-icons/custom-sharkey-icons.woff | Bin 1520 -> 0 bytes .../assets/fonts/sharkey-icons/shark-font.svg | 30 ++++++ .../assets/fonts/sharkey-icons/shark-font.ttf | Bin 0 -> 2652 bytes .../assets/fonts/sharkey-icons/shark-font.woff | Bin 0 -> 1736 bytes .../backend/assets/fonts/sharkey-icons/style.css | 117 +++++++++++++++++---- 7 files changed, 128 insertions(+), 31 deletions(-) delete mode 100644 packages/backend/assets/fonts/sharkey-icons/custom-sharkey-icons.svg delete mode 100644 packages/backend/assets/fonts/sharkey-icons/custom-sharkey-icons.ttf delete mode 100644 packages/backend/assets/fonts/sharkey-icons/custom-sharkey-icons.woff create mode 100644 packages/backend/assets/fonts/sharkey-icons/shark-font.svg create mode 100644 packages/backend/assets/fonts/sharkey-icons/shark-font.ttf create mode 100644 packages/backend/assets/fonts/sharkey-icons/shark-font.woff diff --git a/packages/backend/assets/fonts/sharkey-icons/custom-sharkey-icons.svg b/packages/backend/assets/fonts/sharkey-icons/custom-sharkey-icons.svg deleted file mode 100644 index 777fe5ab05..0000000000 --- a/packages/backend/assets/fonts/sharkey-icons/custom-sharkey-icons.svg +++ /dev/null @@ -1,12 +0,0 @@ - - - - - - - - - - - - diff --git a/packages/backend/assets/fonts/sharkey-icons/custom-sharkey-icons.ttf b/packages/backend/assets/fonts/sharkey-icons/custom-sharkey-icons.ttf deleted file mode 100644 index e48ffbdf73..0000000000 Binary files a/packages/backend/assets/fonts/sharkey-icons/custom-sharkey-icons.ttf and /dev/null differ diff --git a/packages/backend/assets/fonts/sharkey-icons/custom-sharkey-icons.woff b/packages/backend/assets/fonts/sharkey-icons/custom-sharkey-icons.woff deleted file mode 100644 index 2ac8730e3e..0000000000 Binary files a/packages/backend/assets/fonts/sharkey-icons/custom-sharkey-icons.woff and /dev/null differ diff --git a/packages/backend/assets/fonts/sharkey-icons/shark-font.svg b/packages/backend/assets/fonts/sharkey-icons/shark-font.svg new file mode 100644 index 0000000000..b67bd7f7d8 --- /dev/null +++ b/packages/backend/assets/fonts/sharkey-icons/shark-font.svg @@ -0,0 +1,30 @@ + + + + + + + + + + + + + + + + diff --git a/packages/backend/assets/fonts/sharkey-icons/shark-font.ttf b/packages/backend/assets/fonts/sharkey-icons/shark-font.ttf new file mode 100644 index 0000000000..0fdf04df08 Binary files /dev/null and b/packages/backend/assets/fonts/sharkey-icons/shark-font.ttf differ diff --git a/packages/backend/assets/fonts/sharkey-icons/shark-font.woff b/packages/backend/assets/fonts/sharkey-icons/shark-font.woff new file mode 100644 index 0000000000..993666bc3a Binary files /dev/null and b/packages/backend/assets/fonts/sharkey-icons/shark-font.woff differ diff --git a/packages/backend/assets/fonts/sharkey-icons/style.css b/packages/backend/assets/fonts/sharkey-icons/style.css index 4cae7787d4..593337e992 100644 --- a/packages/backend/assets/fonts/sharkey-icons/style.css +++ b/packages/backend/assets/fonts/sharkey-icons/style.css @@ -1,35 +1,114 @@ -@charset "UTF-8"; - @font-face { - font-family: "custom-sharkey-icons"; - src: url("./custom-sharkey-icons.woff") format("woff"), - url("./custom-sharkey-icons.ttf") format("truetype"), - url("./custom-sharkey-icons.svg#custom-sharkey-icons") format("svg"); - font-weight: normal; + font-display: auto; + font-family: "shark-font"; font-style: normal; - font-display: block; + font-weight: normal; + + src: url("./shark-font.woff") format("woff"), url("./shark-font.ttf") format("truetype"), url("./shark-font.svg#shark-font") format("svg"); } .sk-icons { - font-family: "custom-sharkey-icons" !important; - font-style: normal; + display: inline-block; + font-family: "shark-font"; font-weight: normal; + font-style: normal; font-variant: normal; - text-transform: none; + text-rendering: auto; line-height: 1; - speak: none; - -webkit-font-smoothing: antialiased; -moz-osx-font-smoothing: grayscale; + -webkit-font-smoothing: antialiased; +} + +.sk-icons-lg { + font-size: 1.33333em; + line-height: 0.75em; + vertical-align: -0.0667em; +} + +.sk-icons-xs { + font-size: 0.75em; +} + +.sk-icons-sm { + font-size: 0.875em; +} + +.sk-icons-1x { + font-size: 1em; +} + +.sk-icons-2x { + font-size: 2em; +} + +.sk-icons-3x { + font-size: 3em; +} + +.sk-icons-4x { + font-size: 4em; +} + +.sk-icons-5x { + font-size: 5em; } -.sk-icons.sk-shark:before { - content: "\61"; +.sk-icons-6x { + font-size: 6em; } -.sk-icons.sk-misskey:before { - content: "\62"; +.sk-icons-7x { + font-size: 7em; } -.sk-icons.sk-foldermove:before { - content: "\63"; +.sk-icons-8x { + font-size: 8em; } + +.sk-icons-9x { + font-size: 9em; +} + +.sk-icons-10x { + font-size: 10em; +} + +.sk-icons-fw { + text-align: center; + width: 1.25em; +} + +.sk-icons-border { + border: solid 0.08em #eee; + border-radius: 0.1em; + padding: 0.2em 0.25em 0.15em; +} + +.sk-icons-pull-left { + float: left; +} + +.sk-icons-pull-right { + float: right; +} + +.sk-icons.sk-icons-pull-left { + margin-right: 0.3em; +} + +.sk-icons.sk-icons-pull-right { + margin-left: 0.3em; +} + + +.sk-icons.sk-foldermove::before { + content: "\ea01"; +} + +.sk-icons.sk-misskey::before { + content: "\ea02"; +} + +.sk-icons.sk-shark::before { + content: "\ea03"; +} \ No newline at end of file -- cgit v1.2.3-freya From 6d7cebb61c506c155da8ca30a97a263dbe7798ba Mon Sep 17 00:00:00 2001 From: Marie Date: Tue, 6 Aug 2024 15:39:19 +0000 Subject: chore: add icon font to contributing --- CONTRIBUTING.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 2acacd6dfa..1999828755 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -94,6 +94,12 @@ The owner [@syuilo](https://github.com/syuilo) merges the PR into the develop br If your language is not listed in Crowdin, please open an issue. +## Icon Font (Shark Font) +Sharkey has its own Icon Font called Shark Font which can be found at https://activitypub.software/TransFem-org/shark-font +Build Instructions can all be found over there in the `README`. + +If you have an Icon Suggestion or want to add an Icon please open an issue over at that repo. + ![Crowdin](https://d322cqt584bo4o.cloudfront.net/misskey/localized.svg) ## Development -- cgit v1.2.3-freya From 45611e504c074fbb9cbcffc4dffc042967aeedf2 Mon Sep 17 00:00:00 2001 From: Marie Date: Tue, 6 Aug 2024 15:43:15 +0000 Subject: chore: fix positioning in contributing --- CONTRIBUTING.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 1999828755..f75fbd05fd 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -94,13 +94,13 @@ The owner [@syuilo](https://github.com/syuilo) merges the PR into the develop br If your language is not listed in Crowdin, please open an issue. +![Crowdin](https://d322cqt584bo4o.cloudfront.net/misskey/localized.svg) + ## Icon Font (Shark Font) Sharkey has its own Icon Font called Shark Font which can be found at https://activitypub.software/TransFem-org/shark-font Build Instructions can all be found over there in the `README`. -If you have an Icon Suggestion or want to add an Icon please open an issue over at that repo. - -![Crowdin](https://d322cqt584bo4o.cloudfront.net/misskey/localized.svg) +If you have an Icon Suggestion or want to add an Icon please open an issue/merge request over at that repo. ## Development During development, it is useful to use the -- cgit v1.2.3-freya From d79880987506f4cfb8b69c8134fc5beb6083e42c Mon Sep 17 00:00:00 2001 From: Marie Date: Tue, 6 Aug 2024 15:54:18 +0000 Subject: upd: add back in timestamps on src --- packages/backend/assets/fonts/sharkey-icons/style.css | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/backend/assets/fonts/sharkey-icons/style.css b/packages/backend/assets/fonts/sharkey-icons/style.css index 593337e992..7168702e5a 100644 --- a/packages/backend/assets/fonts/sharkey-icons/style.css +++ b/packages/backend/assets/fonts/sharkey-icons/style.css @@ -4,7 +4,7 @@ font-style: normal; font-weight: normal; - src: url("./shark-font.woff") format("woff"), url("./shark-font.ttf") format("truetype"), url("./shark-font.svg#shark-font") format("svg"); + src: url("./shark-font.woff?1722899913909") format("woff"), url("./shark-font.ttf?1722899913909") format("truetype"), url("./shark-font.svg?1722899913909#shark-font") format("svg"); } .sk-icons { -- cgit v1.2.3-freya From 1033436349371f2bf4d0eda9da5794030a9ab7b2 Mon Sep 17 00:00:00 2001 From: Marie Date: Tue, 6 Aug 2024 15:58:56 +0000 Subject: chore: add note about updating css and font files --- CONTRIBUTING.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index f75fbd05fd..d7aaa4f555 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -102,6 +102,10 @@ Build Instructions can all be found over there in the `README`. If you have an Icon Suggestion or want to add an Icon please open an issue/merge request over at that repo. +When Updating the Font make sure to copy **all generated files** from the `dest` folder into `packages/backend/assets/fonts/sharkey-icons` + +For the CSS simply copy the file content and replace the old content in `style.css` and for the WOFF, TTF and SVG simply replace them. + ## Development During development, it is useful to use the -- cgit v1.2.3-freya From e82cc99528f7c9fe9e8e8c82d02a8240b46d6a77 Mon Sep 17 00:00:00 2001 From: Marie Date: Tue, 6 Aug 2024 15:59:24 +0000 Subject: chore: remove space --- CONTRIBUTING.md | 1 - 1 file changed, 1 deletion(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index d7aaa4f555..053fffcaeb 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -103,7 +103,6 @@ Build Instructions can all be found over there in the `README`. If you have an Icon Suggestion or want to add an Icon please open an issue/merge request over at that repo. When Updating the Font make sure to copy **all generated files** from the `dest` folder into `packages/backend/assets/fonts/sharkey-icons` - For the CSS simply copy the file content and replace the old content in `style.css` and for the WOFF, TTF and SVG simply replace them. ## Development -- cgit v1.2.3-freya From 9930c64f2d4a198551cfcff1a7c84b5ab10b54f4 Mon Sep 17 00:00:00 2001 From: Hazel K Date: Mon, 5 Aug 2024 21:19:29 -0400 Subject: Fix timeout comments MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: かっこかり <67428053+kakkokari-gtyih@users.noreply.github.com> --- packages/backend/src/core/RelayService.ts | 2 +- packages/backend/src/core/RoleService.ts | 2 +- packages/backend/src/queue/processors/DeliverProcessorService.ts | 2 +- packages/backend/src/server/NodeinfoServerService.ts | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/packages/backend/src/core/RelayService.ts b/packages/backend/src/core/RelayService.ts index 9caeaf1714..91857dc683 100644 --- a/packages/backend/src/core/RelayService.ts +++ b/packages/backend/src/core/RelayService.ts @@ -35,7 +35,7 @@ export class RelayService { private createSystemUserService: CreateSystemUserService, private apRendererService: ApRendererService, ) { - this.relaysCache = new MemorySingleCache(1000 * 60 * 10); // 10s + this.relaysCache = new MemorySingleCache(1000 * 60 * 10); // 10m } @bindThis diff --git a/packages/backend/src/core/RoleService.ts b/packages/backend/src/core/RoleService.ts index f46aacaef4..2b6089fd3a 100644 --- a/packages/backend/src/core/RoleService.ts +++ b/packages/backend/src/core/RoleService.ts @@ -130,7 +130,7 @@ export class RoleService implements OnApplicationShutdown, OnModuleInit { private fanoutTimelineService: FanoutTimelineService, ) { this.rolesCache = new MemorySingleCache(1000 * 60 * 60); // 1h - this.roleAssignmentByUserIdCache = new MemoryKVCache(1000 * 60 * 5); // 1h + this.roleAssignmentByUserIdCache = new MemoryKVCache(1000 * 60 * 5); // 5m this.redisForSub.on('message', this.onMessage); } diff --git a/packages/backend/src/queue/processors/DeliverProcessorService.ts b/packages/backend/src/queue/processors/DeliverProcessorService.ts index 95477aa2cd..4076e9da90 100644 --- a/packages/backend/src/queue/processors/DeliverProcessorService.ts +++ b/packages/backend/src/queue/processors/DeliverProcessorService.ts @@ -45,7 +45,7 @@ export class DeliverProcessorService { private queueLoggerService: QueueLoggerService, ) { this.logger = this.queueLoggerService.logger.createSubLogger('deliver'); - this.suspendedHostsCache = new MemorySingleCache(1000 * 60 * 60); // 1m + this.suspendedHostsCache = new MemorySingleCache(1000 * 60 * 60); // 1h } @bindThis diff --git a/packages/backend/src/server/NodeinfoServerService.ts b/packages/backend/src/server/NodeinfoServerService.ts index bf12822964..bc8d3c0411 100644 --- a/packages/backend/src/server/NodeinfoServerService.ts +++ b/packages/backend/src/server/NodeinfoServerService.ts @@ -135,7 +135,7 @@ export class NodeinfoServerService { return document; }; - const cache = new MemorySingleCache>>(1000 * 60 * 10); // 10s + const cache = new MemorySingleCache>>(1000 * 60 * 10); // 10m fastify.get(nodeinfo2_1path, async (request, reply) => { const base = await cache.fetch(() => nodeinfo2(21)); -- cgit v1.2.3-freya