diff options
| author | Hazelnoot <acomputerdog@gmail.com> | 2025-04-24 14:23:45 -0400 |
|---|---|---|
| committer | Hazelnoot <acomputerdog@gmail.com> | 2025-04-24 14:23:45 -0400 |
| commit | a4dd19fdd427a5adc8fa80871d1c742aa9708730 (patch) | |
| tree | 3983a0e772922042043026b4af168e8dd3525fb2 /packages/backend/src/core | |
| parent | Merge branch 'develop' into merge/2025-03-24 (diff) | |
| parent | enhance(backend): DB note (userId) インデクス -> (userId, id) 複合イ... (diff) | |
| download | sharkey-a4dd19fdd427a5adc8fa80871d1c742aa9708730.tar.gz sharkey-a4dd19fdd427a5adc8fa80871d1c742aa9708730.tar.bz2 sharkey-a4dd19fdd427a5adc8fa80871d1c742aa9708730.zip | |
merge upstream again
Diffstat (limited to 'packages/backend/src/core')
| -rw-r--r-- | packages/backend/src/core/AccountMoveService.ts | 3 | ||||
| -rw-r--r-- | packages/backend/src/core/AntennaService.ts | 69 | ||||
| -rw-r--r-- | packages/backend/src/core/ChatService.ts | 4 | ||||
| -rw-r--r-- | packages/backend/src/core/FanoutTimelineEndpointService.ts | 2 | ||||
| -rw-r--r-- | packages/backend/src/core/MfmService.ts | 10 | ||||
| -rw-r--r-- | packages/backend/src/core/NoteCreateService.ts | 9 | ||||
| -rw-r--r-- | packages/backend/src/core/PushNotificationService.ts | 1 | ||||
| -rw-r--r-- | packages/backend/src/core/QueueService.ts | 498 | ||||
| -rw-r--r-- | packages/backend/src/core/SystemAccountService.ts | 46 | ||||
| -rw-r--r-- | packages/backend/src/core/WebhookTestService.ts | 12 | ||||
| -rw-r--r-- | packages/backend/src/core/activitypub/ApMfmService.ts | 10 | ||||
| -rw-r--r-- | packages/backend/src/core/activitypub/ApRendererService.ts | 20 | ||||
| -rw-r--r-- | packages/backend/src/core/entities/UserEntityService.ts | 12 |
13 files changed, 586 insertions, 110 deletions
diff --git a/packages/backend/src/core/AccountMoveService.ts b/packages/backend/src/core/AccountMoveService.ts index 5128caff60..7bf33e13c5 100644 --- a/packages/backend/src/core/AccountMoveService.ts +++ b/packages/backend/src/core/AccountMoveService.ts @@ -25,6 +25,7 @@ import InstanceChart from '@/core/chart/charts/instance.js'; import PerUserFollowingChart from '@/core/chart/charts/per-user-following.js'; import { SystemAccountService } from '@/core/SystemAccountService.js'; import { RoleService } from '@/core/RoleService.js'; +import { AntennaService } from '@/core/AntennaService.js'; @Injectable() export class AccountMoveService { @@ -66,6 +67,7 @@ export class AccountMoveService { private queueService: QueueService, private systemAccountService: SystemAccountService, private roleService: RoleService, + private antennaService: AntennaService, ) { } @@ -127,6 +129,7 @@ export class AccountMoveService { this.deleteScheduledNotes(src), this.copyRoles(src, dst), this.updateLists(src, dst), + this.antennaService.onMoveAccount(src, dst), ]); } catch { /* skip if any error happens */ diff --git a/packages/backend/src/core/AntennaService.ts b/packages/backend/src/core/AntennaService.ts index 13e3dcdbd8..cf696e3599 100644 --- a/packages/backend/src/core/AntennaService.ts +++ b/packages/backend/src/core/AntennaService.ts @@ -5,18 +5,20 @@ import { Inject, Injectable } from '@nestjs/common'; import * as Redis from 'ioredis'; -import type { MiAntenna } from '@/models/Antenna.js'; -import type { MiNote } from '@/models/Note.js'; -import type { MiUser } from '@/models/User.js'; +import { In } from 'typeorm'; +import { FanoutTimelineService } from '@/core/FanoutTimelineService.js'; +import type { GlobalEvents } from '@/core/GlobalEventService.js'; import { GlobalEventService } from '@/core/GlobalEventService.js'; +import { UtilityService } from '@/core/UtilityService.js'; +import { bindThis } from '@/decorators.js'; +import { DI } from '@/di-symbols.js'; import * as Acct from '@/misc/acct.js'; import type { Packed } from '@/misc/json-schema.js'; -import { DI } from '@/di-symbols.js'; import type { AntennasRepository, UserListMembershipsRepository } from '@/models/_.js'; -import { UtilityService } from '@/core/UtilityService.js'; -import { bindThis } from '@/decorators.js'; -import type { GlobalEvents } from '@/core/GlobalEventService.js'; -import { FanoutTimelineService } from '@/core/FanoutTimelineService.js'; +import type { MiAntenna } from '@/models/Antenna.js'; +import type { MiNote } from '@/models/Note.js'; +import type { MiUser } from '@/models/User.js'; +import { CacheService } from './CacheService.js'; import type { OnApplicationShutdown } from '@nestjs/common'; @Injectable() @@ -37,6 +39,7 @@ export class AntennaService implements OnApplicationShutdown { @Inject(DI.userListMembershipsRepository) private userListMembershipsRepository: UserListMembershipsRepository, + private cacheService: CacheService, private utilityService: UtilityService, private globalEventService: GlobalEventService, private fanoutTimelineService: FanoutTimelineService, @@ -111,9 +114,6 @@ export class AntennaService implements OnApplicationShutdown { @bindThis public async checkHitAntenna(antenna: MiAntenna, note: (MiNote | Packed<'Note'>), noteUser: { id: MiUser['id']; username: string; host: string | null; isBot: boolean; }): Promise<boolean> { - if (note.visibility === 'specified') return false; - if (note.visibility === 'followers') return false; - if (antenna.excludeNotesInSensitiveChannel && note.channel?.isSensitive) return false; if (antenna.excludeBots && noteUser.isBot) return false; @@ -122,6 +122,18 @@ export class AntennaService implements OnApplicationShutdown { if (!antenna.withReplies && note.replyId != null) return false; + if (note.visibility === 'specified') { + if (note.userId !== antenna.userId) { + if (note.visibleUserIds == null) return false; + if (!note.visibleUserIds.includes(antenna.userId)) return false; + } + } + + if (note.visibility === 'followers') { + const isFollowing = Object.hasOwn(await this.cacheService.userFollowingsCache.fetch(antenna.userId), note.userId); + if (!isFollowing && antenna.userId !== note.userId) return false; + } + if (antenna.src === 'home') { // TODO } else if (antenna.src === 'list') { @@ -213,6 +225,41 @@ export class AntennaService implements OnApplicationShutdown { } @bindThis + public async onMoveAccount(src: MiUser, dst: MiUser): Promise<void> { + // There is a possibility for users to add the srcUser to their antennas, but it's low, so we don't check it. + + // Get MiAntenna[] from cache and filter to select antennas with the src user is in the users list + const srcUserAcct = this.utilityService.getFullApAccount(src.username, src.host).toLowerCase(); + const antennasToMigrate = (await this.getAntennas()).filter(antenna => { + return antenna.users.some(user => { + const { username, host } = Acct.parse(user); + return this.utilityService.getFullApAccount(username, host).toLowerCase() === srcUserAcct; + }); + }); + + if (antennasToMigrate.length === 0) return; + + const antennaIds = antennasToMigrate.map(x => x.id); + + // Update the antennas by appending dst users acct to the users list + const dstUserAcct = '@' + Acct.toString({ username: dst.username, host: dst.host }); + + await this.antennasRepository.createQueryBuilder('antenna') + .update() + .set({ + users: () => 'array_append(antenna.users, :dstUserAcct)', + }) + .where('antenna.id IN (:...antennaIds)', { antennaIds }) + .setParameters({ dstUserAcct }) + .execute(); + + // announce update to event + for (const newAntenna of await this.antennasRepository.findBy({ id: In(antennaIds) })) { + this.globalEventService.publishInternalEvent('antennaUpdated', newAntenna); + } + } + + @bindThis public dispose(): void { this.redisForSub.off('message', this.onRedisMessage); } diff --git a/packages/backend/src/core/ChatService.ts b/packages/backend/src/core/ChatService.ts index b0e8cfb61c..9d294a80cb 100644 --- a/packages/backend/src/core/ChatService.ts +++ b/packages/backend/src/core/ChatService.ts @@ -232,7 +232,7 @@ export class ChatService { const packedMessageForTo = await this.chatEntityService.packMessageDetailed(inserted, toUser); this.globalEventService.publishMainStream(toUser.id, 'newChatMessage', packedMessageForTo); - //this.pushNotificationService.pushNotification(toUser.id, 'newChatMessage', packedMessageForTo); + this.pushNotificationService.pushNotification(toUser.id, 'newChatMessage', packedMessageForTo); }, 3000); } @@ -302,7 +302,7 @@ export class ChatService { if (marker == null) continue; this.globalEventService.publishMainStream(membershipsOtherThanMe[i].userId, 'newChatMessage', packedMessageForTo); - //this.pushNotificationService.pushNotification(membershipsOtherThanMe[i].userId, 'newChatMessage', packedMessageForTo); + this.pushNotificationService.pushNotification(membershipsOtherThanMe[i].userId, 'newChatMessage', packedMessageForTo); } }, 3000); diff --git a/packages/backend/src/core/FanoutTimelineEndpointService.ts b/packages/backend/src/core/FanoutTimelineEndpointService.ts index bd86a80cbd..84ca06ec1e 100644 --- a/packages/backend/src/core/FanoutTimelineEndpointService.ts +++ b/packages/backend/src/core/FanoutTimelineEndpointService.ts @@ -55,7 +55,7 @@ export class FanoutTimelineEndpointService { } @bindThis - private async getMiNotes(ps: TimelineOptions): Promise<MiNote[]> { + async getMiNotes(ps: TimelineOptions): Promise<MiNote[]> { // 呼び出し元と以下の処理をシンプルにするためにdbFallbackを置き換える if (!ps.useDbFallback) ps.dbFallback = () => Promise.resolve([]); diff --git a/packages/backend/src/core/MfmService.ts b/packages/backend/src/core/MfmService.ts index 0ef52ee1a6..1ee3bd2275 100644 --- a/packages/backend/src/core/MfmService.ts +++ b/packages/backend/src/core/MfmService.ts @@ -6,7 +6,7 @@ import { URL } from 'node:url'; import { Inject, Injectable } from '@nestjs/common'; import * as parse5 from 'parse5'; -import { Window } from 'happy-dom'; +import { type Document, type HTMLParagraphElement, Window } from 'happy-dom'; import { DI } from '@/di-symbols.js'; import type { Config } from '@/config.js'; import { intersperse } from '@/misc/prelude/array.js'; @@ -23,6 +23,8 @@ type ChildNode = DefaultTreeAdapterMap['childNode']; const urlRegex = /^https?:\/\/[\w\/:%#@$&?!()\[\]~.,=+\-]+/; const urlRegexFull = /^https?:\/\/[\w\/:%#@$&?!()\[\]~.,=+\-]+$/; +export type Appender = (document: Document, body: HTMLParagraphElement) => void; + @Injectable() export class MfmService { constructor( @@ -343,7 +345,7 @@ export class MfmService { } @bindThis - public toHtml(nodes: mfm.MfmNode[] | null, mentionedRemoteUsers: IMentionedRemoteUsers = []) { + public toHtml(nodes: mfm.MfmNode[] | null, mentionedRemoteUsers: IMentionedRemoteUsers = [], additionalAppenders: Appender[] = []) { if (nodes == null) { return null; } @@ -576,6 +578,10 @@ export class MfmService { appendChildren(nodes, body); + for (const additionalAppender of additionalAppenders) { + additionalAppender(doc, body); + } + const serialized = body.outerHTML; happyDOM.close().catch(err => {}); diff --git a/packages/backend/src/core/NoteCreateService.ts b/packages/backend/src/core/NoteCreateService.ts index 5543cc080d..fd6300483f 100644 --- a/packages/backend/src/core/NoteCreateService.ts +++ b/packages/backend/src/core/NoteCreateService.ts @@ -640,7 +640,14 @@ export class NoteCreateService implements OnApplicationShutdown { }, { jobId: `pollEnd:${note.id}`, delay, - removeOnComplete: true, + removeOnComplete: { + age: 3600 * 24 * 7, // keep up to 7 days + count: 30, + }, + removeOnFail: { + age: 3600 * 24 * 7, // keep up to 7 days + count: 100, + }, }); } diff --git a/packages/backend/src/core/PushNotificationService.ts b/packages/backend/src/core/PushNotificationService.ts index 1479bb00d9..9333c1ebc5 100644 --- a/packages/backend/src/core/PushNotificationService.ts +++ b/packages/backend/src/core/PushNotificationService.ts @@ -22,6 +22,7 @@ type PushNotificationsTypes = { note: Packed<'Note'>; }; 'readAllNotifications': undefined; + newChatMessage: Packed<'ChatMessage'>; }; // Reduce length because push message servers have character limits diff --git a/packages/backend/src/core/QueueService.ts b/packages/backend/src/core/QueueService.ts index 039c47724b..fb0fa8f28d 100644 --- a/packages/backend/src/core/QueueService.ts +++ b/packages/backend/src/core/QueueService.ts @@ -5,6 +5,8 @@ import { randomUUID } from 'node:crypto'; import { Inject, Injectable } from '@nestjs/common'; +import { MetricsTime, type JobType } from 'bullmq'; +import { parse as parseRedisInfo } from 'redis-info'; import type { IActivity } from '@/core/activitypub/type.js'; import type { MiDriveFile } from '@/models/DriveFile.js'; import type { MiWebhook, WebhookEventTypes } from '@/models/Webhook.js'; @@ -40,6 +42,18 @@ import type httpSignature from '@peertube/http-signature'; import type * as Bull from 'bullmq'; import { MiNote } from '@/models/Note.js'; +export const QUEUE_TYPES = [ + 'system', + 'endedPollNotification', + 'deliver', + 'inbox', + 'db', + 'relationship', + 'objectStorage', + 'userWebhookDeliver', + 'systemWebhookDeliver', +] as const; + @Injectable() export class QueueService { constructor( @@ -60,50 +74,58 @@ export class QueueService { this.systemQueue.add('tickCharts', { }, { repeat: { pattern: '55 * * * *' }, - removeOnComplete: true, + removeOnComplete: 10, + removeOnFail: 30, }); this.systemQueue.add('resyncCharts', { }, { repeat: { pattern: '0 0 * * *' }, - removeOnComplete: true, + removeOnComplete: 10, + removeOnFail: 30, }); this.systemQueue.add('cleanCharts', { }, { repeat: { pattern: '0 0 * * *' }, - removeOnComplete: true, + removeOnComplete: 10, + removeOnFail: 30, }); this.systemQueue.add('aggregateRetention', { }, { repeat: { pattern: '0 0 * * *' }, - removeOnComplete: true, + removeOnComplete: 10, + removeOnFail: 30, }); this.systemQueue.add('clean', { }, { repeat: { pattern: '0 0 * * *' }, - removeOnComplete: true, + removeOnComplete: 10, + removeOnFail: 30, }); this.systemQueue.add('checkExpiredMutings', { }, { repeat: { pattern: '*/5 * * * *' }, - removeOnComplete: true, + removeOnComplete: 10, + removeOnFail: 30, }); this.systemQueue.add('bakeBufferedReactions', { }, { repeat: { pattern: '0 0 * * *' }, - removeOnComplete: true, + removeOnComplete: 10, + removeOnFail: 30, }); this.systemQueue.add('checkModeratorsActivity', { }, { // 毎時30分に起動 repeat: { pattern: '30 * * * *' }, - removeOnComplete: true, + removeOnComplete: 10, + removeOnFail: 30, }); } @@ -125,13 +147,21 @@ export class QueueService { isSharedInbox, }; - return this.deliverQueue.add(to, data, { + const label = to.replace('https://', '').replace('/inbox', ''); + + return this.deliverQueue.add(label, data, { attempts: this.config.deliverJobMaxAttempts ?? 12, backoff: { type: 'custom', }, - removeOnComplete: true, - removeOnFail: true, + removeOnComplete: { + age: 3600 * 24 * 7, // keep up to 7 days + count: 30, + }, + removeOnFail: { + age: 3600 * 24 * 7, // keep up to 7 days + count: 100, + }, }); } @@ -153,12 +183,18 @@ export class QueueService { backoff: { type: 'custom', }, - removeOnComplete: true, - removeOnFail: true, + removeOnComplete: { + age: 3600 * 24 * 7, // keep up to 7 days + count: 30, + }, + removeOnFail: { + age: 3600 * 24 * 7, // keep up to 7 days + count: 100, + }, }; await this.deliverQueue.addBulk(Array.from(inboxes.entries(), d => ({ - name: d[0], + name: d[0].replace('https://', '').replace('/inbox', ''), data: { user, content: contentBody, @@ -179,13 +215,21 @@ export class QueueService { signature, }; - return this.inboxQueue.add('', data, { + const label = (activity.id ?? '').replace('https://', '').replace('/activity', ''); + + return this.inboxQueue.add(label, data, { attempts: this.config.inboxJobMaxAttempts ?? 8, backoff: { type: 'custom', }, - removeOnComplete: true, - removeOnFail: true, + removeOnComplete: { + age: 3600 * 24 * 7, // keep up to 7 days + count: 30, + }, + removeOnFail: { + age: 3600 * 24 * 7, // keep up to 7 days + count: 100, + }, }); } @@ -194,8 +238,14 @@ export class QueueService { return this.dbQueue.add('deleteDriveFiles', { user: { id: user.id }, }, { - removeOnComplete: true, - removeOnFail: true, + removeOnComplete: { + age: 3600 * 24 * 7, // keep up to 7 days + count: 30, + }, + removeOnFail: { + age: 3600 * 24 * 7, // keep up to 7 days + count: 100, + }, }); } @@ -204,8 +254,14 @@ export class QueueService { return this.dbQueue.add('exportCustomEmojis', { user: { id: user.id }, }, { - removeOnComplete: true, - removeOnFail: true, + removeOnComplete: { + age: 3600 * 24 * 7, // keep up to 7 days + count: 30, + }, + removeOnFail: { + age: 3600 * 24 * 7, // keep up to 7 days + count: 100, + }, }); } @@ -224,8 +280,14 @@ export class QueueService { return this.dbQueue.add('exportNotes', { user: { id: user.id }, }, { - removeOnComplete: true, - removeOnFail: true, + removeOnComplete: { + age: 3600 * 24 * 7, // keep up to 7 days + count: 30, + }, + removeOnFail: { + age: 3600 * 24 * 7, // keep up to 7 days + count: 100, + }, }); } @@ -234,8 +296,14 @@ export class QueueService { return this.dbQueue.add('exportClips', { user: { id: user.id }, }, { - removeOnComplete: true, - removeOnFail: true, + removeOnComplete: { + age: 3600 * 24 * 7, // keep up to 7 days + count: 30, + }, + removeOnFail: { + age: 3600 * 24 * 7, // keep up to 7 days + count: 100, + }, }); } @@ -244,8 +312,14 @@ export class QueueService { return this.dbQueue.add('exportFavorites', { user: { id: user.id }, }, { - removeOnComplete: true, - removeOnFail: true, + removeOnComplete: { + age: 3600 * 24 * 7, // keep up to 7 days + count: 30, + }, + removeOnFail: { + age: 3600 * 24 * 7, // keep up to 7 days + count: 100, + }, }); } @@ -256,8 +330,14 @@ export class QueueService { excludeMuting, excludeInactive, }, { - removeOnComplete: true, - removeOnFail: true, + removeOnComplete: { + age: 3600 * 24 * 7, // keep up to 7 days + count: 30, + }, + removeOnFail: { + age: 3600 * 24 * 7, // keep up to 7 days + count: 100, + }, }); } @@ -266,8 +346,14 @@ export class QueueService { return this.dbQueue.add('exportMuting', { user: { id: user.id }, }, { - removeOnComplete: true, - removeOnFail: true, + removeOnComplete: { + age: 3600 * 24 * 7, // keep up to 7 days + count: 30, + }, + removeOnFail: { + age: 3600 * 24 * 7, // keep up to 7 days + count: 100, + }, }); } @@ -276,8 +362,14 @@ export class QueueService { return this.dbQueue.add('exportBlocking', { user: { id: user.id }, }, { - removeOnComplete: true, - removeOnFail: true, + removeOnComplete: { + age: 3600 * 24 * 7, // keep up to 7 days + count: 30, + }, + removeOnFail: { + age: 3600 * 24 * 7, // keep up to 7 days + count: 100, + }, }); } @@ -286,8 +378,14 @@ export class QueueService { return this.dbQueue.add('exportUserLists', { user: { id: user.id }, }, { - removeOnComplete: true, - removeOnFail: true, + removeOnComplete: { + age: 3600 * 24 * 7, // keep up to 7 days + count: 30, + }, + removeOnFail: { + age: 3600 * 24 * 7, // keep up to 7 days + count: 100, + }, }); } @@ -296,8 +394,14 @@ export class QueueService { return this.dbQueue.add('exportAntennas', { user: { id: user.id }, }, { - removeOnComplete: true, - removeOnFail: true, + removeOnComplete: { + age: 3600 * 24 * 7, // keep up to 7 days + count: 30, + }, + removeOnFail: { + age: 3600 * 24 * 7, // keep up to 7 days + count: 100, + }, }); } @@ -308,8 +412,14 @@ export class QueueService { fileId: fileId, withReplies, }, { - removeOnComplete: true, - removeOnFail: true, + removeOnComplete: { + age: 3600 * 24 * 7, // keep up to 7 days + count: 30, + }, + removeOnFail: { + age: 3600 * 24 * 7, // keep up to 7 days + count: 100, + }, }); } @@ -373,8 +483,14 @@ export class QueueService { user: { id: user.id }, fileId: fileId, }, { - removeOnComplete: true, - removeOnFail: true, + removeOnComplete: { + age: 3600 * 24 * 7, // keep up to 7 days + count: 30, + }, + removeOnFail: { + age: 3600 * 24 * 7, // keep up to 7 days + count: 100, + }, }); } @@ -384,8 +500,14 @@ export class QueueService { user: { id: user.id }, fileId: fileId, }, { - removeOnComplete: true, - removeOnFail: true, + removeOnComplete: { + age: 3600 * 24 * 7, // keep up to 7 days + count: 30, + }, + removeOnFail: { + age: 3600 * 24 * 7, // keep up to 7 days + count: 100, + }, }); } @@ -405,8 +527,14 @@ export class QueueService { name, data, opts: { - removeOnComplete: true, - removeOnFail: true, + removeOnComplete: { + age: 3600 * 24 * 7, // keep up to 7 days + count: 30, + }, + removeOnFail: { + age: 3600 * 24 * 7, // keep up to 7 days + count: 100, + }, }, }; } @@ -417,8 +545,14 @@ export class QueueService { user: { id: user.id }, fileId: fileId, }, { - removeOnComplete: true, - removeOnFail: true, + removeOnComplete: { + age: 3600 * 24 * 7, // keep up to 7 days + count: 30, + }, + removeOnFail: { + age: 3600 * 24 * 7, // keep up to 7 days + count: 100, + }, }); } @@ -428,8 +562,14 @@ export class QueueService { user: { id: user.id }, fileId: fileId, }, { - removeOnComplete: true, - removeOnFail: true, + removeOnComplete: { + age: 3600 * 24 * 7, // keep up to 7 days + count: 30, + }, + removeOnFail: { + age: 3600 * 24 * 7, // keep up to 7 days + count: 100, + }, }); } @@ -439,8 +579,14 @@ export class QueueService { user: { id: user.id }, antenna, }, { - removeOnComplete: true, - removeOnFail: true, + removeOnComplete: { + age: 3600 * 24 * 7, // keep up to 7 days + count: 30, + }, + removeOnFail: { + age: 3600 * 24 * 7, // keep up to 7 days + count: 100, + }, }); } @@ -450,8 +596,14 @@ export class QueueService { user: { id: user.id }, soft: opts.soft, }, { - removeOnComplete: true, - removeOnFail: true, + removeOnComplete: { + age: 3600 * 24 * 7, // keep up to 7 days + count: 30, + }, + removeOnFail: { + age: 3600 * 24 * 7, // keep up to 7 days + count: 100, + }, }); } @@ -501,8 +653,14 @@ export class QueueService { withReplies: data.withReplies, }, opts: { - removeOnComplete: true, - removeOnFail: true, + removeOnComplete: { + age: 3600 * 24 * 7, // keep up to 7 days + count: 30, + }, + removeOnFail: { + age: 3600 * 24 * 7, // keep up to 7 days + count: 100, + }, ...opts, }, }; @@ -513,16 +671,28 @@ export class QueueService { return this.objectStorageQueue.add('deleteFile', { key: key, }, { - removeOnComplete: true, - removeOnFail: true, + removeOnComplete: { + age: 3600 * 24 * 7, // keep up to 7 days + count: 30, + }, + removeOnFail: { + age: 3600 * 24 * 7, // keep up to 7 days + count: 100, + }, }); } @bindThis public createCleanRemoteFilesJob() { return this.objectStorageQueue.add('cleanRemoteFiles', {}, { - removeOnComplete: true, - removeOnFail: true, + removeOnComplete: { + age: 3600 * 24 * 7, // keep up to 7 days + count: 30, + }, + removeOnFail: { + age: 3600 * 24 * 7, // keep up to 7 days + count: 100, + }, }); } @@ -553,8 +723,14 @@ export class QueueService { backoff: { type: 'custom', }, - removeOnComplete: true, - removeOnFail: true, + removeOnComplete: { + age: 3600 * 24 * 7, // keep up to 7 days + count: 30, + }, + removeOnFail: { + age: 3600 * 24 * 7, // keep up to 7 days + count: 100, + }, }); } @@ -584,21 +760,201 @@ export class QueueService { backoff: { type: 'custom', }, - removeOnComplete: true, - removeOnFail: true, + removeOnComplete: { + age: 3600 * 24 * 7, // keep up to 7 days + count: 30, + }, + removeOnFail: { + age: 3600 * 24 * 7, // keep up to 7 days + count: 100, + }, }); } @bindThis - public destroy() { - this.deliverQueue.once('cleaned', (jobs, status) => { - //deliverLogger.succ(`Cleaned ${jobs.length} ${status} jobs`); - }); - this.deliverQueue.clean(0, 0, 'delayed'); + private getQueue(type: typeof QUEUE_TYPES[number]): Bull.Queue { + switch (type) { + case 'system': return this.systemQueue; + case 'endedPollNotification': return this.endedPollNotificationQueue; + case 'deliver': return this.deliverQueue; + case 'inbox': return this.inboxQueue; + case 'db': return this.dbQueue; + case 'relationship': return this.relationshipQueue; + case 'objectStorage': return this.objectStorageQueue; + case 'userWebhookDeliver': return this.userWebhookDeliverQueue; + case 'systemWebhookDeliver': return this.systemWebhookDeliverQueue; + default: throw new Error(`Unrecognized queue type: ${type}`); + } + } + + @bindThis + public async queueClear(queueType: typeof QUEUE_TYPES[number], state: '*' | 'completed' | 'wait' | 'active' | 'paused' | 'prioritized' | 'delayed' | 'failed') { + const queue = this.getQueue(queueType); + + if (state === '*') { + await Promise.all([ + queue.clean(0, 0, 'completed'), + queue.clean(0, 0, 'wait'), + queue.clean(0, 0, 'active'), + queue.clean(0, 0, 'paused'), + queue.clean(0, 0, 'prioritized'), + queue.clean(0, 0, 'delayed'), + queue.clean(0, 0, 'failed'), + ]); + } else { + await queue.clean(0, 0, state); + } + } - this.inboxQueue.once('cleaned', (jobs, status) => { - //inboxLogger.succ(`Cleaned ${jobs.length} ${status} jobs`); + @bindThis + public async queuePromoteJobs(queueType: typeof QUEUE_TYPES[number]) { + const queue = this.getQueue(queueType); + await queue.promoteJobs(); + } + + @bindThis + public async queueRetryJob(queueType: typeof QUEUE_TYPES[number], jobId: string) { + const queue = this.getQueue(queueType); + const job: Bull.Job | null = await queue.getJob(jobId); + if (job) { + if (job.finishedOn != null) { + await job.retry(); + } else { + await job.promote(); + } + } + } + + @bindThis + public async queueRemoveJob(queueType: typeof QUEUE_TYPES[number], jobId: string) { + const queue = this.getQueue(queueType); + const job: Bull.Job | null = await queue.getJob(jobId); + if (job) { + await job.remove(); + } + } + + @bindThis + private packJobData(job: Bull.Job) { + // eslint-disable-next-line @typescript-eslint/no-unnecessary-condition + const stacktrace = job.stacktrace ? job.stacktrace.filter(Boolean) : []; + stacktrace.reverse(); + + return { + id: job.id, + name: job.name, + data: job.data, + opts: job.opts, + timestamp: job.timestamp, + processedOn: job.processedOn, + processedBy: job.processedBy, + finishedOn: job.finishedOn, + progress: job.progress, + attempts: job.attemptsMade, + delay: job.delay, + failedReason: job.failedReason, + stacktrace: stacktrace, + returnValue: job.returnvalue, + isFailed: !!job.failedReason || (Array.isArray(stacktrace) && stacktrace.length > 0), + }; + } + + @bindThis + public async queueGetJob(queueType: typeof QUEUE_TYPES[number], jobId: string) { + const queue = this.getQueue(queueType); + const job: Bull.Job | null = await queue.getJob(jobId); + if (job) { + return this.packJobData(job); + } else { + throw new Error(`Job not found: ${jobId}`); + } + } + + @bindThis + public async queueGetJobs(queueType: typeof QUEUE_TYPES[number], jobTypes: JobType[], search?: string) { + const RETURN_LIMIT = 100; + const queue = this.getQueue(queueType); + let jobs: Bull.Job[]; + + if (search) { + jobs = await queue.getJobs(jobTypes, 0, 1000); + + jobs = jobs.filter(job => { + const jobString = JSON.stringify(job).toLowerCase(); + return search.toLowerCase().split(' ').every(term => { + return jobString.includes(term); + }); + }); + + jobs = jobs.slice(0, RETURN_LIMIT); + } else { + jobs = await queue.getJobs(jobTypes, 0, RETURN_LIMIT); + } + + return jobs.map(job => this.packJobData(job)); + } + + @bindThis + public async queueGetQueues() { + const fetchings = QUEUE_TYPES.map(async type => { + const queue = this.getQueue(type); + + const counts = await queue.getJobCounts(); + const isPaused = await queue.isPaused(); + const metrics_completed = await queue.getMetrics('completed', 0, MetricsTime.ONE_WEEK); + const metrics_failed = await queue.getMetrics('failed', 0, MetricsTime.ONE_WEEK); + + return { + name: type, + counts: counts, + isPaused, + metrics: { + completed: metrics_completed, + failed: metrics_failed, + }, + }; }); - this.inboxQueue.clean(0, 0, 'delayed'); + + return await Promise.all(fetchings); + } + + @bindThis + public async queueGetQueue(queueType: typeof QUEUE_TYPES[number]) { + const queue = this.getQueue(queueType); + const counts = await queue.getJobCounts(); + const isPaused = await queue.isPaused(); + const metrics_completed = await queue.getMetrics('completed', 0, MetricsTime.ONE_WEEK); + const metrics_failed = await queue.getMetrics('failed', 0, MetricsTime.ONE_WEEK); + const db = parseRedisInfo(await (await queue.client).info()); + + return { + name: queueType, + qualifiedName: queue.qualifiedName, + counts: counts, + isPaused, + metrics: { + completed: metrics_completed, + failed: metrics_failed, + }, + db: { + version: db.redis_version, + mode: db.redis_mode, + runId: db.run_id, + processId: db.process_id, + port: parseInt(db.tcp_port), + os: db.os, + uptime: parseInt(db.uptime_in_seconds), + memory: { + total: parseInt(db.total_system_memory) || parseInt(db.maxmemory), + used: parseInt(db.used_memory), + fragmentationRatio: parseInt(db.mem_fragmentation_ratio), + peak: parseInt(db.used_memory_peak), + }, + clients: { + connected: parseInt(db.connected_clients), + blocked: parseInt(db.blocked_clients), + }, + }, + }; } } diff --git a/packages/backend/src/core/SystemAccountService.ts b/packages/backend/src/core/SystemAccountService.ts index 3f3afe6561..1288dc6ffa 100644 --- a/packages/backend/src/core/SystemAccountService.ts +++ b/packages/backend/src/core/SystemAccountService.ts @@ -5,11 +5,14 @@ import { randomUUID } from 'node:crypto'; import { Inject, Injectable } from '@nestjs/common'; +import type { OnApplicationShutdown } from '@nestjs/common'; import { DataSource, IsNull } from 'typeorm'; +import * as Redis from 'ioredis'; import bcrypt from 'bcryptjs'; import { MiLocalUser, MiUser } from '@/models/User.js'; import { MiSystemAccount, MiUsedUsername, MiUserKeypair, MiUserProfile, type UsersRepository, type SystemAccountsRepository } from '@/models/_.js'; import type { MiMeta, UserProfilesRepository } from '@/models/_.js'; +import type { GlobalEvents } from '@/core/GlobalEventService.js'; import { MemoryKVCache } from '@/misc/cache.js'; import { DI } from '@/di-symbols.js'; import { bindThis } from '@/decorators.js'; @@ -20,10 +23,13 @@ import { genRsaKeyPair } from '@/misc/gen-key-pair.js'; export const SYSTEM_ACCOUNT_TYPES = ['actor', 'relay', 'proxy'] as const; @Injectable() -export class SystemAccountService { +export class SystemAccountService implements OnApplicationShutdown { private cache: MemoryKVCache<MiLocalUser>; constructor( + @Inject(DI.redisForSub) + private redisForSub: Redis.Redis, + @Inject(DI.db) private db: DataSource, @@ -42,6 +48,31 @@ export class SystemAccountService { private idService: IdService, ) { this.cache = new MemoryKVCache<MiLocalUser>(1000 * 60 * 10); // 10m + + this.redisForSub.on('message', this.onMessage); + } + + @bindThis + private async onMessage(_: string, data: string): Promise<void> { + const obj = JSON.parse(data); + + if (obj.channel === 'internal') { + const { type, body } = obj.message as GlobalEvents['internal']['payload']; + switch (type) { + case 'metaUpdated': { + if (body.before != null && body.before.name !== body.after.name) { + for (const account of SYSTEM_ACCOUNT_TYPES) { + await this.updateCorrespondingUserProfile(account, { + name: body.after.name, + }); + } + } + break; + } + default: + break; + } + } } @bindThis @@ -151,7 +182,7 @@ export class SystemAccountService { @bindThis public async updateCorrespondingUserProfile(type: typeof SYSTEM_ACCOUNT_TYPES[number], extra: { - name?: string; + name?: string | null; description?: MiUserProfile['description']; }): Promise<MiLocalUser> { const user = await this.fetch(type); @@ -187,4 +218,15 @@ export class SystemAccountService { public async getProxyActor() { return await this.fetch('proxy'); } + + @bindThis + public dispose(): void { + this.redisForSub.off('message', this.onMessage); + this.cache.dispose(); + } + + @bindThis + public onApplicationShutdown(signal?: string): void { + this.dispose(); + } } diff --git a/packages/backend/src/core/WebhookTestService.ts b/packages/backend/src/core/WebhookTestService.ts index 469b396fb0..2f8cfea7f7 100644 --- a/packages/backend/src/core/WebhookTestService.ts +++ b/packages/backend/src/core/WebhookTestService.ts @@ -431,8 +431,8 @@ export class WebhookTestService { name: user.name, username: user.username, host: user.host, - avatarUrl: user.avatarUrl, - avatarBlurhash: user.avatarBlurhash, + avatarUrl: user.avatarId == null ? null : user.avatarUrl, + avatarBlurhash: user.avatarId == null ? null : user.avatarBlurhash, avatarDecorations: user.avatarDecorations.map(it => ({ id: it.id, angle: it.angle, @@ -464,10 +464,10 @@ export class WebhookTestService { createdAt: new Date().toISOString(), updatedAt: user.updatedAt?.toISOString() ?? null, lastFetchedAt: user.lastFetchedAt?.toISOString() ?? null, - bannerUrl: user.bannerUrl, - bannerBlurhash: user.bannerBlurhash, - backgroundUrl: user.backgroundUrl, - backgroundBlurhash: user.backgroundBlurhash, + bannerUrl: user.bannerId == null ? null : user.bannerUrl, + bannerBlurhash: user.bannerId == null ? null : user.bannerBlurhash, + backgroundUrl: user.backgroundId == null ? null : user.backgroundUrl, + backgroundBlurhash: user.backgroundId == null ? null : user.backgroundBlurhash, listenbrainz: null, isLocked: user.isLocked, isSilenced: false, diff --git a/packages/backend/src/core/activitypub/ApMfmService.ts b/packages/backend/src/core/activitypub/ApMfmService.ts index 318710fa93..c4a948429a 100644 --- a/packages/backend/src/core/activitypub/ApMfmService.ts +++ b/packages/backend/src/core/activitypub/ApMfmService.ts @@ -5,7 +5,7 @@ import { Injectable } from '@nestjs/common'; import * as mfm from '@transfem-org/sfm-js'; -import { MfmService } from '@/core/MfmService.js'; +import { MfmService, Appender } from '@/core/MfmService.js'; import type { MiNote } from '@/models/Note.js'; import { bindThis } from '@/decorators.js'; import { extractApHashtagObjects } from './models/tag.js'; @@ -25,17 +25,17 @@ export class ApMfmService { } @bindThis - public getNoteHtml(note: Pick<MiNote, 'text' | 'mentionedRemoteUsers'>, apAppend?: string) { + public getNoteHtml(note: Pick<MiNote, 'text' | 'mentionedRemoteUsers'>, additionalAppender: Appender[] = []) { let noMisskeyContent = false; - const srcMfm = (note.text ?? '') + (apAppend ?? ''); + const srcMfm = (note.text ?? ''); const parsed = mfm.parse(srcMfm); - if (!apAppend && parsed?.every(n => ['text', 'unicodeEmoji', 'emojiCode', 'mention', 'hashtag', 'url'].includes(n.type))) { + if (!additionalAppender.length && parsed.every(n => ['text', 'unicodeEmoji', 'emojiCode', 'mention', 'hashtag', 'url'].includes(n.type))) { noMisskeyContent = true; } - const content = this.mfmService.toHtml(parsed, note.mentionedRemoteUsers ? JSON.parse(note.mentionedRemoteUsers) : []); + const content = this.mfmService.toHtml(parsed, note.mentionedRemoteUsers ? JSON.parse(note.mentionedRemoteUsers) : [], additionalAppender); return { content, diff --git a/packages/backend/src/core/activitypub/ApRendererService.ts b/packages/backend/src/core/activitypub/ApRendererService.ts index e6eb777f1b..8251bc3b15 100644 --- a/packages/backend/src/core/activitypub/ApRendererService.ts +++ b/packages/backend/src/core/activitypub/ApRendererService.ts @@ -20,7 +20,7 @@ import type { MiEmoji } from '@/models/Emoji.js'; import type { MiPoll } from '@/models/Poll.js'; import type { MiPollVote } from '@/models/PollVote.js'; import { UserKeypairService } from '@/core/UserKeypairService.js'; -import { MfmService } from '@/core/MfmService.js'; +import { MfmService, type Appender } from '@/core/MfmService.js'; import { UserEntityService } from '@/core/entities/UserEntityService.js'; import { DriveFileEntityService } from '@/core/entities/DriveFileEntityService.js'; import type { MiUserKeypair } from '@/models/UserKeypair.js'; @@ -468,10 +468,24 @@ export class ApRendererService { poll = await this.pollsRepository.findOneBy({ noteId: note.id }); } - let apAppend = ''; + const apAppend: Appender[] = []; if (quote) { - apAppend += `\n\nRE: ${quote}`; + // Append quote link as `<br><br><span class="quote-inline">RE: <a href="...">...</a></span>` + // the claas name `quote-inline` is used in non-misskey clients for styling quote notes. + // For compatibility, the span part should be kept as possible. + apAppend.push((doc, body) => { + body.appendChild(doc.createElement('br')); + body.appendChild(doc.createElement('br')); + const span = doc.createElement('span'); + span.className = 'quote-inline'; + span.appendChild(doc.createTextNode('RE: ')); + const link = doc.createElement('a'); + link.setAttribute('href', quote); + link.textContent = quote; + span.appendChild(link); + body.appendChild(span); + }); } let summary = note.cw === '' ? String.fromCharCode(0x200B) : note.cw; diff --git a/packages/backend/src/core/entities/UserEntityService.ts b/packages/backend/src/core/entities/UserEntityService.ts index d11042d73b..16425bb6ce 100644 --- a/packages/backend/src/core/entities/UserEntityService.ts +++ b/packages/backend/src/core/entities/UserEntityService.ts @@ -584,8 +584,8 @@ export class UserEntityService implements OnModuleInit { name: user.name, username: user.username, host: user.host, - avatarUrl: user.avatarUrl ?? this.getIdenticonUrl(user), - avatarBlurhash: user.avatarBlurhash, + avatarUrl: (user.avatarId == null ? null : user.avatarUrl) ?? this.getIdenticonUrl(user), + avatarBlurhash: (user.avatarId == null ? null : user.avatarBlurhash), description: mastoapi ? mastoapi.description : profile ? profile.description : '', createdAt: this.idService.parse(user.id).date.toISOString(), avatarDecorations: user.avatarDecorations.length > 0 ? this.avatarDecorationService.getAll().then(decorations => user.avatarDecorations.filter(ud => decorations.some(d => d.id === ud.id)).map(ud => ({ @@ -643,10 +643,10 @@ export class UserEntityService implements OnModuleInit { : null, updatedAt: user.updatedAt ? user.updatedAt.toISOString() : null, lastFetchedAt: user.lastFetchedAt ? user.lastFetchedAt.toISOString() : null, - bannerUrl: user.bannerUrl, - bannerBlurhash: user.bannerBlurhash, - backgroundUrl: user.backgroundUrl, - backgroundBlurhash: user.backgroundBlurhash, + bannerUrl: user.bannerId == null ? null : user.bannerUrl, + bannerBlurhash: user.bannerId == null ? null : user.bannerBlurhash, + backgroundUrl: user.backgroundUrl == null ? null : user.backgroundUrl, + backgroundBlurhash: user.backgroundBlurhash == null ? null : user.backgroundBlurhash, isLocked: user.isLocked, isSuspended: user.isSuspended, location: profile!.location, |