summaryrefslogtreecommitdiff
path: root/packages/backend/src/core
diff options
context:
space:
mode:
authorHazelnoot <acomputerdog@gmail.com>2025-04-24 14:23:45 -0400
committerHazelnoot <acomputerdog@gmail.com>2025-04-24 14:23:45 -0400
commita4dd19fdd427a5adc8fa80871d1c742aa9708730 (patch)
tree3983a0e772922042043026b4af168e8dd3525fb2 /packages/backend/src/core
parentMerge branch 'develop' into merge/2025-03-24 (diff)
parentenhance(backend): DB note (userId) インデクス -> (userId, id) 複合イ... (diff)
downloadsharkey-a4dd19fdd427a5adc8fa80871d1c742aa9708730.tar.gz
sharkey-a4dd19fdd427a5adc8fa80871d1c742aa9708730.tar.bz2
sharkey-a4dd19fdd427a5adc8fa80871d1c742aa9708730.zip
merge upstream again
Diffstat (limited to 'packages/backend/src/core')
-rw-r--r--packages/backend/src/core/AccountMoveService.ts3
-rw-r--r--packages/backend/src/core/AntennaService.ts69
-rw-r--r--packages/backend/src/core/ChatService.ts4
-rw-r--r--packages/backend/src/core/FanoutTimelineEndpointService.ts2
-rw-r--r--packages/backend/src/core/MfmService.ts10
-rw-r--r--packages/backend/src/core/NoteCreateService.ts9
-rw-r--r--packages/backend/src/core/PushNotificationService.ts1
-rw-r--r--packages/backend/src/core/QueueService.ts498
-rw-r--r--packages/backend/src/core/SystemAccountService.ts46
-rw-r--r--packages/backend/src/core/WebhookTestService.ts12
-rw-r--r--packages/backend/src/core/activitypub/ApMfmService.ts10
-rw-r--r--packages/backend/src/core/activitypub/ApRendererService.ts20
-rw-r--r--packages/backend/src/core/entities/UserEntityService.ts12
13 files changed, 586 insertions, 110 deletions
diff --git a/packages/backend/src/core/AccountMoveService.ts b/packages/backend/src/core/AccountMoveService.ts
index 5128caff60..7bf33e13c5 100644
--- a/packages/backend/src/core/AccountMoveService.ts
+++ b/packages/backend/src/core/AccountMoveService.ts
@@ -25,6 +25,7 @@ import InstanceChart from '@/core/chart/charts/instance.js';
import PerUserFollowingChart from '@/core/chart/charts/per-user-following.js';
import { SystemAccountService } from '@/core/SystemAccountService.js';
import { RoleService } from '@/core/RoleService.js';
+import { AntennaService } from '@/core/AntennaService.js';
@Injectable()
export class AccountMoveService {
@@ -66,6 +67,7 @@ export class AccountMoveService {
private queueService: QueueService,
private systemAccountService: SystemAccountService,
private roleService: RoleService,
+ private antennaService: AntennaService,
) {
}
@@ -127,6 +129,7 @@ export class AccountMoveService {
this.deleteScheduledNotes(src),
this.copyRoles(src, dst),
this.updateLists(src, dst),
+ this.antennaService.onMoveAccount(src, dst),
]);
} catch {
/* skip if any error happens */
diff --git a/packages/backend/src/core/AntennaService.ts b/packages/backend/src/core/AntennaService.ts
index 13e3dcdbd8..cf696e3599 100644
--- a/packages/backend/src/core/AntennaService.ts
+++ b/packages/backend/src/core/AntennaService.ts
@@ -5,18 +5,20 @@
import { Inject, Injectable } from '@nestjs/common';
import * as Redis from 'ioredis';
-import type { MiAntenna } from '@/models/Antenna.js';
-import type { MiNote } from '@/models/Note.js';
-import type { MiUser } from '@/models/User.js';
+import { In } from 'typeorm';
+import { FanoutTimelineService } from '@/core/FanoutTimelineService.js';
+import type { GlobalEvents } from '@/core/GlobalEventService.js';
import { GlobalEventService } from '@/core/GlobalEventService.js';
+import { UtilityService } from '@/core/UtilityService.js';
+import { bindThis } from '@/decorators.js';
+import { DI } from '@/di-symbols.js';
import * as Acct from '@/misc/acct.js';
import type { Packed } from '@/misc/json-schema.js';
-import { DI } from '@/di-symbols.js';
import type { AntennasRepository, UserListMembershipsRepository } from '@/models/_.js';
-import { UtilityService } from '@/core/UtilityService.js';
-import { bindThis } from '@/decorators.js';
-import type { GlobalEvents } from '@/core/GlobalEventService.js';
-import { FanoutTimelineService } from '@/core/FanoutTimelineService.js';
+import type { MiAntenna } from '@/models/Antenna.js';
+import type { MiNote } from '@/models/Note.js';
+import type { MiUser } from '@/models/User.js';
+import { CacheService } from './CacheService.js';
import type { OnApplicationShutdown } from '@nestjs/common';
@Injectable()
@@ -37,6 +39,7 @@ export class AntennaService implements OnApplicationShutdown {
@Inject(DI.userListMembershipsRepository)
private userListMembershipsRepository: UserListMembershipsRepository,
+ private cacheService: CacheService,
private utilityService: UtilityService,
private globalEventService: GlobalEventService,
private fanoutTimelineService: FanoutTimelineService,
@@ -111,9 +114,6 @@ export class AntennaService implements OnApplicationShutdown {
@bindThis
public async checkHitAntenna(antenna: MiAntenna, note: (MiNote | Packed<'Note'>), noteUser: { id: MiUser['id']; username: string; host: string | null; isBot: boolean; }): Promise<boolean> {
- if (note.visibility === 'specified') return false;
- if (note.visibility === 'followers') return false;
-
if (antenna.excludeNotesInSensitiveChannel && note.channel?.isSensitive) return false;
if (antenna.excludeBots && noteUser.isBot) return false;
@@ -122,6 +122,18 @@ export class AntennaService implements OnApplicationShutdown {
if (!antenna.withReplies && note.replyId != null) return false;
+ if (note.visibility === 'specified') {
+ if (note.userId !== antenna.userId) {
+ if (note.visibleUserIds == null) return false;
+ if (!note.visibleUserIds.includes(antenna.userId)) return false;
+ }
+ }
+
+ if (note.visibility === 'followers') {
+ const isFollowing = Object.hasOwn(await this.cacheService.userFollowingsCache.fetch(antenna.userId), note.userId);
+ if (!isFollowing && antenna.userId !== note.userId) return false;
+ }
+
if (antenna.src === 'home') {
// TODO
} else if (antenna.src === 'list') {
@@ -213,6 +225,41 @@ export class AntennaService implements OnApplicationShutdown {
}
@bindThis
+ public async onMoveAccount(src: MiUser, dst: MiUser): Promise<void> {
+ // There is a possibility for users to add the srcUser to their antennas, but it's low, so we don't check it.
+
+ // Get MiAntenna[] from cache and filter to select antennas with the src user is in the users list
+ const srcUserAcct = this.utilityService.getFullApAccount(src.username, src.host).toLowerCase();
+ const antennasToMigrate = (await this.getAntennas()).filter(antenna => {
+ return antenna.users.some(user => {
+ const { username, host } = Acct.parse(user);
+ return this.utilityService.getFullApAccount(username, host).toLowerCase() === srcUserAcct;
+ });
+ });
+
+ if (antennasToMigrate.length === 0) return;
+
+ const antennaIds = antennasToMigrate.map(x => x.id);
+
+ // Update the antennas by appending dst users acct to the users list
+ const dstUserAcct = '@' + Acct.toString({ username: dst.username, host: dst.host });
+
+ await this.antennasRepository.createQueryBuilder('antenna')
+ .update()
+ .set({
+ users: () => 'array_append(antenna.users, :dstUserAcct)',
+ })
+ .where('antenna.id IN (:...antennaIds)', { antennaIds })
+ .setParameters({ dstUserAcct })
+ .execute();
+
+ // announce update to event
+ for (const newAntenna of await this.antennasRepository.findBy({ id: In(antennaIds) })) {
+ this.globalEventService.publishInternalEvent('antennaUpdated', newAntenna);
+ }
+ }
+
+ @bindThis
public dispose(): void {
this.redisForSub.off('message', this.onRedisMessage);
}
diff --git a/packages/backend/src/core/ChatService.ts b/packages/backend/src/core/ChatService.ts
index b0e8cfb61c..9d294a80cb 100644
--- a/packages/backend/src/core/ChatService.ts
+++ b/packages/backend/src/core/ChatService.ts
@@ -232,7 +232,7 @@ export class ChatService {
const packedMessageForTo = await this.chatEntityService.packMessageDetailed(inserted, toUser);
this.globalEventService.publishMainStream(toUser.id, 'newChatMessage', packedMessageForTo);
- //this.pushNotificationService.pushNotification(toUser.id, 'newChatMessage', packedMessageForTo);
+ this.pushNotificationService.pushNotification(toUser.id, 'newChatMessage', packedMessageForTo);
}, 3000);
}
@@ -302,7 +302,7 @@ export class ChatService {
if (marker == null) continue;
this.globalEventService.publishMainStream(membershipsOtherThanMe[i].userId, 'newChatMessage', packedMessageForTo);
- //this.pushNotificationService.pushNotification(membershipsOtherThanMe[i].userId, 'newChatMessage', packedMessageForTo);
+ this.pushNotificationService.pushNotification(membershipsOtherThanMe[i].userId, 'newChatMessage', packedMessageForTo);
}
}, 3000);
diff --git a/packages/backend/src/core/FanoutTimelineEndpointService.ts b/packages/backend/src/core/FanoutTimelineEndpointService.ts
index bd86a80cbd..84ca06ec1e 100644
--- a/packages/backend/src/core/FanoutTimelineEndpointService.ts
+++ b/packages/backend/src/core/FanoutTimelineEndpointService.ts
@@ -55,7 +55,7 @@ export class FanoutTimelineEndpointService {
}
@bindThis
- private async getMiNotes(ps: TimelineOptions): Promise<MiNote[]> {
+ async getMiNotes(ps: TimelineOptions): Promise<MiNote[]> {
// 呼び出し元と以下の処理をシンプルにするためにdbFallbackを置き換える
if (!ps.useDbFallback) ps.dbFallback = () => Promise.resolve([]);
diff --git a/packages/backend/src/core/MfmService.ts b/packages/backend/src/core/MfmService.ts
index 0ef52ee1a6..1ee3bd2275 100644
--- a/packages/backend/src/core/MfmService.ts
+++ b/packages/backend/src/core/MfmService.ts
@@ -6,7 +6,7 @@
import { URL } from 'node:url';
import { Inject, Injectable } from '@nestjs/common';
import * as parse5 from 'parse5';
-import { Window } from 'happy-dom';
+import { type Document, type HTMLParagraphElement, Window } from 'happy-dom';
import { DI } from '@/di-symbols.js';
import type { Config } from '@/config.js';
import { intersperse } from '@/misc/prelude/array.js';
@@ -23,6 +23,8 @@ type ChildNode = DefaultTreeAdapterMap['childNode'];
const urlRegex = /^https?:\/\/[\w\/:%#@$&?!()\[\]~.,=+\-]+/;
const urlRegexFull = /^https?:\/\/[\w\/:%#@$&?!()\[\]~.,=+\-]+$/;
+export type Appender = (document: Document, body: HTMLParagraphElement) => void;
+
@Injectable()
export class MfmService {
constructor(
@@ -343,7 +345,7 @@ export class MfmService {
}
@bindThis
- public toHtml(nodes: mfm.MfmNode[] | null, mentionedRemoteUsers: IMentionedRemoteUsers = []) {
+ public toHtml(nodes: mfm.MfmNode[] | null, mentionedRemoteUsers: IMentionedRemoteUsers = [], additionalAppenders: Appender[] = []) {
if (nodes == null) {
return null;
}
@@ -576,6 +578,10 @@ export class MfmService {
appendChildren(nodes, body);
+ for (const additionalAppender of additionalAppenders) {
+ additionalAppender(doc, body);
+ }
+
const serialized = body.outerHTML;
happyDOM.close().catch(err => {});
diff --git a/packages/backend/src/core/NoteCreateService.ts b/packages/backend/src/core/NoteCreateService.ts
index 5543cc080d..fd6300483f 100644
--- a/packages/backend/src/core/NoteCreateService.ts
+++ b/packages/backend/src/core/NoteCreateService.ts
@@ -640,7 +640,14 @@ export class NoteCreateService implements OnApplicationShutdown {
}, {
jobId: `pollEnd:${note.id}`,
delay,
- removeOnComplete: true,
+ removeOnComplete: {
+ age: 3600 * 24 * 7, // keep up to 7 days
+ count: 30,
+ },
+ removeOnFail: {
+ age: 3600 * 24 * 7, // keep up to 7 days
+ count: 100,
+ },
});
}
diff --git a/packages/backend/src/core/PushNotificationService.ts b/packages/backend/src/core/PushNotificationService.ts
index 1479bb00d9..9333c1ebc5 100644
--- a/packages/backend/src/core/PushNotificationService.ts
+++ b/packages/backend/src/core/PushNotificationService.ts
@@ -22,6 +22,7 @@ type PushNotificationsTypes = {
note: Packed<'Note'>;
};
'readAllNotifications': undefined;
+ newChatMessage: Packed<'ChatMessage'>;
};
// Reduce length because push message servers have character limits
diff --git a/packages/backend/src/core/QueueService.ts b/packages/backend/src/core/QueueService.ts
index 039c47724b..fb0fa8f28d 100644
--- a/packages/backend/src/core/QueueService.ts
+++ b/packages/backend/src/core/QueueService.ts
@@ -5,6 +5,8 @@
import { randomUUID } from 'node:crypto';
import { Inject, Injectable } from '@nestjs/common';
+import { MetricsTime, type JobType } from 'bullmq';
+import { parse as parseRedisInfo } from 'redis-info';
import type { IActivity } from '@/core/activitypub/type.js';
import type { MiDriveFile } from '@/models/DriveFile.js';
import type { MiWebhook, WebhookEventTypes } from '@/models/Webhook.js';
@@ -40,6 +42,18 @@ import type httpSignature from '@peertube/http-signature';
import type * as Bull from 'bullmq';
import { MiNote } from '@/models/Note.js';
+export const QUEUE_TYPES = [
+ 'system',
+ 'endedPollNotification',
+ 'deliver',
+ 'inbox',
+ 'db',
+ 'relationship',
+ 'objectStorage',
+ 'userWebhookDeliver',
+ 'systemWebhookDeliver',
+] as const;
+
@Injectable()
export class QueueService {
constructor(
@@ -60,50 +74,58 @@ export class QueueService {
this.systemQueue.add('tickCharts', {
}, {
repeat: { pattern: '55 * * * *' },
- removeOnComplete: true,
+ removeOnComplete: 10,
+ removeOnFail: 30,
});
this.systemQueue.add('resyncCharts', {
}, {
repeat: { pattern: '0 0 * * *' },
- removeOnComplete: true,
+ removeOnComplete: 10,
+ removeOnFail: 30,
});
this.systemQueue.add('cleanCharts', {
}, {
repeat: { pattern: '0 0 * * *' },
- removeOnComplete: true,
+ removeOnComplete: 10,
+ removeOnFail: 30,
});
this.systemQueue.add('aggregateRetention', {
}, {
repeat: { pattern: '0 0 * * *' },
- removeOnComplete: true,
+ removeOnComplete: 10,
+ removeOnFail: 30,
});
this.systemQueue.add('clean', {
}, {
repeat: { pattern: '0 0 * * *' },
- removeOnComplete: true,
+ removeOnComplete: 10,
+ removeOnFail: 30,
});
this.systemQueue.add('checkExpiredMutings', {
}, {
repeat: { pattern: '*/5 * * * *' },
- removeOnComplete: true,
+ removeOnComplete: 10,
+ removeOnFail: 30,
});
this.systemQueue.add('bakeBufferedReactions', {
}, {
repeat: { pattern: '0 0 * * *' },
- removeOnComplete: true,
+ removeOnComplete: 10,
+ removeOnFail: 30,
});
this.systemQueue.add('checkModeratorsActivity', {
}, {
// 毎時30分に起動
repeat: { pattern: '30 * * * *' },
- removeOnComplete: true,
+ removeOnComplete: 10,
+ removeOnFail: 30,
});
}
@@ -125,13 +147,21 @@ export class QueueService {
isSharedInbox,
};
- return this.deliverQueue.add(to, data, {
+ const label = to.replace('https://', '').replace('/inbox', '');
+
+ return this.deliverQueue.add(label, data, {
attempts: this.config.deliverJobMaxAttempts ?? 12,
backoff: {
type: 'custom',
},
- removeOnComplete: true,
- removeOnFail: true,
+ removeOnComplete: {
+ age: 3600 * 24 * 7, // keep up to 7 days
+ count: 30,
+ },
+ removeOnFail: {
+ age: 3600 * 24 * 7, // keep up to 7 days
+ count: 100,
+ },
});
}
@@ -153,12 +183,18 @@ export class QueueService {
backoff: {
type: 'custom',
},
- removeOnComplete: true,
- removeOnFail: true,
+ removeOnComplete: {
+ age: 3600 * 24 * 7, // keep up to 7 days
+ count: 30,
+ },
+ removeOnFail: {
+ age: 3600 * 24 * 7, // keep up to 7 days
+ count: 100,
+ },
};
await this.deliverQueue.addBulk(Array.from(inboxes.entries(), d => ({
- name: d[0],
+ name: d[0].replace('https://', '').replace('/inbox', ''),
data: {
user,
content: contentBody,
@@ -179,13 +215,21 @@ export class QueueService {
signature,
};
- return this.inboxQueue.add('', data, {
+ const label = (activity.id ?? '').replace('https://', '').replace('/activity', '');
+
+ return this.inboxQueue.add(label, data, {
attempts: this.config.inboxJobMaxAttempts ?? 8,
backoff: {
type: 'custom',
},
- removeOnComplete: true,
- removeOnFail: true,
+ removeOnComplete: {
+ age: 3600 * 24 * 7, // keep up to 7 days
+ count: 30,
+ },
+ removeOnFail: {
+ age: 3600 * 24 * 7, // keep up to 7 days
+ count: 100,
+ },
});
}
@@ -194,8 +238,14 @@ export class QueueService {
return this.dbQueue.add('deleteDriveFiles', {
user: { id: user.id },
}, {
- removeOnComplete: true,
- removeOnFail: true,
+ removeOnComplete: {
+ age: 3600 * 24 * 7, // keep up to 7 days
+ count: 30,
+ },
+ removeOnFail: {
+ age: 3600 * 24 * 7, // keep up to 7 days
+ count: 100,
+ },
});
}
@@ -204,8 +254,14 @@ export class QueueService {
return this.dbQueue.add('exportCustomEmojis', {
user: { id: user.id },
}, {
- removeOnComplete: true,
- removeOnFail: true,
+ removeOnComplete: {
+ age: 3600 * 24 * 7, // keep up to 7 days
+ count: 30,
+ },
+ removeOnFail: {
+ age: 3600 * 24 * 7, // keep up to 7 days
+ count: 100,
+ },
});
}
@@ -224,8 +280,14 @@ export class QueueService {
return this.dbQueue.add('exportNotes', {
user: { id: user.id },
}, {
- removeOnComplete: true,
- removeOnFail: true,
+ removeOnComplete: {
+ age: 3600 * 24 * 7, // keep up to 7 days
+ count: 30,
+ },
+ removeOnFail: {
+ age: 3600 * 24 * 7, // keep up to 7 days
+ count: 100,
+ },
});
}
@@ -234,8 +296,14 @@ export class QueueService {
return this.dbQueue.add('exportClips', {
user: { id: user.id },
}, {
- removeOnComplete: true,
- removeOnFail: true,
+ removeOnComplete: {
+ age: 3600 * 24 * 7, // keep up to 7 days
+ count: 30,
+ },
+ removeOnFail: {
+ age: 3600 * 24 * 7, // keep up to 7 days
+ count: 100,
+ },
});
}
@@ -244,8 +312,14 @@ export class QueueService {
return this.dbQueue.add('exportFavorites', {
user: { id: user.id },
}, {
- removeOnComplete: true,
- removeOnFail: true,
+ removeOnComplete: {
+ age: 3600 * 24 * 7, // keep up to 7 days
+ count: 30,
+ },
+ removeOnFail: {
+ age: 3600 * 24 * 7, // keep up to 7 days
+ count: 100,
+ },
});
}
@@ -256,8 +330,14 @@ export class QueueService {
excludeMuting,
excludeInactive,
}, {
- removeOnComplete: true,
- removeOnFail: true,
+ removeOnComplete: {
+ age: 3600 * 24 * 7, // keep up to 7 days
+ count: 30,
+ },
+ removeOnFail: {
+ age: 3600 * 24 * 7, // keep up to 7 days
+ count: 100,
+ },
});
}
@@ -266,8 +346,14 @@ export class QueueService {
return this.dbQueue.add('exportMuting', {
user: { id: user.id },
}, {
- removeOnComplete: true,
- removeOnFail: true,
+ removeOnComplete: {
+ age: 3600 * 24 * 7, // keep up to 7 days
+ count: 30,
+ },
+ removeOnFail: {
+ age: 3600 * 24 * 7, // keep up to 7 days
+ count: 100,
+ },
});
}
@@ -276,8 +362,14 @@ export class QueueService {
return this.dbQueue.add('exportBlocking', {
user: { id: user.id },
}, {
- removeOnComplete: true,
- removeOnFail: true,
+ removeOnComplete: {
+ age: 3600 * 24 * 7, // keep up to 7 days
+ count: 30,
+ },
+ removeOnFail: {
+ age: 3600 * 24 * 7, // keep up to 7 days
+ count: 100,
+ },
});
}
@@ -286,8 +378,14 @@ export class QueueService {
return this.dbQueue.add('exportUserLists', {
user: { id: user.id },
}, {
- removeOnComplete: true,
- removeOnFail: true,
+ removeOnComplete: {
+ age: 3600 * 24 * 7, // keep up to 7 days
+ count: 30,
+ },
+ removeOnFail: {
+ age: 3600 * 24 * 7, // keep up to 7 days
+ count: 100,
+ },
});
}
@@ -296,8 +394,14 @@ export class QueueService {
return this.dbQueue.add('exportAntennas', {
user: { id: user.id },
}, {
- removeOnComplete: true,
- removeOnFail: true,
+ removeOnComplete: {
+ age: 3600 * 24 * 7, // keep up to 7 days
+ count: 30,
+ },
+ removeOnFail: {
+ age: 3600 * 24 * 7, // keep up to 7 days
+ count: 100,
+ },
});
}
@@ -308,8 +412,14 @@ export class QueueService {
fileId: fileId,
withReplies,
}, {
- removeOnComplete: true,
- removeOnFail: true,
+ removeOnComplete: {
+ age: 3600 * 24 * 7, // keep up to 7 days
+ count: 30,
+ },
+ removeOnFail: {
+ age: 3600 * 24 * 7, // keep up to 7 days
+ count: 100,
+ },
});
}
@@ -373,8 +483,14 @@ export class QueueService {
user: { id: user.id },
fileId: fileId,
}, {
- removeOnComplete: true,
- removeOnFail: true,
+ removeOnComplete: {
+ age: 3600 * 24 * 7, // keep up to 7 days
+ count: 30,
+ },
+ removeOnFail: {
+ age: 3600 * 24 * 7, // keep up to 7 days
+ count: 100,
+ },
});
}
@@ -384,8 +500,14 @@ export class QueueService {
user: { id: user.id },
fileId: fileId,
}, {
- removeOnComplete: true,
- removeOnFail: true,
+ removeOnComplete: {
+ age: 3600 * 24 * 7, // keep up to 7 days
+ count: 30,
+ },
+ removeOnFail: {
+ age: 3600 * 24 * 7, // keep up to 7 days
+ count: 100,
+ },
});
}
@@ -405,8 +527,14 @@ export class QueueService {
name,
data,
opts: {
- removeOnComplete: true,
- removeOnFail: true,
+ removeOnComplete: {
+ age: 3600 * 24 * 7, // keep up to 7 days
+ count: 30,
+ },
+ removeOnFail: {
+ age: 3600 * 24 * 7, // keep up to 7 days
+ count: 100,
+ },
},
};
}
@@ -417,8 +545,14 @@ export class QueueService {
user: { id: user.id },
fileId: fileId,
}, {
- removeOnComplete: true,
- removeOnFail: true,
+ removeOnComplete: {
+ age: 3600 * 24 * 7, // keep up to 7 days
+ count: 30,
+ },
+ removeOnFail: {
+ age: 3600 * 24 * 7, // keep up to 7 days
+ count: 100,
+ },
});
}
@@ -428,8 +562,14 @@ export class QueueService {
user: { id: user.id },
fileId: fileId,
}, {
- removeOnComplete: true,
- removeOnFail: true,
+ removeOnComplete: {
+ age: 3600 * 24 * 7, // keep up to 7 days
+ count: 30,
+ },
+ removeOnFail: {
+ age: 3600 * 24 * 7, // keep up to 7 days
+ count: 100,
+ },
});
}
@@ -439,8 +579,14 @@ export class QueueService {
user: { id: user.id },
antenna,
}, {
- removeOnComplete: true,
- removeOnFail: true,
+ removeOnComplete: {
+ age: 3600 * 24 * 7, // keep up to 7 days
+ count: 30,
+ },
+ removeOnFail: {
+ age: 3600 * 24 * 7, // keep up to 7 days
+ count: 100,
+ },
});
}
@@ -450,8 +596,14 @@ export class QueueService {
user: { id: user.id },
soft: opts.soft,
}, {
- removeOnComplete: true,
- removeOnFail: true,
+ removeOnComplete: {
+ age: 3600 * 24 * 7, // keep up to 7 days
+ count: 30,
+ },
+ removeOnFail: {
+ age: 3600 * 24 * 7, // keep up to 7 days
+ count: 100,
+ },
});
}
@@ -501,8 +653,14 @@ export class QueueService {
withReplies: data.withReplies,
},
opts: {
- removeOnComplete: true,
- removeOnFail: true,
+ removeOnComplete: {
+ age: 3600 * 24 * 7, // keep up to 7 days
+ count: 30,
+ },
+ removeOnFail: {
+ age: 3600 * 24 * 7, // keep up to 7 days
+ count: 100,
+ },
...opts,
},
};
@@ -513,16 +671,28 @@ export class QueueService {
return this.objectStorageQueue.add('deleteFile', {
key: key,
}, {
- removeOnComplete: true,
- removeOnFail: true,
+ removeOnComplete: {
+ age: 3600 * 24 * 7, // keep up to 7 days
+ count: 30,
+ },
+ removeOnFail: {
+ age: 3600 * 24 * 7, // keep up to 7 days
+ count: 100,
+ },
});
}
@bindThis
public createCleanRemoteFilesJob() {
return this.objectStorageQueue.add('cleanRemoteFiles', {}, {
- removeOnComplete: true,
- removeOnFail: true,
+ removeOnComplete: {
+ age: 3600 * 24 * 7, // keep up to 7 days
+ count: 30,
+ },
+ removeOnFail: {
+ age: 3600 * 24 * 7, // keep up to 7 days
+ count: 100,
+ },
});
}
@@ -553,8 +723,14 @@ export class QueueService {
backoff: {
type: 'custom',
},
- removeOnComplete: true,
- removeOnFail: true,
+ removeOnComplete: {
+ age: 3600 * 24 * 7, // keep up to 7 days
+ count: 30,
+ },
+ removeOnFail: {
+ age: 3600 * 24 * 7, // keep up to 7 days
+ count: 100,
+ },
});
}
@@ -584,21 +760,201 @@ export class QueueService {
backoff: {
type: 'custom',
},
- removeOnComplete: true,
- removeOnFail: true,
+ removeOnComplete: {
+ age: 3600 * 24 * 7, // keep up to 7 days
+ count: 30,
+ },
+ removeOnFail: {
+ age: 3600 * 24 * 7, // keep up to 7 days
+ count: 100,
+ },
});
}
@bindThis
- public destroy() {
- this.deliverQueue.once('cleaned', (jobs, status) => {
- //deliverLogger.succ(`Cleaned ${jobs.length} ${status} jobs`);
- });
- this.deliverQueue.clean(0, 0, 'delayed');
+ private getQueue(type: typeof QUEUE_TYPES[number]): Bull.Queue {
+ switch (type) {
+ case 'system': return this.systemQueue;
+ case 'endedPollNotification': return this.endedPollNotificationQueue;
+ case 'deliver': return this.deliverQueue;
+ case 'inbox': return this.inboxQueue;
+ case 'db': return this.dbQueue;
+ case 'relationship': return this.relationshipQueue;
+ case 'objectStorage': return this.objectStorageQueue;
+ case 'userWebhookDeliver': return this.userWebhookDeliverQueue;
+ case 'systemWebhookDeliver': return this.systemWebhookDeliverQueue;
+ default: throw new Error(`Unrecognized queue type: ${type}`);
+ }
+ }
+
+ @bindThis
+ public async queueClear(queueType: typeof QUEUE_TYPES[number], state: '*' | 'completed' | 'wait' | 'active' | 'paused' | 'prioritized' | 'delayed' | 'failed') {
+ const queue = this.getQueue(queueType);
+
+ if (state === '*') {
+ await Promise.all([
+ queue.clean(0, 0, 'completed'),
+ queue.clean(0, 0, 'wait'),
+ queue.clean(0, 0, 'active'),
+ queue.clean(0, 0, 'paused'),
+ queue.clean(0, 0, 'prioritized'),
+ queue.clean(0, 0, 'delayed'),
+ queue.clean(0, 0, 'failed'),
+ ]);
+ } else {
+ await queue.clean(0, 0, state);
+ }
+ }
- this.inboxQueue.once('cleaned', (jobs, status) => {
- //inboxLogger.succ(`Cleaned ${jobs.length} ${status} jobs`);
+ @bindThis
+ public async queuePromoteJobs(queueType: typeof QUEUE_TYPES[number]) {
+ const queue = this.getQueue(queueType);
+ await queue.promoteJobs();
+ }
+
+ @bindThis
+ public async queueRetryJob(queueType: typeof QUEUE_TYPES[number], jobId: string) {
+ const queue = this.getQueue(queueType);
+ const job: Bull.Job | null = await queue.getJob(jobId);
+ if (job) {
+ if (job.finishedOn != null) {
+ await job.retry();
+ } else {
+ await job.promote();
+ }
+ }
+ }
+
+ @bindThis
+ public async queueRemoveJob(queueType: typeof QUEUE_TYPES[number], jobId: string) {
+ const queue = this.getQueue(queueType);
+ const job: Bull.Job | null = await queue.getJob(jobId);
+ if (job) {
+ await job.remove();
+ }
+ }
+
+ @bindThis
+ private packJobData(job: Bull.Job) {
+ // eslint-disable-next-line @typescript-eslint/no-unnecessary-condition
+ const stacktrace = job.stacktrace ? job.stacktrace.filter(Boolean) : [];
+ stacktrace.reverse();
+
+ return {
+ id: job.id,
+ name: job.name,
+ data: job.data,
+ opts: job.opts,
+ timestamp: job.timestamp,
+ processedOn: job.processedOn,
+ processedBy: job.processedBy,
+ finishedOn: job.finishedOn,
+ progress: job.progress,
+ attempts: job.attemptsMade,
+ delay: job.delay,
+ failedReason: job.failedReason,
+ stacktrace: stacktrace,
+ returnValue: job.returnvalue,
+ isFailed: !!job.failedReason || (Array.isArray(stacktrace) && stacktrace.length > 0),
+ };
+ }
+
+ @bindThis
+ public async queueGetJob(queueType: typeof QUEUE_TYPES[number], jobId: string) {
+ const queue = this.getQueue(queueType);
+ const job: Bull.Job | null = await queue.getJob(jobId);
+ if (job) {
+ return this.packJobData(job);
+ } else {
+ throw new Error(`Job not found: ${jobId}`);
+ }
+ }
+
+ @bindThis
+ public async queueGetJobs(queueType: typeof QUEUE_TYPES[number], jobTypes: JobType[], search?: string) {
+ const RETURN_LIMIT = 100;
+ const queue = this.getQueue(queueType);
+ let jobs: Bull.Job[];
+
+ if (search) {
+ jobs = await queue.getJobs(jobTypes, 0, 1000);
+
+ jobs = jobs.filter(job => {
+ const jobString = JSON.stringify(job).toLowerCase();
+ return search.toLowerCase().split(' ').every(term => {
+ return jobString.includes(term);
+ });
+ });
+
+ jobs = jobs.slice(0, RETURN_LIMIT);
+ } else {
+ jobs = await queue.getJobs(jobTypes, 0, RETURN_LIMIT);
+ }
+
+ return jobs.map(job => this.packJobData(job));
+ }
+
+ @bindThis
+ public async queueGetQueues() {
+ const fetchings = QUEUE_TYPES.map(async type => {
+ const queue = this.getQueue(type);
+
+ const counts = await queue.getJobCounts();
+ const isPaused = await queue.isPaused();
+ const metrics_completed = await queue.getMetrics('completed', 0, MetricsTime.ONE_WEEK);
+ const metrics_failed = await queue.getMetrics('failed', 0, MetricsTime.ONE_WEEK);
+
+ return {
+ name: type,
+ counts: counts,
+ isPaused,
+ metrics: {
+ completed: metrics_completed,
+ failed: metrics_failed,
+ },
+ };
});
- this.inboxQueue.clean(0, 0, 'delayed');
+
+ return await Promise.all(fetchings);
+ }
+
+ @bindThis
+ public async queueGetQueue(queueType: typeof QUEUE_TYPES[number]) {
+ const queue = this.getQueue(queueType);
+ const counts = await queue.getJobCounts();
+ const isPaused = await queue.isPaused();
+ const metrics_completed = await queue.getMetrics('completed', 0, MetricsTime.ONE_WEEK);
+ const metrics_failed = await queue.getMetrics('failed', 0, MetricsTime.ONE_WEEK);
+ const db = parseRedisInfo(await (await queue.client).info());
+
+ return {
+ name: queueType,
+ qualifiedName: queue.qualifiedName,
+ counts: counts,
+ isPaused,
+ metrics: {
+ completed: metrics_completed,
+ failed: metrics_failed,
+ },
+ db: {
+ version: db.redis_version,
+ mode: db.redis_mode,
+ runId: db.run_id,
+ processId: db.process_id,
+ port: parseInt(db.tcp_port),
+ os: db.os,
+ uptime: parseInt(db.uptime_in_seconds),
+ memory: {
+ total: parseInt(db.total_system_memory) || parseInt(db.maxmemory),
+ used: parseInt(db.used_memory),
+ fragmentationRatio: parseInt(db.mem_fragmentation_ratio),
+ peak: parseInt(db.used_memory_peak),
+ },
+ clients: {
+ connected: parseInt(db.connected_clients),
+ blocked: parseInt(db.blocked_clients),
+ },
+ },
+ };
}
}
diff --git a/packages/backend/src/core/SystemAccountService.ts b/packages/backend/src/core/SystemAccountService.ts
index 3f3afe6561..1288dc6ffa 100644
--- a/packages/backend/src/core/SystemAccountService.ts
+++ b/packages/backend/src/core/SystemAccountService.ts
@@ -5,11 +5,14 @@
import { randomUUID } from 'node:crypto';
import { Inject, Injectable } from '@nestjs/common';
+import type { OnApplicationShutdown } from '@nestjs/common';
import { DataSource, IsNull } from 'typeorm';
+import * as Redis from 'ioredis';
import bcrypt from 'bcryptjs';
import { MiLocalUser, MiUser } from '@/models/User.js';
import { MiSystemAccount, MiUsedUsername, MiUserKeypair, MiUserProfile, type UsersRepository, type SystemAccountsRepository } from '@/models/_.js';
import type { MiMeta, UserProfilesRepository } from '@/models/_.js';
+import type { GlobalEvents } from '@/core/GlobalEventService.js';
import { MemoryKVCache } from '@/misc/cache.js';
import { DI } from '@/di-symbols.js';
import { bindThis } from '@/decorators.js';
@@ -20,10 +23,13 @@ import { genRsaKeyPair } from '@/misc/gen-key-pair.js';
export const SYSTEM_ACCOUNT_TYPES = ['actor', 'relay', 'proxy'] as const;
@Injectable()
-export class SystemAccountService {
+export class SystemAccountService implements OnApplicationShutdown {
private cache: MemoryKVCache<MiLocalUser>;
constructor(
+ @Inject(DI.redisForSub)
+ private redisForSub: Redis.Redis,
+
@Inject(DI.db)
private db: DataSource,
@@ -42,6 +48,31 @@ export class SystemAccountService {
private idService: IdService,
) {
this.cache = new MemoryKVCache<MiLocalUser>(1000 * 60 * 10); // 10m
+
+ this.redisForSub.on('message', this.onMessage);
+ }
+
+ @bindThis
+ private async onMessage(_: string, data: string): Promise<void> {
+ const obj = JSON.parse(data);
+
+ if (obj.channel === 'internal') {
+ const { type, body } = obj.message as GlobalEvents['internal']['payload'];
+ switch (type) {
+ case 'metaUpdated': {
+ if (body.before != null && body.before.name !== body.after.name) {
+ for (const account of SYSTEM_ACCOUNT_TYPES) {
+ await this.updateCorrespondingUserProfile(account, {
+ name: body.after.name,
+ });
+ }
+ }
+ break;
+ }
+ default:
+ break;
+ }
+ }
}
@bindThis
@@ -151,7 +182,7 @@ export class SystemAccountService {
@bindThis
public async updateCorrespondingUserProfile(type: typeof SYSTEM_ACCOUNT_TYPES[number], extra: {
- name?: string;
+ name?: string | null;
description?: MiUserProfile['description'];
}): Promise<MiLocalUser> {
const user = await this.fetch(type);
@@ -187,4 +218,15 @@ export class SystemAccountService {
public async getProxyActor() {
return await this.fetch('proxy');
}
+
+ @bindThis
+ public dispose(): void {
+ this.redisForSub.off('message', this.onMessage);
+ this.cache.dispose();
+ }
+
+ @bindThis
+ public onApplicationShutdown(signal?: string): void {
+ this.dispose();
+ }
}
diff --git a/packages/backend/src/core/WebhookTestService.ts b/packages/backend/src/core/WebhookTestService.ts
index 469b396fb0..2f8cfea7f7 100644
--- a/packages/backend/src/core/WebhookTestService.ts
+++ b/packages/backend/src/core/WebhookTestService.ts
@@ -431,8 +431,8 @@ export class WebhookTestService {
name: user.name,
username: user.username,
host: user.host,
- avatarUrl: user.avatarUrl,
- avatarBlurhash: user.avatarBlurhash,
+ avatarUrl: user.avatarId == null ? null : user.avatarUrl,
+ avatarBlurhash: user.avatarId == null ? null : user.avatarBlurhash,
avatarDecorations: user.avatarDecorations.map(it => ({
id: it.id,
angle: it.angle,
@@ -464,10 +464,10 @@ export class WebhookTestService {
createdAt: new Date().toISOString(),
updatedAt: user.updatedAt?.toISOString() ?? null,
lastFetchedAt: user.lastFetchedAt?.toISOString() ?? null,
- bannerUrl: user.bannerUrl,
- bannerBlurhash: user.bannerBlurhash,
- backgroundUrl: user.backgroundUrl,
- backgroundBlurhash: user.backgroundBlurhash,
+ bannerUrl: user.bannerId == null ? null : user.bannerUrl,
+ bannerBlurhash: user.bannerId == null ? null : user.bannerBlurhash,
+ backgroundUrl: user.backgroundId == null ? null : user.backgroundUrl,
+ backgroundBlurhash: user.backgroundId == null ? null : user.backgroundBlurhash,
listenbrainz: null,
isLocked: user.isLocked,
isSilenced: false,
diff --git a/packages/backend/src/core/activitypub/ApMfmService.ts b/packages/backend/src/core/activitypub/ApMfmService.ts
index 318710fa93..c4a948429a 100644
--- a/packages/backend/src/core/activitypub/ApMfmService.ts
+++ b/packages/backend/src/core/activitypub/ApMfmService.ts
@@ -5,7 +5,7 @@
import { Injectable } from '@nestjs/common';
import * as mfm from '@transfem-org/sfm-js';
-import { MfmService } from '@/core/MfmService.js';
+import { MfmService, Appender } from '@/core/MfmService.js';
import type { MiNote } from '@/models/Note.js';
import { bindThis } from '@/decorators.js';
import { extractApHashtagObjects } from './models/tag.js';
@@ -25,17 +25,17 @@ export class ApMfmService {
}
@bindThis
- public getNoteHtml(note: Pick<MiNote, 'text' | 'mentionedRemoteUsers'>, apAppend?: string) {
+ public getNoteHtml(note: Pick<MiNote, 'text' | 'mentionedRemoteUsers'>, additionalAppender: Appender[] = []) {
let noMisskeyContent = false;
- const srcMfm = (note.text ?? '') + (apAppend ?? '');
+ const srcMfm = (note.text ?? '');
const parsed = mfm.parse(srcMfm);
- if (!apAppend && parsed?.every(n => ['text', 'unicodeEmoji', 'emojiCode', 'mention', 'hashtag', 'url'].includes(n.type))) {
+ if (!additionalAppender.length && parsed.every(n => ['text', 'unicodeEmoji', 'emojiCode', 'mention', 'hashtag', 'url'].includes(n.type))) {
noMisskeyContent = true;
}
- const content = this.mfmService.toHtml(parsed, note.mentionedRemoteUsers ? JSON.parse(note.mentionedRemoteUsers) : []);
+ const content = this.mfmService.toHtml(parsed, note.mentionedRemoteUsers ? JSON.parse(note.mentionedRemoteUsers) : [], additionalAppender);
return {
content,
diff --git a/packages/backend/src/core/activitypub/ApRendererService.ts b/packages/backend/src/core/activitypub/ApRendererService.ts
index e6eb777f1b..8251bc3b15 100644
--- a/packages/backend/src/core/activitypub/ApRendererService.ts
+++ b/packages/backend/src/core/activitypub/ApRendererService.ts
@@ -20,7 +20,7 @@ import type { MiEmoji } from '@/models/Emoji.js';
import type { MiPoll } from '@/models/Poll.js';
import type { MiPollVote } from '@/models/PollVote.js';
import { UserKeypairService } from '@/core/UserKeypairService.js';
-import { MfmService } from '@/core/MfmService.js';
+import { MfmService, type Appender } from '@/core/MfmService.js';
import { UserEntityService } from '@/core/entities/UserEntityService.js';
import { DriveFileEntityService } from '@/core/entities/DriveFileEntityService.js';
import type { MiUserKeypair } from '@/models/UserKeypair.js';
@@ -468,10 +468,24 @@ export class ApRendererService {
poll = await this.pollsRepository.findOneBy({ noteId: note.id });
}
- let apAppend = '';
+ const apAppend: Appender[] = [];
if (quote) {
- apAppend += `\n\nRE: ${quote}`;
+ // Append quote link as `<br><br><span class="quote-inline">RE: <a href="...">...</a></span>`
+ // the claas name `quote-inline` is used in non-misskey clients for styling quote notes.
+ // For compatibility, the span part should be kept as possible.
+ apAppend.push((doc, body) => {
+ body.appendChild(doc.createElement('br'));
+ body.appendChild(doc.createElement('br'));
+ const span = doc.createElement('span');
+ span.className = 'quote-inline';
+ span.appendChild(doc.createTextNode('RE: '));
+ const link = doc.createElement('a');
+ link.setAttribute('href', quote);
+ link.textContent = quote;
+ span.appendChild(link);
+ body.appendChild(span);
+ });
}
let summary = note.cw === '' ? String.fromCharCode(0x200B) : note.cw;
diff --git a/packages/backend/src/core/entities/UserEntityService.ts b/packages/backend/src/core/entities/UserEntityService.ts
index d11042d73b..16425bb6ce 100644
--- a/packages/backend/src/core/entities/UserEntityService.ts
+++ b/packages/backend/src/core/entities/UserEntityService.ts
@@ -584,8 +584,8 @@ export class UserEntityService implements OnModuleInit {
name: user.name,
username: user.username,
host: user.host,
- avatarUrl: user.avatarUrl ?? this.getIdenticonUrl(user),
- avatarBlurhash: user.avatarBlurhash,
+ avatarUrl: (user.avatarId == null ? null : user.avatarUrl) ?? this.getIdenticonUrl(user),
+ avatarBlurhash: (user.avatarId == null ? null : user.avatarBlurhash),
description: mastoapi ? mastoapi.description : profile ? profile.description : '',
createdAt: this.idService.parse(user.id).date.toISOString(),
avatarDecorations: user.avatarDecorations.length > 0 ? this.avatarDecorationService.getAll().then(decorations => user.avatarDecorations.filter(ud => decorations.some(d => d.id === ud.id)).map(ud => ({
@@ -643,10 +643,10 @@ export class UserEntityService implements OnModuleInit {
: null,
updatedAt: user.updatedAt ? user.updatedAt.toISOString() : null,
lastFetchedAt: user.lastFetchedAt ? user.lastFetchedAt.toISOString() : null,
- bannerUrl: user.bannerUrl,
- bannerBlurhash: user.bannerBlurhash,
- backgroundUrl: user.backgroundUrl,
- backgroundBlurhash: user.backgroundBlurhash,
+ bannerUrl: user.bannerId == null ? null : user.bannerUrl,
+ bannerBlurhash: user.bannerId == null ? null : user.bannerBlurhash,
+ backgroundUrl: user.backgroundUrl == null ? null : user.backgroundUrl,
+ backgroundBlurhash: user.backgroundBlurhash == null ? null : user.backgroundBlurhash,
isLocked: user.isLocked,
isSuspended: user.isSuspended,
location: profile!.location,