diff options
28 files changed, 900 insertions, 14 deletions
diff --git a/.config/ci.yml b/.config/ci.yml index 311a98d8fb..790c4704fa 100644 --- a/.config/ci.yml +++ b/.config/ci.yml @@ -263,12 +263,17 @@ checkActivityPubGetSignature: false # # default: false # disableQueryTruncation: false -# Log settings -# logging: -# sql: -# # Outputs query parameters during SQL execution to the log. -# # default: false -# enableQueryParamLogging: false -# # Disable query truncation. If set to true, the full text of the query will be output to the log. -# # default: false -# disableQueryTruncation: false +# Settings for the activity logger, which records inbound activities to the database. +# Disabled by default due to the large volume of data it saves. +#activityLogging: + # Log activities to the database (default: false) + #enabled: false + + # Save the activity before processing, then update later with the results. + # This has the advantage of capturing activities that cause a hard-crash, but doubles the number of queries used. + # Default: false + #preSave: false + + # How long to save each log entry before deleting it. + # Default: 2592000000 (1 week) + #maxAge: 2592000000 diff --git a/.config/cypress-devcontainer.yml b/.config/cypress-devcontainer.yml index 391bc9998c..9a6f9769e6 100644 --- a/.config/cypress-devcontainer.yml +++ b/.config/cypress-devcontainer.yml @@ -272,3 +272,18 @@ allowedPrivateNetworks: [ # # Disable query truncation. If set to true, the full text of the query will be output to the log. # # default: false # disableQueryTruncation: false + +# Settings for the activity logger, which records inbound activities to the database. +# Disabled by default due to the large volume of data it saves. +#activityLogging: + # Log activities to the database (default: false) + #enabled: false + + # Save the activity before processing, then update later with the results. + # This has the advantage of capturing activities that cause a hard-crash, but doubles the number of queries used. + # Default: false + #preSave: false + + # How long to save each log entry before deleting it. + # Default: 2592000000 (1 week) + #maxAge: 2592000000 diff --git a/.config/docker_example.yml b/.config/docker_example.yml index 1e03e902bf..2d088547ba 100644 --- a/.config/docker_example.yml +++ b/.config/docker_example.yml @@ -345,3 +345,18 @@ checkActivityPubGetSignature: false # # Disable query truncation. If set to true, the full text of the query will be output to the log. # # default: false # disableQueryTruncation: false + +# Settings for the activity logger, which records inbound activities to the database. +# Disabled by default due to the large volume of data it saves. +#activityLogging: + # Log activities to the database (default: false) + #enabled: false + + # Save the activity before processing, then update later with the results. + # This has the advantage of capturing activities that cause a hard-crash, but doubles the number of queries used. + # Default: false + #preSave: false + + # How long to save each log entry before deleting it. + # Default: 2592000000 (1 week) + #maxAge: 2592000000 diff --git a/.config/example.yml b/.config/example.yml index 7d4cd0c659..7bca8642be 100644 --- a/.config/example.yml +++ b/.config/example.yml @@ -383,3 +383,18 @@ checkActivityPubGetSignature: false # # Disable query truncation. If set to true, the full text of the query will be output to the log. # # default: false # disableQueryTruncation: false + +# Settings for the activity logger, which records inbound activities to the database. +# Disabled by default due to the large volume of data it saves. +#activityLogging: + # Log activities to the database (default: false) + #enabled: false + + # Save the activity before processing, then update later with the results. + # This has the advantage of capturing activities that cause a hard-crash, but doubles the number of queries used. + # Default: false + #preSave: false + + # How long to save each log entry before deleting it. + # Default: 2592000000 (1 week) + #maxAge: 2592000000 diff --git a/packages/backend/migration/1731565470048-add-activity-log.js b/packages/backend/migration/1731565470048-add-activity-log.js new file mode 100644 index 0000000000..19c6b336af --- /dev/null +++ b/packages/backend/migration/1731565470048-add-activity-log.js @@ -0,0 +1,28 @@ +/* + * SPDX-FileCopyrightText: hazelnoot and other Sharkey contributors + * SPDX-License-Identifier: AGPL-3.0-only + */ + +export class AddActivityLog1731565470048 { + name = 'AddActivityLog1731565470048' + + async up(queryRunner) { + await queryRunner.query(`CREATE TABLE "activity_context" ("md5" text NOT NULL, "json" jsonb NOT NULL, CONSTRAINT "PK_activity_context" PRIMARY KEY ("md5"))`); + await queryRunner.query(`CREATE INDEX "IDK_activity_context_md5" ON "activity_context" ("md5") `); + await queryRunner.query(`CREATE TABLE "activity_log" ("id" character varying(32) NOT NULL, "at" TIMESTAMP WITH TIME ZONE NOT NULL, "key_id" text NOT NULL, "host" text NOT NULL, "verified" boolean NOT NULL, "accepted" boolean NOT NULL, "result" text NOT NULL, "activity" jsonb NOT NULL, "context_hash" text, "auth_user_id" character varying(32), CONSTRAINT "PK_activity_log" PRIMARY KEY ("id"))`); + await queryRunner.query(`CREATE INDEX "IDX_activity_log_at" ON "activity_log" ("at") `); + await queryRunner.query(`CREATE INDEX "IDX_activity_log_host" ON "activity_log" ("host") `); + await queryRunner.query(`ALTER TABLE "activity_log" ADD CONSTRAINT "FK_activity_log_context_hash" FOREIGN KEY ("context_hash") REFERENCES "activity_context"("md5") ON DELETE CASCADE ON UPDATE NO ACTION`); + await queryRunner.query(`ALTER TABLE "activity_log" ADD CONSTRAINT "FK_activity_log_auth_user_id" FOREIGN KEY ("auth_user_id") REFERENCES "user"("id") ON DELETE CASCADE ON UPDATE NO ACTION`); + } + + async down(queryRunner) { + await queryRunner.query(`ALTER TABLE "activity_log" DROP CONSTRAINT "FK_activity_log_auth_user_id"`); + await queryRunner.query(`ALTER TABLE "activity_log" DROP CONSTRAINT "FK_activity_log_context_hash"`); + await queryRunner.query(`DROP INDEX "public"."IDX_activity_log_host"`); + await queryRunner.query(`DROP INDEX "public"."IDX_activity_log_at"`); + await queryRunner.query(`DROP TABLE "activity_log"`); + await queryRunner.query(`DROP INDEX "public"."IDK_activity_context_md5"`); + await queryRunner.query(`DROP TABLE "activity_context"`); + } +} diff --git a/packages/backend/migration/1731909785724-activity-log-timing.js b/packages/backend/migration/1731909785724-activity-log-timing.js new file mode 100644 index 0000000000..8b72fb8972 --- /dev/null +++ b/packages/backend/migration/1731909785724-activity-log-timing.js @@ -0,0 +1,19 @@ +/* + * SPDX-FileCopyrightText: hazelnoot and other Sharkey contributors + * SPDX-License-Identifier: AGPL-3.0-only + */ + +export class ActivityLogTiming1731909785724 { + name = 'ActivityLogTiming1731909785724' + + async up(queryRunner) { + await queryRunner.query(`ALTER TABLE "activity_log" ADD "duration" double precision NOT NULL DEFAULT '0'`); + await queryRunner.query(`ALTER TABLE "activity_log" ALTER COLUMN "result" DROP NOT NULL`); + } + + async down(queryRunner) { + await queryRunner.query(`UPDATE "activity_log" SET "result" = 'not processed' WHERE "result" IS NULL`); + await queryRunner.query(`ALTER TABLE "activity_log" ALTER COLUMN "result" SET NOT NULL`); + await queryRunner.query(`ALTER TABLE "activity_log" DROP COLUMN "duration"`); + } +} diff --git a/packages/backend/migration/1731910422761-rename-activity-log-indexes.js b/packages/backend/migration/1731910422761-rename-activity-log-indexes.js new file mode 100644 index 0000000000..82d5a796e9 --- /dev/null +++ b/packages/backend/migration/1731910422761-rename-activity-log-indexes.js @@ -0,0 +1,16 @@ +/* + * SPDX-FileCopyrightText: hazelnoot and other Sharkey contributors + * SPDX-License-Identifier: AGPL-3.0-only + */ + +export class RenameActivityLogIndexes1731910422761 { + name = 'RenameActivityLogIndexes1731910422761' + + async up(queryRunner) { + await queryRunner.query(`DROP INDEX "public"."IDK_activity_context_md5"`); + } + + async down(queryRunner) { + await queryRunner.query(`CREATE INDEX "IDK_activity_context_md5" ON "activity_context" ("md5") `); + } +} diff --git a/packages/backend/migration/1731935047347-nullable-activity-log-duration.js b/packages/backend/migration/1731935047347-nullable-activity-log-duration.js new file mode 100644 index 0000000000..2acbd2bca5 --- /dev/null +++ b/packages/backend/migration/1731935047347-nullable-activity-log-duration.js @@ -0,0 +1,20 @@ +/* + * SPDX-FileCopyrightText: hazelnoot and other Sharkey contributors + * SPDX-License-Identifier: AGPL-3.0-only + */ + +export class NullableActivityLogDuration1731935047347 { + name = 'NullableActivityLogDuration1731935047347' + + async up(queryRunner) { + await queryRunner.query(`ALTER TABLE "activity_log" ALTER COLUMN "duration" DROP NOT NULL`); + await queryRunner.query(`ALTER TABLE "activity_log" ALTER COLUMN "duration" DROP DEFAULT`); + await queryRunner.query(`UPDATE "activity_log" SET "duration" = NULL WHERE "duration" = 0`); + } + + async down(queryRunner) { + await queryRunner.query(`UPDATE "activity_log" SET "duration" = 0 WHERE "duration" IS NULL`); + await queryRunner.query(`ALTER TABLE "activity_log" ALTER COLUMN "duration" SET DEFAULT '0'`); + await queryRunner.query(`ALTER TABLE "activity_log" ALTER COLUMN "duration" SET NOT NULL`); + } +} diff --git a/packages/backend/migration/1733756280460-rename_activity_log_to_ap_inbox_log.js b/packages/backend/migration/1733756280460-rename_activity_log_to_ap_inbox_log.js new file mode 100644 index 0000000000..ad25135188 --- /dev/null +++ b/packages/backend/migration/1733756280460-rename_activity_log_to_ap_inbox_log.js @@ -0,0 +1,32 @@ +/* + * SPDX-FileCopyrightText: hazelnoot and other Sharkey contributors + * SPDX-License-Identifier: AGPL-3.0-only + */ + +export class RenameActivityLogToApInboxLog1733756280460 { + name = 'RenameActivityLogToApInboxLog1733756280460' + + async up(queryRunner) { + await queryRunner.query(`ALTER INDEX "IDX_activity_log_at" RENAME TO "IDX_ap_inbox_log_at"`); + await queryRunner.query(`ALTER INDEX "IDX_activity_log_host" RENAME TO "IDX_ap_inbox_log_host"`); + await queryRunner.query(`ALTER TABLE "activity_log" RENAME CONSTRAINT "PK_activity_log" TO "PK_ap_inbox_log"`); + await queryRunner.query(`ALTER TABLE "activity_log" RENAME CONSTRAINT "FK_activity_log_context_hash" TO "FK_ap_inbox_log_context_hash"`); + await queryRunner.query(`ALTER TABLE "activity_log" RENAME CONSTRAINT "FK_activity_log_auth_user_id" TO "FK_ap_inbox_log_auth_user_id"`); + await queryRunner.query(`ALTER TABLE "activity_log" RENAME TO "ap_inbox_log"`); + + await queryRunner.query(`ALTER TABLE "activity_context" RENAME CONSTRAINT "PK_activity_context" TO "PK_ap_context"`); + await queryRunner.query(`ALTER TABLE "activity_context" RENAME TO "ap_context"`); + } + + async down(queryRunner) { + await queryRunner.query(`ALTER TABLE "ap_context" RENAME TO "activity_context"`); + await queryRunner.query(`ALTER TABLE "activity_context" RENAME CONSTRAINT "PK_ap_context" TO "PK_activity_context"`); + + await queryRunner.query(`ALTER TABLE "ap_inbox_log" RENAME TO "activity_log"`); + await queryRunner.query(`ALTER TABLE "activity_log" RENAME CONSTRAINT "FK_ap_inbox_log_auth_user_id" TO "FK_activity_log_auth_user_id"`); + await queryRunner.query(`ALTER TABLE "activity_log" RENAME CONSTRAINT "FK_ap_inbox_log_context_hash" TO "FK_activity_log_context_hash"`); + await queryRunner.query(`ALTER TABLE "activity_log" RENAME CONSTRAINT "PK_ap_inbox_log" TO "PK_activity_log"`); + await queryRunner.query(`ALTER INDEX "IDX_ap_inbox_log_host" RENAME TO "IDX_activity_log_host"`); + await queryRunner.query(`ALTER INDEX "IDX_ap_inbox_log_at" RENAME TO "IDX_activity_log_at"`); + } +} diff --git a/packages/backend/migration/1738293576355-create_ap_fetch_log.js b/packages/backend/migration/1738293576355-create_ap_fetch_log.js new file mode 100644 index 0000000000..4371f50b4a --- /dev/null +++ b/packages/backend/migration/1738293576355-create_ap_fetch_log.js @@ -0,0 +1,19 @@ +export class CreateApFetchLog1738293576355 { + name = 'CreateApFetchLog1738293576355' + + async up(queryRunner) { + await queryRunner.query(`CREATE TABLE "ap_fetch_log" ("id" character varying(32) NOT NULL, "at" TIMESTAMP WITH TIME ZONE NOT NULL, "duration" double precision, "host" text NOT NULL, "request_uri" text NOT NULL, "object_uri" text, "accepted" boolean, "result" text, "object" jsonb, "context_hash" text, CONSTRAINT "PK_ap_fetch_log" PRIMARY KEY ("id"))`); + await queryRunner.query(`CREATE INDEX "IDX_ap_fetch_log_at" ON "ap_fetch_log" ("at") `); + await queryRunner.query(`CREATE INDEX "IDX_ap_fetch_log_host" ON "ap_fetch_log" ("host") `); + await queryRunner.query(`CREATE INDEX "IDX_ap_fetch_log_object_uri" ON "ap_fetch_log" ("object_uri") `); + await queryRunner.query(`ALTER TABLE "ap_fetch_log" ADD CONSTRAINT "FK_ap_fetch_log_context_hash" FOREIGN KEY ("context_hash") REFERENCES "ap_context"("md5") ON DELETE CASCADE ON UPDATE NO ACTION`); + } + + async down(queryRunner) { + await queryRunner.query(`ALTER TABLE "ap_fetch_log" DROP CONSTRAINT "FK_ap_fetch_log_context_hash"`); + await queryRunner.query(`DROP INDEX "public"."IDX_ap_fetch_log_object_uri"`); + await queryRunner.query(`DROP INDEX "public"."IDX_ap_fetch_log_host"`); + await queryRunner.query(`DROP INDEX "public"."IDX_ap_fetch_log_at"`); + await queryRunner.query(`DROP TABLE "ap_fetch_log"`); + } +} diff --git a/packages/backend/src/boot/common.ts b/packages/backend/src/boot/common.ts index ad59a55688..2f97980e9a 100644 --- a/packages/backend/src/boot/common.ts +++ b/packages/backend/src/boot/common.ts @@ -13,6 +13,7 @@ import { ServerStatsService } from '@/daemons/ServerStatsService.js'; import { ServerService } from '@/server/ServerService.js'; import { MainModule } from '@/MainModule.js'; import { envOption } from '@/env.js'; +import { ApLogCleanupService } from '@/daemons/ApLogCleanupService.js'; export async function server() { const app = await NestFactory.createApplicationContext(MainModule, { @@ -28,6 +29,7 @@ export async function server() { if (!envOption.noDaemons) { app.get(QueueStatsService).start(); app.get(ServerStatsService).start(); + app.get(ApLogCleanupService).start(); } return app; diff --git a/packages/backend/src/config.ts b/packages/backend/src/config.ts index d35befdc2b..24f3c472a4 100644 --- a/packages/backend/src/config.ts +++ b/packages/backend/src/config.ts @@ -129,6 +129,12 @@ type Source = { enableQueryParamLogging? : boolean, } } + + activityLogging?: { + enabled?: boolean; + preSave?: boolean; + maxAge?: number; + }; }; export type Config = { @@ -238,6 +244,12 @@ export type Config = { pidFile: string; filePermissionBits?: string; + + activityLogging: { + enabled: boolean; + preSave: boolean; + maxAge: number; + }; }; export type FulltextSearchProvider = 'sqlLike' | 'sqlPgroonga' | 'meilisearch'; @@ -380,6 +392,11 @@ export function loadConfig(): Config { pidFile: config.pidFile, filePermissionBits: config.filePermissionBits, logging: config.logging, + activityLogging: { + enabled: config.activityLogging?.enabled ?? false, + preSave: config.activityLogging?.preSave ?? false, + maxAge: config.activityLogging?.maxAge ?? (1000 * 60 * 60 * 24 * 30), + }, }; } @@ -531,4 +548,5 @@ function applyEnvOverrides(config: Source) { _apply_top(['import', ['downloadTimeout', 'maxFileSize']]); _apply_top([['signToActivityPubGet', 'checkActivityPubGetSignature', 'setupPassword']]); _apply_top(['logging', 'sql', ['disableQueryTruncation', 'enableQueryParamLogging']]); + _apply_top(['activityLogging', ['enabled', 'preSave', 'maxAge']]); } diff --git a/packages/backend/src/core/ApLogService.ts b/packages/backend/src/core/ApLogService.ts new file mode 100644 index 0000000000..096ec21de7 --- /dev/null +++ b/packages/backend/src/core/ApLogService.ts @@ -0,0 +1,207 @@ +/* + * SPDX-FileCopyrightText: hazelnoot and other Sharkey contributors + * SPDX-License-Identifier: AGPL-3.0-only + */ + +import { createHash } from 'crypto'; +import { Inject, Injectable } from '@nestjs/common'; +import { In, LessThan } from 'typeorm'; +import { DI } from '@/di-symbols.js'; +import { SkApFetchLog, SkApInboxLog, SkApContext } from '@/models/_.js'; +import type { ApContextsRepository, ApFetchLogsRepository, ApInboxLogsRepository } from '@/models/_.js'; +import type { Config } from '@/config.js'; +import { JsonValue } from '@/misc/json-value.js'; +import { UtilityService } from '@/core/UtilityService.js'; +import { IdService } from '@/core/IdService.js'; +import { IActivity, IObject } from './activitypub/type.js'; + +@Injectable() +export class ApLogService { + constructor( + @Inject(DI.config) + private readonly config: Config, + + @Inject(DI.apContextsRepository) + private apContextsRepository: ApContextsRepository, + + @Inject(DI.apInboxLogsRepository) + private readonly apInboxLogsRepository: ApInboxLogsRepository, + + @Inject(DI.apFetchLogsRepository) + private readonly apFetchLogsRepository: ApFetchLogsRepository, + + private readonly utilityService: UtilityService, + private readonly idService: IdService, + ) {} + + /** + * Creates an inbox log from an activity, and saves it if pre-save is enabled. + */ + public async createInboxLog(data: Partial<SkApInboxLog> & { + activity: IActivity, + keyId: string, + }): Promise<SkApInboxLog> { + const { object: activity, context, contextHash } = extractObjectContext(data.activity); + const host = this.utilityService.extractDbHost(data.keyId); + + const log = new SkApInboxLog({ + id: this.idService.gen(), + at: new Date(), + verified: false, + accepted: false, + host, + ...data, + activity, + context, + contextHash, + }); + + if (this.config.activityLogging.preSave) { + await this.saveInboxLog(log); + } + + return log; + } + + /** + * Saves or finalizes an inbox log. + */ + public async saveInboxLog(log: SkApInboxLog): Promise<SkApInboxLog> { + if (log.context) { + await this.saveContext(log.context); + } + + // Will be UPDATE with preSave, and INSERT without. + await this.apInboxLogsRepository.upsert(log, ['id']); + return log; + } + + /** + * Creates a fetch log from an activity, and saves it if pre-save is enabled. + */ + public async createFetchLog(data: Partial<SkApFetchLog> & { + requestUri: string + host: string, + }): Promise<SkApFetchLog> { + const log = new SkApFetchLog({ + id: this.idService.gen(), + at: new Date(), + accepted: false, + ...data, + }); + + if (this.config.activityLogging.preSave) { + await this.saveFetchLog(log); + } + + return log; + } + + /** + * Saves or finalizes a fetch log. + */ + public async saveFetchLog(log: SkApFetchLog): Promise<SkApFetchLog> { + if (log.context) { + await this.saveContext(log.context); + } + + // Will be UPDATE with preSave, and INSERT without. + await this.apFetchLogsRepository.upsert(log, ['id']); + return log; + } + + private async saveContext(context: SkApContext): Promise<void> { + // https://stackoverflow.com/a/47064558 + await this.apContextsRepository + .createQueryBuilder('activity_context') + .insert() + .into(SkApContext) + .values(context) + .orIgnore('md5') + .execute(); + } + + /** + * Deletes all logged copies of an object or objects + * @param objectUris URIs / AP IDs of the objects to delete + */ + public async deleteObjectLogs(objectUris: string | string[]): Promise<number> { + if (Array.isArray(objectUris)) { + const logsDeleted = await this.apFetchLogsRepository.delete({ + objectUri: In(objectUris), + }); + return logsDeleted.affected ?? 0; + } else { + const logsDeleted = await this.apFetchLogsRepository.delete({ + objectUri: objectUris, + }); + return logsDeleted.affected ?? 0; + } + } + + /** + * Deletes all expired AP logs and garbage-collects the AP context cache. + * Returns the total number of deleted rows. + */ + public async deleteExpiredLogs(): Promise<number> { + // This is the date in UTC of the oldest log to KEEP + const oldestAllowed = new Date(Date.now() - this.config.activityLogging.maxAge); + + // Delete all logs older than the threshold. + const inboxDeleted = await this.deleteExpiredInboxLogs(oldestAllowed); + const fetchDeleted = await this.deleteExpiredFetchLogs(oldestAllowed); + + return inboxDeleted + fetchDeleted; + } + + private async deleteExpiredInboxLogs(oldestAllowed: Date): Promise<number> { + const { affected } = await this.apInboxLogsRepository.delete({ + at: LessThan(oldestAllowed), + }); + + return affected ?? 0; + } + + private async deleteExpiredFetchLogs(oldestAllowed: Date): Promise<number> { + const { affected } = await this.apFetchLogsRepository.delete({ + at: LessThan(oldestAllowed), + }); + + return affected ?? 0; + } +} + +export function extractObjectContext<T extends IObject>(input: T) { + const object = Object.assign({}, input, { '@context': undefined }) as Omit<T, '@context'>; + const { context, contextHash } = parseContext(input['@context']); + + return { object, context, contextHash }; +} + +export function parseContext(input: JsonValue | undefined): { contextHash: string | null, context: SkApContext | null } { + // Empty contexts are excluded for easier querying + if (input == null) { + return { + contextHash: null, + context: null, + }; + } + + const contextHash = createHash('md5').update(JSON.stringify(input)).digest('base64'); + const context = new SkApContext({ + md5: contextHash, + json: input, + }); + return { contextHash, context }; +} + +export function calculateDurationSince(startTime: bigint): number { + // Calculate the processing time with correct rounding and decimals. + // 1. Truncate nanoseconds to microseconds + // 2. Scale to 1/10 millisecond ticks. + // 3. Round to nearest tick. + // 4. Sale to milliseconds + // Example: 123,456,789 ns -> 123,456 us -> 12,345.6 ticks -> 12,346 ticks -> 123.46 ms + const endTime = process.hrtime.bigint(); + return Math.round(Number((endTime - startTime) / 1000n) / 10) / 100; +} diff --git a/packages/backend/src/core/CoreModule.ts b/packages/backend/src/core/CoreModule.ts index 8c9f419c44..47be6967d7 100644 --- a/packages/backend/src/core/CoreModule.ts +++ b/packages/backend/src/core/CoreModule.ts @@ -157,6 +157,7 @@ import { QueueService } from './QueueService.js'; import { LoggerService } from './LoggerService.js'; import { SponsorsService } from './SponsorsService.js'; import type { Provider } from '@nestjs/common'; +import { ApLogService } from '@/core/ApLogService.js'; //#region 文字列ベースでのinjection用(循環参照対応のため) const $LoggerService: Provider = { provide: 'LoggerService', useExisting: LoggerService }; @@ -166,6 +167,7 @@ const $AccountMoveService: Provider = { provide: 'AccountMoveService', useExisti const $AccountUpdateService: Provider = { provide: 'AccountUpdateService', useExisting: AccountUpdateService }; const $AnnouncementService: Provider = { provide: 'AnnouncementService', useExisting: AnnouncementService }; const $AntennaService: Provider = { provide: 'AntennaService', useExisting: AntennaService }; +const $ApLogService: Provider = { provide: 'ApLogService', useExisting: ApLogService }; const $AppLockService: Provider = { provide: 'AppLockService', useExisting: AppLockService }; const $AchievementService: Provider = { provide: 'AchievementService', useExisting: AchievementService }; const $AvatarDecorationService: Provider = { provide: 'AvatarDecorationService', useExisting: AvatarDecorationService }; @@ -322,6 +324,7 @@ const $SponsorsService: Provider = { provide: 'SponsorsService', useExisting: Sp AccountUpdateService, AnnouncementService, AntennaService, + ApLogService, AppLockService, AchievementService, AvatarDecorationService, @@ -474,6 +477,7 @@ const $SponsorsService: Provider = { provide: 'SponsorsService', useExisting: Sp $AccountUpdateService, $AnnouncementService, $AntennaService, + $ApLogService, $AppLockService, $AchievementService, $AvatarDecorationService, @@ -627,6 +631,7 @@ const $SponsorsService: Provider = { provide: 'SponsorsService', useExisting: Sp AccountUpdateService, AnnouncementService, AntennaService, + ApLogService, AppLockService, AchievementService, AvatarDecorationService, @@ -778,6 +783,7 @@ const $SponsorsService: Provider = { provide: 'SponsorsService', useExisting: Sp $AccountUpdateService, $AnnouncementService, $AntennaService, + $ApLogService, $AppLockService, $AchievementService, $AvatarDecorationService, diff --git a/packages/backend/src/core/NoteDeleteService.ts b/packages/backend/src/core/NoteDeleteService.ts index b51a3143c9..cb258a4f5a 100644 --- a/packages/backend/src/core/NoteDeleteService.ts +++ b/packages/backend/src/core/NoteDeleteService.ts @@ -24,9 +24,14 @@ import { SearchService } from '@/core/SearchService.js'; import { ModerationLogService } from '@/core/ModerationLogService.js'; import { isQuote, isRenote } from '@/misc/is-renote.js'; import { LatestNoteService } from '@/core/LatestNoteService.js'; +import { ApLogService } from '@/core/ApLogService.js'; +import Logger from '@/logger.js'; +import { LoggerService } from './LoggerService.js'; @Injectable() export class NoteDeleteService { + private readonly logger: Logger; + constructor( @Inject(DI.config) private config: Config, @@ -55,7 +60,11 @@ export class NoteDeleteService { private perUserNotesChart: PerUserNotesChart, private instanceChart: InstanceChart, private latestNoteService: LatestNoteService, - ) {} + private readonly apLogService: ApLogService, + loggerService: LoggerService, + ) { + this.logger = loggerService.getLogger('note-delete-service'); + } /** * 投稿を削除します。 @@ -156,6 +165,11 @@ export class NoteDeleteService { note: note, }); } + + if (note.uri) { + this.apLogService.deleteObjectLogs(note.uri) + .catch(err => this.logger.error(err, `Failed to delete AP logs for note '${note.uri}'`)); + } } @bindThis diff --git a/packages/backend/src/core/activitypub/ApResolverService.ts b/packages/backend/src/core/activitypub/ApResolverService.ts index a0c3a4846c..410803609c 100644 --- a/packages/backend/src/core/activitypub/ApResolverService.ts +++ b/packages/backend/src/core/activitypub/ApResolverService.ts @@ -7,7 +7,7 @@ import { Inject, Injectable } from '@nestjs/common'; import { IsNull, Not } from 'typeorm'; import type { MiLocalUser, MiRemoteUser } from '@/models/User.js'; import { InstanceActorService } from '@/core/InstanceActorService.js'; -import type { NotesRepository, PollsRepository, NoteReactionsRepository, UsersRepository, FollowRequestsRepository, MiMeta } from '@/models/_.js'; +import type { NotesRepository, PollsRepository, NoteReactionsRepository, UsersRepository, FollowRequestsRepository, MiMeta, SkApFetchLog } from '@/models/_.js'; import type { Config } from '@/config.js'; import { HttpRequestService } from '@/core/HttpRequestService.js'; import { DI } from '@/di-symbols.js'; @@ -17,7 +17,8 @@ import { LoggerService } from '@/core/LoggerService.js'; import type Logger from '@/logger.js'; import { fromTuple } from '@/misc/from-tuple.js'; import { IdentifiableError } from '@/misc/identifiable-error.js'; -import { isCollectionOrOrderedCollection } from './type.js'; +import { ApLogService, calculateDurationSince, extractObjectContext } from '@/core/ApLogService.js'; +import { getNullableApId, isCollectionOrOrderedCollection } from './type.js'; import { ApDbResolverService } from './ApDbResolverService.js'; import { ApRendererService } from './ApRendererService.js'; import { ApRequestService } from './ApRequestService.js'; @@ -43,6 +44,7 @@ export class Resolver { private apRendererService: ApRendererService, private apDbResolverService: ApDbResolverService, private loggerService: LoggerService, + private readonly apLogService: ApLogService, private recursionLimit = 256, ) { this.history = new Set(); @@ -81,6 +83,44 @@ export class Resolver { return value; } + const host = this.utilityService.extractDbHost(value); + if (this.config.activityLogging.enabled && !this.utilityService.isSelfHost(host)) { + return await this._resolveLogged(value, host); + } else { + return await this._resolve(value, host); + } + } + + private async _resolveLogged(requestUri: string, host: string): Promise<IObject> { + const startTime = process.hrtime.bigint(); + + const log = await this.apLogService.createFetchLog({ + host: host, + requestUri, + }); + + try { + const result = await this._resolve(requestUri, host, log); + + log.accepted = true; + log.result = 'ok'; + + return result; + } catch (err) { + log.accepted = false; + log.result = String(err); + + throw err; + } finally { + log.duration = calculateDurationSince(startTime); + + // Save or finalize asynchronously + this.apLogService.saveFetchLog(log) + .catch(err => this.logger.error('Failed to record AP object fetch:', err)); + } + } + + private async _resolve(value: string, host: string, log?: SkApFetchLog): Promise<IObject> { if (value.includes('#')) { // URLs with fragment parts cannot be resolved correctly because // the fragment part does not get transmitted over HTTP(S). @@ -98,7 +138,6 @@ export class Resolver { this.history.add(value); - const host = this.utilityService.extractDbHost(value); if (this.utilityService.isSelfHost(host)) { return await this.resolveLocal(value); } @@ -115,6 +154,20 @@ export class Resolver { ? await this.apRequestService.signedGet(value, this.user) as IObject : await this.httpRequestService.getActivityJson(value)) as IObject; + if (log) { + const { object: objectOnly, context, contextHash } = extractObjectContext(object); + const objectUri = getNullableApId(object); + + if (objectUri) { + log.objectUri = objectUri; + log.host = this.utilityService.extractDbHost(objectUri); + } + + log.object = objectOnly; + log.context = context; + log.contextHash = contextHash; + } + if ( Array.isArray(object['@context']) ? !(object['@context'] as unknown[]).includes('https://www.w3.org/ns/activitystreams') : @@ -232,6 +285,7 @@ export class ApResolverService { private apRendererService: ApRendererService, private apDbResolverService: ApDbResolverService, private loggerService: LoggerService, + private readonly apLogService: ApLogService, ) { } @@ -252,6 +306,7 @@ export class ApResolverService { this.apRendererService, this.apDbResolverService, this.loggerService, + this.apLogService, ); } } diff --git a/packages/backend/src/daemons/ApLogCleanupService.ts b/packages/backend/src/daemons/ApLogCleanupService.ts new file mode 100644 index 0000000000..61f76b4e2c --- /dev/null +++ b/packages/backend/src/daemons/ApLogCleanupService.ts @@ -0,0 +1,62 @@ +/* + * SPDX-FileCopyrightText: hazelnoot and other Sharkey contributors + * SPDX-License-Identifier: AGPL-3.0-only + */ + +import { Injectable, type OnApplicationShutdown } from '@nestjs/common'; +import { bindThis } from '@/decorators.js'; +import { LoggerService } from '@/core/LoggerService.js'; +import Logger from '@/logger.js'; +import { ApLogService } from '@/core/ApLogService.js'; + +// 10 minutes +export const scanInterval = 1000 * 60 * 10; + +@Injectable() +export class ApLogCleanupService implements OnApplicationShutdown { + private readonly logger: Logger; + private scanTimer: NodeJS.Timeout | null = null; + + constructor( + private readonly apLogService: ApLogService, + loggerService: LoggerService, + ) { + this.logger = loggerService.getLogger('activity-log-cleanup'); + } + + @bindThis + public async start(): Promise<void> { + // Just in case start() gets called multiple times. + this.dispose(); + + // Prune at startup, in case the server was rebooted during the interval. + // noinspection ES6MissingAwait + this.tick(); + + // Prune on a regular interval for the lifetime of the server. + this.scanTimer = setInterval(this.tick, scanInterval); + } + + @bindThis + private async tick(): Promise<void> { + try { + const affected = await this.apLogService.deleteExpiredLogs(); + this.logger.info(`Activity Log cleanup complete; removed ${affected} expired logs.`); + } catch (err) { + this.logger.error('Activity Log cleanup failed:', err as Error); + } + } + + @bindThis + public onApplicationShutdown(): void { + this.dispose(); + } + + @bindThis + public dispose(): void { + if (this.scanTimer) { + clearInterval(this.scanTimer); + this.scanTimer = null; + } + } +} diff --git a/packages/backend/src/daemons/DaemonModule.ts b/packages/backend/src/daemons/DaemonModule.ts index a67907e6dd..ea71875f19 100644 --- a/packages/backend/src/daemons/DaemonModule.ts +++ b/packages/backend/src/daemons/DaemonModule.ts @@ -8,6 +8,7 @@ import { CoreModule } from '@/core/CoreModule.js'; import { GlobalModule } from '@/GlobalModule.js'; import { QueueStatsService } from './QueueStatsService.js'; import { ServerStatsService } from './ServerStatsService.js'; +import { ApLogCleanupService } from './ApLogCleanupService.js'; @Module({ imports: [ @@ -17,10 +18,12 @@ import { ServerStatsService } from './ServerStatsService.js'; providers: [ QueueStatsService, ServerStatsService, + ApLogCleanupService, ], exports: [ QueueStatsService, ServerStatsService, + ApLogCleanupService, ], }) export class DaemonModule {} diff --git a/packages/backend/src/di-symbols.ts b/packages/backend/src/di-symbols.ts index 296cc4815b..9f4ef5e2e9 100644 --- a/packages/backend/src/di-symbols.ts +++ b/packages/backend/src/di-symbols.ts @@ -22,6 +22,9 @@ export const DI = { appsRepository: Symbol('appsRepository'), avatarDecorationsRepository: Symbol('avatarDecorationsRepository'), latestNotesRepository: Symbol('latestNotesRepository'), + apContextsRepository: Symbol('apContextsRepository'), + apFetchLogsRepository: Symbol('apFetchLogsRepository'), + apInboxLogsRepository: Symbol('apInboxLogsRepository'), noteFavoritesRepository: Symbol('noteFavoritesRepository'), noteThreadMutingsRepository: Symbol('noteThreadMutingsRepository'), noteReactionsRepository: Symbol('noteReactionsRepository'), diff --git a/packages/backend/src/models/RepositoryModule.ts b/packages/backend/src/models/RepositoryModule.ts index 3a1158a42a..78510ba588 100644 --- a/packages/backend/src/models/RepositoryModule.ts +++ b/packages/backend/src/models/RepositoryModule.ts @@ -80,7 +80,10 @@ import { MiUserPublickey, MiUserSecurityKey, MiWebhook, - NoteEdit + NoteEdit, + SkApContext, + SkApFetchLog, + SkApInboxLog, } from './_.js'; import type { DataSource } from 'typeorm'; @@ -126,6 +129,24 @@ const $latestNotesRepository: Provider = { inject: [DI.db], }; +const $apContextRepository: Provider = { + provide: DI.apContextsRepository, + useFactory: (db: DataSource) => db.getRepository(SkApContext).extend(miRepository as MiRepository<SkApContext>), + inject: [DI.db], +}; + +const $apFetchLogsRepository: Provider = { + provide: DI.apFetchLogsRepository, + useFactory: (db: DataSource) => db.getRepository(SkApFetchLog).extend(miRepository as MiRepository<SkApFetchLog>), + inject: [DI.db], +}; + +const $apInboxLogsRepository: Provider = { + provide: DI.apInboxLogsRepository, + useFactory: (db: DataSource) => db.getRepository(SkApInboxLog).extend(miRepository as MiRepository<SkApInboxLog>), + inject: [DI.db], +}; + const $noteFavoritesRepository: Provider = { provide: DI.noteFavoritesRepository, useFactory: (db: DataSource) => db.getRepository(MiNoteFavorite).extend(miRepository as MiRepository<MiNoteFavorite>), @@ -526,6 +547,9 @@ const $noteScheduleRepository: Provider = { $appsRepository, $avatarDecorationsRepository, $latestNotesRepository, + $apContextRepository, + $apFetchLogsRepository, + $apInboxLogsRepository, $noteFavoritesRepository, $noteThreadMutingsRepository, $noteReactionsRepository, @@ -600,6 +624,9 @@ const $noteScheduleRepository: Provider = { $appsRepository, $avatarDecorationsRepository, $latestNotesRepository, + $apContextRepository, + $apFetchLogsRepository, + $apInboxLogsRepository, $noteFavoritesRepository, $noteThreadMutingsRepository, $noteReactionsRepository, diff --git a/packages/backend/src/models/SkApContext.ts b/packages/backend/src/models/SkApContext.ts new file mode 100644 index 0000000000..ff4c6d6fbf --- /dev/null +++ b/packages/backend/src/models/SkApContext.ts @@ -0,0 +1,25 @@ +/* + * SPDX-FileCopyrightText: hazelnoot and other Sharkey contributors + * SPDX-License-Identifier: AGPL-3.0-only + */ + +import { Column, PrimaryColumn, Entity } from 'typeorm'; + +@Entity('ap_context') +export class SkApContext { + @PrimaryColumn('text', { + primaryKeyConstraintName: 'PK_ap_context', + }) + public md5: string; + + @Column('jsonb') + // https://github.com/typeorm/typeorm/issues/8559 + // eslint-disable-next-line @typescript-eslint/no-explicit-any + public json: any; + + constructor(data?: Partial<SkApContext>) { + if (data) { + Object.assign(this, data); + } + } +} diff --git a/packages/backend/src/models/SkApFetchLog.ts b/packages/backend/src/models/SkApFetchLog.ts new file mode 100644 index 0000000000..1e7d861b6c --- /dev/null +++ b/packages/backend/src/models/SkApFetchLog.ts @@ -0,0 +1,89 @@ +/* + * SPDX-FileCopyrightText: hazelnoot and other Sharkey contributors + * SPDX-License-Identifier: AGPL-3.0-only + */ + +import { Column, Index, JoinColumn, ManyToOne, PrimaryColumn, Entity } from 'typeorm'; +import { SkApContext } from '@/models/SkApContext.js'; +import { id } from './util/id.js'; + +/** + * Records objects fetched via AP + */ +@Entity('ap_fetch_log') +export class SkApFetchLog { + @PrimaryColumn({ + ...id(), + primaryKeyConstraintName: 'PK_ap_fetch_log', + }) + public id: string; + + @Index('IDX_ap_fetch_log_at') + @Column('timestamptz') + public at: Date; + + /** + * Processing duration in milliseconds + */ + @Column('double precision', { nullable: true }) + public duration: number | null = null; + + /** + * DB hostname extracted from responseUri, or requestUri if fetch is incomplete + */ + @Index('IDX_ap_fetch_log_host') + @Column('text') + public host: string; + + /** + * Original requested URI + */ + @Column('text', { + name: 'request_uri', + }) + public requestUri: string; + + /** + * Canonical URI / object ID, taken from the final payload + */ + @Column('text', { + name: 'object_uri', + nullable: true, + }) + @Index('IDX_ap_fetch_log_object_uri') + public objectUri: string | null = null; + + @Column('boolean', { nullable: true }) + public accepted: boolean | null = null; + + @Column('text', { nullable: true }) + public result: string | null = null; + + @Column('jsonb', { nullable: true }) + // https://github.com/typeorm/typeorm/issues/8559 + // eslint-disable-next-line @typescript-eslint/no-explicit-any + public object: any | null = null; + + @Column({ + type: 'text', + name: 'context_hash', + nullable: true, + }) + public contextHash: string | null; + + @ManyToOne(() => SkApContext, { + onDelete: 'CASCADE', + nullable: true, + }) + @JoinColumn({ + name: 'context_hash', + foreignKeyConstraintName: 'FK_ap_fetch_log_context_hash', + }) + public context: SkApContext | null; + + constructor(data?: Partial<SkApFetchLog>) { + if (data) { + Object.assign(this, data); + } + } +} diff --git a/packages/backend/src/models/SkApInboxLog.ts b/packages/backend/src/models/SkApInboxLog.ts new file mode 100644 index 0000000000..867094405c --- /dev/null +++ b/packages/backend/src/models/SkApInboxLog.ts @@ -0,0 +1,109 @@ +/* + * SPDX-FileCopyrightText: hazelnoot and other Sharkey contributors + * SPDX-License-Identifier: AGPL-3.0-only + */ + +import { Column, Entity, Index, JoinColumn, ManyToOne, PrimaryColumn } from 'typeorm'; +import { SkApContext } from '@/models/SkApContext.js'; +import { MiUser } from '@/models/_.js'; +import { id } from './util/id.js'; + +/** + * Records activities received in the inbox + */ +@Entity('ap_inbox_log') +export class SkApInboxLog { + @PrimaryColumn({ + ...id(), + primaryKeyConstraintName: 'PK_ap_inbox_log', + }) + public id: string; + + @Index('IDX_ap_inbox_log_at') + @Column('timestamptz') + public at: Date; + + /** + * Processing duration in milliseconds + */ + @Column('double precision', { nullable: true }) + public duration: number | null = null; + + /** + * Key ID that was used to sign this request. + * Untrusted unless verified is true. + */ + @Column({ + type: 'text', + name: 'key_id', + }) + public keyId: string; + + /** + * Instance that the activity was sent from. + * Untrusted unless verified is true. + */ + @Index('IDX_ap_inbox_log_host') + @Column('text') + public host: string; + + @Column('boolean') + public verified: boolean; + + @Column('boolean') + public accepted: boolean; + + @Column('text', { nullable: true }) + public result: string | null = null; + + @Column('jsonb') + // https://github.com/typeorm/typeorm/issues/8559 + // eslint-disable-next-line @typescript-eslint/no-explicit-any + public activity: any; + + @Column({ + type: 'text', + name: 'context_hash', + nullable: true, + }) + public contextHash: string | null; + + @ManyToOne(() => SkApContext, { + onDelete: 'CASCADE', + nullable: true, + }) + @JoinColumn({ + name: 'context_hash', + foreignKeyConstraintName: 'FK_ap_inbox_log_context_hash', + }) + public context: SkApContext | null; + + /** + * ID of the user who signed this request. + */ + @Column({ + ...id(), + name: 'auth_user_id', + nullable: true, + }) + public authUserId: string | null; + + /** + * User who signed this request. + */ + @ManyToOne(() => MiUser, { + onDelete: 'CASCADE', + nullable: true, + }) + @JoinColumn({ + name: 'auth_user_id', + foreignKeyConstraintName: 'FK_ap_inbox_log_auth_user_id', + }) + public authUser: MiUser | null; + + constructor(data?: Partial<SkApInboxLog>) { + if (data) { + Object.assign(this, data); + } + } +} diff --git a/packages/backend/src/models/_.ts b/packages/backend/src/models/_.ts index 9a4ebfc90f..4bd6e78ef4 100644 --- a/packages/backend/src/models/_.ts +++ b/packages/backend/src/models/_.ts @@ -82,6 +82,9 @@ import { NoteEdit } from '@/models/NoteEdit.js'; import { MiBubbleGameRecord } from '@/models/BubbleGameRecord.js'; import { MiReversiGame } from '@/models/ReversiGame.js'; import { MiNoteSchedule } from '@/models/NoteSchedule.js'; +import { SkApInboxLog } from '@/models/SkApInboxLog.js'; +import { SkApFetchLog } from '@/models/SkApFetchLog.js'; +import { SkApContext } from '@/models/SkApContext.js'; import type { QueryDeepPartialEntity } from 'typeorm/query-builder/QueryPartialEntity.js'; export interface MiRepository<T extends ObjectLiteral> { @@ -129,6 +132,9 @@ export const miRepository = { export { SkLatestNote, + SkApContext, + SkApFetchLog, + SkApInboxLog, MiAbuseUserReport, MiAbuseReportNotificationRecipient, MiAccessToken, @@ -229,6 +235,9 @@ export type HashtagsRepository = Repository<MiHashtag> & MiRepository<MiHashtag> export type InstancesRepository = Repository<MiInstance> & MiRepository<MiInstance>; export type MetasRepository = Repository<MiMeta> & MiRepository<MiMeta>; export type LatestNotesRepository = Repository<SkLatestNote> & MiRepository<SkLatestNote>; +export type ApContextsRepository = Repository<SkApContext> & MiRepository<SkApContext>; +export type ApFetchLogsRepository = Repository<SkApFetchLog> & MiRepository<SkApFetchLog>; +export type ApInboxLogsRepository = Repository<SkApInboxLog> & MiRepository<SkApInboxLog>; export type ModerationLogsRepository = Repository<MiModerationLog> & MiRepository<MiModerationLog>; export type MutingsRepository = Repository<MiMuting> & MiRepository<MiMuting>; export type RenoteMutingsRepository = Repository<MiRenoteMuting> & MiRepository<MiRenoteMuting>; diff --git a/packages/backend/src/postgres.ts b/packages/backend/src/postgres.ts index 98405052c6..1a5fdc8412 100644 --- a/packages/backend/src/postgres.ts +++ b/packages/backend/src/postgres.ts @@ -85,6 +85,9 @@ import { Config } from '@/config.js'; import MisskeyLogger from '@/logger.js'; import { bindThis } from '@/decorators.js'; import { SkLatestNote } from '@/models/LatestNote.js'; +import { SkApContext } from '@/models/SkApContext.js'; +import { SkApFetchLog } from '@/models/SkApFetchLog.js'; +import { SkApInboxLog } from '@/models/SkApInboxLog.js'; pg.types.setTypeParser(20, Number); @@ -171,6 +174,9 @@ class MyCustomLogger implements Logger { export const entities = [ SkLatestNote, + SkApContext, + SkApFetchLog, + SkApInboxLog, MiAnnouncement, MiAnnouncementRead, MiMeta, diff --git a/packages/backend/src/queue/processors/DeleteAccountProcessorService.ts b/packages/backend/src/queue/processors/DeleteAccountProcessorService.ts index e350b97f53..66bed72f18 100644 --- a/packages/backend/src/queue/processors/DeleteAccountProcessorService.ts +++ b/packages/backend/src/queue/processors/DeleteAccountProcessorService.ts @@ -15,6 +15,7 @@ import type { MiNoteReaction } from '@/models/NoteReaction.js'; import { EmailService } from '@/core/EmailService.js'; import { bindThis } from '@/decorators.js'; import { SearchService } from '@/core/SearchService.js'; +import { ApLogService } from '@/core/ApLogService.js'; import { ReactionService } from '@/core/ReactionService.js'; import { QueueLoggerService } from '../QueueLoggerService.js'; import type * as Bull from 'bullmq'; @@ -45,6 +46,7 @@ export class DeleteAccountProcessorService { private queueLoggerService: QueueLoggerService, private searchService: SearchService, private reactionService: ReactionService, + private readonly apLogService: ApLogService, ) { this.logger = this.queueLoggerService.logger.createSubLogger('delete-account'); } @@ -84,6 +86,13 @@ export class DeleteAccountProcessorService { for (const note of notes) { await this.searchService.unindexNote(note); } + + // Delete note AP logs + const noteUris = notes.map(n => n.uri).filter(u => !!u) as string[]; + if (noteUris.length > 0) { + await this.apLogService.deleteObjectLogs(noteUris) + .catch(err => this.logger.error(err, `Failed to delete AP logs for notes of user '${user.uri ?? user.id}'`)); + } } this.logger.succ('All of notes deleted'); @@ -149,6 +158,13 @@ export class DeleteAccountProcessorService { this.logger.succ('All of files deleted'); } + { // Delete actor logs + if (user.uri) { + await this.apLogService.deleteObjectLogs(user.uri) + .catch(err => this.logger.error(err, `Failed to delete AP logs for user '${user.uri}'`)); + } + } + { // Send email notification const profile = await this.userProfilesRepository.findOneByOrFail({ userId: user.id }); if (profile.email && profile.emailVerified) { diff --git a/packages/backend/src/queue/processors/InboxProcessorService.ts b/packages/backend/src/queue/processors/InboxProcessorService.ts index 87d4bf52fa..35a0bf095d 100644 --- a/packages/backend/src/queue/processors/InboxProcessorService.ts +++ b/packages/backend/src/queue/processors/InboxProcessorService.ts @@ -29,6 +29,9 @@ import { CollapsedQueue } from '@/misc/collapsed-queue.js'; import { MiNote } from '@/models/Note.js'; import { MiMeta } from '@/models/Meta.js'; import { DI } from '@/di-symbols.js'; +import { SkApInboxLog } from '@/models/_.js'; +import type { Config } from '@/config.js'; +import { ApLogService, calculateDurationSince } from '@/core/ApLogService.js'; import { QueueLoggerService } from '../QueueLoggerService.js'; import type { InboxJobData } from '../types.js'; @@ -46,6 +49,9 @@ export class InboxProcessorService implements OnApplicationShutdown { @Inject(DI.meta) private meta: MiMeta, + @Inject(DI.config) + private config: Config, + private utilityService: UtilityService, private apInboxService: ApInboxService, private federatedInstanceService: FederatedInstanceService, @@ -57,6 +63,7 @@ export class InboxProcessorService implements OnApplicationShutdown { private apRequestChart: ApRequestChart, private federationChart: FederationChart, private queueLoggerService: QueueLoggerService, + private readonly apLogService: ApLogService, ) { this.logger = this.queueLoggerService.logger.createSubLogger('inbox'); this.updateInstanceQueue = new CollapsedQueue(process.env.NODE_ENV !== 'test' ? 60 * 1000 * 5 : 0, this.collapseUpdateInstanceJobs, this.performUpdateInstance); @@ -64,6 +71,41 @@ export class InboxProcessorService implements OnApplicationShutdown { @bindThis public async process(job: Bull.Job<InboxJobData>): Promise<string> { + if (this.config.activityLogging.enabled) { + return await this._processLogged(job); + } else { + return await this._process(job); + } + } + + private async _processLogged(job: Bull.Job<InboxJobData>): Promise<string> { + const startTime = process.hrtime.bigint(); + const activity = job.data.activity; + const keyId = job.data.signature.keyId; + const log = await this.apLogService.createInboxLog({ activity, keyId }); + + try { + const result = await this._process(job, log); + + log.accepted = result.startsWith('ok'); + log.result = result; + + return result; + } catch (err) { + log.accepted = false; + log.result = String(err); + + throw err; + } finally { + log.duration = calculateDurationSince(startTime); + + // Save or finalize asynchronously + this.apLogService.saveInboxLog(log) + .catch(err => this.logger.error('Failed to record AP activity:', err)); + } + } + + private async _process(job: Bull.Job<InboxJobData>, log?: SkApInboxLog): Promise<string> { const signature = job.data.signature; // HTTP-signature let activity = job.data.activity; @@ -197,6 +239,13 @@ export class InboxProcessorService implements OnApplicationShutdown { delete activity.id; } + // Record verified user in log + if (log) { + log.verified = true; + log.authUser = authUser.user; + log.authUserId = authUser.user.id; + } + this.apRequestChart.inbox(); this.federationChart.inbox(authUser.user.host); diff --git a/packages/backend/test/misc/mock-resolver.ts b/packages/backend/test/misc/mock-resolver.ts index c8f3db8aac..791b04e6e2 100644 --- a/packages/backend/test/misc/mock-resolver.ts +++ b/packages/backend/test/misc/mock-resolver.ts @@ -23,6 +23,7 @@ import type { PollsRepository, UsersRepository, } from '@/models/_.js'; +import { ApLogService } from '@/core/ApLogService.js'; type MockResponse = { type: string; @@ -49,6 +50,7 @@ export class MockResolver extends Resolver { {} as ApRendererService, {} as ApDbResolverService, loggerService, + {} as ApLogService, ); } |