From 81944b3bdf49cf95294adcefc265a568b921dee0 Mon Sep 17 00:00:00 2001 From: Hazelnoot Date: Thu, 30 Jan 2025 22:36:19 -0500 Subject: implement AP fetch logs --- packages/backend/src/core/ApLogService.ts | 189 ++++++++++++++++++++++++++++++ 1 file changed, 189 insertions(+) create mode 100644 packages/backend/src/core/ApLogService.ts (limited to 'packages/backend/src/core/ApLogService.ts') diff --git a/packages/backend/src/core/ApLogService.ts b/packages/backend/src/core/ApLogService.ts new file mode 100644 index 0000000000..362eba24be --- /dev/null +++ b/packages/backend/src/core/ApLogService.ts @@ -0,0 +1,189 @@ +/* + * SPDX-FileCopyrightText: hazelnoot and other Sharkey contributors + * SPDX-License-Identifier: AGPL-3.0-only + */ + +import { createHash } from 'crypto'; +import { Inject, Injectable } from '@nestjs/common'; +import { LessThan } from 'typeorm'; +import { DI } from '@/di-symbols.js'; +import { SkApFetchLog, SkApInboxLog, SkApContext } from '@/models/_.js'; +import type { ApContextsRepository, ApFetchLogsRepository, ApInboxLogsRepository } from '@/models/_.js'; +import type { Config } from '@/config.js'; +import { JsonValue } from '@/misc/json-value.js'; +import { UtilityService } from '@/core/UtilityService.js'; +import { IdService } from '@/core/IdService.js'; +import { IActivity, IObject } from './activitypub/type.js'; + +@Injectable() +export class ApLogService { + constructor( + @Inject(DI.config) + private readonly config: Config, + + @Inject(DI.apContextsRepository) + private apContextsRepository: ApContextsRepository, + + @Inject(DI.apInboxLogsRepository) + private readonly apInboxLogsRepository: ApInboxLogsRepository, + + @Inject(DI.apFetchLogsRepository) + private readonly apFetchLogsRepository: ApFetchLogsRepository, + + private readonly utilityService: UtilityService, + private readonly idService: IdService, + ) {} + + /** + * Creates an inbox log from an activity, and saves it if pre-save is enabled. + */ + public async createInboxLog(data: Partial & { + activity: IActivity, + keyId: string, + }): Promise { + const { object: activity, context, contextHash } = extractObjectContext(data.activity); + const host = this.utilityService.extractDbHost(data.keyId); + + const log = new SkApInboxLog({ + id: this.idService.gen(), + at: new Date(), + verified: false, + accepted: false, + host, + ...data, + activity, + context, + contextHash, + }); + + if (this.config.activityLogging.preSave) { + await this.saveInboxLog(log); + } + + return log; + } + + /** + * Saves or finalizes an inbox log. + */ + public async saveInboxLog(log: SkApInboxLog): Promise { + if (log.context) { + await this.saveContext(log.context); + } + + // Will be UPDATE with preSave, and INSERT without. + await this.apInboxLogsRepository.upsert(log, ['id']); + return log; + } + + /** + * Creates a fetch log from an activity, and saves it if pre-save is enabled. + */ + public async createFetchLog(data: Partial & { + requestUri: string + host: string, + }): Promise { + const log = new SkApFetchLog({ + id: this.idService.gen(), + at: new Date(), + accepted: false, + ...data, + }); + + if (this.config.activityLogging.preSave) { + await this.saveFetchLog(log); + } + + return log; + } + + /** + * Saves or finalizes a fetch log. + */ + public async saveFetchLog(log: SkApFetchLog): Promise { + if (log.context) { + await this.saveContext(log.context); + } + + // Will be UPDATE with preSave, and INSERT without. + await this.apFetchLogsRepository.upsert(log, ['id']); + return log; + } + + private async saveContext(context: SkApContext): Promise { + // https://stackoverflow.com/a/47064558 + await this.apContextsRepository + .createQueryBuilder('activity_context') + .insert() + .into(SkApContext) + .values(context) + .orIgnore('md5') + .execute(); + } + + /** + * Deletes all expired AP logs and garbage-collects the AP context cache. + * Returns the total number of deleted rows. + */ + public async deleteExpiredLogs(): Promise { + // This is the date in UTC of the oldest log to KEEP + const oldestAllowed = new Date(Date.now() - this.config.activityLogging.maxAge); + + // Delete all logs older than the threshold. + const inboxDeleted = await this.deleteExpiredInboxLogs(oldestAllowed); + const fetchDeleted = await this.deleteExpiredFetchLogs(oldestAllowed); + + return inboxDeleted + fetchDeleted; + } + + private async deleteExpiredInboxLogs(oldestAllowed: Date): Promise { + const { affected } = await this.apInboxLogsRepository.delete({ + at: LessThan(oldestAllowed), + }); + + return affected ?? 0; + } + + private async deleteExpiredFetchLogs(oldestAllowed: Date): Promise { + const { affected } = await this.apFetchLogsRepository.delete({ + at: LessThan(oldestAllowed), + }); + + return affected ?? 0; + } +} + +export function extractObjectContext(input: T) { + const object = Object.assign({}, input, { '@context': undefined }) as Omit; + const { context, contextHash } = parseContext(input['@context']); + + return { object, context, contextHash }; +} + +export function parseContext(input: JsonValue | undefined): { contextHash: string | null, context: SkApContext | null } { + // Empty contexts are excluded for easier querying + if (input == null) { + return { + contextHash: null, + context: null, + }; + } + + const contextHash = createHash('md5').update(JSON.stringify(input)).digest('base64'); + const context = new SkApContext({ + md5: contextHash, + json: input, + }); + return { contextHash, context }; +} + +export function calculateDurationSince(startTime: bigint): number { + // Calculate the processing time with correct rounding and decimals. + // 1. Truncate nanoseconds to microseconds + // 2. Scale to 1/10 millisecond ticks. + // 3. Round to nearest tick. + // 4. Sale to milliseconds + // Example: 123,456,789 ns -> 123,456 us -> 12,345.6 ticks -> 12,346 ticks -> 123.46 ms + const endTime = process.hrtime.bigint(); + return Math.round(Number((endTime - startTime) / 1000n) / 10) / 100; +} -- cgit v1.2.3-freya From 9de5ecae518973e29f63bde51062aa0ed7eadb11 Mon Sep 17 00:00:00 2001 From: Hazelnoot Date: Sun, 2 Feb 2025 23:19:41 -0500 Subject: delete fetch logs when a note or user is deleted --- packages/backend/src/core/ApLogService.ts | 20 +++++++++++++++++++- packages/backend/src/core/NoteDeleteService.ts | 16 +++++++++++++++- .../processors/DeleteAccountProcessorService.ts | 16 ++++++++++++++++ 3 files changed, 50 insertions(+), 2 deletions(-) (limited to 'packages/backend/src/core/ApLogService.ts') diff --git a/packages/backend/src/core/ApLogService.ts b/packages/backend/src/core/ApLogService.ts index 362eba24be..096ec21de7 100644 --- a/packages/backend/src/core/ApLogService.ts +++ b/packages/backend/src/core/ApLogService.ts @@ -5,7 +5,7 @@ import { createHash } from 'crypto'; import { Inject, Injectable } from '@nestjs/common'; -import { LessThan } from 'typeorm'; +import { In, LessThan } from 'typeorm'; import { DI } from '@/di-symbols.js'; import { SkApFetchLog, SkApInboxLog, SkApContext } from '@/models/_.js'; import type { ApContextsRepository, ApFetchLogsRepository, ApInboxLogsRepository } from '@/models/_.js'; @@ -121,6 +121,24 @@ export class ApLogService { .execute(); } + /** + * Deletes all logged copies of an object or objects + * @param objectUris URIs / AP IDs of the objects to delete + */ + public async deleteObjectLogs(objectUris: string | string[]): Promise { + if (Array.isArray(objectUris)) { + const logsDeleted = await this.apFetchLogsRepository.delete({ + objectUri: In(objectUris), + }); + return logsDeleted.affected ?? 0; + } else { + const logsDeleted = await this.apFetchLogsRepository.delete({ + objectUri: objectUris, + }); + return logsDeleted.affected ?? 0; + } + } + /** * Deletes all expired AP logs and garbage-collects the AP context cache. * Returns the total number of deleted rows. diff --git a/packages/backend/src/core/NoteDeleteService.ts b/packages/backend/src/core/NoteDeleteService.ts index b51a3143c9..cb258a4f5a 100644 --- a/packages/backend/src/core/NoteDeleteService.ts +++ b/packages/backend/src/core/NoteDeleteService.ts @@ -24,9 +24,14 @@ import { SearchService } from '@/core/SearchService.js'; import { ModerationLogService } from '@/core/ModerationLogService.js'; import { isQuote, isRenote } from '@/misc/is-renote.js'; import { LatestNoteService } from '@/core/LatestNoteService.js'; +import { ApLogService } from '@/core/ApLogService.js'; +import Logger from '@/logger.js'; +import { LoggerService } from './LoggerService.js'; @Injectable() export class NoteDeleteService { + private readonly logger: Logger; + constructor( @Inject(DI.config) private config: Config, @@ -55,7 +60,11 @@ export class NoteDeleteService { private perUserNotesChart: PerUserNotesChart, private instanceChart: InstanceChart, private latestNoteService: LatestNoteService, - ) {} + private readonly apLogService: ApLogService, + loggerService: LoggerService, + ) { + this.logger = loggerService.getLogger('note-delete-service'); + } /** * 投稿を削除します。 @@ -156,6 +165,11 @@ export class NoteDeleteService { note: note, }); } + + if (note.uri) { + this.apLogService.deleteObjectLogs(note.uri) + .catch(err => this.logger.error(err, `Failed to delete AP logs for note '${note.uri}'`)); + } } @bindThis diff --git a/packages/backend/src/queue/processors/DeleteAccountProcessorService.ts b/packages/backend/src/queue/processors/DeleteAccountProcessorService.ts index e350b97f53..66bed72f18 100644 --- a/packages/backend/src/queue/processors/DeleteAccountProcessorService.ts +++ b/packages/backend/src/queue/processors/DeleteAccountProcessorService.ts @@ -15,6 +15,7 @@ import type { MiNoteReaction } from '@/models/NoteReaction.js'; import { EmailService } from '@/core/EmailService.js'; import { bindThis } from '@/decorators.js'; import { SearchService } from '@/core/SearchService.js'; +import { ApLogService } from '@/core/ApLogService.js'; import { ReactionService } from '@/core/ReactionService.js'; import { QueueLoggerService } from '../QueueLoggerService.js'; import type * as Bull from 'bullmq'; @@ -45,6 +46,7 @@ export class DeleteAccountProcessorService { private queueLoggerService: QueueLoggerService, private searchService: SearchService, private reactionService: ReactionService, + private readonly apLogService: ApLogService, ) { this.logger = this.queueLoggerService.logger.createSubLogger('delete-account'); } @@ -84,6 +86,13 @@ export class DeleteAccountProcessorService { for (const note of notes) { await this.searchService.unindexNote(note); } + + // Delete note AP logs + const noteUris = notes.map(n => n.uri).filter(u => !!u) as string[]; + if (noteUris.length > 0) { + await this.apLogService.deleteObjectLogs(noteUris) + .catch(err => this.logger.error(err, `Failed to delete AP logs for notes of user '${user.uri ?? user.id}'`)); + } } this.logger.succ('All of notes deleted'); @@ -149,6 +158,13 @@ export class DeleteAccountProcessorService { this.logger.succ('All of files deleted'); } + { // Delete actor logs + if (user.uri) { + await this.apLogService.deleteObjectLogs(user.uri) + .catch(err => this.logger.error(err, `Failed to delete AP logs for user '${user.uri}'`)); + } + } + { // Send email notification const profile = await this.userProfilesRepository.findOneByOrFail({ userId: user.id }); if (profile.email && profile.emailVerified) { -- cgit v1.2.3-freya