diff options
| author | Julia <julia@insertdomain.name> | 2025-03-02 19:54:32 +0000 |
|---|---|---|
| committer | Julia <julia@insertdomain.name> | 2025-03-02 19:54:32 +0000 |
| commit | 9e13c375c5ef4103ad5ee87fea583b154e9e16f3 (patch) | |
| tree | fe9e7b1a474e22fb0c37bd68cfd260f7ba39be74 /packages/backend/src/core | |
| parent | merge: pin corepack version (!885) (diff) | |
| parent | bump version (diff) | |
| download | sharkey-9e13c375c5ef4103ad5ee87fea583b154e9e16f3.tar.gz sharkey-9e13c375c5ef4103ad5ee87fea583b154e9e16f3.tar.bz2 sharkey-9e13c375c5ef4103ad5ee87fea583b154e9e16f3.zip | |
merge: 2025.2.2 (!927)
View MR for information: https://activitypub.software/TransFem-org/Sharkey/-/merge_requests/927
Approved-by: Marie <github@yuugi.dev>
Approved-by: Julia <julia@insertdomain.name>
Diffstat (limited to 'packages/backend/src/core')
43 files changed, 1930 insertions, 653 deletions
diff --git a/packages/backend/src/core/AbuseReportNotificationService.ts b/packages/backend/src/core/AbuseReportNotificationService.ts index 742e2621fd..9bca795479 100644 --- a/packages/backend/src/core/AbuseReportNotificationService.ts +++ b/packages/backend/src/core/AbuseReportNotificationService.ts @@ -160,22 +160,22 @@ export class AbuseReportNotificationService implements OnApplicationShutdown { }; }); - const recipientWebhookIds = await this.fetchWebhookRecipients() - .then(it => it - .filter(it => it.isActive && it.systemWebhookId && it.method === 'webhook') - .map(it => it.systemWebhookId) - .filter(x => x != null)); - for (const webhookId of recipientWebhookIds) { - await Promise.all( - convertedReports.map(it => { - return this.systemWebhookService.enqueueSystemWebhook( - webhookId, - type, - it, - ); - }), - ); - } + const inactiveRecipients = await this.fetchWebhookRecipients() + .then(it => it.filter(it => !it.isActive)); + const withoutWebhookIds = inactiveRecipients + .map(it => it.systemWebhookId) + .filter(x => x != null); + return Promise.all( + convertedReports.map(it => { + return this.systemWebhookService.enqueueSystemWebhook( + type, + it, + { + excludes: withoutWebhookIds, + }, + ); + }), + ); } /** diff --git a/packages/backend/src/core/AccountMoveService.ts b/packages/backend/src/core/AccountMoveService.ts index 24d11f29ff..e24fefb4b5 100644 --- a/packages/backend/src/core/AccountMoveService.ts +++ b/packages/backend/src/core/AccountMoveService.ts @@ -9,7 +9,7 @@ import { IsNull, In, MoreThan, Not } from 'typeorm'; import { bindThis } from '@/decorators.js'; import { DI } from '@/di-symbols.js'; import type { MiLocalUser, MiRemoteUser, MiUser } from '@/models/User.js'; -import type { BlockingsRepository, FollowingsRepository, InstancesRepository, MiMeta, MutingsRepository, UserListMembershipsRepository, UsersRepository } from '@/models/_.js'; +import type { BlockingsRepository, FollowingsRepository, InstancesRepository, MiMeta, MutingsRepository, UserListMembershipsRepository, UsersRepository, NoteScheduleRepository, MiNoteSchedule } from '@/models/_.js'; import type { RelationshipJobData, ThinUser } from '@/queue/types.js'; import { IdService } from '@/core/IdService.js'; @@ -49,6 +49,9 @@ export class AccountMoveService { @Inject(DI.instancesRepository) private instancesRepository: InstancesRepository, + @Inject(DI.noteScheduleRepository) + private noteScheduleRepository: NoteScheduleRepository, + private userEntityService: UserEntityService, private idService: IdService, private apPersonService: ApPersonService, @@ -119,6 +122,7 @@ export class AccountMoveService { await Promise.all([ this.copyBlocking(src, dst), this.copyMutings(src, dst), + this.deleteScheduledNotes(src), this.updateLists(src, dst), ]); } catch { @@ -201,6 +205,21 @@ export class AccountMoveService { await this.mutingsRepository.insert(arrayToInsert); } + @bindThis + public async deleteScheduledNotes(src: ThinUser): Promise<void> { + const scheduledNotes = await this.noteScheduleRepository.findBy({ + userId: src.id, + }) as MiNoteSchedule[]; + + for (const note of scheduledNotes) { + await this.queueService.ScheduleNotePostQueue.remove(`schedNote:${note.id}`); + } + + await this.noteScheduleRepository.delete({ + userId: src.id, + }); + } + /** * Update lists while moving accounts. * - No removal of the old account from the lists diff --git a/packages/backend/src/core/ApLogService.ts b/packages/backend/src/core/ApLogService.ts new file mode 100644 index 0000000000..096ec21de7 --- /dev/null +++ b/packages/backend/src/core/ApLogService.ts @@ -0,0 +1,207 @@ +/* + * SPDX-FileCopyrightText: hazelnoot and other Sharkey contributors + * SPDX-License-Identifier: AGPL-3.0-only + */ + +import { createHash } from 'crypto'; +import { Inject, Injectable } from '@nestjs/common'; +import { In, LessThan } from 'typeorm'; +import { DI } from '@/di-symbols.js'; +import { SkApFetchLog, SkApInboxLog, SkApContext } from '@/models/_.js'; +import type { ApContextsRepository, ApFetchLogsRepository, ApInboxLogsRepository } from '@/models/_.js'; +import type { Config } from '@/config.js'; +import { JsonValue } from '@/misc/json-value.js'; +import { UtilityService } from '@/core/UtilityService.js'; +import { IdService } from '@/core/IdService.js'; +import { IActivity, IObject } from './activitypub/type.js'; + +@Injectable() +export class ApLogService { + constructor( + @Inject(DI.config) + private readonly config: Config, + + @Inject(DI.apContextsRepository) + private apContextsRepository: ApContextsRepository, + + @Inject(DI.apInboxLogsRepository) + private readonly apInboxLogsRepository: ApInboxLogsRepository, + + @Inject(DI.apFetchLogsRepository) + private readonly apFetchLogsRepository: ApFetchLogsRepository, + + private readonly utilityService: UtilityService, + private readonly idService: IdService, + ) {} + + /** + * Creates an inbox log from an activity, and saves it if pre-save is enabled. + */ + public async createInboxLog(data: Partial<SkApInboxLog> & { + activity: IActivity, + keyId: string, + }): Promise<SkApInboxLog> { + const { object: activity, context, contextHash } = extractObjectContext(data.activity); + const host = this.utilityService.extractDbHost(data.keyId); + + const log = new SkApInboxLog({ + id: this.idService.gen(), + at: new Date(), + verified: false, + accepted: false, + host, + ...data, + activity, + context, + contextHash, + }); + + if (this.config.activityLogging.preSave) { + await this.saveInboxLog(log); + } + + return log; + } + + /** + * Saves or finalizes an inbox log. + */ + public async saveInboxLog(log: SkApInboxLog): Promise<SkApInboxLog> { + if (log.context) { + await this.saveContext(log.context); + } + + // Will be UPDATE with preSave, and INSERT without. + await this.apInboxLogsRepository.upsert(log, ['id']); + return log; + } + + /** + * Creates a fetch log from an activity, and saves it if pre-save is enabled. + */ + public async createFetchLog(data: Partial<SkApFetchLog> & { + requestUri: string + host: string, + }): Promise<SkApFetchLog> { + const log = new SkApFetchLog({ + id: this.idService.gen(), + at: new Date(), + accepted: false, + ...data, + }); + + if (this.config.activityLogging.preSave) { + await this.saveFetchLog(log); + } + + return log; + } + + /** + * Saves or finalizes a fetch log. + */ + public async saveFetchLog(log: SkApFetchLog): Promise<SkApFetchLog> { + if (log.context) { + await this.saveContext(log.context); + } + + // Will be UPDATE with preSave, and INSERT without. + await this.apFetchLogsRepository.upsert(log, ['id']); + return log; + } + + private async saveContext(context: SkApContext): Promise<void> { + // https://stackoverflow.com/a/47064558 + await this.apContextsRepository + .createQueryBuilder('activity_context') + .insert() + .into(SkApContext) + .values(context) + .orIgnore('md5') + .execute(); + } + + /** + * Deletes all logged copies of an object or objects + * @param objectUris URIs / AP IDs of the objects to delete + */ + public async deleteObjectLogs(objectUris: string | string[]): Promise<number> { + if (Array.isArray(objectUris)) { + const logsDeleted = await this.apFetchLogsRepository.delete({ + objectUri: In(objectUris), + }); + return logsDeleted.affected ?? 0; + } else { + const logsDeleted = await this.apFetchLogsRepository.delete({ + objectUri: objectUris, + }); + return logsDeleted.affected ?? 0; + } + } + + /** + * Deletes all expired AP logs and garbage-collects the AP context cache. + * Returns the total number of deleted rows. + */ + public async deleteExpiredLogs(): Promise<number> { + // This is the date in UTC of the oldest log to KEEP + const oldestAllowed = new Date(Date.now() - this.config.activityLogging.maxAge); + + // Delete all logs older than the threshold. + const inboxDeleted = await this.deleteExpiredInboxLogs(oldestAllowed); + const fetchDeleted = await this.deleteExpiredFetchLogs(oldestAllowed); + + return inboxDeleted + fetchDeleted; + } + + private async deleteExpiredInboxLogs(oldestAllowed: Date): Promise<number> { + const { affected } = await this.apInboxLogsRepository.delete({ + at: LessThan(oldestAllowed), + }); + + return affected ?? 0; + } + + private async deleteExpiredFetchLogs(oldestAllowed: Date): Promise<number> { + const { affected } = await this.apFetchLogsRepository.delete({ + at: LessThan(oldestAllowed), + }); + + return affected ?? 0; + } +} + +export function extractObjectContext<T extends IObject>(input: T) { + const object = Object.assign({}, input, { '@context': undefined }) as Omit<T, '@context'>; + const { context, contextHash } = parseContext(input['@context']); + + return { object, context, contextHash }; +} + +export function parseContext(input: JsonValue | undefined): { contextHash: string | null, context: SkApContext | null } { + // Empty contexts are excluded for easier querying + if (input == null) { + return { + contextHash: null, + context: null, + }; + } + + const contextHash = createHash('md5').update(JSON.stringify(input)).digest('base64'); + const context = new SkApContext({ + md5: contextHash, + json: input, + }); + return { contextHash, context }; +} + +export function calculateDurationSince(startTime: bigint): number { + // Calculate the processing time with correct rounding and decimals. + // 1. Truncate nanoseconds to microseconds + // 2. Scale to 1/10 millisecond ticks. + // 3. Round to nearest tick. + // 4. Sale to milliseconds + // Example: 123,456,789 ns -> 123,456 us -> 12,345.6 ticks -> 12,346 ticks -> 123.46 ms + const endTime = process.hrtime.bigint(); + return Math.round(Number((endTime - startTime) / 1000n) / 10) / 100; +} diff --git a/packages/backend/src/core/CaptchaService.ts b/packages/backend/src/core/CaptchaService.ts index 5b1ab00cfe..d17101ac97 100644 --- a/packages/backend/src/core/CaptchaService.ts +++ b/packages/backend/src/core/CaptchaService.ts @@ -6,6 +6,69 @@ import { Injectable } from '@nestjs/common'; import { HttpRequestService } from '@/core/HttpRequestService.js'; import { bindThis } from '@/decorators.js'; +import { MetaService } from '@/core/MetaService.js'; +import { MiMeta } from '@/models/Meta.js'; +import Logger from '@/logger.js'; +import { LoggerService } from './LoggerService.js'; + +export const supportedCaptchaProviders = ['none', 'hcaptcha', 'mcaptcha', 'recaptcha', 'turnstile', 'fc', 'testcaptcha'] as const; +export type CaptchaProvider = typeof supportedCaptchaProviders[number]; + +export const captchaErrorCodes = { + invalidProvider: Symbol('invalidProvider'), + invalidParameters: Symbol('invalidParameters'), + noResponseProvided: Symbol('noResponseProvided'), + requestFailed: Symbol('requestFailed'), + verificationFailed: Symbol('verificationFailed'), + unknown: Symbol('unknown'), +} as const; +export type CaptchaErrorCode = typeof captchaErrorCodes[keyof typeof captchaErrorCodes]; + +export type CaptchaSetting = { + provider: CaptchaProvider; + hcaptcha: { + siteKey: string | null; + secretKey: string | null; + } + mcaptcha: { + siteKey: string | null; + secretKey: string | null; + instanceUrl: string | null; + } + recaptcha: { + siteKey: string | null; + secretKey: string | null; + } + turnstile: { + siteKey: string | null; + secretKey: string | null; + } + fc: { + siteKey: string | null; + secretKey: string | null; + } +} + +export class CaptchaError extends Error { + public readonly code: CaptchaErrorCode; + public readonly cause?: unknown; + + constructor(code: CaptchaErrorCode, message: string, cause?: unknown) { + super(message); + this.code = code; + this.cause = cause; + this.name = 'CaptchaError'; + } +} + +export type CaptchaSaveSuccess = { + success: true; +} +export type CaptchaSaveFailure = { + success: false; + error: CaptchaError; +} +export type CaptchaSaveResult = CaptchaSaveSuccess | CaptchaSaveFailure; type CaptchaResponse = { success: boolean; @@ -15,9 +78,14 @@ type CaptchaResponse = { @Injectable() export class CaptchaService { + private readonly logger: Logger; + constructor( private httpRequestService: HttpRequestService, + private metaService: MetaService, + loggerService: LoggerService, ) { + this.logger = loggerService.getLogger('captcha'); } @bindThis @@ -45,39 +113,39 @@ export class CaptchaService { @bindThis public async verifyRecaptcha(secret: string, response: string | null | undefined): Promise<void> { if (response == null) { - throw new Error('recaptcha-failed: no response provided'); + throw new CaptchaError(captchaErrorCodes.noResponseProvided, 'recaptcha-failed: no response provided'); } const result = await this.getCaptchaResponse('https://www.recaptcha.net/recaptcha/api/siteverify', secret, response).catch(err => { - throw new Error(`recaptcha-request-failed: ${err}`); + throw new CaptchaError(captchaErrorCodes.requestFailed, `recaptcha-request-failed: ${err}`); }); if (result.success !== true) { const errorCodes = result['error-codes'] ? result['error-codes'].join(', ') : ''; - throw new Error(`recaptcha-failed: ${errorCodes}`); + throw new CaptchaError(captchaErrorCodes.verificationFailed, `recaptcha-failed: ${errorCodes}`); } } @bindThis public async verifyHcaptcha(secret: string, response: string | null | undefined): Promise<void> { if (response == null) { - throw new Error('hcaptcha-failed: no response provided'); + throw new CaptchaError(captchaErrorCodes.noResponseProvided, 'hcaptcha-failed: no response provided'); } const result = await this.getCaptchaResponse('https://hcaptcha.com/siteverify', secret, response).catch(err => { - throw new Error(`hcaptcha-request-failed: ${err}`); + throw new CaptchaError(captchaErrorCodes.requestFailed, `hcaptcha-request-failed: ${err}`); }); if (result.success !== true) { const errorCodes = result['error-codes'] ? result['error-codes'].join(', ') : ''; - throw new Error(`hcaptcha-failed: ${errorCodes}`); + throw new CaptchaError(captchaErrorCodes.verificationFailed, `hcaptcha-failed: ${errorCodes}`); } } @bindThis public async verifyFriendlyCaptcha(secret: string, response: string | null | undefined): Promise<void> { if (response == null) { - throw new Error('frc-failed: no response provided'); + throw new CaptchaError(captchaErrorCodes.noResponseProvided, 'frc-failed: no response provided'); } const result = await this.httpRequestService.send('https://api.friendlycaptcha.com/api/v1/siteverify', { @@ -89,17 +157,17 @@ export class CaptchaService { headers: { 'Content-Type': 'application/json', }, - }); + }, { throwErrorWhenResponseNotOk: false }); if (result.status !== 200) { - throw new Error('frc-failed: frc didn\'t return 200 OK'); + throw new CaptchaError(captchaErrorCodes.requestFailed, `frc-request-failed: ${result.status}`); } const resp = await result.json() as CaptchaResponse; if (resp.success !== true) { const errorCodes = resp['errors'] ? resp['errors'].join(', ') : ''; - throw new Error(`frc-failed: ${errorCodes}`); + throw new CaptchaError(captchaErrorCodes.verificationFailed, `frc-failed: ${errorCodes}`); } } @@ -107,7 +175,7 @@ export class CaptchaService { @bindThis public async verifyMcaptcha(secret: string, siteKey: string, instanceHost: string, response: string | null | undefined): Promise<void> { if (response == null) { - throw new Error('mcaptcha-failed: no response provided'); + throw new CaptchaError(captchaErrorCodes.noResponseProvided, 'mcaptcha-failed: no response provided'); } const endpointUrl = new URL('/api/v1/pow/siteverify', instanceHost); @@ -121,46 +189,272 @@ export class CaptchaService { headers: { 'Content-Type': 'application/json', }, - }); + }, { throwErrorWhenResponseNotOk: false }); if (result.status !== 200) { - throw new Error('mcaptcha-failed: mcaptcha didn\'t return 200 OK'); + throw new CaptchaError(captchaErrorCodes.requestFailed, 'mcaptcha-failed: mcaptcha didn\'t return 200 OK'); } const resp = (await result.json()) as { valid: boolean }; if (!resp.valid) { - throw new Error('mcaptcha-request-failed'); + throw new CaptchaError(captchaErrorCodes.verificationFailed, 'mcaptcha-request-failed'); } } @bindThis public async verifyTurnstile(secret: string, response: string | null | undefined): Promise<void> { if (response == null) { - throw new Error('turnstile-failed: no response provided'); + throw new CaptchaError(captchaErrorCodes.noResponseProvided, 'turnstile-failed: no response provided'); } const result = await this.getCaptchaResponse('https://challenges.cloudflare.com/turnstile/v0/siteverify', secret, response).catch(err => { - throw new Error(`turnstile-request-failed: ${err}`); + throw new CaptchaError(captchaErrorCodes.requestFailed, `turnstile-request-failed: ${err}`); }); if (result.success !== true) { const errorCodes = result['error-codes'] ? result['error-codes'].join(', ') : ''; - throw new Error(`turnstile-failed: ${errorCodes}`); + throw new CaptchaError(captchaErrorCodes.verificationFailed, `turnstile-failed: ${errorCodes}`); } } @bindThis public async verifyTestcaptcha(response: string | null | undefined): Promise<void> { if (response == null) { - throw new Error('testcaptcha-failed: no response provided'); + throw new CaptchaError(captchaErrorCodes.noResponseProvided, 'testcaptcha-failed: no response provided'); } const success = response === 'testcaptcha-passed'; if (!success) { - throw new Error('testcaptcha-failed'); + throw new CaptchaError(captchaErrorCodes.verificationFailed, 'testcaptcha-failed'); + } + } + + @bindThis + public async get(): Promise<CaptchaSetting> { + const meta = await this.metaService.fetch(true); + + let provider: CaptchaProvider; + switch (true) { + case meta.enableHcaptcha: { + provider = 'hcaptcha'; + break; + } + case meta.enableMcaptcha: { + provider = 'mcaptcha'; + break; + } + case meta.enableRecaptcha: { + provider = 'recaptcha'; + break; + } + case meta.enableTurnstile: { + provider = 'turnstile'; + break; + } + case meta.enableTestcaptcha: { + provider = 'testcaptcha'; + break; + } + case meta.enableFC: { + provider = 'fc'; + break; + } + default: { + provider = 'none'; + break; + } + } + + return { + provider: provider, + hcaptcha: { + siteKey: meta.hcaptchaSiteKey, + secretKey: meta.hcaptchaSecretKey, + }, + mcaptcha: { + siteKey: meta.mcaptchaSitekey, + secretKey: meta.mcaptchaSecretKey, + instanceUrl: meta.mcaptchaInstanceUrl, + }, + recaptcha: { + siteKey: meta.recaptchaSiteKey, + secretKey: meta.recaptchaSecretKey, + }, + turnstile: { + siteKey: meta.turnstileSiteKey, + secretKey: meta.turnstileSecretKey, + }, + fc: { + siteKey: meta.fcSiteKey, + secretKey: meta.fcSecretKey, + }, + }; + } + + /** + * captchaの設定を更新します. その際、フロントエンド側で受け取ったcaptchaからの戻り値を検証し、passした場合のみ設定を更新します. + * 実際の検証処理はサービス内で定義されている各captchaプロバイダの検証関数に委譲します. + * + * @param provider 検証するcaptchaのプロバイダ + * @param params + * @param params.sitekey hcaptcha, recaptcha, turnstile, mcaptchaの場合に指定するsitekey. それ以外のプロバイダでは無視されます + * @param params.secret hcaptcha, recaptcha, turnstile, mcaptchaの場合に指定するsecret. それ以外のプロバイダでは無視されます + * @param params.instanceUrl mcaptchaの場合に指定するインスタンスのURL. それ以外のプロバイダでは無視されます + * @param params.captchaResult フロントエンド側で受け取ったcaptchaプロバイダからの戻り値. この値を使ってサーバサイドでの検証を行います + * @see verifyHcaptcha + * @see verifyMcaptcha + * @see verifyRecaptcha + * @see verifyTurnstile + * @see verifyTestcaptcha + */ + @bindThis + public async save( + provider: CaptchaProvider, + params?: { + sitekey?: string | null; + secret?: string | null; + instanceUrl?: string | null; + captchaResult?: string | null; + }, + ): Promise<CaptchaSaveResult> { + if (!supportedCaptchaProviders.includes(provider)) { + return { + success: false, + error: new CaptchaError(captchaErrorCodes.invalidProvider, `Invalid captcha provider: ${provider}`), + }; } + + const operation = { + none: async () => { + await this.updateMeta(provider, params); + }, + hcaptcha: async () => { + if (!params?.secret || !params.captchaResult) { + throw new CaptchaError(captchaErrorCodes.invalidParameters, 'hcaptcha-failed: secret and captureResult are required'); + } + + await this.verifyHcaptcha(params.secret, params.captchaResult); + await this.updateMeta(provider, params); + }, + mcaptcha: async () => { + if (!params?.secret || !params.sitekey || !params.instanceUrl || !params.captchaResult) { + throw new CaptchaError(captchaErrorCodes.invalidParameters, 'mcaptcha-failed: secret, sitekey, instanceUrl and captureResult are required'); + } + + await this.verifyMcaptcha(params.secret, params.sitekey, params.instanceUrl, params.captchaResult); + await this.updateMeta(provider, params); + }, + recaptcha: async () => { + if (!params?.secret || !params.captchaResult) { + throw new CaptchaError(captchaErrorCodes.invalidParameters, 'recaptcha-failed: secret and captureResult are required'); + } + + await this.verifyRecaptcha(params.secret, params.captchaResult); + await this.updateMeta(provider, params); + }, + turnstile: async () => { + if (!params?.secret || !params.captchaResult) { + throw new CaptchaError(captchaErrorCodes.invalidParameters, 'turnstile-failed: secret and captureResult are required'); + } + + await this.verifyTurnstile(params.secret, params.captchaResult); + await this.updateMeta(provider, params); + }, + testcaptcha: async () => { + if (!params?.captchaResult) { + throw new CaptchaError(captchaErrorCodes.invalidParameters, 'turnstile-failed: captureResult are required'); + } + + await this.verifyTestcaptcha(params.captchaResult); + await this.updateMeta(provider, params); + }, + fc: async () => { + if (!params?.secret || !params.captchaResult) { + throw new CaptchaError(captchaErrorCodes.invalidParameters, 'frc-failed: secret and captureResult are required'); + } + + await this.verifyFriendlyCaptcha(params.captchaResult, params.captchaResult); + await this.updateMeta(provider, params); + }, + }[provider]; + + return operation() + .then(() => ({ success: true }) as CaptchaSaveSuccess) + .catch(err => { + this.logger.info(err); + const error = err instanceof CaptchaError + ? err + : new CaptchaError(captchaErrorCodes.unknown, `unknown error: ${err}`); + return { + success: false, + error, + }; + }); + } + + @bindThis + private async updateMeta( + provider: CaptchaProvider, + params?: { + sitekey?: string | null; + secret?: string | null; + instanceUrl?: string | null; + }, + ) { + const metaPartial: Partial< + Pick< + MiMeta, + ('enableHcaptcha' | 'hcaptchaSiteKey' | 'hcaptchaSecretKey') | + ('enableMcaptcha' | 'mcaptchaSitekey' | 'mcaptchaSecretKey' | 'mcaptchaInstanceUrl') | + ('enableRecaptcha' | 'recaptchaSiteKey' | 'recaptchaSecretKey') | + ('enableTurnstile' | 'turnstileSiteKey' | 'turnstileSecretKey') | + ('enableTestcaptcha' | 'enableFC' | 'fcSiteKey' | 'fcSecretKey') + > + > = { + enableHcaptcha: provider === 'hcaptcha', + enableMcaptcha: provider === 'mcaptcha', + enableRecaptcha: provider === 'recaptcha', + enableTurnstile: provider === 'turnstile', + enableTestcaptcha: provider === 'testcaptcha', + enableFC: provider === 'fc', + }; + + const updateIfNotUndefined = <K extends keyof typeof metaPartial>(key: K, value: typeof metaPartial[K]) => { + if (value !== undefined) { + metaPartial[key] = value; + } + }; + switch (provider) { + case 'hcaptcha': { + updateIfNotUndefined('hcaptchaSiteKey', params?.sitekey); + updateIfNotUndefined('hcaptchaSecretKey', params?.secret); + break; + } + case 'mcaptcha': { + updateIfNotUndefined('mcaptchaSitekey', params?.sitekey); + updateIfNotUndefined('mcaptchaSecretKey', params?.secret); + updateIfNotUndefined('mcaptchaInstanceUrl', params?.instanceUrl); + break; + } + case 'recaptcha': { + updateIfNotUndefined('recaptchaSiteKey', params?.sitekey); + updateIfNotUndefined('recaptchaSecretKey', params?.secret); + break; + } + case 'turnstile': { + updateIfNotUndefined('turnstileSiteKey', params?.sitekey); + updateIfNotUndefined('turnstileSecretKey', params?.secret); + break; + } + case 'fc': { + updateIfNotUndefined('fcSiteKey', params?.sitekey); + updateIfNotUndefined('fcSecretKey', params?.secret); + } + } + + await this.metaService.update(metaPartial); } } diff --git a/packages/backend/src/core/CoreModule.ts b/packages/backend/src/core/CoreModule.ts index 141f905d7f..3c35dfc4ff 100644 --- a/packages/backend/src/core/CoreModule.ts +++ b/packages/backend/src/core/CoreModule.ts @@ -17,6 +17,8 @@ import { WebhookTestService } from '@/core/WebhookTestService.js'; import { FlashService } from '@/core/FlashService.js'; import { TimeService } from '@/core/TimeService.js'; import { EnvService } from '@/core/EnvService.js'; +import { ApUtilityService } from '@/core/activitypub/ApUtilityService.js'; +import { ApLogService } from '@/core/ApLogService.js'; import { AccountMoveService } from './AccountMoveService.js'; import { AccountUpdateService } from './AccountUpdateService.js'; import { AnnouncementService } from './AnnouncementService.js'; @@ -166,6 +168,7 @@ const $AccountMoveService: Provider = { provide: 'AccountMoveService', useExisti const $AccountUpdateService: Provider = { provide: 'AccountUpdateService', useExisting: AccountUpdateService }; const $AnnouncementService: Provider = { provide: 'AnnouncementService', useExisting: AnnouncementService }; const $AntennaService: Provider = { provide: 'AntennaService', useExisting: AntennaService }; +const $ApLogService: Provider = { provide: 'ApLogService', useExisting: ApLogService }; const $AppLockService: Provider = { provide: 'AppLockService', useExisting: AppLockService }; const $AchievementService: Provider = { provide: 'AchievementService', useExisting: AchievementService }; const $AvatarDecorationService: Provider = { provide: 'AvatarDecorationService', useExisting: AvatarDecorationService }; @@ -232,6 +235,8 @@ const $FanoutTimelineEndpointService: Provider = { provide: 'FanoutTimelineEndpo const $ChannelFollowingService: Provider = { provide: 'ChannelFollowingService', useExisting: ChannelFollowingService }; const $RegistryApiService: Provider = { provide: 'RegistryApiService', useExisting: RegistryApiService }; const $ReversiService: Provider = { provide: 'ReversiService', useExisting: ReversiService }; +const $TimeService: Provider = { provide: 'TimeService', useExisting: TimeService }; +const $EnvService: Provider = { provide: 'EnvService', useExisting: EnvService }; const $ChartLoggerService: Provider = { provide: 'ChartLoggerService', useExisting: ChartLoggerService }; const $FederationChart: Provider = { provide: 'FederationChart', useExisting: FederationChart }; @@ -304,6 +309,7 @@ const $ApMentionService: Provider = { provide: 'ApMentionService', useExisting: const $ApNoteService: Provider = { provide: 'ApNoteService', useExisting: ApNoteService }; const $ApPersonService: Provider = { provide: 'ApPersonService', useExisting: ApPersonService }; const $ApQuestionService: Provider = { provide: 'ApQuestionService', useExisting: ApQuestionService }; +const $ApUtilityService: Provider = { provide: 'ApUtilityService', useExisting: ApUtilityService }; //#endregion const $SponsorsService: Provider = { provide: 'SponsorsService', useExisting: SponsorsService }; @@ -320,6 +326,7 @@ const $SponsorsService: Provider = { provide: 'SponsorsService', useExisting: Sp AccountUpdateService, AnnouncementService, AntennaService, + ApLogService, AppLockService, AchievementService, AvatarDecorationService, @@ -460,6 +467,7 @@ const $SponsorsService: Provider = { provide: 'SponsorsService', useExisting: Sp ApNoteService, ApPersonService, ApQuestionService, + ApUtilityService, QueueService, SponsorsService, @@ -472,6 +480,7 @@ const $SponsorsService: Provider = { provide: 'SponsorsService', useExisting: Sp $AccountUpdateService, $AnnouncementService, $AntennaService, + $ApLogService, $AppLockService, $AchievementService, $AvatarDecorationService, @@ -538,6 +547,8 @@ const $SponsorsService: Provider = { provide: 'SponsorsService', useExisting: Sp $ChannelFollowingService, $RegistryApiService, $ReversiService, + $TimeService, + $EnvService, $ChartLoggerService, $FederationChart, @@ -610,6 +621,7 @@ const $SponsorsService: Provider = { provide: 'SponsorsService', useExisting: Sp $ApNoteService, $ApPersonService, $ApQuestionService, + $ApUtilityService, //#endregion $SponsorsService, @@ -623,6 +635,7 @@ const $SponsorsService: Provider = { provide: 'SponsorsService', useExisting: Sp AccountUpdateService, AnnouncementService, AntennaService, + ApLogService, AppLockService, AchievementService, AvatarDecorationService, @@ -762,6 +775,7 @@ const $SponsorsService: Provider = { provide: 'SponsorsService', useExisting: Sp ApNoteService, ApPersonService, ApQuestionService, + ApUtilityService, QueueService, SponsorsService, @@ -774,6 +788,7 @@ const $SponsorsService: Provider = { provide: 'SponsorsService', useExisting: Sp $AccountUpdateService, $AnnouncementService, $AntennaService, + $ApLogService, $AppLockService, $AchievementService, $AvatarDecorationService, @@ -839,6 +854,8 @@ const $SponsorsService: Provider = { provide: 'SponsorsService', useExisting: Sp $ChannelFollowingService, $RegistryApiService, $ReversiService, + $TimeService, + $EnvService, $FederationChart, $NotesChart, @@ -910,6 +927,7 @@ const $SponsorsService: Provider = { provide: 'SponsorsService', useExisting: Sp $ApNoteService, $ApPersonService, $ApQuestionService, + $ApUtilityService, //#endregion $SponsorsService, diff --git a/packages/backend/src/core/CustomEmojiService.ts b/packages/backend/src/core/CustomEmojiService.ts index cc33fb5c0b..2e4eddf797 100644 --- a/packages/backend/src/core/CustomEmojiService.ts +++ b/packages/backend/src/core/CustomEmojiService.ts @@ -4,19 +4,18 @@ */ import { Inject, Injectable, OnApplicationShutdown } from '@nestjs/common'; -import { In, IsNull } from 'typeorm'; import * as Redis from 'ioredis'; -import { DI } from '@/di-symbols.js'; -import { IdService } from '@/core/IdService.js'; +import { In, IsNull } from 'typeorm'; import { EmojiEntityService } from '@/core/entities/EmojiEntityService.js'; import { GlobalEventService } from '@/core/GlobalEventService.js'; -import type { MiDriveFile } from '@/models/DriveFile.js'; -import type { MiEmoji } from '@/models/Emoji.js'; -import type { DriveFilesRepository, EmojisRepository, MiRole, MiUser } from '@/models/_.js'; +import { IdService } from '@/core/IdService.js'; +import { UtilityService } from '@/core/UtilityService.js'; import { bindThis } from '@/decorators.js'; +import { DI } from '@/di-symbols.js'; import { MemoryKVCache, RedisSingleCache } from '@/misc/cache.js'; -import { UtilityService } from '@/core/UtilityService.js'; -import { query } from '@/misc/prelude/url.js'; +import { sqlLikeEscape } from '@/misc/sql-like-escape.js'; +import type { DriveFilesRepository, EmojisRepository, MiRole, MiUser } from '@/models/_.js'; +import type { MiEmoji } from '@/models/Emoji.js'; import type { Serialized } from '@/types.js'; import { ModerationLogService } from '@/core/ModerationLogService.js'; import type { Config } from '@/config.js'; @@ -24,6 +23,42 @@ import { DriveService } from './DriveService.js'; const parseEmojiStrRegexp = /^([-\w]+)(?:@([\w.-]+))?$/; +export const fetchEmojisHostTypes = [ + 'local', + 'remote', + 'all', +] as const; +export type FetchEmojisHostTypes = typeof fetchEmojisHostTypes[number]; +export const fetchEmojisSortKeys = [ + '+id', + '-id', + '+updatedAt', + '-updatedAt', + '+name', + '-name', + '+host', + '-host', + '+uri', + '-uri', + '+publicUrl', + '-publicUrl', + '+type', + '-type', + '+aliases', + '-aliases', + '+category', + '-category', + '+license', + '-license', + '+isSensitive', + '-isSensitive', + '+localOnly', + '-localOnly', + '+roleIdsThatCanBeUsedThisEmojiAsReaction', + '-roleIdsThatCanBeUsedThisEmojiAsReaction', +] as const; +export type FetchEmojisSortKeys = typeof fetchEmojisSortKeys[number]; + @Injectable() export class CustomEmojiService implements OnApplicationShutdown { private emojisCache: MemoryKVCache<MiEmoji | null>; @@ -32,16 +67,12 @@ export class CustomEmojiService implements OnApplicationShutdown { constructor( @Inject(DI.redis) private redisClient: Redis.Redis, - @Inject(DI.config) private config: Config, - @Inject(DI.emojisRepository) private emojisRepository: EmojisRepository, - @Inject(DI.driveFilesRepository) private driveFilesRepository: DriveFilesRepository, - private utilityService: UtilityService, private idService: IdService, private emojiEntityService: EmojiEntityService, @@ -67,7 +98,9 @@ export class CustomEmojiService implements OnApplicationShutdown { @bindThis public async add(data: { - driveFile: MiDriveFile; + originalUrl: string; + publicUrl: string; + fileType: string; name: string; category: string | null; aliases: string[]; @@ -84,9 +117,9 @@ export class CustomEmojiService implements OnApplicationShutdown { category: data.category, host: data.host, aliases: data.aliases, - originalUrl: data.driveFile.url, - publicUrl: data.driveFile.webpublicUrl ?? data.driveFile.url, - type: data.driveFile.webpublicType ?? data.driveFile.type, + originalUrl: data.originalUrl, + publicUrl: data.publicUrl, + type: data.fileType, license: data.license, isSensitive: data.isSensitive, localOnly: data.localOnly, @@ -115,7 +148,9 @@ export class CustomEmojiService implements OnApplicationShutdown { public async update(data: ( { id: MiEmoji['id'], name?: string; } | { name: string; id?: MiEmoji['id'], } ) & { - driveFile?: MiDriveFile; + originalUrl?: string; + publicUrl?: string; + fileType?: string; category?: string | null; aliases?: string[]; license?: string | null; @@ -148,18 +183,22 @@ export class CustomEmojiService implements OnApplicationShutdown { license: data.license, isSensitive: data.isSensitive, localOnly: data.localOnly, - originalUrl: data.driveFile != null ? data.driveFile.url : undefined, - publicUrl: data.driveFile != null ? (data.driveFile.webpublicUrl ?? data.driveFile.url) : undefined, - type: data.driveFile != null ? (data.driveFile.webpublicType ?? data.driveFile.type) : undefined, + originalUrl: data.originalUrl, + publicUrl: data.publicUrl, + type: data.fileType, roleIdsThatCanBeUsedThisEmojiAsReaction: data.roleIdsThatCanBeUsedThisEmojiAsReaction ?? undefined, }); this.localEmojisCache.refresh(); - if (data.driveFile != null) { - const file = await this.driveFilesRepository.findOneBy({ url: emoji.originalUrl, userHost: emoji.host ? emoji.host : IsNull() }); - if (file && file.id !== data.driveFile.id) { - await this.driveService.deleteFile(file, false, moderator ? moderator : undefined); + // If we're changing the file, then we need to delete the old one + if (data.originalUrl != null && data.originalUrl !== emoji.originalUrl) { + const oldFile = await this.driveFilesRepository.findOneBy({ url: emoji.originalUrl, userHost: emoji.host ? emoji.host : IsNull() }); + const newFile = await this.driveFilesRepository.findOneBy({ url: data.originalUrl, userHost: emoji.host ? emoji.host : IsNull() }); + + // But DON'T delete if this is the same file reference, otherwise we'll break the emoji! + if (oldFile && newFile && oldFile.id !== newFile.id) { + await this.driveService.deleteFile(oldFile, false, moderator ? moderator : undefined); } } @@ -336,7 +375,7 @@ export class CustomEmojiService implements OnApplicationShutdown { @bindThis private normalizeHost(src: string | undefined, noteUserHost: string | null): string | null { - // クエリに使うホスト + // クエリに使うホスト let host = src === '.' ? null // .はローカルホスト (ここがマッチするのはリアクションのみ) : src === undefined ? noteUserHost // ノートなどでホスト省略表記の場合はローカルホスト (ここがリアクションにマッチすることはない) : this.utilityService.isSelfHost(src) ? null // 自ホスト指定 @@ -445,6 +484,151 @@ export class CustomEmojiService implements OnApplicationShutdown { } @bindThis + public async fetchEmojis( + params?: { + query?: { + updatedAtFrom?: string; + updatedAtTo?: string; + name?: string; + host?: string; + uri?: string; + publicUrl?: string; + type?: string; + aliases?: string; + category?: string; + license?: string; + isSensitive?: boolean; + localOnly?: boolean; + hostType?: FetchEmojisHostTypes; + roleIds?: string[]; + }, + sinceId?: string; + untilId?: string; + }, + opts?: { + limit?: number; + page?: number; + sortKeys?: FetchEmojisSortKeys[] + }, + ) { + function multipleWordsToQuery(words: string) { + return words.split(/\s/).filter(x => x.length > 0).map(x => `%${sqlLikeEscape(x)}%`); + } + + const builder = this.emojisRepository.createQueryBuilder('emoji'); + if (params?.query) { + const q = params.query; + if (q.updatedAtFrom) { + // noIndexScan + builder.andWhere('CAST(emoji.updatedAt AS DATE) >= :updatedAtFrom', { updatedAtFrom: q.updatedAtFrom }); + } + if (q.updatedAtTo) { + // noIndexScan + builder.andWhere('CAST(emoji.updatedAt AS DATE) <= :updatedAtTo', { updatedAtTo: q.updatedAtTo }); + } + if (q.name) { + builder.andWhere('emoji.name ~~ ANY(ARRAY[:...name])', { name: multipleWordsToQuery(q.name) }); + } + + switch (true) { + case q.hostType === 'local': { + builder.andWhere('emoji.host IS NULL'); + break; + } + case q.hostType === 'remote': { + if (q.host) { + // noIndexScan + builder.andWhere('emoji.host ~~ ANY(ARRAY[:...host])', { host: multipleWordsToQuery(q.host) }); + } else { + builder.andWhere('emoji.host IS NOT NULL'); + } + break; + } + } + + if (q.uri) { + // noIndexScan + builder.andWhere('emoji.uri ~~ ANY(ARRAY[:...uri])', { uri: multipleWordsToQuery(q.uri) }); + } + if (q.publicUrl) { + // noIndexScan + builder.andWhere('emoji.publicUrl ~~ ANY(ARRAY[:...publicUrl])', { publicUrl: multipleWordsToQuery(q.publicUrl) }); + } + if (q.type) { + // noIndexScan + builder.andWhere('emoji.type ~~ ANY(ARRAY[:...type])', { type: multipleWordsToQuery(q.type) }); + } + if (q.aliases) { + // noIndexScan + const subQueryBuilder = builder.subQuery() + .select('COUNT(0)', 'count') + .from( + sq2 => sq2 + .select('unnest(subEmoji.aliases)', 'alias') + .addSelect('subEmoji.id', 'id') + .from('emoji', 'subEmoji'), + 'aliasTable', + ) + .where('"emoji"."id" = "aliasTable"."id"') + .andWhere('"aliasTable"."alias" ~~ ANY(ARRAY[:...aliases])', { aliases: multipleWordsToQuery(q.aliases) }); + + builder.andWhere(`(${subQueryBuilder.getQuery()}) > 0`); + } + if (q.category) { + builder.andWhere('emoji.category ~~ ANY(ARRAY[:...category])', { category: multipleWordsToQuery(q.category) }); + } + if (q.license) { + // noIndexScan + builder.andWhere('emoji.license ~~ ANY(ARRAY[:...license])', { license: multipleWordsToQuery(q.license) }); + } + if (q.isSensitive != null) { + // noIndexScan + builder.andWhere('emoji.isSensitive = :isSensitive', { isSensitive: q.isSensitive }); + } + if (q.localOnly != null) { + // noIndexScan + builder.andWhere('emoji.localOnly = :localOnly', { localOnly: q.localOnly }); + } + if (q.roleIds && q.roleIds.length > 0) { + builder.andWhere('emoji.roleIdsThatCanBeUsedThisEmojiAsReaction && ARRAY[:...roleIds]::VARCHAR[]', { roleIds: q.roleIds }); + } + } + + if (params?.sinceId) { + builder.andWhere('emoji.id > :sinceId', { sinceId: params.sinceId }); + } + if (params?.untilId) { + builder.andWhere('emoji.id < :untilId', { untilId: params.untilId }); + } + + if (opts?.sortKeys && opts.sortKeys.length > 0) { + for (const sortKey of opts.sortKeys) { + const direction = sortKey.startsWith('-') ? 'DESC' : 'ASC'; + const key = sortKey.replace(/^[+-]/, ''); + builder.addOrderBy(`emoji.${key}`, direction); + } + } else { + builder.addOrderBy('emoji.id', 'DESC'); + } + + const limit = opts?.limit ?? 10; + if (opts?.page) { + builder.skip((opts.page - 1) * limit); + } + + builder.take(limit); + + const [emojis, count] = await builder.getManyAndCount(); + + return { + emojis, + count: (count > limit ? emojis.length : count), + allCount: count, + allPages: Math.ceil(count / limit), + }; + } + + @bindThis public dispose(): void { this.emojisCache.dispose(); } diff --git a/packages/backend/src/core/DriveService.ts b/packages/backend/src/core/DriveService.ts index 086f2f94d5..a65059b417 100644 --- a/packages/backend/src/core/DriveService.ts +++ b/packages/backend/src/core/DriveService.ts @@ -37,6 +37,7 @@ import { InternalStorageService } from '@/core/InternalStorageService.js'; import { DriveFileEntityService } from '@/core/entities/DriveFileEntityService.js'; import { UserEntityService } from '@/core/entities/UserEntityService.js'; import { FileInfoService } from '@/core/FileInfoService.js'; +import type { FileInfo } from '@/core/FileInfoService.js'; import { bindThis } from '@/decorators.js'; import { RoleService } from '@/core/RoleService.js'; import { correctFilename } from '@/misc/correct-filename.js'; @@ -139,15 +140,18 @@ export class DriveService { /*** * Save file + * @param file * @param path Path for original * @param name Name for original (should be extention corrected) - * @param type Content-Type for original - * @param hash Hash for original - * @param size Size for original + * @param info File metadata */ @bindThis - private async save(file: MiDriveFile, path: string, name: string, type: string, hash: string, size: number): Promise<MiDriveFile> { - // thunbnail, webpublic を必要なら生成 + private async save(file: MiDriveFile, path: string, name: string, info: FileInfo): Promise<MiDriveFile> { + const type = info.type.mime; + const hash = info.md5; + const size = info.size; + + // thunbnail, webpublic を必要なら生成 const alts = await this.generateAlts(path, type, !file.uri); if (this.meta.useObjectStorage) { @@ -223,9 +227,11 @@ export class DriveService { return await this.driveFilesRepository.insertOne(file); } else { // use internal storage - const accessKey = randomUUID(); - const thumbnailAccessKey = 'thumbnail-' + randomUUID(); - const webpublicAccessKey = 'webpublic-' + randomUUID(); + const ext = FILE_TYPE_BROWSERSAFE.includes(type) ? info.type.ext : null; + + const accessKey = makeFileKey(ext); + const thumbnailAccessKey = makeFileKey(ext, 'thumbnail'); + const webpublicAccessKey = makeFileKey(ext, 'webpublic'); // Ugly type is just to help TS figure out that 2nd / 3rd promises are optional. const promises: [Promise<string>, ...(Promise<string> | undefined)[]] = [ @@ -514,7 +520,7 @@ export class DriveService { // If usage limit exceeded if (driveCapacity < usage + info.size) { if (isLocalUser) { - throw new IdentifiableError('c6244ed2-a39a-4e1c-bf93-f0fbd7764fa6', 'No free space.'); + throw new IdentifiableError('c6244ed2-a39a-4e1c-bf93-f0fbd7764fa6', 'No free space.', true); } await this.expireOldFile(await this.usersRepository.findOneByOrFail({ id: user.id }) as MiRemoteUser, driveCapacity - info.size); } @@ -616,7 +622,7 @@ export class DriveService { } } } else { - file = await (this.save(file, path, detectedName, info.type.mime, info.md5, info.size)); + file = await (this.save(file, path, detectedName, info)); } this.registerLogger.succ(`drive file has been created ${file.id}`); @@ -862,3 +868,16 @@ export class DriveService { } } } + +function makeFileKey(ext: string | null, prefix?: string): string { + const parts: string[] = [randomUUID()]; + + if (prefix) { + parts.unshift(prefix, '-'); + } + if (ext) { + parts.push('.', ext); + } + + return parts.join(''); +} diff --git a/packages/backend/src/core/FederatedInstanceService.ts b/packages/backend/src/core/FederatedInstanceService.ts index fca3ad847a..3f7ed99348 100644 --- a/packages/backend/src/core/FederatedInstanceService.ts +++ b/packages/backend/src/core/FederatedInstanceService.ts @@ -5,6 +5,7 @@ import { Inject, Injectable, OnApplicationShutdown } from '@nestjs/common'; import * as Redis from 'ioredis'; +import { QueryFailedError } from 'typeorm'; import type { InstancesRepository } from '@/models/_.js'; import type { MiInstance } from '@/models/Instance.js'; import { MemoryKVCache, RedisKVCache } from '@/misc/cache.js'; @@ -12,7 +13,6 @@ import { IdService } from '@/core/IdService.js'; import { DI } from '@/di-symbols.js'; import { UtilityService } from '@/core/UtilityService.js'; import { bindThis } from '@/decorators.js'; -import { QueryFailedError } from 'typeorm'; import { isDuplicateKeyValueError } from '@/misc/is-duplicate-key-value-error.js'; @Injectable() diff --git a/packages/backend/src/core/FetchInstanceMetadataService.ts b/packages/backend/src/core/FetchInstanceMetadataService.ts index 987999bce7..ce3af7c774 100644 --- a/packages/backend/src/core/FetchInstanceMetadataService.ts +++ b/packages/backend/src/core/FetchInstanceMetadataService.ts @@ -181,7 +181,7 @@ export class FetchInstanceMetadataService { } @bindThis - private async fetchDom(instance: MiInstance): Promise<DOMWindow['document']> { + private async fetchDom(instance: MiInstance): Promise<Document> { this.logger.info(`Fetching HTML of ${instance.host} ...`); const url = 'https://' + instance.host; @@ -206,7 +206,7 @@ export class FetchInstanceMetadataService { } @bindThis - private async fetchFaviconUrl(instance: MiInstance, doc: DOMWindow['document'] | null): Promise<string | null> { + private async fetchFaviconUrl(instance: MiInstance, doc: Document | null): Promise<string | null> { const url = 'https://' + instance.host; if (doc) { @@ -232,7 +232,7 @@ export class FetchInstanceMetadataService { } @bindThis - private async fetchIconUrl(instance: MiInstance, doc: DOMWindow['document'] | null, manifest: Record<string, any> | null): Promise<string | null> { + private async fetchIconUrl(instance: MiInstance, doc: Document | null, manifest: Record<string, any> | null): Promise<string | null> { if (manifest && manifest.icons && manifest.icons.length > 0 && manifest.icons[0].src) { const url = 'https://' + instance.host; return (new URL(manifest.icons[0].src, url)).href; @@ -261,7 +261,7 @@ export class FetchInstanceMetadataService { } @bindThis - private async getThemeColor(info: NodeInfo | null, doc: DOMWindow['document'] | null, manifest: Record<string, any> | null): Promise<string | null> { + private async getThemeColor(info: NodeInfo | null, doc: Document | null, manifest: Record<string, any> | null): Promise<string | null> { const themeColor = info?.metadata?.themeColor ?? doc?.querySelector('meta[name="theme-color"]')?.getAttribute('content') ?? manifest?.theme_color; if (themeColor) { @@ -273,7 +273,7 @@ export class FetchInstanceMetadataService { } @bindThis - private async getSiteName(info: NodeInfo | null, doc: DOMWindow['document'] | null, manifest: Record<string, any> | null): Promise<string | null> { + private async getSiteName(info: NodeInfo | null, doc: Document | null, manifest: Record<string, any> | null): Promise<string | null> { if (info && info.metadata) { if (typeof info.metadata.nodeName === 'string') { return info.metadata.nodeName; @@ -298,7 +298,7 @@ export class FetchInstanceMetadataService { } @bindThis - private async getDescription(info: NodeInfo | null, doc: DOMWindow['document'] | null, manifest: Record<string, any> | null): Promise<string | null> { + private async getDescription(info: NodeInfo | null, doc: Document | null, manifest: Record<string, any> | null): Promise<string | null> { if (info && info.metadata) { if (typeof info.metadata.nodeDescription === 'string') { return info.metadata.nodeDescription; diff --git a/packages/backend/src/core/HttpRequestService.ts b/packages/backend/src/core/HttpRequestService.ts index 083153940a..19992a7597 100644 --- a/packages/backend/src/core/HttpRequestService.ts +++ b/packages/backend/src/core/HttpRequestService.ts @@ -16,8 +16,8 @@ import type { Config } from '@/config.js'; import { StatusError } from '@/misc/status-error.js'; import { bindThis } from '@/decorators.js'; import { validateContentTypeSetAsActivityPub } from '@/core/activitypub/misc/validator.js'; -import { assertActivityMatchesUrls } from '@/core/activitypub/misc/check-against-url.js'; -import type { IObject } from '@/core/activitypub/type.js'; +import { IObject } from '@/core/activitypub/type.js'; +import { ApUtilityService } from './activitypub/ApUtilityService.js'; import type { Response } from 'node-fetch'; import type { URL } from 'node:url'; @@ -145,6 +145,7 @@ export class HttpRequestService { constructor( @Inject(DI.config) private config: Config, + private readonly apUtilityService: ApUtilityService, ) { const cache = new CacheableLookup({ maxTtl: 3600, // 1hours @@ -198,6 +199,7 @@ export class HttpRequestService { * Get agent by URL * @param url URL * @param bypassProxy Allways bypass proxy + * @param isLocalAddressAllowed */ @bindThis public getAgentByUrl(url: URL, bypassProxy = false, isLocalAddressAllowed = false): http.Agent | https.Agent { @@ -229,10 +231,11 @@ export class HttpRequestService { validators: [validateContentTypeSetAsActivityPub], }); - const finalUrl = res.url; // redirects may have been involved const activity = await res.json() as IObject; - assertActivityMatchesUrls(activity, [finalUrl]); + // Make sure the object ID matches the final URL (which is where it actually exists). + // The caller (ApResolverService) will verify the ID against the original / entry URL, which ensures that all three match. + this.apUtilityService.assertIdMatchesUrlAuthority(activity, res.url); return activity; } diff --git a/packages/backend/src/core/MfmService.ts b/packages/backend/src/core/MfmService.ts index 42676d6f98..6c2f673217 100644 --- a/packages/backend/src/core/MfmService.ts +++ b/packages/backend/src/core/MfmService.ts @@ -179,6 +179,40 @@ export class MfmService { break; } + // this is here only to catch upstream changes! + case 'ruby--': { + let ruby: [string, string][] = []; + for (const child of node.childNodes) { + if (child.nodeName === 'rp') { + continue; + } + if (treeAdapter.isTextNode(child) && !/\s|\[|\]/.test(child.value)) { + ruby.push([child.value, '']); + continue; + } + if (child.nodeName === 'rt' && ruby.length > 0) { + const rt = getText(child); + if (/\s|\[|\]/.test(rt)) { + // If any space is included in rt, it is treated as a normal text + ruby = []; + appendChildren(node.childNodes); + break; + } else { + ruby.at(-1)![1] = rt; + continue; + } + } + // If any other element is included in ruby, it is treated as a normal text + ruby = []; + appendChildren(node.childNodes); + break; + } + for (const [base, rt] of ruby) { + text += `$[ruby ${base} ${rt}]`; + } + break; + } + // block code (<pre><code>) case 'pre': { if (node.childNodes.length === 1 && node.childNodes[0].nodeName === 'code') { @@ -230,6 +264,75 @@ export class MfmService { break; } + case 'rp': break; + case 'rt': { + appendChildren(node.childNodes); + break; + } + case 'ruby': { + if (node.childNodes) { + /* + we get: + ``` + <ruby> + some text <rp>(</rp> <rt>annotation</rt> <rp>)</rp> + more text <rt>more annotation<rt> + </ruby> + ``` + + and we want to produce: + ``` + $[ruby $[group some text] annotation] + $[ruby $[group more text] more annotation] + ``` + + that `group` is a hack, because when the `ruby` render + sees just text inside the `$[ruby]`, it splits on + whitespace, considers the first "word" to be the main + content, and the rest the annotation + + with that `group`, we force it to consider the whole + group as the main content + + (note that the `rp` are to be ignored, they only exist + for browsers who don't understand ruby) + */ + let nonRtNodes = []; + // scan children, ignore `rp`, split on `rt` + for (const child of node.childNodes) { + if (treeAdapter.isTextNode(child)) { + nonRtNodes.push(child); + continue; + } + if (!treeAdapter.isElementNode(child)) { + continue; + } + if (child.nodeName === 'rp') { + continue; + } + if (child.nodeName === 'rt') { + // the only case in which we don't need a `$[group ]` + // is when both sides of the ruby are simple words + const needsGroup = nonRtNodes.length > 1 || + /\s|\[|\]/.test(getText(nonRtNodes[0])) || + /\s|\[|\]/.test(getText(child)); + text += '$[ruby '; + if (needsGroup) text += '$[group '; + appendChildren(nonRtNodes); + if (needsGroup) text += ']'; + text += ' '; + analyze(child); + text += ']'; + nonRtNodes = []; + continue; + } + nonRtNodes.push(child); + } + appendChildren(nonRtNodes); + } + break; + } + default: // includes inline elements { appendChildren(node.childNodes); @@ -348,6 +451,14 @@ export class MfmService { } } + // hack for ruby, should never be needed because we should + // never send this out to other instances + case 'group': { + const el = doc.createElement('span'); + appendChildren(node.children, el); + return el; + } + default: { return fnDefault(node); } @@ -526,11 +637,65 @@ export class MfmService { }, async fn(node) { - const el = doc.createElement('span'); - el.textContent = '*'; - await appendChildren(node.children, el); - el.textContent += '*'; - return el; + switch (node.props.name) { + case 'group': { // hack for ruby + const el = doc.createElement('span'); + await appendChildren(node.children, el); + return el; + } + case 'ruby': { + if (node.children.length === 1) { + const child = node.children[0]; + const text = child.type === 'text' ? child.props.text : ''; + const rubyEl = doc.createElement('ruby'); + const rtEl = doc.createElement('rt'); + + const rpStartEl = doc.createElement('rp'); + rpStartEl.appendChild(doc.createTextNode('(')); + const rpEndEl = doc.createElement('rp'); + rpEndEl.appendChild(doc.createTextNode(')')); + + rubyEl.appendChild(doc.createTextNode(text.split(' ')[0])); + rtEl.appendChild(doc.createTextNode(text.split(' ')[1])); + rubyEl.appendChild(rpStartEl); + rubyEl.appendChild(rtEl); + rubyEl.appendChild(rpEndEl); + return rubyEl; + } else { + const rt = node.children.at(-1); + + if (!rt) { + const el = doc.createElement('span'); + await appendChildren(node.children, el); + return el; + } + + const text = rt.type === 'text' ? rt.props.text : ''; + const rubyEl = doc.createElement('ruby'); + const rtEl = doc.createElement('rt'); + + const rpStartEl = doc.createElement('rp'); + rpStartEl.appendChild(doc.createTextNode('(')); + const rpEndEl = doc.createElement('rp'); + rpEndEl.appendChild(doc.createTextNode(')')); + + await appendChildren(node.children.slice(0, node.children.length - 1), rubyEl); + rtEl.appendChild(doc.createTextNode(text.trim())); + rubyEl.appendChild(rpStartEl); + rubyEl.appendChild(rtEl); + rubyEl.appendChild(rpEndEl); + return rubyEl; + } + } + + default: { + const el = doc.createElement('span'); + el.textContent = '*'; + await appendChildren(node.children, el); + el.textContent += '*'; + return el; + } + } }, blockCode(node) { diff --git a/packages/backend/src/core/NoteCreateService.ts b/packages/backend/src/core/NoteCreateService.ts index 96bb30a0d6..df31cb4247 100644 --- a/packages/backend/src/core/NoteCreateService.ts +++ b/packages/backend/src/core/NoteCreateService.ts @@ -144,6 +144,7 @@ type Option = { uri?: string | null; url?: string | null; app?: MiApp | null; + processErrors?: string[] | null; }; export type PureRenoteOption = Option & { renote: MiNote } & ({ text?: null } | { cw?: null } | { reply?: null } | { poll?: null } | { files?: null | [] }); @@ -228,7 +229,7 @@ export class NoteCreateService implements OnApplicationShutdown { } @bindThis - public async create(user: { + public async create(user: MiUser & { id: MiUser['id']; username: MiUser['username']; host: MiUser['host']; @@ -309,6 +310,9 @@ export class NoteCreateService implements OnApplicationShutdown { } } + // Check quote permissions + await this.checkQuotePermissions(data, user); + // Check blocking if (this.isRenote(data) && !this.isQuote(data)) { if (data.renote.userHost === null) { @@ -435,7 +439,7 @@ export class NoteCreateService implements OnApplicationShutdown { } @bindThis - public async import(user: { + public async import(user: MiUser & { id: MiUser['id']; username: MiUser['username']; host: MiUser['host']; @@ -482,14 +486,15 @@ export class NoteCreateService implements OnApplicationShutdown { renoteUserId: data.renote ? data.renote.userId : null, renoteUserHost: data.renote ? data.renote.userHost : null, userHost: user.host, + processErrors: data.processErrors, }); // should really not happen, but better safe than sorry if (data.reply?.id === insert.id) { - throw new Error("A note can't reply to itself"); + throw new Error('A note can\'t reply to itself'); } if (data.renote?.id === insert.id) { - throw new Error("A note can't renote itself"); + throw new Error('A note can\'t renote itself'); } if (data.uri != null) insert.uri = data.uri; @@ -552,7 +557,7 @@ export class NoteCreateService implements OnApplicationShutdown { } @bindThis - private async postNoteCreated(note: MiNote, user: { + private async postNoteCreated(note: MiNote, user: MiUser & { id: MiUser['id']; username: MiUser['username']; host: MiUser['host']; @@ -678,14 +683,7 @@ export class NoteCreateService implements OnApplicationShutdown { this.roleService.addNoteToRoleTimeline(noteObj); - this.webhookService.getActiveWebhooks().then(webhooks => { - webhooks = webhooks.filter(x => x.userId === user.id && x.on.includes('note')); - for (const webhook of webhooks) { - this.queueService.userWebhookDeliver(webhook, 'note', { - note: noteObj, - }); - } - }); + this.webhookService.enqueueUserWebhook(user.id, 'note', { note: noteObj }); const nm = new NotificationManager(this.mutingsRepository, this.notificationService, user, note); @@ -717,13 +715,7 @@ export class NoteCreateService implements OnApplicationShutdown { if (!isThreadMuted && !muted) { nm.push(data.reply.userId, 'reply'); this.globalEventService.publishMainStream(data.reply.userId, 'reply', noteObj); - - const webhooks = (await this.webhookService.getActiveWebhooks()).filter(x => x.userId === data.reply!.userId && x.on.includes('reply')); - for (const webhook of webhooks) { - this.queueService.userWebhookDeliver(webhook, 'reply', { - note: noteObj, - }); - } + this.webhookService.enqueueUserWebhook(data.reply.userId, 'reply', { note: noteObj }); } } } @@ -757,22 +749,16 @@ export class NoteCreateService implements OnApplicationShutdown { // Publish event if ((user.id !== data.renote.userId) && data.renote.userHost === null) { this.globalEventService.publishMainStream(data.renote.userId, 'renote', noteObj); - - const webhooks = (await this.webhookService.getActiveWebhooks()).filter(x => x.userId === data.renote!.userId && x.on.includes('renote')); - for (const webhook of webhooks) { - this.queueService.userWebhookDeliver(webhook, 'renote', { - note: noteObj, - }); - } + this.webhookService.enqueueUserWebhook(data.renote.userId, 'renote', { note: noteObj }); } } nm.notify(); //#region AP deliver - if (this.userEntityService.isLocalUser(user)) { + if (!data.localOnly && this.userEntityService.isLocalUser(user)) { (async () => { - const noteActivity = await this.renderNoteOrRenoteActivity(data, note); + const noteActivity = await this.renderNoteOrRenoteActivity(data, note, user); const dm = this.apDeliverManagerService.createDeliverManager(user, noteActivity); // メンションされたリモートユーザーに配送 @@ -905,13 +891,7 @@ export class NoteCreateService implements OnApplicationShutdown { }); this.globalEventService.publishMainStream(u.id, 'mention', detailPackedNote); - - const webhooks = (await this.webhookService.getActiveWebhooks()).filter(x => x.userId === u.id && x.on.includes('mention')); - for (const webhook of webhooks) { - this.queueService.userWebhookDeliver(webhook, 'mention', { - note: detailPackedNote, - }); - } + this.webhookService.enqueueUserWebhook(u.id, 'mention', { note: detailPackedNote }); // Create notification nm.push(u.id, 'mention'); @@ -924,12 +904,12 @@ export class NoteCreateService implements OnApplicationShutdown { } @bindThis - private async renderNoteOrRenoteActivity(data: Option, note: MiNote) { + private async renderNoteOrRenoteActivity(data: Option, note: MiNote, user: MiUser) { if (data.localOnly) return null; const content = this.isRenote(data) && !this.isQuote(data) ? this.apRendererService.renderAnnounce(data.renote.uri ? data.renote.uri : `${this.config.url}/notes/${data.renote.id}`, note) - : this.apRendererService.renderCreate(await this.apRendererService.renderNote(note, false), note); + : this.apRendererService.renderCreate(await this.apRendererService.renderNote(note, user, false), note); return this.apRendererService.addContext(content); } @@ -1172,4 +1152,29 @@ export class NoteCreateService implements OnApplicationShutdown { public async onApplicationShutdown(signal?: string | undefined): Promise<void> { await this.dispose(); } + + @bindThis + public async checkQuotePermissions(data: Option, user: MiUser): Promise<void> { + // Not a quote + if (!this.isRenote(data) || !this.isQuote(data)) return; + + // User cannot quote + if (user.rejectQuotes) { + if (user.host == null) { + throw new IdentifiableError('1c0ea108-d1e3-4e8e-aa3f-4d2487626153', 'QUOTE_DISABLED_FOR_USER'); + } else { + (data as Option).renote = null; + (data.processErrors ??= []).push('quoteUnavailable'); + } + } + + // Instance cannot quote + if (user.host) { + const instance = await this.federatedInstanceService.fetch(user.host); + if (instance?.rejectQuotes) { + (data as Option).renote = null; + (data.processErrors ??= []).push('quoteUnavailable'); + } + } + } } diff --git a/packages/backend/src/core/NoteDeleteService.ts b/packages/backend/src/core/NoteDeleteService.ts index b51a3143c9..1f94e65809 100644 --- a/packages/backend/src/core/NoteDeleteService.ts +++ b/packages/backend/src/core/NoteDeleteService.ts @@ -24,9 +24,14 @@ import { SearchService } from '@/core/SearchService.js'; import { ModerationLogService } from '@/core/ModerationLogService.js'; import { isQuote, isRenote } from '@/misc/is-renote.js'; import { LatestNoteService } from '@/core/LatestNoteService.js'; +import { ApLogService } from '@/core/ApLogService.js'; +import Logger from '@/logger.js'; +import { LoggerService } from './LoggerService.js'; @Injectable() export class NoteDeleteService { + private readonly logger: Logger; + constructor( @Inject(DI.config) private config: Config, @@ -55,7 +60,11 @@ export class NoteDeleteService { private perUserNotesChart: PerUserNotesChart, private instanceChart: InstanceChart, private latestNoteService: LatestNoteService, - ) {} + private readonly apLogService: ApLogService, + loggerService: LoggerService, + ) { + this.logger = loggerService.getLogger('note-delete-service'); + } /** * 投稿を削除します。 @@ -153,9 +162,13 @@ export class NoteDeleteService { noteUserId: note.userId, noteUserUsername: user.username, noteUserHost: user.host, - note: note, }); } + + if (note.uri) { + this.apLogService.deleteObjectLogs(note.uri) + .catch(err => this.logger.error(err, `Failed to delete AP logs for note '${note.uri}'`)); + } } @bindThis diff --git a/packages/backend/src/core/NoteEditService.ts b/packages/backend/src/core/NoteEditService.ts index f1c7bcbea5..7851af86b7 100644 --- a/packages/backend/src/core/NoteEditService.ts +++ b/packages/backend/src/core/NoteEditService.ts @@ -140,6 +140,7 @@ type Option = { app?: MiApp | null; updatedAt?: Date | null; editcount?: boolean | null; + processErrors?: string[] | null; }; @Injectable() @@ -224,7 +225,7 @@ export class NoteEditService implements OnApplicationShutdown { } @bindThis - public async edit(user: { + public async edit(user: MiUser & { id: MiUser['id']; username: MiUser['username']; host: MiUser['host']; @@ -309,7 +310,7 @@ export class NoteEditService implements OnApplicationShutdown { if (this.isRenote(data)) { if (data.renote.id === oldnote.id) { - throw new Error("A note can't renote itself"); + throw new Error('A note can\'t renote itself'); } switch (data.renote.visibility) { @@ -337,6 +338,9 @@ export class NoteEditService implements OnApplicationShutdown { } } + // Check quote permissions + await this.noteCreateService.checkQuotePermissions(data, user); + // Check blocking if (this.isRenote(data) && !this.isQuote(data)) { if (data.renote.userHost === null) { @@ -529,6 +533,7 @@ export class NoteEditService implements OnApplicationShutdown { if (data.uri != null) note.uri = data.uri; if (data.url != null) note.url = data.url; + if (data.processErrors !== undefined) note.processErrors = data.processErrors; if (mentionedUsers.length > 0) { note.mentions = mentionedUsers.map(u => u.id); @@ -584,7 +589,7 @@ export class NoteEditService implements OnApplicationShutdown { } @bindThis - private async postNoteEdited(note: MiNote, oldNote: MiNote, user: { + private async postNoteEdited(note: MiNote, oldNote: MiNote, user: MiUser & { id: MiUser['id']; username: MiUser['username']; host: MiUser['host']; @@ -664,14 +669,7 @@ export class NoteEditService implements OnApplicationShutdown { this.roleService.addNoteToRoleTimeline(noteObj); - this.webhookService.getActiveWebhooks().then(webhooks => { - webhooks = webhooks.filter(x => x.userId === user.id && x.on.includes('note')); - for (const webhook of webhooks) { - this.queueService.userWebhookDeliver(webhook, 'note', { - note: noteObj, - }); - } - }); + this.webhookService.enqueueUserWebhook(user.id, 'note', { note: noteObj }); const nm = new NotificationManager(this.mutingsRepository, this.notificationService, user, note); @@ -700,12 +698,7 @@ export class NoteEditService implements OnApplicationShutdown { nm.push(data.reply.userId, 'edited'); this.globalEventService.publishMainStream(data.reply.userId, 'edited', noteObj); - const webhooks = (await this.webhookService.getActiveWebhooks()).filter(x => x.userId === data.reply!.userId && x.on.includes('edited')); - for (const webhook of webhooks) { - this.queueService.userWebhookDeliver(webhook, 'edited', { - note: noteObj, - }); - } + this.webhookService.enqueueUserWebhook(data.reply.userId, 'reply', { note: noteObj }); } } } @@ -713,9 +706,9 @@ export class NoteEditService implements OnApplicationShutdown { nm.notify(); //#region AP deliver - if (this.userEntityService.isLocalUser(user)) { + if (!data.localOnly && this.userEntityService.isLocalUser(user)) { (async () => { - const noteActivity = await this.renderNoteOrRenoteActivity(data, note); + const noteActivity = await this.renderNoteOrRenoteActivity(data, note, user); const dm = this.apDeliverManagerService.createDeliverManager(user, noteActivity); // メンションされたリモートユーザーに配送 @@ -810,6 +803,7 @@ export class NoteEditService implements OnApplicationShutdown { (note.files != null && note.files.length > 0); } + // TODO why is this unused? @bindThis private async createMentionedEvents(mentionedUsers: MinimumUser[], note: MiNote, nm: NotificationManager) { for (const u of mentionedUsers.filter(u => this.userEntityService.isLocalUser(u))) { @@ -837,13 +831,7 @@ export class NoteEditService implements OnApplicationShutdown { }); this.globalEventService.publishMainStream(u.id, 'edited', detailPackedNote); - - const webhooks = (await this.webhookService.getActiveWebhooks()).filter(x => x.userId === u.id && x.on.includes('edited')); - for (const webhook of webhooks) { - this.queueService.userWebhookDeliver(webhook, 'edited', { - note: detailPackedNote, - }); - } + this.webhookService.enqueueUserWebhook(u.id, 'edited', { note: detailPackedNote }); // Create notification nm.push(u.id, 'edited'); @@ -851,14 +839,12 @@ export class NoteEditService implements OnApplicationShutdown { } @bindThis - private async renderNoteOrRenoteActivity(data: Option, note: MiNote) { + private async renderNoteOrRenoteActivity(data: Option, note: MiNote, user: MiUser) { if (data.localOnly) return null; - const user = await this.usersRepository.findOneBy({ id: note.userId }); - if (user == null) throw new Error('user not found'); const content = this.isRenote(data) && !this.isQuote(data) ? this.apRendererService.renderAnnounce(data.renote.uri ? data.renote.uri : `${this.config.url}/notes/${data.renote.id}`, note) - : this.apRendererService.renderUpdate(await this.apRendererService.renderUpNote(note, false), user); + : this.apRendererService.renderUpdate(await this.apRendererService.renderUpNote(note, user, false), user); return this.apRendererService.addContext(content); } diff --git a/packages/backend/src/core/PollService.ts b/packages/backend/src/core/PollService.ts index 6c96ab16cf..d6364613bd 100644 --- a/packages/backend/src/core/PollService.ts +++ b/packages/backend/src/core/PollService.ts @@ -100,7 +100,7 @@ export class PollService { if (user == null) throw new Error('note not found'); if (this.userEntityService.isLocalUser(user)) { - const content = this.apRendererService.addContext(this.apRendererService.renderUpdate(await this.apRendererService.renderNote(note, false), user)); + const content = this.apRendererService.addContext(this.apRendererService.renderUpdate(await this.apRendererService.renderNote(note, user, false), user)); this.apDeliverManagerService.deliverToFollowers(user, content); this.relayService.deliverToRelays(user, content); } diff --git a/packages/backend/src/core/S3Service.ts b/packages/backend/src/core/S3Service.ts index bb2a463354..37721d2bf1 100644 --- a/packages/backend/src/core/S3Service.ts +++ b/packages/backend/src/core/S3Service.ts @@ -28,7 +28,7 @@ export class S3Service { ? `${meta.objectStorageUseSSL ? 'https' : 'http'}://${meta.objectStorageEndpoint}` : `${meta.objectStorageUseSSL ? 'https' : 'http'}://example.net`; // dummy url to select http(s) agent - const agent = this.httpRequestService.getAgentByUrl(new URL(u), !meta.objectStorageUseProxy); + const agent = this.httpRequestService.getAgentByUrl(new URL(u), !meta.objectStorageUseProxy, true); const handlerOption: NodeHttpHandlerOptions = {}; if (meta.objectStorageUseSSL) { handlerOption.httpsAgent = agent as https.Agent; diff --git a/packages/backend/src/core/SearchService.ts b/packages/backend/src/core/SearchService.ts index 6dc3e85fc8..4782a6c7b0 100644 --- a/packages/backend/src/core/SearchService.ts +++ b/packages/backend/src/core/SearchService.ts @@ -6,16 +6,17 @@ import { Inject, Injectable } from '@nestjs/common'; import { In } from 'typeorm'; import { DI } from '@/di-symbols.js'; -import type { Config } from '@/config.js'; +import { type Config, FulltextSearchProvider } from '@/config.js'; import { bindThis } from '@/decorators.js'; import { MiNote } from '@/models/Note.js'; -import { MiUser } from '@/models/_.js'; import type { NotesRepository } from '@/models/_.js'; +import { MiUser } from '@/models/_.js'; import { sqlLikeEscape } from '@/misc/sql-like-escape.js'; import { isUserRelated } from '@/misc/is-user-related.js'; import { CacheService } from '@/core/CacheService.js'; import { QueryService } from '@/core/QueryService.js'; import { IdService } from '@/core/IdService.js'; +import { LoggerService } from '@/core/LoggerService.js'; import type { Index, MeiliSearch } from 'meilisearch'; type K = string; @@ -27,12 +28,81 @@ type Q = { op: '<', k: K, v: number } | { op: '>=', k: K, v: number } | { op: '<=', k: K, v: number } | - { op: 'is null', k: K} | - { op: 'is not null', k: K} | + { op: 'is null', k: K } | + { op: 'is not null', k: K } | { op: 'and', qs: Q[] } | { op: 'or', qs: Q[] } | { op: 'not', q: Q }; +const fileTypes = { + image: [ + 'image/webp', + 'image/png', + 'image/jpeg', + 'image/avif', + 'image/apng', + 'image/gif', + ], + video: [ + 'video/mp4', + 'video/webm', + 'video/mpeg', + 'video/x-m4v', + ], + audio: [ + 'audio/mpeg', + 'audio/flac', + 'audio/wav', + 'audio/aac', + 'audio/webm', + 'audio/opus', + 'audio/ogg', + 'audio/x-m4a', + 'audio/mod', + 'audio/s3m', + 'audio/xm', + 'audio/it', + 'audio/x-mod', + 'audio/x-s3m', + 'audio/x-xm', + 'audio/x-it', + ], + // Keep in sync with frontend-shared/js/const.ts + module: [ + 'audio/mod', + 'audio/x-mod', + 'audio/s3m', + 'audio/x-s3m', + 'audio/xm', + 'audio/x-xm', + 'audio/it', + 'audio/x-it', + ], + flash: [ + 'application/x-shockwave-flash', + 'application/vnd.adobe.flash.movie', + ], +}; + +// Make sure to regenerate misskey-js and check search.note.vue after changing these +export const fileTypeCategories = ['image', 'video', 'audio', 'module', 'flash', null] as const; +export type FileTypeCategory = typeof fileTypeCategories[number]; + +export type SearchOpts = { + userId?: MiNote['userId'] | null; + channelId?: MiNote['channelId'] | null; + host?: string | null; + filetype?: FileTypeCategory; + order?: string | null; + disableMeili?: boolean | null; +}; + +export type SearchPagination = { + untilId?: MiNote['id']; + sinceId?: MiNote['id']; + limit: number; +}; + function compileValue(value: V): string { if (typeof value === 'string') { return `'${value}'`; // TODO: escape @@ -64,7 +134,8 @@ function compileQuery(q: Q): string { @Injectable() export class SearchService { private readonly meilisearchIndexScope: 'local' | 'global' | string[] = 'local'; - private meilisearchNoteIndex: Index | null = null; + private readonly meilisearchNoteIndex: Index | null = null; + private readonly provider: FulltextSearchProvider; constructor( @Inject(DI.config) @@ -79,6 +150,7 @@ export class SearchService { private cacheService: CacheService, private queryService: QueryService, private idService: IdService, + private loggerService: LoggerService, ) { if (meilisearch) { this.meilisearchNoteIndex = meilisearch.index(`${this.config.meilisearch?.index}---notes`); @@ -110,189 +182,198 @@ export class SearchService { if (this.config.meilisearch?.scope) { this.meilisearchIndexScope = this.config.meilisearch.scope; } + + this.provider = config.fulltextSearch?.provider ?? 'sqlLike'; + this.loggerService.getLogger('SearchService').info(`-- Provider: ${this.provider}`); } @bindThis public async indexNote(note: MiNote): Promise<void> { + if (!this.meilisearch) return; if (note.text == null && note.cw == null) return; if (!['home', 'public'].includes(note.visibility)) return; - if (this.meilisearch) { - switch (this.meilisearchIndexScope) { - case 'global': - break; + switch (this.meilisearchIndexScope) { + case 'global': + break; - case 'local': - if (note.userHost == null) break; - return; + case 'local': + if (note.userHost == null) break; + return; - default: { - if (note.userHost == null) break; - if (this.meilisearchIndexScope.includes(note.userHost)) break; - return; - } + default: { + if (note.userHost == null) break; + if (this.meilisearchIndexScope.includes(note.userHost)) break; + return; } - - await this.meilisearchNoteIndex?.addDocuments([{ - id: note.id, - createdAt: this.idService.parse(note.id).date.getTime(), - userId: note.userId, - userHost: note.userHost, - channelId: note.channelId, - cw: note.cw, - text: note.text, - tags: note.tags, - attachedFileTypes: note.attachedFileTypes, - }], { - primaryKey: 'id', - }); } + + await this.meilisearchNoteIndex?.addDocuments([{ + id: note.id, + createdAt: this.idService.parse(note.id).date.getTime(), + userId: note.userId, + userHost: note.userHost, + channelId: note.channelId, + cw: note.cw, + text: note.text, + tags: note.tags, + attachedFileTypes: note.attachedFileTypes, + }], { + primaryKey: 'id', + }); } @bindThis public async unindexNote(note: MiNote): Promise<void> { + if (!this.meilisearch) return; if (!['home', 'public'].includes(note.visibility)) return; - if (this.meilisearch) { - this.meilisearchNoteIndex!.deleteDocument(note.id); - } + await this.meilisearchNoteIndex?.deleteDocument(note.id); } @bindThis - public async searchNote(q: string, me: MiUser | null, opts: { - userId?: MiNote['userId'] | null; - channelId?: MiNote['channelId'] | null; - host?: string | null; - filetype?: string | null; - order?: string | null; - disableMeili?: boolean | null; - }, pagination: { - untilId?: MiNote['id']; - sinceId?: MiNote['id']; - limit?: number; - }): Promise<MiNote[]> { - if (this.meilisearch && !opts.disableMeili) { - const filter: Q = { - op: 'and', - qs: [], - }; - if (pagination.untilId) filter.qs.push({ op: '<', k: 'createdAt', v: this.idService.parse(pagination.untilId).date.getTime() }); - if (pagination.sinceId) filter.qs.push({ op: '>', k: 'createdAt', v: this.idService.parse(pagination.sinceId).date.getTime() }); - if (opts.userId) filter.qs.push({ op: '=', k: 'userId', v: opts.userId }); - if (opts.channelId) filter.qs.push({ op: '=', k: 'channelId', v: opts.channelId }); - if (opts.host) { - if (opts.host === '.') { - filter.qs.push({ op: 'is null', k: 'userHost' }); - } else { - filter.qs.push({ op: '=', k: 'userHost', v: opts.host }); - } + public async searchNote( + q: string, + me: MiUser | null, + opts: SearchOpts, + pagination: SearchPagination, + ): Promise<MiNote[]> { + switch (this.provider) { + case 'sqlLike': + case 'sqlPgroonga': + case 'sqlTsvector': { + // ほとんど内容に差がないのでsqlLikeとsqlPgroongaを同じ処理にしている. + // 今後の拡張で差が出る用であれば関数を分ける. + return this.searchNoteByLike(q, me, opts, pagination); } - if (opts.filetype) { - if (opts.filetype === 'image') { - filter.qs.push({ op: 'or', qs: [ - { op: '=', k: 'attachedFileTypes', v: 'image/webp' }, - { op: '=', k: 'attachedFileTypes', v: 'image/png' }, - { op: '=', k: 'attachedFileTypes', v: 'image/jpeg' }, - { op: '=', k: 'attachedFileTypes', v: 'image/avif' }, - { op: '=', k: 'attachedFileTypes', v: 'image/apng' }, - { op: '=', k: 'attachedFileTypes', v: 'image/gif' }, - ] }); - } else if (opts.filetype === 'video') { - filter.qs.push({ op: 'or', qs: [ - { op: '=', k: 'attachedFileTypes', v: 'video/mp4' }, - { op: '=', k: 'attachedFileTypes', v: 'video/webm' }, - { op: '=', k: 'attachedFileTypes', v: 'video/mpeg' }, - { op: '=', k: 'attachedFileTypes', v: 'video/x-m4v' }, - ] }); - } else if (opts.filetype === 'audio') { - filter.qs.push({ op: 'or', qs: [ - { op: '=', k: 'attachedFileTypes', v: 'audio/mpeg' }, - { op: '=', k: 'attachedFileTypes', v: 'audio/flac' }, - { op: '=', k: 'attachedFileTypes', v: 'audio/wav' }, - { op: '=', k: 'attachedFileTypes', v: 'audio/aac' }, - { op: '=', k: 'attachedFileTypes', v: 'audio/webm' }, - { op: '=', k: 'attachedFileTypes', v: 'audio/opus' }, - { op: '=', k: 'attachedFileTypes', v: 'audio/ogg' }, - { op: '=', k: 'attachedFileTypes', v: 'audio/x-m4a' }, - { op: '=', k: 'attachedFileTypes', v: 'audio/mod' }, - { op: '=', k: 'attachedFileTypes', v: 'audio/s3m' }, - { op: '=', k: 'attachedFileTypes', v: 'audio/xm' }, - { op: '=', k: 'attachedFileTypes', v: 'audio/it' }, - { op: '=', k: 'attachedFileTypes', v: 'audio/x-mod' }, - { op: '=', k: 'attachedFileTypes', v: 'audio/x-s3m' }, - { op: '=', k: 'attachedFileTypes', v: 'audio/x-xm' }, - { op: '=', k: 'attachedFileTypes', v: 'audio/x-it' }, - ] }); - } + case 'meilisearch': { + return this.searchNoteByMeiliSearch(q, me, opts, pagination); } - const res = await this.meilisearchNoteIndex!.search(q, { - sort: [`createdAt:${opts.order ? opts.order : 'desc'}`], - matchingStrategy: 'all', - attributesToRetrieve: ['id', 'createdAt'], - filter: compileQuery(filter), - limit: pagination.limit, - }); - if (res.hits.length === 0) return []; - const [ - userIdsWhoMeMuting, - userIdsWhoBlockingMe, - ] = me ? await Promise.all([ - this.cacheService.userMutingsCache.fetch(me.id), - this.cacheService.userBlockedCache.fetch(me.id), - ]) : [new Set<string>(), new Set<string>()]; - const notes = (await this.notesRepository.findBy({ - id: In(res.hits.map(x => x.id)), - })).filter(note => { - if (me && isUserRelated(note, userIdsWhoBlockingMe)) return false; - if (me && isUserRelated(note, userIdsWhoMeMuting)) return false; - return true; - }); - return notes.sort((a, b) => a.id > b.id ? -1 : 1); + default: { + // eslint-disable-next-line @typescript-eslint/no-unused-vars + const typeCheck: never = this.provider; + return []; + } + } + } + + @bindThis + private async searchNoteByLike( + q: string, + me: MiUser | null, + opts: SearchOpts, + pagination: SearchPagination, + ): Promise<MiNote[]> { + const query = this.queryService.makePaginationQuery(this.notesRepository.createQueryBuilder('note'), pagination.sinceId, pagination.untilId); + + if (opts.userId) { + query.andWhere('note.userId = :userId', { userId: opts.userId }); + } else if (opts.channelId) { + query.andWhere('note.channelId = :channelId', { channelId: opts.channelId }); + } + + query + .innerJoinAndSelect('note.user', 'user') + .leftJoinAndSelect('note.reply', 'reply') + .leftJoinAndSelect('note.renote', 'renote') + .leftJoinAndSelect('reply.user', 'replyUser') + .leftJoinAndSelect('renote.user', 'renoteUser'); + + if (this.config.fulltextSearch?.provider === 'sqlPgroonga') { + query.andWhere('note.text &@~ :q', { q }); + } else if (this.config.fulltextSearch?.provider === 'sqlTsvector') { + query.andWhere('note.tsvector_embedding @@ websearch_to_tsquery(:q)', { q }); } else { - const query = this.queryService.makePaginationQuery(this.notesRepository.createQueryBuilder('note'), pagination.sinceId, pagination.untilId); + query.andWhere('note.text ILIKE :q', { q: `%${ sqlLikeEscape(q) }%` }); + } - if (opts.userId) { - query.andWhere('note.userId = :userId', { userId: opts.userId }); - } else if (opts.channelId) { - query.andWhere('note.channelId = :channelId', { channelId: opts.channelId }); + if (opts.host) { + if (opts.host === '.') { + query.andWhere('note.userHost IS NULL'); + } else { + query.andWhere('note.userHost = :host', { host: opts.host }); } + } - query - .andWhere('note.text ILIKE :q', { q: `%${ sqlLikeEscape(q) }%` }) - .innerJoinAndSelect('note.user', 'user') - .leftJoinAndSelect('note.reply', 'reply') - .leftJoinAndSelect('note.renote', 'renote') - .leftJoinAndSelect('reply.user', 'replyUser') - .leftJoinAndSelect('renote.user', 'renoteUser'); + if (opts.filetype) { + query.andWhere('note."attachedFileTypes" && :types', { types: fileTypes[opts.filetype] }); + } - if (opts.host) { - if (opts.host === '.') { - query.andWhere('user.host IS NULL'); - } else { - query.andWhere('user.host = :host', { host: opts.host }); - } - } + this.queryService.generateVisibilityQuery(query, me); + if (me) this.queryService.generateMutedUserQuery(query, me); + if (me) this.queryService.generateBlockedUserQuery(query, me); + + return await query.limit(pagination.limit).getMany(); + } - if (opts.filetype) { - /* this is very ugly, but the "correct" solution would - be `and exists (select 1 from - unnest(note."attachedFileTypes") x(t) where t like - :type)` and I can't find a way to get TypeORM to - generate that; this hack works because `~*` is - "regexp match, ignoring case" and the stringified - version of an array of varchars (which is what - `attachedFileTypes` is) looks like `{foo,bar}`, so - we're looking for opts.filetype as the first half of - a MIME type, either at start of the array (after the - `{`) or later (after a `,`) */ - query.andWhere(`note."attachedFileTypes"::varchar ~* :type`, { type: `[{,]${opts.filetype}/` }); + @bindThis + private async searchNoteByMeiliSearch( + q: string, + me: MiUser | null, + opts: SearchOpts, + pagination: SearchPagination, + ): Promise<MiNote[]> { + if (!this.meilisearch || !this.meilisearchNoteIndex) { + throw new Error('MeiliSearch is not available'); + } + + const filter: Q = { + op: 'and', + qs: [], + }; + if (pagination.untilId) filter.qs.push({ + op: '<', + k: 'createdAt', + v: this.idService.parse(pagination.untilId).date.getTime(), + }); + if (pagination.sinceId) filter.qs.push({ + op: '>', + k: 'createdAt', + v: this.idService.parse(pagination.sinceId).date.getTime(), + }); + if (opts.userId) filter.qs.push({ op: '=', k: 'userId', v: opts.userId }); + if (opts.channelId) filter.qs.push({ op: '=', k: 'channelId', v: opts.channelId }); + if (opts.host) { + if (opts.host === '.') { + filter.qs.push({ op: 'is null', k: 'userHost' }); + } else { + filter.qs.push({ op: '=', k: 'userHost', v: opts.host }); } + } - this.queryService.generateVisibilityQuery(query, me); - if (me) this.queryService.generateMutedUserQuery(query, me); - if (me) this.queryService.generateBlockedUserQuery(query, me); + if (opts.filetype) { + const filters = fileTypes[opts.filetype].map(mime => ({ op: '=' as const, k: 'attachedFileTypes', v: mime })); + filter.qs.push({ op: 'or', qs: filters }); + } - return await query.limit(pagination.limit).getMany(); + const res = await this.meilisearchNoteIndex.search(q, { + sort: [`createdAt:${opts.order ? opts.order : 'desc'}`], + matchingStrategy: 'all', + attributesToRetrieve: ['id', 'createdAt'], + filter: compileQuery(filter), + limit: pagination.limit, + }); + if (res.hits.length === 0) { + return []; } + + const [ + userIdsWhoMeMuting, + userIdsWhoBlockingMe, + ] = me + ? await Promise.all([ + this.cacheService.userMutingsCache.fetch(me.id), + this.cacheService.userBlockedCache.fetch(me.id), + ]) + : [new Set<string>(), new Set<string>()]; + const notes = (await this.notesRepository.findBy({ + id: In(res.hits.map(x => x.id)), + })).filter(note => { + if (me && isUserRelated(note, userIdsWhoBlockingMe)) return false; + if (me && isUserRelated(note, userIdsWhoMeMuting)) return false; + return true; + }); + + return notes.sort((a, b) => a.id > b.id ? -1 : 1); } } diff --git a/packages/backend/src/core/SignupService.ts b/packages/backend/src/core/SignupService.ts index 0ad448e95f..9fc0c2b34a 100644 --- a/packages/backend/src/core/SignupService.ts +++ b/packages/backend/src/core/SignupService.ts @@ -162,3 +162,4 @@ export class SignupService { return { account, secret }; } } + diff --git a/packages/backend/src/core/SystemWebhookService.ts b/packages/backend/src/core/SystemWebhookService.ts index de00169612..8239490adc 100644 --- a/packages/backend/src/core/SystemWebhookService.ts +++ b/packages/backend/src/core/SystemWebhookService.ts @@ -50,7 +50,6 @@ export type SystemWebhookPayload<T extends SystemWebhookEventType> = @Injectable() export class SystemWebhookService implements OnApplicationShutdown { - private logger: Logger; private activeSystemWebhooksFetched = false; private activeSystemWebhooks: MiSystemWebhook[] = []; @@ -62,11 +61,9 @@ export class SystemWebhookService implements OnApplicationShutdown { private idService: IdService, private queueService: QueueService, private moderationLogService: ModerationLogService, - private loggerService: LoggerService, private globalEventService: GlobalEventService, ) { this.redisForSub.on('message', this.onMessage); - this.logger = this.loggerService.getLogger('webhook'); } @bindThis @@ -193,28 +190,24 @@ export class SystemWebhookService implements OnApplicationShutdown { /** * SystemWebhook をWebhook配送キューに追加する * @see QueueService.systemWebhookDeliver - * // TODO: contentの型を厳格化する */ @bindThis public async enqueueSystemWebhook<T extends SystemWebhookEventType>( - webhook: MiSystemWebhook | MiSystemWebhook['id'], type: T, content: SystemWebhookPayload<T>, + opts?: { + excludes?: MiSystemWebhook['id'][]; + }, ) { - const webhookEntity = typeof webhook === 'string' - ? (await this.fetchActiveSystemWebhooks()).find(a => a.id === webhook) - : webhook; - if (!webhookEntity || !webhookEntity.isActive) { - this.logger.info(`SystemWebhook is not active or not found : ${webhook}`); - return; - } - - if (!webhookEntity.on.includes(type)) { - this.logger.info(`SystemWebhook ${webhookEntity.id} is not listening to ${type}`); - return; - } - - return this.queueService.systemWebhookDeliver(webhookEntity, type, content); + const webhooks = await this.fetchActiveSystemWebhooks() + .then(webhooks => { + return webhooks.filter(webhook => !opts?.excludes?.includes(webhook.id) && webhook.on.includes(type)); + }); + return Promise.all( + webhooks.map(webhook => { + return this.queueService.systemWebhookDeliver(webhook, type, content); + }), + ); } @bindThis diff --git a/packages/backend/src/core/UserBlockingService.ts b/packages/backend/src/core/UserBlockingService.ts index 2f1310b8ef..8da1bb2092 100644 --- a/packages/backend/src/core/UserBlockingService.ts +++ b/packages/backend/src/core/UserBlockingService.ts @@ -118,13 +118,7 @@ export class UserBlockingService implements OnModuleInit { schema: 'UserDetailedNotMe', }).then(async packed => { this.globalEventService.publishMainStream(follower.id, 'unfollow', packed); - - const webhooks = (await this.webhookService.getActiveWebhooks()).filter(x => x.userId === follower.id && x.on.includes('unfollow')); - for (const webhook of webhooks) { - this.queueService.userWebhookDeliver(webhook, 'unfollow', { - user: packed, - }); - } + this.webhookService.enqueueUserWebhook(follower.id, 'unfollow', { user: packed }); }); } diff --git a/packages/backend/src/core/UserFollowingService.ts b/packages/backend/src/core/UserFollowingService.ts index 8963003057..b98ca97ec9 100644 --- a/packages/backend/src/core/UserFollowingService.ts +++ b/packages/backend/src/core/UserFollowingService.ts @@ -333,13 +333,7 @@ export class UserFollowingService implements OnModuleInit { schema: 'UserDetailedNotMe', }).then(async packed => { this.globalEventService.publishMainStream(follower.id, 'follow', packed); - - const webhooks = (await this.webhookService.getActiveWebhooks()).filter(x => x.userId === follower.id && x.on.includes('follow')); - for (const webhook of webhooks) { - this.queueService.userWebhookDeliver(webhook, 'follow', { - user: packed, - }); - } + this.webhookService.enqueueUserWebhook(follower.id, 'follow', { user: packed }); }); } @@ -347,13 +341,7 @@ export class UserFollowingService implements OnModuleInit { if (this.userEntityService.isLocalUser(followee)) { this.userEntityService.pack(follower.id, followee).then(async packed => { this.globalEventService.publishMainStream(followee.id, 'followed', packed); - - const webhooks = (await this.webhookService.getActiveWebhooks()).filter(x => x.userId === followee.id && x.on.includes('followed')); - for (const webhook of webhooks) { - this.queueService.userWebhookDeliver(webhook, 'followed', { - user: packed, - }); - } + this.webhookService.enqueueUserWebhook(followee.id, 'followed', { user: packed }); }); // 通知を作成 @@ -400,13 +388,7 @@ export class UserFollowingService implements OnModuleInit { schema: 'UserDetailedNotMe', }).then(async packed => { this.globalEventService.publishMainStream(follower.id, 'unfollow', packed); - - const webhooks = (await this.webhookService.getActiveWebhooks()).filter(x => x.userId === follower.id && x.on.includes('unfollow')); - for (const webhook of webhooks) { - this.queueService.userWebhookDeliver(webhook, 'unfollow', { - user: packed, - }); - } + this.webhookService.enqueueUserWebhook(follower.id, 'unfollow', { user: packed }); }); } @@ -744,13 +726,7 @@ export class UserFollowingService implements OnModuleInit { }); this.globalEventService.publishMainStream(follower.id, 'unfollow', packedFollowee); - - const webhooks = (await this.webhookService.getActiveWebhooks()).filter(x => x.userId === follower.id && x.on.includes('unfollow')); - for (const webhook of webhooks) { - this.queueService.userWebhookDeliver(webhook, 'unfollow', { - user: packedFollowee, - }); - } + this.webhookService.enqueueUserWebhook(follower.id, 'unfollow', { user: packedFollowee }); } @bindThis diff --git a/packages/backend/src/core/UserListService.ts b/packages/backend/src/core/UserListService.ts index 6333356fe9..4f4d59a02c 100644 --- a/packages/backend/src/core/UserListService.ts +++ b/packages/backend/src/core/UserListService.ts @@ -58,7 +58,7 @@ export class UserListService implements OnApplicationShutdown, OnModuleInit { } async onModuleInit() { - this.roleService = this.moduleRef.get(RoleService.name); + this.roleService = this.moduleRef.get('RoleService'); } @bindThis diff --git a/packages/backend/src/core/UserService.ts b/packages/backend/src/core/UserService.ts index 9b1961c631..1f471513f3 100644 --- a/packages/backend/src/core/UserService.ts +++ b/packages/backend/src/core/UserService.ts @@ -63,13 +63,6 @@ export class UserService { @bindThis public async notifySystemWebhook(user: MiUser, type: 'userCreated') { const packedUser = await this.userEntityService.pack(user, null, { schema: 'UserLite' }); - const recipientWebhookIds = await this.systemWebhookService.fetchSystemWebhooks({ isActive: true, on: [type] }); - for (const webhookId of recipientWebhookIds) { - await this.systemWebhookService.enqueueSystemWebhook( - webhookId, - type, - packedUser, - ); - } + return this.systemWebhookService.enqueueSystemWebhook(type, packedUser); } } diff --git a/packages/backend/src/core/UserWebhookService.ts b/packages/backend/src/core/UserWebhookService.ts index 911efdf768..08db4c9afc 100644 --- a/packages/backend/src/core/UserWebhookService.ts +++ b/packages/backend/src/core/UserWebhookService.ts @@ -5,13 +5,14 @@ import { Inject, Injectable } from '@nestjs/common'; import * as Redis from 'ioredis'; -import { type WebhooksRepository } from '@/models/_.js'; +import { MiUser, type WebhooksRepository } from '@/models/_.js'; import { MiWebhook, WebhookEventTypes } from '@/models/Webhook.js'; import { DI } from '@/di-symbols.js'; import { bindThis } from '@/decorators.js'; import { GlobalEvents } from '@/core/GlobalEventService.js'; -import type { OnApplicationShutdown } from '@nestjs/common'; import type { Packed } from '@/misc/json-schema.js'; +import { QueueService } from '@/core/QueueService.js'; +import type { OnApplicationShutdown } from '@nestjs/common'; export type UserWebhookPayload<T extends WebhookEventTypes> = T extends 'note' | 'reply' | 'renote' |'mention' | 'edited' ? { @@ -34,6 +35,7 @@ export class UserWebhookService implements OnApplicationShutdown { private redisForSub: Redis.Redis, @Inject(DI.webhooksRepository) private webhooksRepository: WebhooksRepository, + private queueService: QueueService, ) { this.redisForSub.on('message', this.onMessage); } @@ -75,6 +77,25 @@ export class UserWebhookService implements OnApplicationShutdown { return query.getMany(); } + /** + * UserWebhook をWebhook配送キューに追加する + * @see QueueService.userWebhookDeliver + */ + @bindThis + public async enqueueUserWebhook<T extends WebhookEventTypes>( + userId: MiUser['id'], + type: T, + content: UserWebhookPayload<T>, + ) { + const webhooks = await this.getActiveWebhooks() + .then(webhooks => webhooks.filter(webhook => webhook.userId === userId && webhook.on.includes(type))); + return Promise.all( + webhooks.map(webhook => { + return this.queueService.userWebhookDeliver(webhook, type, content); + }), + ); + } + @bindThis private async onMessage(_: string, data: string): Promise<void> { const obj = JSON.parse(data); diff --git a/packages/backend/src/core/UtilityService.ts b/packages/backend/src/core/UtilityService.ts index f905914022..81eaa5f95d 100644 --- a/packages/backend/src/core/UtilityService.ts +++ b/packages/backend/src/core/UtilityService.ts @@ -3,8 +3,7 @@ * SPDX-License-Identifier: AGPL-3.0-only */ -import { URL } from 'node:url'; -import punycode from 'punycode/punycode.js'; +import { URL, domainToASCII } from 'node:url'; import { Inject, Injectable } from '@nestjs/common'; import RE2 from 're2'; import psl from 'psl'; @@ -107,13 +106,13 @@ export class UtilityService { @bindThis public toPuny(host: string): string { - return punycode.toASCII(host.toLowerCase()); + return domainToASCII(host.toLowerCase()); } @bindThis public toPunyNullable(host: string | null | undefined): string | null { if (host == null) return null; - return punycode.toASCII(host.toLowerCase()); + return domainToASCII(host.toLowerCase()); } @bindThis diff --git a/packages/backend/src/core/WebAuthnService.ts b/packages/backend/src/core/WebAuthnService.ts index ad53192f18..ed75e4f467 100644 --- a/packages/backend/src/core/WebAuthnService.ts +++ b/packages/backend/src/core/WebAuthnService.ts @@ -189,14 +189,12 @@ export class WebAuthnService { */ @bindThis public async verifySignInWithPasskeyAuthentication(context: string, response: AuthenticationResponseJSON): Promise<MiUser['id'] | null> { - const challenge = await this.redisClient.get(`webauthn:challenge:${context}`); + const challenge = await this.redisClient.getdel(`webauthn:challenge:${context}`); if (!challenge) { throw new IdentifiableError('2d16e51c-007b-4edd-afd2-f7dd02c947f6', `challenge '${context}' not found`); } - await this.redisClient.del(`webauthn:challenge:${context}`); - const key = await this.userSecurityKeysRepository.findOneBy({ id: response.id, }); diff --git a/packages/backend/src/core/WebhookTestService.ts b/packages/backend/src/core/WebhookTestService.ts index dfe7a259c4..2e50f4472f 100644 --- a/packages/backend/src/core/WebhookTestService.ts +++ b/packages/backend/src/core/WebhookTestService.ts @@ -99,6 +99,8 @@ function generateDummyUser(override?: Partial<MiUser>): MiUser { signupReason: null, noindex: false, enableRss: true, + mandatoryCW: null, + rejectQuotes: false, ...override, }; } @@ -142,6 +144,7 @@ function generateDummyNote(override?: Partial<MiNote>): MiNote { renoteUserId: null, renoteUserHost: null, updatedAt: null, + processErrors: [], ...override, }; } @@ -216,6 +219,7 @@ function toPackedUserLite(user: MiUser, override?: Packed<'UserLite'>): Packed<' isSystem: false, isSilenced: user.isSilenced, enableRss: true, + mandatoryCW: null, ...override, }; } diff --git a/packages/backend/src/core/activitypub/ApInboxService.ts b/packages/backend/src/core/activitypub/ApInboxService.ts index 278c97f907..1eef85aeef 100644 --- a/packages/backend/src/core/activitypub/ApInboxService.ts +++ b/packages/backend/src/core/activitypub/ApInboxService.ts @@ -363,10 +363,12 @@ export class ApInboxService { this.logger.info(`Creating the (Re)Note: ${uri}`); const activityAudience = await this.apAudienceService.parseAudience(actor, activity.to, activity.cc, resolver); - const createdAt = activity.published ? new Date(activity.published) : null; + let createdAt = activity.published ? new Date(activity.published) : null; - if (createdAt && createdAt < this.idService.parse(renote.id).date) { - return 'skip: malformed createdAt'; + const renoteDate = this.idService.parse(renote.id).date; + if (createdAt && createdAt < renoteDate) { + this.logger.warn(`Correcting invalid publish time for Announce "${uri}"`); + createdAt = renoteDate; } await this.noteCreateService.create(actor, { diff --git a/packages/backend/src/core/activitypub/ApRendererService.ts b/packages/backend/src/core/activitypub/ApRendererService.ts index fb706a775f..cb9b74f6d7 100644 --- a/packages/backend/src/core/activitypub/ApRendererService.ts +++ b/packages/backend/src/core/activitypub/ApRendererService.ts @@ -28,6 +28,7 @@ import type { UsersRepository, UserProfilesRepository, NotesRepository, DriveFil import { bindThis } from '@/decorators.js'; import { CustomEmojiService } from '@/core/CustomEmojiService.js'; import { IdService } from '@/core/IdService.js'; +import { appendContentWarning } from '@/misc/append-content-warning.js'; import { JsonLdService } from './JsonLdService.js'; import { ApMfmService } from './ApMfmService.js'; import { CONTEXT } from './misc/contexts.js'; @@ -192,6 +193,9 @@ export class ApRendererService { // || emoji.originalUrl してるのは後方互換性のため(publicUrlはstringなので??はだめ) url: emoji.publicUrl || emoji.originalUrl, }, + _misskey_license: { + freeText: emoji.license, + }, }; } @@ -336,7 +340,7 @@ export class ApRendererService { } @bindThis - public async renderNote(note: MiNote, dive = true): Promise<IPost> { + public async renderNote(note: MiNote, author: MiUser, dive = true): Promise<IPost> { const getPromisedFiles = async (ids: string[]): Promise<MiDriveFile[]> => { if (ids.length === 0) return []; const items = await this.driveFilesRepository.findBy({ id: In(ids) }); @@ -350,14 +354,14 @@ export class ApRendererService { inReplyToNote = await this.notesRepository.findOneBy({ id: note.replyId }); if (inReplyToNote != null) { - const inReplyToUserExist = await this.usersRepository.exists({ where: { id: inReplyToNote.userId } }); + const inReplyToUser = await this.usersRepository.findOneBy({ id: inReplyToNote.userId }); - if (inReplyToUserExist) { + if (inReplyToUser) { if (inReplyToNote.uri) { inReplyTo = inReplyToNote.uri; } else { if (dive) { - inReplyTo = await this.renderNote(inReplyToNote, false); + inReplyTo = await this.renderNote(inReplyToNote, inReplyToUser, false); } else { inReplyTo = `${this.config.url}/notes/${inReplyToNote.id}`; } @@ -420,7 +424,12 @@ export class ApRendererService { apAppend += `\n\nRE: ${quote}`; } - const summary = note.cw === '' ? String.fromCharCode(0x200B) : note.cw; + let summary = note.cw === '' ? String.fromCharCode(0x200B) : note.cw; + + // Apply mandatory CW, if applicable + if (author.mandatoryCW) { + summary = appendContentWarning(summary, author.mandatoryCW); + } const { content } = this.apMfmService.getNoteHtml(note, apAppend); @@ -633,7 +642,7 @@ export class ApRendererService { } @bindThis - public async renderUpNote(note: MiNote, dive = true): Promise<IPost> { + public async renderUpNote(note: MiNote, author: MiUser, dive = true): Promise<IPost> { const getPromisedFiles = async (ids: string[]): Promise<MiDriveFile[]> => { if (ids.length === 0) return []; const items = await this.driveFilesRepository.findBy({ id: In(ids) }); @@ -647,14 +656,14 @@ export class ApRendererService { inReplyToNote = await this.notesRepository.findOneBy({ id: note.replyId }); if (inReplyToNote != null) { - const inReplyToUserExist = await this.usersRepository.exists({ where: { id: inReplyToNote.userId } }); + const inReplyToUser = await this.usersRepository.findOneBy({ id: inReplyToNote.userId }); - if (inReplyToUserExist) { + if (inReplyToUser) { if (inReplyToNote.uri) { inReplyTo = inReplyToNote.uri; } else { if (dive) { - inReplyTo = await this.renderUpNote(inReplyToNote, false); + inReplyTo = await this.renderUpNote(inReplyToNote, inReplyToUser, false); } else { inReplyTo = `${this.config.url}/notes/${inReplyToNote.id}`; } @@ -717,7 +726,12 @@ export class ApRendererService { apAppend += `\n\nRE: ${quote}`; } - const summary = note.cw === '' ? String.fromCharCode(0x200B) : note.cw; + let summary = note.cw === '' ? String.fromCharCode(0x200B) : note.cw; + + // Apply mandatory CW, if applicable + if (author.mandatoryCW) { + summary = appendContentWarning(summary, author.mandatoryCW); + } const { content } = this.apMfmService.getNoteHtml(note, apAppend); diff --git a/packages/backend/src/core/activitypub/ApRequestService.ts b/packages/backend/src/core/activitypub/ApRequestService.ts index 8036c9638f..b63d4eb2ab 100644 --- a/packages/backend/src/core/activitypub/ApRequestService.ts +++ b/packages/backend/src/core/activitypub/ApRequestService.ts @@ -11,13 +11,12 @@ import { DI } from '@/di-symbols.js'; import type { Config } from '@/config.js'; import type { MiUser } from '@/models/User.js'; import { UserKeypairService } from '@/core/UserKeypairService.js'; -import { UtilityService } from '@/core/UtilityService.js'; +import { ApUtilityService } from '@/core/activitypub/ApUtilityService.js'; import { HttpRequestService } from '@/core/HttpRequestService.js'; import { LoggerService } from '@/core/LoggerService.js'; import { bindThis } from '@/decorators.js'; import type Logger from '@/logger.js'; import { validateContentTypeSetAsActivityPub } from '@/core/activitypub/misc/validator.js'; -import { assertActivityMatchesUrls } from '@/core/activitypub/misc/check-against-url.js'; import type { IObject } from './type.js'; type Request = { @@ -148,7 +147,7 @@ export class ApRequestService { private userKeypairService: UserKeypairService, private httpRequestService: HttpRequestService, private loggerService: LoggerService, - private utilityService: UtilityService, + private readonly apUtilityService: ApUtilityService, ) { // eslint-disable-next-line @typescript-eslint/no-unnecessary-condition this.logger = this.loggerService?.getLogger('ap-request'); // なぜか TypeError: Cannot read properties of undefined (reading 'getLogger') と言われる @@ -183,9 +182,10 @@ export class ApRequestService { * Get AP object with http-signature * @param user http-signature user * @param url URL to fetch + * @param followAlternate */ @bindThis - public async signedGet(url: string, user: { id: MiUser['id'] }, followAlternate?: boolean): Promise<unknown> { + public async signedGet(url: string, user: { id: MiUser['id'] }, followAlternate?: boolean): Promise<IObject> { const _followAlternate = followAlternate ?? true; const keypair = await this.userKeypairService.getUserKeypair(user.id); @@ -239,13 +239,22 @@ export class ApRequestService { try { document.documentElement.innerHTML = html; - const alternate = document.querySelector('head > link[rel="alternate"][type="application/activity+json"]'); + // Search for any matching value in priority order: + // 1. Type=AP > Type=none > Type=anything + // 2. Alternate > Canonical + // 3. Page order (fallback) + const alternate = + document.querySelector('head > link[href][rel="alternate"][type="application/activity+json"]') ?? + document.querySelector('head > link[href][rel="canonical"][type="application/activity+json"]') ?? + document.querySelector('head > link[href][rel="alternate"]:not([type])') ?? + document.querySelector('head > link[href][rel="canonical"]:not([type])') ?? + document.querySelector('head > link[href][rel="alternate"]') ?? + document.querySelector('head > link[href][rel="canonical"]'); + if (alternate) { const href = alternate.getAttribute('href'); - if (href) { - if (this.utilityService.punyHostPSLDomain(url) === this.utilityService.punyHostPSLDomain(href)) { - return await this.signedGet(href, user, false); - } + if (href && this.apUtilityService.haveSameAuthority(url, href)) { + return await this.signedGet(href, user, false); } } } catch (e) { @@ -258,10 +267,11 @@ export class ApRequestService { validateContentTypeSetAsActivityPub(res); - const finalUrl = res.url; // redirects may have been involved const activity = await res.json() as IObject; - assertActivityMatchesUrls(activity, [finalUrl]); + // Make sure the object ID matches the final URL (which is where it actually exists). + // The caller (ApResolverService) will verify the ID against the original / entry URL, which ensures that all three match. + this.apUtilityService.assertIdMatchesUrlAuthority(activity, res.url); return activity; } diff --git a/packages/backend/src/core/activitypub/ApResolverService.ts b/packages/backend/src/core/activitypub/ApResolverService.ts index c82a9be3b1..f9ccf10fa7 100644 --- a/packages/backend/src/core/activitypub/ApResolverService.ts +++ b/packages/backend/src/core/activitypub/ApResolverService.ts @@ -5,10 +5,9 @@ import { Inject, Injectable } from '@nestjs/common'; import { IsNull, Not } from 'typeorm'; -import { UnrecoverableError } from 'bullmq'; import type { MiLocalUser, MiRemoteUser } from '@/models/User.js'; import { InstanceActorService } from '@/core/InstanceActorService.js'; -import type { NotesRepository, PollsRepository, NoteReactionsRepository, UsersRepository, FollowRequestsRepository, MiMeta } from '@/models/_.js'; +import type { NotesRepository, PollsRepository, NoteReactionsRepository, UsersRepository, FollowRequestsRepository, MiMeta, SkApFetchLog } from '@/models/_.js'; import type { Config } from '@/config.js'; import { HttpRequestService } from '@/core/HttpRequestService.js'; import { DI } from '@/di-symbols.js'; @@ -17,7 +16,10 @@ import { bindThis } from '@/decorators.js'; import { LoggerService } from '@/core/LoggerService.js'; import type Logger from '@/logger.js'; import { fromTuple } from '@/misc/from-tuple.js'; -import { isCollectionOrOrderedCollection } from './type.js'; +import { IdentifiableError } from '@/misc/identifiable-error.js'; +import { ApLogService, calculateDurationSince, extractObjectContext } from '@/core/ApLogService.js'; +import { ApUtilityService } from '@/core/activitypub/ApUtilityService.js'; +import { getApId, getNullableApId, isCollectionOrOrderedCollection } from './type.js'; import { ApDbResolverService } from './ApDbResolverService.js'; import { ApRendererService } from './ApRendererService.js'; import { ApRequestService } from './ApRequestService.js'; @@ -43,6 +45,8 @@ export class Resolver { private apRendererService: ApRendererService, private apDbResolverService: ApDbResolverService, private loggerService: LoggerService, + private readonly apLogService: ApLogService, + private readonly apUtilityService: ApUtilityService, private recursionLimit = 256, ) { this.history = new Set(); @@ -68,7 +72,7 @@ export class Resolver { if (isCollectionOrOrderedCollection(collection)) { return collection; } else { - throw new UnrecoverableError(`unrecognized collection type: ${collection.type}`); + throw new IdentifiableError('f100eccf-f347-43fb-9b45-96a0831fb635', `unrecognized collection type: ${collection.type}`); } } @@ -81,30 +85,67 @@ export class Resolver { return value; } + const host = this.utilityService.extractDbHost(value); + if (this.config.activityLogging.enabled && !this.utilityService.isSelfHost(host)) { + return await this._resolveLogged(value, host); + } else { + return await this._resolve(value, host); + } + } + + private async _resolveLogged(requestUri: string, host: string): Promise<IObject> { + const startTime = process.hrtime.bigint(); + + const log = await this.apLogService.createFetchLog({ + host: host, + requestUri, + }); + + try { + const result = await this._resolve(requestUri, host, log); + + log.accepted = true; + log.result = 'ok'; + + return result; + } catch (err) { + log.accepted = false; + log.result = String(err); + + throw err; + } finally { + log.duration = calculateDurationSince(startTime); + + // Save or finalize asynchronously + this.apLogService.saveFetchLog(log) + .catch(err => this.logger.error('Failed to record AP object fetch:', err)); + } + } + + private async _resolve(value: string, host: string, log?: SkApFetchLog): Promise<IObject> { if (value.includes('#')) { // URLs with fragment parts cannot be resolved correctly because // the fragment part does not get transmitted over HTTP(S). // Avoid strange behaviour by not trying to resolve these at all. - throw new UnrecoverableError(`cannot resolve URL with fragment: ${value}`); + throw new IdentifiableError('b94fd5b1-0e3b-4678-9df2-dad4cd515ab2', `cannot resolve URL with fragment: ${value}`); } if (this.history.has(value)) { - throw new Error(`cannot resolve already resolved URL: ${value}`); + throw new IdentifiableError('0dc86cf6-7cd6-4e56-b1e6-5903d62d7ea5', `cannot resolve already resolved URL: ${value}`); } if (this.history.size > this.recursionLimit) { - throw new Error(`hit recursion limit: ${value}`); + throw new IdentifiableError('d592da9f-822f-4d91-83d7-4ceefabcf3d2', `hit recursion limit: ${value}`); } this.history.add(value); - const host = this.utilityService.extractDbHost(value); if (this.utilityService.isSelfHost(host)) { return await this.resolveLocal(value); } if (!this.utilityService.isFederationAllowedHost(host)) { - throw new UnrecoverableError(`cannot fetch AP object ${value}: blocked instance ${host}`); + throw new IdentifiableError('09d79f9e-64f1-4316-9cfa-e75c4d091574', `cannot fetch AP object ${value}: blocked instance ${host}`); } if (this.config.signToActivityPubGet && !this.user) { @@ -115,32 +156,42 @@ export class Resolver { ? await this.apRequestService.signedGet(value, this.user) as IObject : await this.httpRequestService.getActivityJson(value)) as IObject; + if (log) { + const { object: objectOnly, context, contextHash } = extractObjectContext(object); + const objectUri = getNullableApId(object); + + if (objectUri) { + log.objectUri = objectUri; + log.host = this.utilityService.extractDbHost(objectUri); + } + + log.object = objectOnly; + log.context = context; + log.contextHash = contextHash; + } + if ( Array.isArray(object['@context']) ? !(object['@context'] as unknown[]).includes('https://www.w3.org/ns/activitystreams') : object['@context'] !== 'https://www.w3.org/ns/activitystreams' ) { - throw new UnrecoverableError(`invalid AP object ${value}: does not have ActivityStreams context`); + throw new IdentifiableError('72180409-793c-4973-868e-5a118eb5519b', `invalid AP object ${value}: does not have ActivityStreams context`); } - // Since redirects are allowed, we cannot safely validate an anonymous object. - // Reject any responses without an ID, as all other checks depend on that value. - if (object.id == null) { - throw new UnrecoverableError(`invalid AP object ${value}: missing id`); - } + // The object ID is already validated to match the final URL's authority by signedGet / getActivityJson. + // We only need to validate that it also matches the original URL's authority, in case of redirects. + const objectId = getApId(object); // We allow some limited cross-domain redirects, which means the host may have changed during fetch. // Additional checks are needed to validate the scope of cross-domain redirects. - const finalHost = this.utilityService.extractDbHost(object.id); + const finalHost = this.utilityService.extractDbHost(objectId); if (finalHost !== host) { // Make sure the redirect stayed within the same authority. - if (this.utilityService.punyHostPSLDomain(object.id) !== this.utilityService.punyHostPSLDomain(value)) { - throw new UnrecoverableError(`invalid AP object ${value}: id ${object.id} has different host`); - } + this.apUtilityService.assertIdMatchesUrlAuthority(object, value); // Check if the redirect bounce from [allowed domain] to [blocked domain]. if (!this.utilityService.isFederationAllowedHost(finalHost)) { - throw new UnrecoverableError(`cannot fetch AP object ${value}: redirected to blocked instance ${finalHost}`); + throw new IdentifiableError('0a72bf24-2d9b-4f1d-886b-15aaa31adeda', `cannot fetch AP object ${value}: redirected to blocked instance ${finalHost}`); } } @@ -150,17 +201,18 @@ export class Resolver { @bindThis private resolveLocal(url: string): Promise<IObject> { const parsed = this.apDbResolverService.parseUri(url); - if (!parsed.local) throw new UnrecoverableError(`resolveLocal - not a local URL: ${url}`); + if (!parsed.local) throw new IdentifiableError('02b40cd0-fa92-4b0c-acc9-fb2ada952ab8', `resolveLocal - not a local URL: ${url}`); switch (parsed.type) { case 'notes': return this.notesRepository.findOneByOrFail({ id: parsed.id }) .then(async note => { + const author = await this.usersRepository.findOneByOrFail({ id: note.userId }); if (parsed.rest === 'activity') { // this refers to the create activity and not the note itself - return this.apRendererService.addContext(this.apRendererService.renderCreate(await this.apRendererService.renderNote(note), note)); + return this.apRendererService.addContext(this.apRendererService.renderCreate(await this.apRendererService.renderNote(note, author), note)); } else { - return this.apRendererService.renderNote(note); + return this.apRendererService.renderNote(note, author); } }); case 'users': @@ -179,7 +231,7 @@ export class Resolver { case 'follows': return this.followRequestsRepository.findOneBy({ id: parsed.id }) .then(async followRequest => { - if (followRequest == null) throw new UnrecoverableError(`resolveLocal - invalid follow request ID ${parsed.id}: ${url}`); + if (followRequest == null) throw new IdentifiableError('a9d946e5-d276-47f8-95fb-f04230289bb0', `resolveLocal - invalid follow request ID ${parsed.id}: ${url}`); const [follower, followee] = await Promise.all([ this.usersRepository.findOneBy({ id: followRequest.followerId, @@ -191,12 +243,12 @@ export class Resolver { }), ]); if (follower == null || followee == null) { - throw new Error(`resolveLocal - follower or followee does not exist: ${url}`); + throw new IdentifiableError('06ae3170-1796-4d93-a697-2611ea6d83b6', `resolveLocal - follower or followee does not exist: ${url}`); } return this.apRendererService.addContext(this.apRendererService.renderFollow(follower as MiLocalUser | MiRemoteUser, followee as MiLocalUser | MiRemoteUser, url)); }); default: - throw new UnrecoverableError(`resolveLocal: type ${parsed.type} unhandled: ${url}`); + throw new IdentifiableError('7a5d2fc0-94bc-4db6-b8b8-1bf24a2e23d0', `resolveLocal: type ${parsed.type} unhandled: ${url}`); } } } @@ -232,6 +284,8 @@ export class ApResolverService { private apRendererService: ApRendererService, private apDbResolverService: ApDbResolverService, private loggerService: LoggerService, + private readonly apLogService: ApLogService, + private readonly apUtilityService: ApUtilityService, ) { } @@ -252,6 +306,8 @@ export class ApResolverService { this.apRendererService, this.apDbResolverService, this.loggerService, + this.apLogService, + this.apUtilityService, ); } } diff --git a/packages/backend/src/core/activitypub/ApUtilityService.ts b/packages/backend/src/core/activitypub/ApUtilityService.ts new file mode 100644 index 0000000000..ae6e4997e4 --- /dev/null +++ b/packages/backend/src/core/activitypub/ApUtilityService.ts @@ -0,0 +1,108 @@ +/* + * SPDX-FileCopyrightText: hazelnoot and other Sharkey contributors + * SPDX-License-Identifier: AGPL-3.0-only + */ + +import { Injectable } from '@nestjs/common'; +import { UtilityService } from '@/core/UtilityService.js'; +import { IdentifiableError } from '@/misc/identifiable-error.js'; +import { toArray } from '@/misc/prelude/array.js'; +import { EnvService } from '@/core/EnvService.js'; +import { getApId, getOneApHrefNullable, IObject } from './type.js'; + +@Injectable() +export class ApUtilityService { + constructor( + private readonly utilityService: UtilityService, + private readonly envService: EnvService, + ) {} + + /** + * Verifies that the object's ID has the same authority as the provided URL. + * Returns on success, throws on any validation error. + */ + public assertIdMatchesUrlAuthority(object: IObject, url: string): void { + // This throws if the ID is missing or invalid, but that's ok. + // Anonymous objects are impossible to verify, so we don't allow fetching them. + const id = getApId(object); + + // Make sure the object ID matches the final URL (which is where it actually exists). + // The caller (ApResolverService) will verify the ID against the original / entry URL, which ensures that all three match. + if (!this.haveSameAuthority(url, id)) { + throw new IdentifiableError('fd93c2fa-69a8-440f-880b-bf178e0ec877', `invalid AP object ${url}: id ${id} has different host authority`); + } + } + + /** + * Checks if two URLs have the same host authority + */ + public haveSameAuthority(url1: string, url2: string): boolean { + if (url1 === url2) return true; + + const authority1 = this.utilityService.punyHostPSLDomain(url1); + const authority2 = this.utilityService.punyHostPSLDomain(url2); + return authority1 === authority2; + } + + /** + * Finds the "best" URL for a given AP object. + * The list of URLs is first filtered via findSameAuthorityUrl, then further filtered based on mediaType, and finally sorted to select the best one. + * @throws {IdentifiableError} if object does not have an ID + * @returns the best URL, or null if none were found + */ + public findBestObjectUrl(object: IObject): string | null { + const targetUrl = getApId(object); + const targetAuthority = this.utilityService.punyHostPSLDomain(targetUrl); + + const rawUrls = toArray(object.url); + const acceptableUrls = rawUrls + .map(raw => ({ + url: getOneApHrefNullable(raw), + type: typeof(raw) === 'object' + ? raw.mediaType?.toLowerCase() + : undefined, + })) + .filter(({ url, type }) => { + if (!url) return false; + if (!this.checkHttps(url)) return false; + if (!isAcceptableUrlType(type)) return false; + + const urlAuthority = this.utilityService.punyHostPSLDomain(url); + return urlAuthority === targetAuthority; + }) + .sort((a, b) => { + return rankUrlType(a.type) - rankUrlType(b.type); + }); + + return acceptableUrls[0]?.url ?? null; + } + + /** + * Checks if the URL contains HTTPS. + * Additionally, allows HTTP in non-production environments. + * Based on check-https.ts. + */ + private checkHttps(url: string): boolean { + const isNonProd = this.envService.env.NODE_ENV !== 'production'; + + // noinspection HttpUrlsUsage + return url.startsWith('https://') || (url.startsWith('http://') && isNonProd); + } +} + +function isAcceptableUrlType(type: string | undefined): boolean { + if (!type) return true; + if (type.startsWith('text/')) return true; + if (type.startsWith('application/ld+json')) return true; + if (type.startsWith('application/activity+json')) return true; + return false; +} + +function rankUrlType(type: string | undefined): number { + if (!type) return 2; + if (type === 'text/html') return 0; + if (type.startsWith('text/')) return 1; + if (type.startsWith('application/ld+json')) return 3; + if (type.startsWith('application/activity+json')) return 4; + return 5; +} diff --git a/packages/backend/src/core/activitypub/misc/check-against-url.ts b/packages/backend/src/core/activitypub/misc/check-against-url.ts deleted file mode 100644 index edfab5a216..0000000000 --- a/packages/backend/src/core/activitypub/misc/check-against-url.ts +++ /dev/null @@ -1,31 +0,0 @@ -/* - * SPDX-FileCopyrightText: dakkar and sharkey-project - * SPDX-License-Identifier: AGPL-3.0-only - */ - -import { UnrecoverableError } from 'bullmq'; -import type { IObject } from '../type.js'; - -function getHrefsFrom(one: IObject | string | undefined | (IObject | string | undefined)[]): (string | undefined)[] { - if (Array.isArray(one)) { - return one.flatMap(h => getHrefsFrom(h)); - } - return [ - typeof(one) === 'object' ? one.href : one, - ]; -} - -export function assertActivityMatchesUrls(activity: IObject, urls: string[]) { - const expectedUrls = new Set(urls - .filter(u => URL.canParse(u)) - .map(u => new URL(u).href), - ); - - const actualUrls = [activity.id, ...getHrefsFrom(activity.url)] - .filter(u => u && URL.canParse(u)) - .map(u => new URL(u as string).href); - - if (!actualUrls.some(u => expectedUrls.has(u))) { - throw new UnrecoverableError(`bad Activity: neither id(${activity.id}) nor url(${JSON.stringify(activity.url)}) match location(${urls})`); - } -} diff --git a/packages/backend/src/core/activitypub/misc/contexts.ts b/packages/backend/src/core/activitypub/misc/contexts.ts index d7b6fc6589..5c0b8ffcbb 100644 --- a/packages/backend/src/core/activitypub/misc/contexts.ts +++ b/packages/backend/src/core/activitypub/misc/contexts.ts @@ -561,6 +561,11 @@ const extension_context_definition = { '_misskey_requireSigninToViewContents': 'misskey:_misskey_requireSigninToViewContents', '_misskey_makeNotesFollowersOnlyBefore': 'misskey:_misskey_makeNotesFollowersOnlyBefore', '_misskey_makeNotesHiddenBefore': 'misskey:_misskey_makeNotesHiddenBefore', + '_misskey_license': 'misskey:_misskey_license', + 'freeText': { + '@id': 'misskey:freeText', + '@type': 'schema:text', + }, 'isCat': 'misskey:isCat', // Firefish firefish: 'https://joinfirefish.org/ns#', diff --git a/packages/backend/src/core/activitypub/models/ApNoteService.ts b/packages/backend/src/core/activitypub/models/ApNoteService.ts index e4c4fe54b5..63f9887a8d 100644 --- a/packages/backend/src/core/activitypub/models/ApNoteService.ts +++ b/packages/backend/src/core/activitypub/models/ApNoteService.ts @@ -25,12 +25,14 @@ import { UtilityService } from '@/core/UtilityService.js'; import { bindThis } from '@/decorators.js'; import { checkHttps } from '@/misc/check-https.js'; import { IdentifiableError } from '@/misc/identifiable-error.js'; -import { getOneApId, getApId, getOneApHrefNullable, validPost, isEmoji, getApType } from '../type.js'; +import { isRetryableError } from '@/misc/is-retryable-error.js'; +import { getOneApId, getApId, validPost, isEmoji, getApType, isApObject, isDocument, IApDocument } from '../type.js'; import { ApLoggerService } from '../ApLoggerService.js'; import { ApMfmService } from '../ApMfmService.js'; import { ApDbResolverService } from '../ApDbResolverService.js'; import { ApResolverService } from '../ApResolverService.js'; import { ApAudienceService } from '../ApAudienceService.js'; +import { ApUtilityService } from '../ApUtilityService.js'; import { ApPersonService } from './ApPersonService.js'; import { extractApHashtags } from './tag.js'; import { ApMentionService } from './ApMentionService.js'; @@ -81,6 +83,7 @@ export class ApNoteService { private noteEditService: NoteEditService, private apDbResolverService: ApDbResolverService, private apLoggerService: ApLoggerService, + private readonly apUtilityService: ApUtilityService, ) { this.logger = this.apLoggerService.logger; } @@ -91,7 +94,6 @@ export class ApNoteService { uri: string, actor?: MiRemoteUser, user?: MiRemoteUser, - note?: MiNote, ): Error | null { const expectHost = this.utilityService.extractDbHost(uri); const apType = getApType(object); @@ -123,13 +125,6 @@ export class ApNoteService { } } - if (note) { - const url = (object.url) ? getOneApId(object.url) : note.url; - if (url && url !== note.url) { - return new IdentifiableError('d450b8a9-48e4-4dab-ae36-f4db763fda7c', `invalid Note: updated url does not match original url. updated url: ${url}, original url: ${note.url}`); - } - } - return null; } @@ -185,17 +180,7 @@ export class ApNoteService { throw new UnrecoverableError(`unexpected schema of note.id ${note.id} in ${entryUri}`); } - const url = getOneApHrefNullable(note.url); - - if (url != null) { - if (!checkHttps(url)) { - throw new UnrecoverableError(`unexpected schema of note.url ${url} in ${entryUri}`); - } - - if (this.utilityService.punyHostPSLDomain(url) !== this.utilityService.punyHostPSLDomain(note.id)) { - throw new UnrecoverableError(`note url <> uri host mismatch: ${url} <> ${note.id} in ${entryUri}`); - } - } + const url = this.apUtilityService.findBestObjectUrl(note); this.logger.info(`Creating the Note: ${note.id}`); @@ -270,6 +255,14 @@ export class ApNoteService { if (file) files.push(file); } + // Some software (Peertube) attaches a thumbnail under "icon" instead of "attachment" + const icon = getBestIcon(note); + if (icon) { + icon.sensitive ??= note.sensitive; + const file = await this.apImageService.resolveImage(actor, icon); + if (file) files.push(file); + } + // リプライ const reply: MiNote | null = note.inReplyTo ? await this.resolveNote(note.inReplyTo, { resolver }) @@ -288,44 +281,8 @@ export class ApNoteService { : null; // 引用 - let quote: MiNote | undefined | null = null; - - if (note._misskey_quote ?? note.quoteUrl ?? note.quoteUri) { - const tryResolveNote = async (uri: unknown): Promise< - | { status: 'ok'; res: MiNote } - | { status: 'permerror' | 'temperror' } - > => { - if (typeof uri !== 'string' || !/^https?:/.test(uri)) { - this.logger.warn(`Failed to resolve quote ${uri} for note ${entryUri}: URI is invalid`); - return { status: 'permerror' }; - } - try { - const res = await this.resolveNote(uri, { resolver }); - if (res == null) { - this.logger.warn(`Failed to resolve quote ${uri} for note ${entryUri}: resolution error`); - return { status: 'permerror' }; - } - return { status: 'ok', res }; - } catch (e) { - const error = e instanceof Error ? `${e.name}: ${e.message}` : String(e); - this.logger.warn(`Failed to resolve quote ${uri} for note ${entryUri}: ${error}`); - - return { - status: (e instanceof StatusError && !e.isRetryable) ? 'permerror' : 'temperror', - }; - } - }; - - const uris = unique([note._misskey_quote, note.quoteUrl, note.quoteUri].filter(x => x != null)); - const results = await Promise.all(uris.map(tryResolveNote)); - - quote = results.filter((x): x is { status: 'ok', res: MiNote } => x.status === 'ok').map(x => x.res).at(0); - if (!quote) { - if (results.some(x => x.status === 'temperror')) { - throw new Error(`temporary error resolving quote for ${entryUri}`); - } - } - } + const quote = await this.getQuote(note, entryUri, resolver); + const processErrors = quote === null ? ['quoteUnavailable'] : null; // vote if (reply && reply.hasPoll) { @@ -361,7 +318,8 @@ export class ApNoteService { createdAt: note.published ? new Date(note.published) : null, files, reply, - renote: quote, + renote: quote ?? null, + processErrors, name: note.name, cw, text, @@ -411,7 +369,7 @@ export class ApNoteService { const object = await resolver.resolve(value); const entryUri = getApId(value); - const err = this.validateNote(object, entryUri, actor, user, updatedNote); + const err = this.validateNote(object, entryUri, actor, user); if (err) { this.logger.error(err.message, { resolver: { history: resolver.getHistory() }, @@ -437,17 +395,7 @@ export class ApNoteService { throw new UnrecoverableError(`unexpected schema of note.id ${note.id} in ${noteUri}`); } - const url = getOneApHrefNullable(note.url); - - if (url != null) { - if (!checkHttps(url)) { - throw new UnrecoverableError(`unexpected schema of note.url ${url} in ${noteUri}`); - } - - if (this.utilityService.punyHostPSLDomain(url) !== this.utilityService.punyHostPSLDomain(note.id)) { - throw new UnrecoverableError(`note url <> id host mismatch: ${url} <> ${note.id} in ${noteUri}`); - } - } + const url = this.apUtilityService.findBestObjectUrl(note); this.logger.info(`Creating the Note: ${note.id}`); @@ -504,6 +452,14 @@ export class ApNoteService { if (file) files.push(file); } + // Some software (Peertube) attaches a thumbnail under "icon" instead of "attachment" + const icon = getBestIcon(note); + if (icon) { + icon.sensitive ??= note.sensitive; + const file = await this.apImageService.resolveImage(actor, icon); + if (file) files.push(file); + } + // リプライ const reply: MiNote | null = note.inReplyTo ? await this.resolveNote(note.inReplyTo, { resolver }) @@ -522,44 +478,8 @@ export class ApNoteService { : null; // 引用 - let quote: MiNote | undefined | null = null; - - if (note._misskey_quote ?? note.quoteUrl ?? note.quoteUri) { - const tryResolveNote = async (uri: unknown): Promise< - | { status: 'ok'; res: MiNote } - | { status: 'permerror' | 'temperror' } - > => { - if (typeof uri !== 'string' || !/^https?:/.test(uri)) { - this.logger.warn(`Failed to resolve quote ${uri} for note ${entryUri}: URI is invalid`); - return { status: 'permerror' }; - } - try { - const res = await this.resolveNote(uri, { resolver }); - if (res == null) { - this.logger.warn(`Failed to resolve quote ${uri} for note ${entryUri}: resolution error`); - return { status: 'permerror' }; - } - return { status: 'ok', res }; - } catch (e) { - const error = e instanceof Error ? `${e.name}: ${e.message}` : String(e); - this.logger.warn(`Failed to resolve quote ${uri} for note ${entryUri}: ${error}`); - - return { - status: (e instanceof StatusError && !e.isRetryable) ? 'permerror' : 'temperror', - }; - } - }; - - const uris = unique([note._misskey_quote, note.quoteUrl, note.quoteUri].filter(x => x != null)); - const results = await Promise.all(uris.map(tryResolveNote)); - - quote = results.filter((x): x is { status: 'ok', res: MiNote } => x.status === 'ok').map(x => x.res).at(0); - if (!quote) { - if (results.some(x => x.status === 'temperror')) { - throw new Error(`temporary error resolving quote for ${entryUri}`); - } - } - } + const quote = await this.getQuote(note, entryUri, resolver); + const processErrors = quote === null ? ['quoteUnavailable'] : null; // vote if (reply && reply.hasPoll) { @@ -595,7 +515,8 @@ export class ApNoteService { createdAt: note.published ? new Date(note.published) : null, files, reply, - renote: quote, + renote: quote ?? null, + processErrors, name: note.name, cw, text, @@ -690,6 +611,8 @@ export class ApNoteService { originalUrl: tag.icon.url, publicUrl: tag.icon.url, updatedAt: new Date(), + // _misskey_license が存在しなければ `null` + license: (tag._misskey_license?.freeText ?? null), }); const emoji = await this.emojisRepository.findOneBy({ host, name }); @@ -711,7 +634,87 @@ export class ApNoteService { publicUrl: tag.icon.url, updatedAt: new Date(), aliases: [], + // _misskey_license が存在しなければ `null` + license: (tag._misskey_license?.freeText ?? null) }); })); } + + /** + * Fetches the note's quoted post. + * On success - returns the note. + * On skip (no quote) - returns undefined. + * On permanent error - returns null. + * On temporary error - throws an exception. + */ + private async getQuote(note: IPost, entryUri: string, resolver: Resolver): Promise<MiNote | null | undefined> { + const quoteUris = new Set<string>(); + if (note._misskey_quote) quoteUris.add(note._misskey_quote); + if (note.quoteUrl) quoteUris.add(note.quoteUrl); + if (note.quoteUri) quoteUris.add(note.quoteUri); + + // No quote, return undefined + if (quoteUris.size < 1) return undefined; + + /** + * Attempts to resolve a quote by URI. + * Returns the note if successful, true if there's a retryable error, and false if there's a permanent error. + */ + const resolveQuote = async (uri: unknown): Promise<MiNote | boolean> => { + if (typeof(uri) !== 'string' || !/^https?:/.test(uri)) { + this.logger.warn(`Failed to resolve quote "${uri}" for note "${entryUri}": URI is invalid`); + return false; + } + + try { + const quote = await this.resolveNote(uri, { resolver }); + + if (quote == null) { + this.logger.warn(`Failed to resolve quote "${uri}" for note "${entryUri}": request error`); + return false; + } + + return quote; + } catch (e) { + if (e instanceof Error) { + this.logger.warn(`Failed to resolve quote "${uri}" for note "${entryUri}":`, e); + } else { + this.logger.warn(`Failed to resolve quote "${uri}" for note "${entryUri}": ${e}`); + } + + return isRetryableError(e); + } + }; + + const results = await Promise.all(Array.from(quoteUris).map(u => resolveQuote(u))); + + // Success - return the quote + const quote = results.find(r => typeof(r) === 'object'); + if (quote) return quote; + + // Temporary / retryable error - throw error + const tempError = results.find(r => r === true); + if (tempError) throw new Error(`temporary error resolving quote for "${entryUri}"`); + + // Permanent error - return null + return null; + } +} + +function getBestIcon(note: IObject): IObject | null { + const icons: IObject[] = toArray(note.icon); + if (icons.length < 2) { + return icons[0] ?? null; + } + + return icons.reduce((best, i) => { + if (!isApObject(i)) return best; + if (!isDocument(i)) return best; + if (!best) return i; + if (!best.width || !best.height) return i; + if (!i.width || !i.height) return best; + if (i.width > best.width) return i; + if (i.height > best.height) return i; + return best; + }, null as IApDocument | null) ?? null; } diff --git a/packages/backend/src/core/activitypub/models/ApPersonService.ts b/packages/backend/src/core/activitypub/models/ApPersonService.ts index 5c71dbc626..da29a3c527 100644 --- a/packages/backend/src/core/activitypub/models/ApPersonService.ts +++ b/packages/backend/src/core/activitypub/models/ApPersonService.ts @@ -39,8 +39,8 @@ import { bindThis } from '@/decorators.js'; import { RoleService } from '@/core/RoleService.js'; import { DriveFileEntityService } from '@/core/entities/DriveFileEntityService.js'; import type { AccountMoveService } from '@/core/AccountMoveService.js'; -import { checkHttps } from '@/misc/check-https.js'; -import { getApId, getApType, getOneApHrefNullable, isActor, isCollection, isCollectionOrOrderedCollection, isPropertyValue } from '../type.js'; +import { ApUtilityService } from '@/core/activitypub/ApUtilityService.js'; +import { getApId, getApType, isActor, isCollection, isCollectionOrOrderedCollection, isPropertyValue } from '../type.js'; import { extractApHashtags } from './tag.js'; import type { OnModuleInit } from '@nestjs/common'; import type { ApNoteService } from './ApNoteService.js'; @@ -106,6 +106,7 @@ export class ApPersonService implements OnModuleInit { private followingsRepository: FollowingsRepository, private roleService: RoleService, + private readonly apUtilityService: ApUtilityService, ) { } @@ -346,21 +347,11 @@ export class ApPersonService implements OnModuleInit { const bday = person['vcard:bday']?.match(/^\d{4}-\d{2}-\d{2}/); - const url = getOneApHrefNullable(person.url); - if (person.id == null) { throw new UnrecoverableError(`Refusing to create person without id: ${uri}`); } - if (url != null) { - if (!checkHttps(url)) { - throw new UnrecoverableError(`unexpected schema of person url ${url} in ${uri}`); - } - - if (this.utilityService.punyHostPSLDomain(url) !== this.utilityService.punyHostPSLDomain(person.id)) { - throw new UnrecoverableError(`person url <> uri host mismatch: ${url} <> ${person.id} in ${uri}`); - } - } + const url = this.apUtilityService.findBestObjectUrl(person); // Create user let user: MiRemoteUser | null = null; @@ -398,7 +389,7 @@ export class ApPersonService implements OnModuleInit { alsoKnownAs: person.alsoKnownAs, // We use "!== false" to handle incorrect types, missing / null values, and "default to true" logic. hideOnlineStatus: person.hideOnlineStatus !== false, - isExplorable: person.discoverable, + isExplorable: person.discoverable !== false, username: person.preferredUsername, approved: true, usernameLower: person.preferredUsername?.toLowerCase(), @@ -447,7 +438,7 @@ export class ApPersonService implements OnModuleInit { await transactionalEntityManager.save(new MiUserPublickey({ userId: user.id, keyId: person.publicKey.id, - keyPem: person.publicKey.publicKeyPem, + keyPem: person.publicKey.publicKeyPem.trim(), })); } }); @@ -566,21 +557,11 @@ export class ApPersonService implements OnModuleInit { const bday = person['vcard:bday']?.match(/^\d{4}-\d{2}-\d{2}/); - const url = getOneApHrefNullable(person.url); - if (person.id == null) { throw new UnrecoverableError(`Refusing to update person without id: ${uri}`); } - if (url != null) { - if (!checkHttps(url)) { - throw new UnrecoverableError(`unexpected schema of person url ${url} in ${uri}`); - } - - if (this.utilityService.punyHostPSLDomain(url) !== this.utilityService.punyHostPSLDomain(person.id)) { - throw new UnrecoverableError(`person url <> uri host mismatch: ${url} <> ${person.id} in ${uri}`); - } - } + const url = this.apUtilityService.findBestObjectUrl(person); const updates = { lastFetchedAt: new Date(), @@ -602,7 +583,7 @@ export class ApPersonService implements OnModuleInit { alsoKnownAs: person.alsoKnownAs ?? null, // We use "!== false" to handle incorrect types, missing / null values, and "default to true" logic. hideOnlineStatus: person.hideOnlineStatus !== false, - isExplorable: person.discoverable, + isExplorable: person.discoverable !== false, ...(await this.resolveAvatarAndBanner(exist, person.icon, person.image, person.backgroundUrl).catch(() => ({}))), } as Partial<MiRemoteUser> & Pick<MiRemoteUser, 'isBot' | 'isCat' | 'speakAsCat' | 'isLocked' | 'movedToUri' | 'alsoKnownAs' | 'isExplorable'>; diff --git a/packages/backend/src/core/activitypub/type.ts b/packages/backend/src/core/activitypub/type.ts index d67f8cf62e..d8e7b3c9c3 100644 --- a/packages/backend/src/core/activitypub/type.ts +++ b/packages/backend/src/core/activitypub/type.ts @@ -3,7 +3,7 @@ * SPDX-License-Identifier: AGPL-3.0-only */ -import { UnrecoverableError } from 'bullmq'; +import { IdentifiableError } from '@/misc/identifiable-error.js'; import { fromTuple } from '@/misc/from-tuple.js'; export type Obj = { [x: string]: any }; @@ -65,7 +65,7 @@ export function getApId(value: string | IObject | [string | IObject]): string { if (typeof value === 'string') return value; if (typeof value.id === 'string') return value.id; - throw new UnrecoverableError('cannot determine id'); + throw new IdentifiableError('ad2dc287-75c1-44c4-839d-3d2e64576675', `invalid AP object ${value}: missing id`); } /** @@ -202,7 +202,7 @@ export interface IActor extends IObject { manuallyApprovesFollowers?: boolean; movedTo?: string; alsoKnownAs?: string[]; - discoverable?: boolean; + discoverable?: boolean | null; inbox: string; sharedInbox?: string; // 後方互換性のため publicKey?: { @@ -270,6 +270,11 @@ export interface IApEmoji extends IObject { type: 'Emoji'; name: string; updated: string; + // Misskey拡張。後方互換性のためにoptional。 + // 将来の拡張性を考慮してobjectにしている + _misskey_license?: { + freeText: string | null; + }; } export const isEmoji = (object: IObject): object is IApEmoji => @@ -285,6 +290,8 @@ export const validDocumentTypes = ['Audio', 'Document', 'Image', 'Page', 'Video' export interface IApDocument extends IObject { type: 'Audio' | 'Document' | 'Image' | 'Page' | 'Video'; + width?: number; + height?: number; } export const isDocument = (object: IObject): object is IApDocument => { diff --git a/packages/backend/src/core/chart/ChartManagementService.ts b/packages/backend/src/core/chart/ChartManagementService.ts index ef65af2432..81495c8a6c 100644 --- a/packages/backend/src/core/chart/ChartManagementService.ts +++ b/packages/backend/src/core/chart/ChartManagementService.ts @@ -41,7 +41,7 @@ export class ChartManagementService implements OnApplicationShutdown { private perUserFollowingChart: PerUserFollowingChart, private perUserDriveChart: PerUserDriveChart, private apRequestChart: ApRequestChart, - private chartLoggerService: ChartLoggerService, + chartLoggerService: ChartLoggerService, ) { this.charts = [ this.federationChart, diff --git a/packages/backend/src/core/entities/EmojiEntityService.ts b/packages/backend/src/core/entities/EmojiEntityService.ts index 841bd731c0..490d3f2511 100644 --- a/packages/backend/src/core/entities/EmojiEntityService.ts +++ b/packages/backend/src/core/entities/EmojiEntityService.ts @@ -4,10 +4,10 @@ */ import { Inject, Injectable } from '@nestjs/common'; +import { In } from 'typeorm'; import { DI } from '@/di-symbols.js'; -import type { EmojisRepository } from '@/models/_.js'; +import type { EmojisRepository, MiRole, RolesRepository } from '@/models/_.js'; import type { Packed } from '@/misc/json-schema.js'; -import type { } from '@/models/Blocking.js'; import type { MiEmoji } from '@/models/Emoji.js'; import { bindThis } from '@/decorators.js'; @@ -16,6 +16,8 @@ export class EmojiEntityService { constructor( @Inject(DI.emojisRepository) private emojisRepository: EmojisRepository, + @Inject(DI.rolesRepository) + private rolesRepository: RolesRepository, ) { } @@ -68,8 +70,90 @@ export class EmojiEntityService { @bindThis public packDetailedMany( emojis: any[], - ) { + ): Promise<Packed<'EmojiDetailed'>[]> { return Promise.all(emojis.map(x => this.packDetailed(x))); } + + @bindThis + public async packDetailedAdmin( + src: MiEmoji['id'] | MiEmoji, + hint?: { + roles?: Map<MiRole['id'], MiRole> + }, + ): Promise<Packed<'EmojiDetailedAdmin'>> { + const emoji = typeof src === 'object' ? src : await this.emojisRepository.findOneByOrFail({ id: src }); + + const roles = Array.of<MiRole>(); + if (emoji.roleIdsThatCanBeUsedThisEmojiAsReaction.length > 0) { + if (hint?.roles) { + const hintRoles = hint.roles; + roles.push( + ...emoji.roleIdsThatCanBeUsedThisEmojiAsReaction + .filter(x => hintRoles.has(x)) + .map(x => hintRoles.get(x)!), + ); + } else { + roles.push( + ...await this.rolesRepository.findBy({ id: In(emoji.roleIdsThatCanBeUsedThisEmojiAsReaction) }), + ); + } + + roles.sort((a, b) => { + if (a.displayOrder !== b.displayOrder) { + return b.displayOrder - a.displayOrder; + } + + return a.id.localeCompare(b.id); + }); + } + + return { + id: emoji.id, + updatedAt: emoji.updatedAt?.toISOString() ?? null, + name: emoji.name, + host: emoji.host, + uri: emoji.uri, + type: emoji.type, + aliases: emoji.aliases, + category: emoji.category, + publicUrl: emoji.publicUrl, + originalUrl: emoji.originalUrl, + license: emoji.license, + localOnly: emoji.localOnly, + isSensitive: emoji.isSensitive, + roleIdsThatCanBeUsedThisEmojiAsReaction: roles.map(it => ({ id: it.id, name: it.name })), + }; + } + + @bindThis + public async packDetailedAdminMany( + emojis: MiEmoji['id'][] | MiEmoji[], + hint?: { + roles?: Map<MiRole['id'], MiRole> + }, + ): Promise<Packed<'EmojiDetailedAdmin'>[]> { + // IDのみの要素をピックアップし、DBからレコードを取り出して他の値を補完する + const emojiEntities = emojis.filter(x => typeof x === 'object') as MiEmoji[]; + const emojiIdOnlyList = emojis.filter(x => typeof x === 'string') as string[]; + if (emojiIdOnlyList.length > 0) { + emojiEntities.push(...await this.emojisRepository.findBy({ id: In(emojiIdOnlyList) })); + } + + // 特定ロール専用の絵文字である場合、そのロール情報をあらかじめまとめて取得しておく(pack側で都度取得も出来るが負荷が高いので) + let hintRoles: Map<MiRole['id'], MiRole>; + if (hint?.roles) { + hintRoles = hint.roles; + } else { + const roles = Array.of<MiRole>(); + const roleIds = [...new Set(emojiEntities.flatMap(x => x.roleIdsThatCanBeUsedThisEmojiAsReaction))]; + if (roleIds.length > 0) { + roles.push(...await this.rolesRepository.findBy({ id: In(roleIds) })); + } + + hintRoles = new Map(roles.map(x => [x.id, x])); + } + + return Promise.all(emojis.map(x => this.packDetailedAdmin(x, { roles: hintRoles }))); + } } diff --git a/packages/backend/src/core/entities/InstanceEntityService.ts b/packages/backend/src/core/entities/InstanceEntityService.ts index 63e5923255..fcc9bed3bd 100644 --- a/packages/backend/src/core/entities/InstanceEntityService.ts +++ b/packages/backend/src/core/entities/InstanceEntityService.ts @@ -60,6 +60,7 @@ export class InstanceEntityService { latestRequestReceivedAt: instance.latestRequestReceivedAt ? instance.latestRequestReceivedAt.toISOString() : null, isNSFW: instance.isNSFW, rejectReports: instance.rejectReports, + rejectQuotes: instance.rejectQuotes, moderationNote: iAmModerator ? instance.moderationNote : null, }; } diff --git a/packages/backend/src/core/entities/MetaEntityService.ts b/packages/backend/src/core/entities/MetaEntityService.ts index 7d7b4cbd81..84d591ce7a 100644 --- a/packages/backend/src/core/entities/MetaEntityService.ts +++ b/packages/backend/src/core/entities/MetaEntityService.ts @@ -95,6 +95,7 @@ export class MetaEntityService { mcaptchaInstanceUrl: instance.mcaptchaInstanceUrl, enableRecaptcha: instance.enableRecaptcha, enableAchievements: instance.enableAchievements, + robotsTxt: instance.robotsTxt, recaptchaSiteKey: instance.recaptchaSiteKey, enableTurnstile: instance.enableTurnstile, turnstileSiteKey: instance.turnstileSiteKey, @@ -144,6 +145,7 @@ export class MetaEntityService { enableUrlPreview: instance.urlPreviewEnabled, noteSearchableScope: (this.config.meilisearch == null || this.config.meilisearch.scope !== 'local') ? 'global' : 'local', maxFileSize: this.config.maxFileSize, + federation: this.meta.federation, }; return packed; @@ -184,3 +186,4 @@ export class MetaEntityService { return packDetailed; } } + diff --git a/packages/backend/src/core/entities/NoteEntityService.ts b/packages/backend/src/core/entities/NoteEntityService.ts index eb6b353752..537677ed34 100644 --- a/packages/backend/src/core/entities/NoteEntityService.ts +++ b/packages/backend/src/core/entities/NoteEntityService.ts @@ -23,7 +23,6 @@ import type { CustomEmojiService } from '../CustomEmojiService.js'; import type { ReactionService } from '../ReactionService.js'; import type { UserEntityService } from './UserEntityService.js'; import type { DriveFileEntityService } from './DriveFileEntityService.js'; -import type { Config } from '@/config.js'; // is-renote.tsとよしなにリンク function isPureRenote(note: MiNote): note is MiNote & { renoteId: MiNote['id']; renote: MiNote } { @@ -42,8 +41,14 @@ function getAppearNoteIds(notes: MiNote[]): Set<string> { for (const note of notes) { if (isPureRenote(note)) { appearNoteIds.add(note.renoteId); + if (note.renote?.replyId) { + appearNoteIds.add(note.renote.replyId); + } } else { appearNoteIds.add(note.id); + if (note.replyId) { + appearNoteIds.add(note.replyId); + } } } return appearNoteIds; @@ -69,9 +74,6 @@ export class NoteEntityService implements OnModuleInit { @Inject(DI.usersRepository) private usersRepository: UsersRepository, - @Inject(DI.config) - private config: Config, - @Inject(DI.notesRepository) private notesRepository: NotesRepository, @@ -110,8 +112,7 @@ export class NoteEntityService implements OnModuleInit { } @bindThis - private async hideNote(packedNote: Packed<'Note'>, meId: MiUser['id'] | null): Promise<void> { - // FIXME: このvisibility変更処理が当関数にあるのは若干不自然かもしれない(関数名を treatVisibility とかに変える手もある) + private treatVisibility(packedNote: Packed<'Note'>): Packed<'Note'>['visibility'] { if (packedNote.visibility === 'public' || packedNote.visibility === 'home') { const followersOnlyBefore = packedNote.user.makeNotesFollowersOnlyBefore; if ((followersOnlyBefore != null) @@ -123,7 +124,11 @@ export class NoteEntityService implements OnModuleInit { packedNote.visibility = 'followers'; } } + return packedNote.visibility; + } + @bindThis + public async hideNote(packedNote: Packed<'Note'>, meId: MiUser['id'] | null): Promise<void> { if (meId === packedNote.userId) return; // TODO: isVisibleForMe を使うようにしても良さそう(型違うけど) @@ -272,7 +277,9 @@ export class NoteEntityService implements OnModuleInit { const reaction = _hint_.myReactions.get(note.id); if (reaction) { return this.reactionService.convertLegacyReaction(reaction); - } else { + } else if (reaction === null) { + // the hints explicitly say this note has no reactions from + // this user return undefined; } } @@ -483,6 +490,7 @@ export class NoteEntityService implements OnModuleInit { ...(opts.detail ? { clippedCount: note.clippedCount, + processErrors: note.processErrors, reply: note.replyId ? this.pack(note.reply ?? note.replyId, me, { detail: false, @@ -500,6 +508,8 @@ export class NoteEntityService implements OnModuleInit { } : {}), }); + this.treatVisibility(packed); + if (!opts.skipHide) { await this.hideNote(packed, meId); } @@ -525,44 +535,39 @@ export class NoteEntityService implements OnModuleInit { if (meId) { const idsNeedFetchMyReaction = new Set<MiNote['id']>(); - // パフォーマンスのためノートが作成されてから2秒以上経っていない場合はリアクションを取得しない - const oldId = this.idService.gen(Date.now() - 2000); - + const targetNotes: MiNote[] = []; for (const note of notes) { if (isPureRenote(note)) { - const reactionsCount = Object.values(this.reactionsBufferingService.mergeReactions(note.renote.reactions, bufferedReactions?.get(note.renote.id)?.deltas ?? {})).reduce((a, b) => a + b, 0); - if (reactionsCount === 0) { - myReactionsMap.set(note.renote.id, null); - } else if (reactionsCount <= note.renote.reactionAndUserPairCache.length + (bufferedReactions?.get(note.renote.id)?.pairs.length ?? 0)) { - const pairInBuffer = bufferedReactions?.get(note.renote.id)?.pairs.find(p => p[0] === meId); - if (pairInBuffer) { - myReactionsMap.set(note.renote.id, pairInBuffer[1]); - } else { - const pair = note.renote.reactionAndUserPairCache.find(p => p.startsWith(meId)); - myReactionsMap.set(note.renote.id, pair ? pair.split('/')[1] : null); - } - } else { - idsNeedFetchMyReaction.add(note.renote.id); + // we may need to fetch 'my reaction' for renote target. + targetNotes.push(note.renote); + if (note.renote.reply) { + // idem if the renote is also a reply. + targetNotes.push(note.renote.reply); } } else { - if (note.id < oldId) { - const reactionsCount = Object.values(this.reactionsBufferingService.mergeReactions(note.reactions, bufferedReactions?.get(note.id)?.deltas ?? {})).reduce((a, b) => a + b, 0); - if (reactionsCount === 0) { - myReactionsMap.set(note.id, null); - } else if (reactionsCount <= note.reactionAndUserPairCache.length + (bufferedReactions?.get(note.id)?.pairs.length ?? 0)) { - const pairInBuffer = bufferedReactions?.get(note.id)?.pairs.find(p => p[0] === meId); - if (pairInBuffer) { - myReactionsMap.set(note.id, pairInBuffer[1]); - } else { - const pair = note.reactionAndUserPairCache.find(p => p.startsWith(meId)); - myReactionsMap.set(note.id, pair ? pair.split('/')[1] : null); - } - } else { - idsNeedFetchMyReaction.add(note.id); - } + if (note.reply) { + // idem for OP of a regular reply. + targetNotes.push(note.reply); + } + + targetNotes.push(note); + } + } + + for (const note of targetNotes) { + const reactionsCount = Object.values(this.reactionsBufferingService.mergeReactions(note.reactions, bufferedReactions?.get(note.id)?.deltas ?? {})).reduce((a, b) => a + b, 0); + if (reactionsCount === 0) { + myReactionsMap.set(note.id, null); + } else if (reactionsCount <= note.reactionAndUserPairCache.length + (bufferedReactions?.get(note.id)?.pairs.length ?? 0)) { + const pairInBuffer = bufferedReactions?.get(note.id)?.pairs.find(p => p[0] === meId); + if (pairInBuffer) { + myReactionsMap.set(note.id, pairInBuffer[1]); } else { - myReactionsMap.set(note.id, null); + const pair = note.reactionAndUserPairCache.find(p => p.startsWith(meId)); + myReactionsMap.set(note.id, pair ? pair.split('/')[1] : null); } + } else { + idsNeedFetchMyReaction.add(note.id); } } diff --git a/packages/backend/src/core/entities/UserEntityService.ts b/packages/backend/src/core/entities/UserEntityService.ts index 6bfe865038..96fef863a0 100644 --- a/packages/backend/src/core/entities/UserEntityService.ts +++ b/packages/backend/src/core/entities/UserEntityService.ts @@ -49,11 +49,13 @@ import { IdService } from '@/core/IdService.js'; import type { AnnouncementService } from '@/core/AnnouncementService.js'; import type { CustomEmojiService } from '@/core/CustomEmojiService.js'; import { AvatarDecorationService } from '@/core/AvatarDecorationService.js'; +import { isSystemAccount } from '@/misc/is-system-account.js'; import type { OnModuleInit } from '@nestjs/common'; import type { NoteEntityService } from './NoteEntityService.js'; import type { DriveFileEntityService } from './DriveFileEntityService.js'; import type { PageEntityService } from './PageEntityService.js'; -import { isSystemAccount } from '@/misc/is-system-account.js'; + +/* eslint-disable @typescript-eslint/no-non-null-assertion */ const Ajv = _Ajv.default; const ajv = new Ajv(); @@ -81,6 +83,8 @@ export type UserRelation = { isBlocked: boolean isMuted: boolean isRenoteMuted: boolean + isInstanceMuted?: boolean + memo?: string | null } @Injectable() @@ -180,6 +184,9 @@ export class UserEntityService implements OnModuleInit { isBlocked, isMuted, isRenoteMuted, + host, + memo, + mutedInstances, ] = await Promise.all([ this.followingsRepository.findOneBy({ followerId: me, @@ -227,8 +234,25 @@ export class UserEntityService implements OnModuleInit { muteeId: target, }, }), + this.usersRepository.createQueryBuilder('u') + .select('u.host') + .where({ id: target }) + .getRawOne<{ u_host: string }>() + .then(it => it?.u_host ?? null), + this.userMemosRepository.createQueryBuilder('m') + .select('m.memo') + .where({ userId: me, targetUserId: target }) + .getRawOne<{ m_memo: string | null }>() + .then(it => it?.m_memo ?? null), + this.userProfilesRepository.createQueryBuilder('p') + .select('p.mutedInstances') + .where({ userId: me }) + .getRawOne<{ p_mutedInstances: string[] }>() + .then(it => it?.p_mutedInstances ?? []), ]); + const isInstanceMuted = !!host && mutedInstances.includes(host); + return { id: target, following, @@ -240,6 +264,8 @@ export class UserEntityService implements OnModuleInit { isBlocked, isMuted, isRenoteMuted, + isInstanceMuted, + memo, }; } @@ -254,6 +280,9 @@ export class UserEntityService implements OnModuleInit { blockees, muters, renoteMuters, + hosts, + memos, + mutedInstances, ] = await Promise.all([ this.followingsRepository.findBy({ followerId: me }) .then(f => new Map(f.map(it => [it.followeeId, it]))), @@ -292,6 +321,27 @@ export class UserEntityService implements OnModuleInit { .where('m.muterId = :me', { me }) .getRawMany<{ m_muteeId: string }>() .then(it => it.map(it => it.m_muteeId)), + this.usersRepository.createQueryBuilder('u') + .select(['u.id', 'u.host']) + .where({ id: In(targets) } ) + .getRawMany<{ m_id: string, m_host: string }>() + .then(it => it.reduce((map, it) => { + map[it.m_id] = it.m_host; + return map; + }, {} as Record<string, string>)), + this.userMemosRepository.createQueryBuilder('m') + .select(['m.targetUserId', 'm.memo']) + .where({ userId: me, targetUserId: In(targets) }) + .getRawMany<{ m_targetUserId: string, m_memo: string | null }>() + .then(it => it.reduce((map, it) => { + map[it.m_targetUserId] = it.m_memo; + return map; + }, {} as Record<string, string | null>)), + this.userProfilesRepository.createQueryBuilder('p') + .select('p.mutedInstances') + .where({ userId: me }) + .getRawOne<{ p_mutedInstances: string[] }>() + .then(it => it?.p_mutedInstances ?? []), ]); return new Map( @@ -311,6 +361,8 @@ export class UserEntityService implements OnModuleInit { isBlocked: blockees.includes(target), isMuted: muters.includes(target), isRenoteMuted: renoteMuters.includes(target), + isInstanceMuted: mutedInstances.includes(hosts[target]), + memo: memos[target] ?? null, }, ]; }), @@ -540,6 +592,8 @@ export class UserEntityService implements OnModuleInit { isCat: user.isCat, noindex: user.noindex, enableRss: user.enableRss, + mandatoryCW: user.mandatoryCW, + rejectQuotes: user.rejectQuotes, isSilenced: user.isSilenced || this.roleService.getUserPolicies(user.id).then(r => !r.canPublicNote), speakAsCat: user.speakAsCat ?? false, approved: user.approved, @@ -669,6 +723,8 @@ export class UserEntityService implements OnModuleInit { achievements: profile!.achievements, loggedInDays: profile!.loggedInDates.length, policies: this.roleService.getUserPolicies(user.id), + defaultCW: profile!.defaultCW, + defaultCWPriority: profile!.defaultCWPriority, } : {}), ...(opts.includeSecrets ? { |