diff options
Diffstat (limited to 'packages/backend/src/core')
29 files changed, 987 insertions, 364 deletions
diff --git a/packages/backend/src/core/AbuseReportNotificationService.ts b/packages/backend/src/core/AbuseReportNotificationService.ts index 742e2621fd..9bca795479 100644 --- a/packages/backend/src/core/AbuseReportNotificationService.ts +++ b/packages/backend/src/core/AbuseReportNotificationService.ts @@ -160,22 +160,22 @@ export class AbuseReportNotificationService implements OnApplicationShutdown { }; }); - const recipientWebhookIds = await this.fetchWebhookRecipients() - .then(it => it - .filter(it => it.isActive && it.systemWebhookId && it.method === 'webhook') - .map(it => it.systemWebhookId) - .filter(x => x != null)); - for (const webhookId of recipientWebhookIds) { - await Promise.all( - convertedReports.map(it => { - return this.systemWebhookService.enqueueSystemWebhook( - webhookId, - type, - it, - ); - }), - ); - } + const inactiveRecipients = await this.fetchWebhookRecipients() + .then(it => it.filter(it => !it.isActive)); + const withoutWebhookIds = inactiveRecipients + .map(it => it.systemWebhookId) + .filter(x => x != null); + return Promise.all( + convertedReports.map(it => { + return this.systemWebhookService.enqueueSystemWebhook( + type, + it, + { + excludes: withoutWebhookIds, + }, + ); + }), + ); } /** diff --git a/packages/backend/src/core/CaptchaService.ts b/packages/backend/src/core/CaptchaService.ts index 5b1ab00cfe..d17101ac97 100644 --- a/packages/backend/src/core/CaptchaService.ts +++ b/packages/backend/src/core/CaptchaService.ts @@ -6,6 +6,69 @@ import { Injectable } from '@nestjs/common'; import { HttpRequestService } from '@/core/HttpRequestService.js'; import { bindThis } from '@/decorators.js'; +import { MetaService } from '@/core/MetaService.js'; +import { MiMeta } from '@/models/Meta.js'; +import Logger from '@/logger.js'; +import { LoggerService } from './LoggerService.js'; + +export const supportedCaptchaProviders = ['none', 'hcaptcha', 'mcaptcha', 'recaptcha', 'turnstile', 'fc', 'testcaptcha'] as const; +export type CaptchaProvider = typeof supportedCaptchaProviders[number]; + +export const captchaErrorCodes = { + invalidProvider: Symbol('invalidProvider'), + invalidParameters: Symbol('invalidParameters'), + noResponseProvided: Symbol('noResponseProvided'), + requestFailed: Symbol('requestFailed'), + verificationFailed: Symbol('verificationFailed'), + unknown: Symbol('unknown'), +} as const; +export type CaptchaErrorCode = typeof captchaErrorCodes[keyof typeof captchaErrorCodes]; + +export type CaptchaSetting = { + provider: CaptchaProvider; + hcaptcha: { + siteKey: string | null; + secretKey: string | null; + } + mcaptcha: { + siteKey: string | null; + secretKey: string | null; + instanceUrl: string | null; + } + recaptcha: { + siteKey: string | null; + secretKey: string | null; + } + turnstile: { + siteKey: string | null; + secretKey: string | null; + } + fc: { + siteKey: string | null; + secretKey: string | null; + } +} + +export class CaptchaError extends Error { + public readonly code: CaptchaErrorCode; + public readonly cause?: unknown; + + constructor(code: CaptchaErrorCode, message: string, cause?: unknown) { + super(message); + this.code = code; + this.cause = cause; + this.name = 'CaptchaError'; + } +} + +export type CaptchaSaveSuccess = { + success: true; +} +export type CaptchaSaveFailure = { + success: false; + error: CaptchaError; +} +export type CaptchaSaveResult = CaptchaSaveSuccess | CaptchaSaveFailure; type CaptchaResponse = { success: boolean; @@ -15,9 +78,14 @@ type CaptchaResponse = { @Injectable() export class CaptchaService { + private readonly logger: Logger; + constructor( private httpRequestService: HttpRequestService, + private metaService: MetaService, + loggerService: LoggerService, ) { + this.logger = loggerService.getLogger('captcha'); } @bindThis @@ -45,39 +113,39 @@ export class CaptchaService { @bindThis public async verifyRecaptcha(secret: string, response: string | null | undefined): Promise<void> { if (response == null) { - throw new Error('recaptcha-failed: no response provided'); + throw new CaptchaError(captchaErrorCodes.noResponseProvided, 'recaptcha-failed: no response provided'); } const result = await this.getCaptchaResponse('https://www.recaptcha.net/recaptcha/api/siteverify', secret, response).catch(err => { - throw new Error(`recaptcha-request-failed: ${err}`); + throw new CaptchaError(captchaErrorCodes.requestFailed, `recaptcha-request-failed: ${err}`); }); if (result.success !== true) { const errorCodes = result['error-codes'] ? result['error-codes'].join(', ') : ''; - throw new Error(`recaptcha-failed: ${errorCodes}`); + throw new CaptchaError(captchaErrorCodes.verificationFailed, `recaptcha-failed: ${errorCodes}`); } } @bindThis public async verifyHcaptcha(secret: string, response: string | null | undefined): Promise<void> { if (response == null) { - throw new Error('hcaptcha-failed: no response provided'); + throw new CaptchaError(captchaErrorCodes.noResponseProvided, 'hcaptcha-failed: no response provided'); } const result = await this.getCaptchaResponse('https://hcaptcha.com/siteverify', secret, response).catch(err => { - throw new Error(`hcaptcha-request-failed: ${err}`); + throw new CaptchaError(captchaErrorCodes.requestFailed, `hcaptcha-request-failed: ${err}`); }); if (result.success !== true) { const errorCodes = result['error-codes'] ? result['error-codes'].join(', ') : ''; - throw new Error(`hcaptcha-failed: ${errorCodes}`); + throw new CaptchaError(captchaErrorCodes.verificationFailed, `hcaptcha-failed: ${errorCodes}`); } } @bindThis public async verifyFriendlyCaptcha(secret: string, response: string | null | undefined): Promise<void> { if (response == null) { - throw new Error('frc-failed: no response provided'); + throw new CaptchaError(captchaErrorCodes.noResponseProvided, 'frc-failed: no response provided'); } const result = await this.httpRequestService.send('https://api.friendlycaptcha.com/api/v1/siteverify', { @@ -89,17 +157,17 @@ export class CaptchaService { headers: { 'Content-Type': 'application/json', }, - }); + }, { throwErrorWhenResponseNotOk: false }); if (result.status !== 200) { - throw new Error('frc-failed: frc didn\'t return 200 OK'); + throw new CaptchaError(captchaErrorCodes.requestFailed, `frc-request-failed: ${result.status}`); } const resp = await result.json() as CaptchaResponse; if (resp.success !== true) { const errorCodes = resp['errors'] ? resp['errors'].join(', ') : ''; - throw new Error(`frc-failed: ${errorCodes}`); + throw new CaptchaError(captchaErrorCodes.verificationFailed, `frc-failed: ${errorCodes}`); } } @@ -107,7 +175,7 @@ export class CaptchaService { @bindThis public async verifyMcaptcha(secret: string, siteKey: string, instanceHost: string, response: string | null | undefined): Promise<void> { if (response == null) { - throw new Error('mcaptcha-failed: no response provided'); + throw new CaptchaError(captchaErrorCodes.noResponseProvided, 'mcaptcha-failed: no response provided'); } const endpointUrl = new URL('/api/v1/pow/siteverify', instanceHost); @@ -121,46 +189,272 @@ export class CaptchaService { headers: { 'Content-Type': 'application/json', }, - }); + }, { throwErrorWhenResponseNotOk: false }); if (result.status !== 200) { - throw new Error('mcaptcha-failed: mcaptcha didn\'t return 200 OK'); + throw new CaptchaError(captchaErrorCodes.requestFailed, 'mcaptcha-failed: mcaptcha didn\'t return 200 OK'); } const resp = (await result.json()) as { valid: boolean }; if (!resp.valid) { - throw new Error('mcaptcha-request-failed'); + throw new CaptchaError(captchaErrorCodes.verificationFailed, 'mcaptcha-request-failed'); } } @bindThis public async verifyTurnstile(secret: string, response: string | null | undefined): Promise<void> { if (response == null) { - throw new Error('turnstile-failed: no response provided'); + throw new CaptchaError(captchaErrorCodes.noResponseProvided, 'turnstile-failed: no response provided'); } const result = await this.getCaptchaResponse('https://challenges.cloudflare.com/turnstile/v0/siteverify', secret, response).catch(err => { - throw new Error(`turnstile-request-failed: ${err}`); + throw new CaptchaError(captchaErrorCodes.requestFailed, `turnstile-request-failed: ${err}`); }); if (result.success !== true) { const errorCodes = result['error-codes'] ? result['error-codes'].join(', ') : ''; - throw new Error(`turnstile-failed: ${errorCodes}`); + throw new CaptchaError(captchaErrorCodes.verificationFailed, `turnstile-failed: ${errorCodes}`); } } @bindThis public async verifyTestcaptcha(response: string | null | undefined): Promise<void> { if (response == null) { - throw new Error('testcaptcha-failed: no response provided'); + throw new CaptchaError(captchaErrorCodes.noResponseProvided, 'testcaptcha-failed: no response provided'); } const success = response === 'testcaptcha-passed'; if (!success) { - throw new Error('testcaptcha-failed'); + throw new CaptchaError(captchaErrorCodes.verificationFailed, 'testcaptcha-failed'); + } + } + + @bindThis + public async get(): Promise<CaptchaSetting> { + const meta = await this.metaService.fetch(true); + + let provider: CaptchaProvider; + switch (true) { + case meta.enableHcaptcha: { + provider = 'hcaptcha'; + break; + } + case meta.enableMcaptcha: { + provider = 'mcaptcha'; + break; + } + case meta.enableRecaptcha: { + provider = 'recaptcha'; + break; + } + case meta.enableTurnstile: { + provider = 'turnstile'; + break; + } + case meta.enableTestcaptcha: { + provider = 'testcaptcha'; + break; + } + case meta.enableFC: { + provider = 'fc'; + break; + } + default: { + provider = 'none'; + break; + } + } + + return { + provider: provider, + hcaptcha: { + siteKey: meta.hcaptchaSiteKey, + secretKey: meta.hcaptchaSecretKey, + }, + mcaptcha: { + siteKey: meta.mcaptchaSitekey, + secretKey: meta.mcaptchaSecretKey, + instanceUrl: meta.mcaptchaInstanceUrl, + }, + recaptcha: { + siteKey: meta.recaptchaSiteKey, + secretKey: meta.recaptchaSecretKey, + }, + turnstile: { + siteKey: meta.turnstileSiteKey, + secretKey: meta.turnstileSecretKey, + }, + fc: { + siteKey: meta.fcSiteKey, + secretKey: meta.fcSecretKey, + }, + }; + } + + /** + * captchaの設定を更新します. その際、フロントエンド側で受け取ったcaptchaからの戻り値を検証し、passした場合のみ設定を更新します. + * 実際の検証処理はサービス内で定義されている各captchaプロバイダの検証関数に委譲します. + * + * @param provider 検証するcaptchaのプロバイダ + * @param params + * @param params.sitekey hcaptcha, recaptcha, turnstile, mcaptchaの場合に指定するsitekey. それ以外のプロバイダでは無視されます + * @param params.secret hcaptcha, recaptcha, turnstile, mcaptchaの場合に指定するsecret. それ以外のプロバイダでは無視されます + * @param params.instanceUrl mcaptchaの場合に指定するインスタンスのURL. それ以外のプロバイダでは無視されます + * @param params.captchaResult フロントエンド側で受け取ったcaptchaプロバイダからの戻り値. この値を使ってサーバサイドでの検証を行います + * @see verifyHcaptcha + * @see verifyMcaptcha + * @see verifyRecaptcha + * @see verifyTurnstile + * @see verifyTestcaptcha + */ + @bindThis + public async save( + provider: CaptchaProvider, + params?: { + sitekey?: string | null; + secret?: string | null; + instanceUrl?: string | null; + captchaResult?: string | null; + }, + ): Promise<CaptchaSaveResult> { + if (!supportedCaptchaProviders.includes(provider)) { + return { + success: false, + error: new CaptchaError(captchaErrorCodes.invalidProvider, `Invalid captcha provider: ${provider}`), + }; } + + const operation = { + none: async () => { + await this.updateMeta(provider, params); + }, + hcaptcha: async () => { + if (!params?.secret || !params.captchaResult) { + throw new CaptchaError(captchaErrorCodes.invalidParameters, 'hcaptcha-failed: secret and captureResult are required'); + } + + await this.verifyHcaptcha(params.secret, params.captchaResult); + await this.updateMeta(provider, params); + }, + mcaptcha: async () => { + if (!params?.secret || !params.sitekey || !params.instanceUrl || !params.captchaResult) { + throw new CaptchaError(captchaErrorCodes.invalidParameters, 'mcaptcha-failed: secret, sitekey, instanceUrl and captureResult are required'); + } + + await this.verifyMcaptcha(params.secret, params.sitekey, params.instanceUrl, params.captchaResult); + await this.updateMeta(provider, params); + }, + recaptcha: async () => { + if (!params?.secret || !params.captchaResult) { + throw new CaptchaError(captchaErrorCodes.invalidParameters, 'recaptcha-failed: secret and captureResult are required'); + } + + await this.verifyRecaptcha(params.secret, params.captchaResult); + await this.updateMeta(provider, params); + }, + turnstile: async () => { + if (!params?.secret || !params.captchaResult) { + throw new CaptchaError(captchaErrorCodes.invalidParameters, 'turnstile-failed: secret and captureResult are required'); + } + + await this.verifyTurnstile(params.secret, params.captchaResult); + await this.updateMeta(provider, params); + }, + testcaptcha: async () => { + if (!params?.captchaResult) { + throw new CaptchaError(captchaErrorCodes.invalidParameters, 'turnstile-failed: captureResult are required'); + } + + await this.verifyTestcaptcha(params.captchaResult); + await this.updateMeta(provider, params); + }, + fc: async () => { + if (!params?.secret || !params.captchaResult) { + throw new CaptchaError(captchaErrorCodes.invalidParameters, 'frc-failed: secret and captureResult are required'); + } + + await this.verifyFriendlyCaptcha(params.captchaResult, params.captchaResult); + await this.updateMeta(provider, params); + }, + }[provider]; + + return operation() + .then(() => ({ success: true }) as CaptchaSaveSuccess) + .catch(err => { + this.logger.info(err); + const error = err instanceof CaptchaError + ? err + : new CaptchaError(captchaErrorCodes.unknown, `unknown error: ${err}`); + return { + success: false, + error, + }; + }); + } + + @bindThis + private async updateMeta( + provider: CaptchaProvider, + params?: { + sitekey?: string | null; + secret?: string | null; + instanceUrl?: string | null; + }, + ) { + const metaPartial: Partial< + Pick< + MiMeta, + ('enableHcaptcha' | 'hcaptchaSiteKey' | 'hcaptchaSecretKey') | + ('enableMcaptcha' | 'mcaptchaSitekey' | 'mcaptchaSecretKey' | 'mcaptchaInstanceUrl') | + ('enableRecaptcha' | 'recaptchaSiteKey' | 'recaptchaSecretKey') | + ('enableTurnstile' | 'turnstileSiteKey' | 'turnstileSecretKey') | + ('enableTestcaptcha' | 'enableFC' | 'fcSiteKey' | 'fcSecretKey') + > + > = { + enableHcaptcha: provider === 'hcaptcha', + enableMcaptcha: provider === 'mcaptcha', + enableRecaptcha: provider === 'recaptcha', + enableTurnstile: provider === 'turnstile', + enableTestcaptcha: provider === 'testcaptcha', + enableFC: provider === 'fc', + }; + + const updateIfNotUndefined = <K extends keyof typeof metaPartial>(key: K, value: typeof metaPartial[K]) => { + if (value !== undefined) { + metaPartial[key] = value; + } + }; + switch (provider) { + case 'hcaptcha': { + updateIfNotUndefined('hcaptchaSiteKey', params?.sitekey); + updateIfNotUndefined('hcaptchaSecretKey', params?.secret); + break; + } + case 'mcaptcha': { + updateIfNotUndefined('mcaptchaSitekey', params?.sitekey); + updateIfNotUndefined('mcaptchaSecretKey', params?.secret); + updateIfNotUndefined('mcaptchaInstanceUrl', params?.instanceUrl); + break; + } + case 'recaptcha': { + updateIfNotUndefined('recaptchaSiteKey', params?.sitekey); + updateIfNotUndefined('recaptchaSecretKey', params?.secret); + break; + } + case 'turnstile': { + updateIfNotUndefined('turnstileSiteKey', params?.sitekey); + updateIfNotUndefined('turnstileSecretKey', params?.secret); + break; + } + case 'fc': { + updateIfNotUndefined('fcSiteKey', params?.sitekey); + updateIfNotUndefined('fcSecretKey', params?.secret); + } + } + + await this.metaService.update(metaPartial); } } diff --git a/packages/backend/src/core/CustomEmojiService.ts b/packages/backend/src/core/CustomEmojiService.ts index cc33fb5c0b..165d58d1b2 100644 --- a/packages/backend/src/core/CustomEmojiService.ts +++ b/packages/backend/src/core/CustomEmojiService.ts @@ -4,19 +4,18 @@ */ import { Inject, Injectable, OnApplicationShutdown } from '@nestjs/common'; -import { In, IsNull } from 'typeorm'; import * as Redis from 'ioredis'; -import { DI } from '@/di-symbols.js'; -import { IdService } from '@/core/IdService.js'; +import { In, IsNull } from 'typeorm'; import { EmojiEntityService } from '@/core/entities/EmojiEntityService.js'; import { GlobalEventService } from '@/core/GlobalEventService.js'; -import type { MiDriveFile } from '@/models/DriveFile.js'; -import type { MiEmoji } from '@/models/Emoji.js'; -import type { DriveFilesRepository, EmojisRepository, MiRole, MiUser } from '@/models/_.js'; +import { IdService } from '@/core/IdService.js'; +import { UtilityService } from '@/core/UtilityService.js'; import { bindThis } from '@/decorators.js'; +import { DI } from '@/di-symbols.js'; import { MemoryKVCache, RedisSingleCache } from '@/misc/cache.js'; -import { UtilityService } from '@/core/UtilityService.js'; -import { query } from '@/misc/prelude/url.js'; +import { sqlLikeEscape } from '@/misc/sql-like-escape.js'; +import type { DriveFilesRepository, EmojisRepository, MiRole, MiUser } from '@/models/_.js'; +import type { MiEmoji } from '@/models/Emoji.js'; import type { Serialized } from '@/types.js'; import { ModerationLogService } from '@/core/ModerationLogService.js'; import type { Config } from '@/config.js'; @@ -24,6 +23,42 @@ import { DriveService } from './DriveService.js'; const parseEmojiStrRegexp = /^([-\w]+)(?:@([\w.-]+))?$/; +export const fetchEmojisHostTypes = [ + 'local', + 'remote', + 'all', +] as const; +export type FetchEmojisHostTypes = typeof fetchEmojisHostTypes[number]; +export const fetchEmojisSortKeys = [ + '+id', + '-id', + '+updatedAt', + '-updatedAt', + '+name', + '-name', + '+host', + '-host', + '+uri', + '-uri', + '+publicUrl', + '-publicUrl', + '+type', + '-type', + '+aliases', + '-aliases', + '+category', + '-category', + '+license', + '-license', + '+isSensitive', + '-isSensitive', + '+localOnly', + '-localOnly', + '+roleIdsThatCanBeUsedThisEmojiAsReaction', + '-roleIdsThatCanBeUsedThisEmojiAsReaction', +] as const; +export type FetchEmojisSortKeys = typeof fetchEmojisSortKeys[number]; + @Injectable() export class CustomEmojiService implements OnApplicationShutdown { private emojisCache: MemoryKVCache<MiEmoji | null>; @@ -32,16 +67,12 @@ export class CustomEmojiService implements OnApplicationShutdown { constructor( @Inject(DI.redis) private redisClient: Redis.Redis, - @Inject(DI.config) private config: Config, - @Inject(DI.emojisRepository) private emojisRepository: EmojisRepository, - @Inject(DI.driveFilesRepository) private driveFilesRepository: DriveFilesRepository, - private utilityService: UtilityService, private idService: IdService, private emojiEntityService: EmojiEntityService, @@ -67,7 +98,9 @@ export class CustomEmojiService implements OnApplicationShutdown { @bindThis public async add(data: { - driveFile: MiDriveFile; + originalUrl: string; + publicUrl: string; + fileType: string; name: string; category: string | null; aliases: string[]; @@ -84,9 +117,9 @@ export class CustomEmojiService implements OnApplicationShutdown { category: data.category, host: data.host, aliases: data.aliases, - originalUrl: data.driveFile.url, - publicUrl: data.driveFile.webpublicUrl ?? data.driveFile.url, - type: data.driveFile.webpublicType ?? data.driveFile.type, + originalUrl: data.originalUrl, + publicUrl: data.publicUrl, + type: data.fileType, license: data.license, isSensitive: data.isSensitive, localOnly: data.localOnly, @@ -115,7 +148,9 @@ export class CustomEmojiService implements OnApplicationShutdown { public async update(data: ( { id: MiEmoji['id'], name?: string; } | { name: string; id?: MiEmoji['id'], } ) & { - driveFile?: MiDriveFile; + originalUrl?: string; + publicUrl?: string; + fileType?: string; category?: string | null; aliases?: string[]; license?: string | null; @@ -148,18 +183,22 @@ export class CustomEmojiService implements OnApplicationShutdown { license: data.license, isSensitive: data.isSensitive, localOnly: data.localOnly, - originalUrl: data.driveFile != null ? data.driveFile.url : undefined, - publicUrl: data.driveFile != null ? (data.driveFile.webpublicUrl ?? data.driveFile.url) : undefined, - type: data.driveFile != null ? (data.driveFile.webpublicType ?? data.driveFile.type) : undefined, + originalUrl: data.originalUrl, + publicUrl: data.publicUrl, + type: data.fileType, roleIdsThatCanBeUsedThisEmojiAsReaction: data.roleIdsThatCanBeUsedThisEmojiAsReaction ?? undefined, }); this.localEmojisCache.refresh(); - if (data.driveFile != null) { - const file = await this.driveFilesRepository.findOneBy({ url: emoji.originalUrl, userHost: emoji.host ? emoji.host : IsNull() }); - if (file && file.id !== data.driveFile.id) { - await this.driveService.deleteFile(file, false, moderator ? moderator : undefined); + // If we're changing the file, then we need to delete the old one + if (data.originalUrl != null && data.originalUrl !== emoji.originalUrl) { + const oldFile = await this.driveFilesRepository.findOneBy({ url: emoji.originalUrl, userHost: emoji.host ? emoji.host : IsNull() }); + const newFile = await this.driveFilesRepository.findOneBy({ url: data.originalUrl, userHost: emoji.host ? emoji.host : IsNull() }); + + // But DON'T delete if this is the same file reference, otherwise we'll break the emoji! + if (oldFile && newFile && oldFile.id !== newFile.id) { + await this.driveService.deleteFile(oldFile, false, moderator ? moderator : undefined); } } @@ -336,7 +375,7 @@ export class CustomEmojiService implements OnApplicationShutdown { @bindThis private normalizeHost(src: string | undefined, noteUserHost: string | null): string | null { - // クエリに使うホスト + // クエリに使うホスト let host = src === '.' ? null // .はローカルホスト (ここがマッチするのはリアクションのみ) : src === undefined ? noteUserHost // ノートなどでホスト省略表記の場合はローカルホスト (ここがリアクションにマッチすることはない) : this.utilityService.isSelfHost(src) ? null // 自ホスト指定 @@ -445,6 +484,151 @@ export class CustomEmojiService implements OnApplicationShutdown { } @bindThis + public async fetchEmojis( + params?: { + query?: { + updatedAtFrom?: string; + updatedAtTo?: string; + name?: string; + host?: string; + uri?: string; + publicUrl?: string; + type?: string; + aliases?: string; + category?: string; + license?: string; + isSensitive?: boolean; + localOnly?: boolean; + hostType?: FetchEmojisHostTypes; + roleIds?: string[]; + }, + sinceId?: string; + untilId?: string; + }, + opts?: { + limit?: number; + page?: number; + sortKeys?: FetchEmojisSortKeys[] + }, + ) { + function multipleWordsToQuery(words: string) { + return words.split(/\s/).filter(x => x.length > 0).map(x => `%${sqlLikeEscape(x)}%`); + } + + const builder = this.emojisRepository.createQueryBuilder('emoji'); + if (params?.query) { + const q = params.query; + if (q.updatedAtFrom) { + // noIndexScan + builder.andWhere('CAST(emoji.updatedAt AS DATE) >= :updateAtFrom', { updateAtFrom: q.updatedAtFrom }); + } + if (q.updatedAtTo) { + // noIndexScan + builder.andWhere('CAST(emoji.updatedAt AS DATE) <= :updateAtTo', { updateAtTo: q.updatedAtTo }); + } + if (q.name) { + builder.andWhere('emoji.name ~~ ANY(ARRAY[:...name])', { name: multipleWordsToQuery(q.name) }); + } + + switch (true) { + case q.hostType === 'local': { + builder.andWhere('emoji.host IS NULL'); + break; + } + case q.hostType === 'remote': { + if (q.host) { + // noIndexScan + builder.andWhere('emoji.host ~~ ANY(ARRAY[:...host])', { host: multipleWordsToQuery(q.host) }); + } else { + builder.andWhere('emoji.host IS NOT NULL'); + } + break; + } + } + + if (q.uri) { + // noIndexScan + builder.andWhere('emoji.uri ~~ ANY(ARRAY[:...uri])', { uri: multipleWordsToQuery(q.uri) }); + } + if (q.publicUrl) { + // noIndexScan + builder.andWhere('emoji.publicUrl ~~ ANY(ARRAY[:...publicUrl])', { publicUrl: multipleWordsToQuery(q.publicUrl) }); + } + if (q.type) { + // noIndexScan + builder.andWhere('emoji.type ~~ ANY(ARRAY[:...type])', { type: multipleWordsToQuery(q.type) }); + } + if (q.aliases) { + // noIndexScan + const subQueryBuilder = builder.subQuery() + .select('COUNT(0)', 'count') + .from( + sq2 => sq2 + .select('unnest(subEmoji.aliases)', 'alias') + .addSelect('subEmoji.id', 'id') + .from('emoji', 'subEmoji'), + 'aliasTable', + ) + .where('"emoji"."id" = "aliasTable"."id"') + .andWhere('"aliasTable"."alias" ~~ ANY(ARRAY[:...aliases])', { aliases: multipleWordsToQuery(q.aliases) }); + + builder.andWhere(`(${subQueryBuilder.getQuery()}) > 0`); + } + if (q.category) { + builder.andWhere('emoji.category ~~ ANY(ARRAY[:...category])', { category: multipleWordsToQuery(q.category) }); + } + if (q.license) { + // noIndexScan + builder.andWhere('emoji.license ~~ ANY(ARRAY[:...license])', { license: multipleWordsToQuery(q.license) }); + } + if (q.isSensitive != null) { + // noIndexScan + builder.andWhere('emoji.isSensitive = :isSensitive', { isSensitive: q.isSensitive }); + } + if (q.localOnly != null) { + // noIndexScan + builder.andWhere('emoji.localOnly = :localOnly', { localOnly: q.localOnly }); + } + if (q.roleIds && q.roleIds.length > 0) { + builder.andWhere('emoji.roleIdsThatCanBeUsedThisEmojiAsReaction && ARRAY[:...roleIds]::VARCHAR[]', { roleIds: q.roleIds }); + } + } + + if (params?.sinceId) { + builder.andWhere('emoji.id > :sinceId', { sinceId: params.sinceId }); + } + if (params?.untilId) { + builder.andWhere('emoji.id < :untilId', { untilId: params.untilId }); + } + + if (opts?.sortKeys && opts.sortKeys.length > 0) { + for (const sortKey of opts.sortKeys) { + const direction = sortKey.startsWith('-') ? 'DESC' : 'ASC'; + const key = sortKey.replace(/^[+-]/, ''); + builder.addOrderBy(`emoji.${key}`, direction); + } + } else { + builder.addOrderBy('emoji.id', 'DESC'); + } + + const limit = opts?.limit ?? 10; + if (opts?.page) { + builder.skip((opts.page - 1) * limit); + } + + builder.take(limit); + + const [emojis, count] = await builder.getManyAndCount(); + + return { + emojis, + count: (count > limit ? emojis.length : count), + allCount: count, + allPages: Math.ceil(count / limit), + }; + } + + @bindThis public dispose(): void { this.emojisCache.dispose(); } diff --git a/packages/backend/src/core/DriveService.ts b/packages/backend/src/core/DriveService.ts index 734ce6b88f..a65059b417 100644 --- a/packages/backend/src/core/DriveService.ts +++ b/packages/backend/src/core/DriveService.ts @@ -520,7 +520,7 @@ export class DriveService { // If usage limit exceeded if (driveCapacity < usage + info.size) { if (isLocalUser) { - throw new IdentifiableError('c6244ed2-a39a-4e1c-bf93-f0fbd7764fa6', 'No free space.'); + throw new IdentifiableError('c6244ed2-a39a-4e1c-bf93-f0fbd7764fa6', 'No free space.', true); } await this.expireOldFile(await this.usersRepository.findOneByOrFail({ id: user.id }) as MiRemoteUser, driveCapacity - info.size); } diff --git a/packages/backend/src/core/FederatedInstanceService.ts b/packages/backend/src/core/FederatedInstanceService.ts index fca3ad847a..3f7ed99348 100644 --- a/packages/backend/src/core/FederatedInstanceService.ts +++ b/packages/backend/src/core/FederatedInstanceService.ts @@ -5,6 +5,7 @@ import { Inject, Injectable, OnApplicationShutdown } from '@nestjs/common'; import * as Redis from 'ioredis'; +import { QueryFailedError } from 'typeorm'; import type { InstancesRepository } from '@/models/_.js'; import type { MiInstance } from '@/models/Instance.js'; import { MemoryKVCache, RedisKVCache } from '@/misc/cache.js'; @@ -12,7 +13,6 @@ import { IdService } from '@/core/IdService.js'; import { DI } from '@/di-symbols.js'; import { UtilityService } from '@/core/UtilityService.js'; import { bindThis } from '@/decorators.js'; -import { QueryFailedError } from 'typeorm'; import { isDuplicateKeyValueError } from '@/misc/is-duplicate-key-value-error.js'; @Injectable() diff --git a/packages/backend/src/core/FetchInstanceMetadataService.ts b/packages/backend/src/core/FetchInstanceMetadataService.ts index 987999bce7..ce3af7c774 100644 --- a/packages/backend/src/core/FetchInstanceMetadataService.ts +++ b/packages/backend/src/core/FetchInstanceMetadataService.ts @@ -181,7 +181,7 @@ export class FetchInstanceMetadataService { } @bindThis - private async fetchDom(instance: MiInstance): Promise<DOMWindow['document']> { + private async fetchDom(instance: MiInstance): Promise<Document> { this.logger.info(`Fetching HTML of ${instance.host} ...`); const url = 'https://' + instance.host; @@ -206,7 +206,7 @@ export class FetchInstanceMetadataService { } @bindThis - private async fetchFaviconUrl(instance: MiInstance, doc: DOMWindow['document'] | null): Promise<string | null> { + private async fetchFaviconUrl(instance: MiInstance, doc: Document | null): Promise<string | null> { const url = 'https://' + instance.host; if (doc) { @@ -232,7 +232,7 @@ export class FetchInstanceMetadataService { } @bindThis - private async fetchIconUrl(instance: MiInstance, doc: DOMWindow['document'] | null, manifest: Record<string, any> | null): Promise<string | null> { + private async fetchIconUrl(instance: MiInstance, doc: Document | null, manifest: Record<string, any> | null): Promise<string | null> { if (manifest && manifest.icons && manifest.icons.length > 0 && manifest.icons[0].src) { const url = 'https://' + instance.host; return (new URL(manifest.icons[0].src, url)).href; @@ -261,7 +261,7 @@ export class FetchInstanceMetadataService { } @bindThis - private async getThemeColor(info: NodeInfo | null, doc: DOMWindow['document'] | null, manifest: Record<string, any> | null): Promise<string | null> { + private async getThemeColor(info: NodeInfo | null, doc: Document | null, manifest: Record<string, any> | null): Promise<string | null> { const themeColor = info?.metadata?.themeColor ?? doc?.querySelector('meta[name="theme-color"]')?.getAttribute('content') ?? manifest?.theme_color; if (themeColor) { @@ -273,7 +273,7 @@ export class FetchInstanceMetadataService { } @bindThis - private async getSiteName(info: NodeInfo | null, doc: DOMWindow['document'] | null, manifest: Record<string, any> | null): Promise<string | null> { + private async getSiteName(info: NodeInfo | null, doc: Document | null, manifest: Record<string, any> | null): Promise<string | null> { if (info && info.metadata) { if (typeof info.metadata.nodeName === 'string') { return info.metadata.nodeName; @@ -298,7 +298,7 @@ export class FetchInstanceMetadataService { } @bindThis - private async getDescription(info: NodeInfo | null, doc: DOMWindow['document'] | null, manifest: Record<string, any> | null): Promise<string | null> { + private async getDescription(info: NodeInfo | null, doc: Document | null, manifest: Record<string, any> | null): Promise<string | null> { if (info && info.metadata) { if (typeof info.metadata.nodeDescription === 'string') { return info.metadata.nodeDescription; diff --git a/packages/backend/src/core/MfmService.ts b/packages/backend/src/core/MfmService.ts index dc47e38562..6c2f673217 100644 --- a/packages/backend/src/core/MfmService.ts +++ b/packages/backend/src/core/MfmService.ts @@ -179,6 +179,40 @@ export class MfmService { break; } + // this is here only to catch upstream changes! + case 'ruby--': { + let ruby: [string, string][] = []; + for (const child of node.childNodes) { + if (child.nodeName === 'rp') { + continue; + } + if (treeAdapter.isTextNode(child) && !/\s|\[|\]/.test(child.value)) { + ruby.push([child.value, '']); + continue; + } + if (child.nodeName === 'rt' && ruby.length > 0) { + const rt = getText(child); + if (/\s|\[|\]/.test(rt)) { + // If any space is included in rt, it is treated as a normal text + ruby = []; + appendChildren(node.childNodes); + break; + } else { + ruby.at(-1)![1] = rt; + continue; + } + } + // If any other element is included in ruby, it is treated as a normal text + ruby = []; + appendChildren(node.childNodes); + break; + } + for (const [base, rt] of ruby) { + text += `$[ruby ${base} ${rt}]`; + } + break; + } + // block code (<pre><code>) case 'pre': { if (node.childNodes.length === 1 && node.childNodes[0].nodeName === 'code') { @@ -277,16 +311,24 @@ export class MfmService { continue; } if (child.nodeName === 'rt') { - text += '$[ruby $[group '; + // the only case in which we don't need a `$[group ]` + // is when both sides of the ruby are simple words + const needsGroup = nonRtNodes.length > 1 || + /\s|\[|\]/.test(getText(nonRtNodes[0])) || + /\s|\[|\]/.test(getText(child)); + text += '$[ruby '; + if (needsGroup) text += '$[group '; appendChildren(nonRtNodes); - text += '] '; + if (needsGroup) text += ']'; + text += ' '; analyze(child); - text += '] '; + text += ']'; nonRtNodes = []; continue; } nonRtNodes.push(child); } + appendChildren(nonRtNodes); } break; } diff --git a/packages/backend/src/core/NoteCreateService.ts b/packages/backend/src/core/NoteCreateService.ts index 96bb30a0d6..3bfced1d80 100644 --- a/packages/backend/src/core/NoteCreateService.ts +++ b/packages/backend/src/core/NoteCreateService.ts @@ -678,14 +678,7 @@ export class NoteCreateService implements OnApplicationShutdown { this.roleService.addNoteToRoleTimeline(noteObj); - this.webhookService.getActiveWebhooks().then(webhooks => { - webhooks = webhooks.filter(x => x.userId === user.id && x.on.includes('note')); - for (const webhook of webhooks) { - this.queueService.userWebhookDeliver(webhook, 'note', { - note: noteObj, - }); - } - }); + this.webhookService.enqueueUserWebhook(user.id, 'note', { note: noteObj }); const nm = new NotificationManager(this.mutingsRepository, this.notificationService, user, note); @@ -717,13 +710,7 @@ export class NoteCreateService implements OnApplicationShutdown { if (!isThreadMuted && !muted) { nm.push(data.reply.userId, 'reply'); this.globalEventService.publishMainStream(data.reply.userId, 'reply', noteObj); - - const webhooks = (await this.webhookService.getActiveWebhooks()).filter(x => x.userId === data.reply!.userId && x.on.includes('reply')); - for (const webhook of webhooks) { - this.queueService.userWebhookDeliver(webhook, 'reply', { - note: noteObj, - }); - } + this.webhookService.enqueueUserWebhook(data.reply.userId, 'reply', { note: noteObj }); } } } @@ -757,20 +744,14 @@ export class NoteCreateService implements OnApplicationShutdown { // Publish event if ((user.id !== data.renote.userId) && data.renote.userHost === null) { this.globalEventService.publishMainStream(data.renote.userId, 'renote', noteObj); - - const webhooks = (await this.webhookService.getActiveWebhooks()).filter(x => x.userId === data.renote!.userId && x.on.includes('renote')); - for (const webhook of webhooks) { - this.queueService.userWebhookDeliver(webhook, 'renote', { - note: noteObj, - }); - } + this.webhookService.enqueueUserWebhook(data.renote.userId, 'renote', { note: noteObj }); } } nm.notify(); //#region AP deliver - if (this.userEntityService.isLocalUser(user)) { + if (!data.localOnly && this.userEntityService.isLocalUser(user)) { (async () => { const noteActivity = await this.renderNoteOrRenoteActivity(data, note); const dm = this.apDeliverManagerService.createDeliverManager(user, noteActivity); @@ -905,13 +886,7 @@ export class NoteCreateService implements OnApplicationShutdown { }); this.globalEventService.publishMainStream(u.id, 'mention', detailPackedNote); - - const webhooks = (await this.webhookService.getActiveWebhooks()).filter(x => x.userId === u.id && x.on.includes('mention')); - for (const webhook of webhooks) { - this.queueService.userWebhookDeliver(webhook, 'mention', { - note: detailPackedNote, - }); - } + this.webhookService.enqueueUserWebhook(u.id, 'mention', { note: detailPackedNote }); // Create notification nm.push(u.id, 'mention'); diff --git a/packages/backend/src/core/NoteEditService.ts b/packages/backend/src/core/NoteEditService.ts index f1c7bcbea5..453ad5d9d0 100644 --- a/packages/backend/src/core/NoteEditService.ts +++ b/packages/backend/src/core/NoteEditService.ts @@ -664,14 +664,7 @@ export class NoteEditService implements OnApplicationShutdown { this.roleService.addNoteToRoleTimeline(noteObj); - this.webhookService.getActiveWebhooks().then(webhooks => { - webhooks = webhooks.filter(x => x.userId === user.id && x.on.includes('note')); - for (const webhook of webhooks) { - this.queueService.userWebhookDeliver(webhook, 'note', { - note: noteObj, - }); - } - }); + this.webhookService.enqueueUserWebhook(user.id, 'note', { note: noteObj }); const nm = new NotificationManager(this.mutingsRepository, this.notificationService, user, note); @@ -700,12 +693,7 @@ export class NoteEditService implements OnApplicationShutdown { nm.push(data.reply.userId, 'edited'); this.globalEventService.publishMainStream(data.reply.userId, 'edited', noteObj); - const webhooks = (await this.webhookService.getActiveWebhooks()).filter(x => x.userId === data.reply!.userId && x.on.includes('edited')); - for (const webhook of webhooks) { - this.queueService.userWebhookDeliver(webhook, 'edited', { - note: noteObj, - }); - } + this.webhookService.enqueueUserWebhook(data.reply.userId, 'reply', { note: noteObj }); } } } @@ -713,7 +701,7 @@ export class NoteEditService implements OnApplicationShutdown { nm.notify(); //#region AP deliver - if (this.userEntityService.isLocalUser(user)) { + if (!data.localOnly && this.userEntityService.isLocalUser(user)) { (async () => { const noteActivity = await this.renderNoteOrRenoteActivity(data, note); const dm = this.apDeliverManagerService.createDeliverManager(user, noteActivity); @@ -810,6 +798,7 @@ export class NoteEditService implements OnApplicationShutdown { (note.files != null && note.files.length > 0); } + // TODO why is this unused? @bindThis private async createMentionedEvents(mentionedUsers: MinimumUser[], note: MiNote, nm: NotificationManager) { for (const u of mentionedUsers.filter(u => this.userEntityService.isLocalUser(u))) { @@ -837,13 +826,7 @@ export class NoteEditService implements OnApplicationShutdown { }); this.globalEventService.publishMainStream(u.id, 'edited', detailPackedNote); - - const webhooks = (await this.webhookService.getActiveWebhooks()).filter(x => x.userId === u.id && x.on.includes('edited')); - for (const webhook of webhooks) { - this.queueService.userWebhookDeliver(webhook, 'edited', { - note: detailPackedNote, - }); - } + this.webhookService.enqueueUserWebhook(u.id, 'edited', { note: detailPackedNote }); // Create notification nm.push(u.id, 'edited'); diff --git a/packages/backend/src/core/S3Service.ts b/packages/backend/src/core/S3Service.ts index bb2a463354..37721d2bf1 100644 --- a/packages/backend/src/core/S3Service.ts +++ b/packages/backend/src/core/S3Service.ts @@ -28,7 +28,7 @@ export class S3Service { ? `${meta.objectStorageUseSSL ? 'https' : 'http'}://${meta.objectStorageEndpoint}` : `${meta.objectStorageUseSSL ? 'https' : 'http'}://example.net`; // dummy url to select http(s) agent - const agent = this.httpRequestService.getAgentByUrl(new URL(u), !meta.objectStorageUseProxy); + const agent = this.httpRequestService.getAgentByUrl(new URL(u), !meta.objectStorageUseProxy, true); const handlerOption: NodeHttpHandlerOptions = {}; if (meta.objectStorageUseSSL) { handlerOption.httpsAgent = agent as https.Agent; diff --git a/packages/backend/src/core/SearchService.ts b/packages/backend/src/core/SearchService.ts index e8a98ca617..6e46fb798c 100644 --- a/packages/backend/src/core/SearchService.ts +++ b/packages/backend/src/core/SearchService.ts @@ -6,16 +6,17 @@ import { Inject, Injectable } from '@nestjs/common'; import { In } from 'typeorm'; import { DI } from '@/di-symbols.js'; -import type { Config } from '@/config.js'; +import { type Config, FulltextSearchProvider } from '@/config.js'; import { bindThis } from '@/decorators.js'; import { MiNote } from '@/models/Note.js'; -import { MiUser } from '@/models/_.js'; import type { NotesRepository } from '@/models/_.js'; +import { MiUser } from '@/models/_.js'; import { sqlLikeEscape } from '@/misc/sql-like-escape.js'; import { isUserRelated } from '@/misc/is-user-related.js'; import { CacheService } from '@/core/CacheService.js'; import { QueryService } from '@/core/QueryService.js'; import { IdService } from '@/core/IdService.js'; +import { LoggerService } from '@/core/LoggerService.js'; import type { Index, MeiliSearch } from 'meilisearch'; type K = string; @@ -27,40 +28,12 @@ type Q = { op: '<', k: K, v: number } | { op: '>=', k: K, v: number } | { op: '<=', k: K, v: number } | - { op: 'is null', k: K} | - { op: 'is not null', k: K} | + { op: 'is null', k: K } | + { op: 'is not null', k: K } | { op: 'and', qs: Q[] } | { op: 'or', qs: Q[] } | { op: 'not', q: Q }; -function compileValue(value: V): string { - if (typeof value === 'string') { - return `'${value}'`; // TODO: escape - } else if (typeof value === 'number') { - return value.toString(); - } else if (typeof value === 'boolean') { - return value.toString(); - } - throw new Error('unrecognized value'); -} - -function compileQuery(q: Q): string { - switch (q.op) { - case '=': return `(${q.k} = ${compileValue(q.v)})`; - case '!=': return `(${q.k} != ${compileValue(q.v)})`; - case '>': return `(${q.k} > ${compileValue(q.v)})`; - case '<': return `(${q.k} < ${compileValue(q.v)})`; - case '>=': return `(${q.k} >= ${compileValue(q.v)})`; - case '<=': return `(${q.k} <= ${compileValue(q.v)})`; - case 'and': return q.qs.length === 0 ? '' : `(${ q.qs.map(_q => compileQuery(_q)).join(' AND ') })`; - case 'or': return q.qs.length === 0 ? '' : `(${ q.qs.map(_q => compileQuery(_q)).join(' OR ') })`; - case 'is null': return `(${q.k} IS NULL)`; - case 'is not null': return `(${q.k} IS NOT NULL)`; - case 'not': return `(NOT ${compileQuery(q.q)})`; - default: throw new Error('unrecognized query operator'); - } -} - const fileTypes = { image: [ 'image/webp', @@ -115,10 +88,54 @@ const fileTypes = { export const fileTypeCategories = ['image', 'video', 'audio', 'module', 'flash', null] as const; export type FileTypeCategory = typeof fileTypeCategories[number]; +export type SearchOpts = { + userId?: MiNote['userId'] | null; + channelId?: MiNote['channelId'] | null; + host?: string | null; + filetype?: FileTypeCategory; + order?: string | null; + disableMeili?: boolean | null; +}; + +export type SearchPagination = { + untilId?: MiNote['id']; + sinceId?: MiNote['id']; + limit: number; +}; + +function compileValue(value: V): string { + if (typeof value === 'string') { + return `'${value}'`; // TODO: escape + } else if (typeof value === 'number') { + return value.toString(); + } else if (typeof value === 'boolean') { + return value.toString(); + } + throw new Error('unrecognized value'); +} + +function compileQuery(q: Q): string { + switch (q.op) { + case '=': return `(${q.k} = ${compileValue(q.v)})`; + case '!=': return `(${q.k} != ${compileValue(q.v)})`; + case '>': return `(${q.k} > ${compileValue(q.v)})`; + case '<': return `(${q.k} < ${compileValue(q.v)})`; + case '>=': return `(${q.k} >= ${compileValue(q.v)})`; + case '<=': return `(${q.k} <= ${compileValue(q.v)})`; + case 'and': return q.qs.length === 0 ? '' : `(${ q.qs.map(_q => compileQuery(_q)).join(' AND ') })`; + case 'or': return q.qs.length === 0 ? '' : `(${ q.qs.map(_q => compileQuery(_q)).join(' OR ') })`; + case 'is null': return `(${q.k} IS NULL)`; + case 'is not null': return `(${q.k} IS NOT NULL)`; + case 'not': return `(NOT ${compileQuery(q.q)})`; + default: throw new Error('unrecognized query operator'); + } +} + @Injectable() export class SearchService { private readonly meilisearchIndexScope: 'local' | 'global' | string[] = 'local'; - private meilisearchNoteIndex: Index | null = null; + private readonly meilisearchNoteIndex: Index | null = null; + private readonly provider: FulltextSearchProvider; constructor( @Inject(DI.config) @@ -133,6 +150,7 @@ export class SearchService { private cacheService: CacheService, private queryService: QueryService, private idService: IdService, + private loggerService: LoggerService, ) { if (meilisearch) { this.meilisearchNoteIndex = meilisearch.index(`${this.config.meilisearch?.index}---notes`); @@ -164,144 +182,195 @@ export class SearchService { if (this.config.meilisearch?.scope) { this.meilisearchIndexScope = this.config.meilisearch.scope; } + + this.provider = config.fulltextSearch?.provider ?? 'sqlLike'; + this.loggerService.getLogger('SearchService').info(`-- Provider: ${this.provider}`); } @bindThis public async indexNote(note: MiNote): Promise<void> { + if (!this.meilisearch) return; if (note.text == null && note.cw == null) return; if (!['home', 'public'].includes(note.visibility)) return; - if (this.meilisearch) { - switch (this.meilisearchIndexScope) { - case 'global': - break; + switch (this.meilisearchIndexScope) { + case 'global': + break; - case 'local': - if (note.userHost == null) break; - return; + case 'local': + if (note.userHost == null) break; + return; - default: { - if (note.userHost == null) break; - if (this.meilisearchIndexScope.includes(note.userHost)) break; - return; - } + default: { + if (note.userHost == null) break; + if (this.meilisearchIndexScope.includes(note.userHost)) break; + return; } - - await this.meilisearchNoteIndex?.addDocuments([{ - id: note.id, - createdAt: this.idService.parse(note.id).date.getTime(), - userId: note.userId, - userHost: note.userHost, - channelId: note.channelId, - cw: note.cw, - text: note.text, - tags: note.tags, - attachedFileTypes: note.attachedFileTypes, - }], { - primaryKey: 'id', - }); } + + await this.meilisearchNoteIndex?.addDocuments([{ + id: note.id, + createdAt: this.idService.parse(note.id).date.getTime(), + userId: note.userId, + userHost: note.userHost, + channelId: note.channelId, + cw: note.cw, + text: note.text, + tags: note.tags, + attachedFileTypes: note.attachedFileTypes, + }], { + primaryKey: 'id', + }); } @bindThis public async unindexNote(note: MiNote): Promise<void> { + if (!this.meilisearch) return; if (!['home', 'public'].includes(note.visibility)) return; - if (this.meilisearch) { - this.meilisearchNoteIndex!.deleteDocument(note.id); - } + await this.meilisearchNoteIndex?.deleteDocument(note.id); } @bindThis - public async searchNote(q: string, me: MiUser | null, opts: { - userId?: MiNote['userId'] | null; - channelId?: MiNote['channelId'] | null; - host?: string | null; - filetype?: FileTypeCategory; - order?: string | null; - disableMeili?: boolean | null; - }, pagination: { - untilId?: MiNote['id']; - sinceId?: MiNote['id']; - limit?: number; - }): Promise<MiNote[]> { - if (this.meilisearch && !opts.disableMeili) { - const filter: Q = { - op: 'and', - qs: [], - }; - if (pagination.untilId) filter.qs.push({ op: '<', k: 'createdAt', v: this.idService.parse(pagination.untilId).date.getTime() }); - if (pagination.sinceId) filter.qs.push({ op: '>', k: 'createdAt', v: this.idService.parse(pagination.sinceId).date.getTime() }); - if (opts.userId) filter.qs.push({ op: '=', k: 'userId', v: opts.userId }); - if (opts.channelId) filter.qs.push({ op: '=', k: 'channelId', v: opts.channelId }); - if (opts.host) { - if (opts.host === '.') { - filter.qs.push({ op: 'is null', k: 'userHost' }); - } else { - filter.qs.push({ op: '=', k: 'userHost', v: opts.host }); - } + public async searchNote( + q: string, + me: MiUser | null, + opts: SearchOpts, + pagination: SearchPagination, + ): Promise<MiNote[]> { + switch (this.provider) { + case 'sqlLike': + case 'sqlPgroonga': { + // ほとんど内容に差がないのでsqlLikeとsqlPgroongaを同じ処理にしている. + // 今後の拡張で差が出る用であれば関数を分ける. + return this.searchNoteByLike(q, me, opts, pagination); } - if (opts.filetype) { - const filters = fileTypes[opts.filetype].map(mime => ({ op: '=' as const, k: 'attachedFileTypes', v: mime })); - filter.qs.push({ op: 'or', qs: filters }); + case 'meilisearch': { + return this.searchNoteByMeiliSearch(q, me, opts, pagination); } - const res = await this.meilisearchNoteIndex!.search(q, { - sort: [`createdAt:${opts.order ? opts.order : 'desc'}`], - matchingStrategy: 'all', - attributesToRetrieve: ['id', 'createdAt'], - filter: compileQuery(filter), - limit: pagination.limit, - }); - if (res.hits.length === 0) return []; - const [ - userIdsWhoMeMuting, - userIdsWhoBlockingMe, - ] = me ? await Promise.all([ - this.cacheService.userMutingsCache.fetch(me.id), - this.cacheService.userBlockedCache.fetch(me.id), - ]) : [new Set<string>(), new Set<string>()]; - const notes = (await this.notesRepository.findBy({ - id: In(res.hits.map(x => x.id)), - })).filter(note => { - if (me && isUserRelated(note, userIdsWhoBlockingMe)) return false; - if (me && isUserRelated(note, userIdsWhoMeMuting)) return false; - return true; - }); - return notes.sort((a, b) => a.id > b.id ? -1 : 1); + default: { + // eslint-disable-next-line @typescript-eslint/no-unused-vars + const typeCheck: never = this.provider; + return []; + } + } + } + + @bindThis + private async searchNoteByLike( + q: string, + me: MiUser | null, + opts: SearchOpts, + pagination: SearchPagination, + ): Promise<MiNote[]> { + const query = this.queryService.makePaginationQuery(this.notesRepository.createQueryBuilder('note'), pagination.sinceId, pagination.untilId); + + if (opts.userId) { + query.andWhere('note.userId = :userId', { userId: opts.userId }); + } else if (opts.channelId) { + query.andWhere('note.channelId = :channelId', { channelId: opts.channelId }); + } + + query + .innerJoinAndSelect('note.user', 'user') + .leftJoinAndSelect('note.reply', 'reply') + .leftJoinAndSelect('note.renote', 'renote') + .leftJoinAndSelect('reply.user', 'replyUser') + .leftJoinAndSelect('renote.user', 'renoteUser'); + + if (this.config.fulltextSearch?.provider === 'sqlPgroonga') { + query.andWhere('note.text &@~ :q', { q }); } else { - const query = this.queryService.makePaginationQuery(this.notesRepository.createQueryBuilder('note'), pagination.sinceId, pagination.untilId); + query.andWhere('note.text ILIKE :q', { q: `%${ sqlLikeEscape(q) }%` }); + } - if (opts.userId) { - query.andWhere('note.userId = :userId', { userId: opts.userId }); - } else if (opts.channelId) { - query.andWhere('note.channelId = :channelId', { channelId: opts.channelId }); + if (opts.host) { + if (opts.host === '.') { + query.andWhere('note.userHost IS NULL'); + } else { + query.andWhere('note.userHost = :host', { host: opts.host }); } + } - query - .andWhere('note.text ILIKE :q', { q: `%${ sqlLikeEscape(q) }%` }) - .innerJoinAndSelect('note.user', 'user') - .leftJoinAndSelect('note.reply', 'reply') - .leftJoinAndSelect('note.renote', 'renote') - .leftJoinAndSelect('reply.user', 'replyUser') - .leftJoinAndSelect('renote.user', 'renoteUser'); + if (opts.filetype) { + query.andWhere('note."attachedFileTypes" && :types', { types: fileTypes[opts.filetype] }); + } - if (opts.host) { - if (opts.host === '.') { - query.andWhere('note.userHost IS NULL'); - } else { - query.andWhere('note.userHost = :host', { host: opts.host }); - } - } + this.queryService.generateVisibilityQuery(query, me); + if (me) this.queryService.generateMutedUserQuery(query, me); + if (me) this.queryService.generateBlockedUserQuery(query, me); - if (opts.filetype) { - query.andWhere('note."attachedFileTypes" && :types', { types: fileTypes[opts.filetype] }); + return await query.limit(pagination.limit).getMany(); + } + + @bindThis + private async searchNoteByMeiliSearch( + q: string, + me: MiUser | null, + opts: SearchOpts, + pagination: SearchPagination, + ): Promise<MiNote[]> { + if (!this.meilisearch || !this.meilisearchNoteIndex) { + throw new Error('MeiliSearch is not available'); + } + + const filter: Q = { + op: 'and', + qs: [], + }; + if (pagination.untilId) filter.qs.push({ + op: '<', + k: 'createdAt', + v: this.idService.parse(pagination.untilId).date.getTime(), + }); + if (pagination.sinceId) filter.qs.push({ + op: '>', + k: 'createdAt', + v: this.idService.parse(pagination.sinceId).date.getTime(), + }); + if (opts.userId) filter.qs.push({ op: '=', k: 'userId', v: opts.userId }); + if (opts.channelId) filter.qs.push({ op: '=', k: 'channelId', v: opts.channelId }); + if (opts.host) { + if (opts.host === '.') { + filter.qs.push({ op: 'is null', k: 'userHost' }); + } else { + filter.qs.push({ op: '=', k: 'userHost', v: opts.host }); } + } - this.queryService.generateVisibilityQuery(query, me); - if (me) this.queryService.generateMutedUserQuery(query, me); - if (me) this.queryService.generateBlockedUserQuery(query, me); + if (opts.filetype) { + const filters = fileTypes[opts.filetype].map(mime => ({ op: '=' as const, k: 'attachedFileTypes', v: mime })); + filter.qs.push({ op: 'or', qs: filters }); + } - return await query.limit(pagination.limit).getMany(); + const res = await this.meilisearchNoteIndex.search(q, { + sort: [`createdAt:${opts.order ? opts.order : 'desc'}`], + matchingStrategy: 'all', + attributesToRetrieve: ['id', 'createdAt'], + filter: compileQuery(filter), + limit: pagination.limit, + }); + if (res.hits.length === 0) { + return []; } + + const [ + userIdsWhoMeMuting, + userIdsWhoBlockingMe, + ] = me + ? await Promise.all([ + this.cacheService.userMutingsCache.fetch(me.id), + this.cacheService.userBlockedCache.fetch(me.id), + ]) + : [new Set<string>(), new Set<string>()]; + const notes = (await this.notesRepository.findBy({ + id: In(res.hits.map(x => x.id)), + })).filter(note => { + if (me && isUserRelated(note, userIdsWhoBlockingMe)) return false; + if (me && isUserRelated(note, userIdsWhoMeMuting)) return false; + return true; + }); + + return notes.sort((a, b) => a.id > b.id ? -1 : 1); } } diff --git a/packages/backend/src/core/SignupService.ts b/packages/backend/src/core/SignupService.ts index 0ad448e95f..9fc0c2b34a 100644 --- a/packages/backend/src/core/SignupService.ts +++ b/packages/backend/src/core/SignupService.ts @@ -162,3 +162,4 @@ export class SignupService { return { account, secret }; } } + diff --git a/packages/backend/src/core/SystemWebhookService.ts b/packages/backend/src/core/SystemWebhookService.ts index de00169612..8239490adc 100644 --- a/packages/backend/src/core/SystemWebhookService.ts +++ b/packages/backend/src/core/SystemWebhookService.ts @@ -50,7 +50,6 @@ export type SystemWebhookPayload<T extends SystemWebhookEventType> = @Injectable() export class SystemWebhookService implements OnApplicationShutdown { - private logger: Logger; private activeSystemWebhooksFetched = false; private activeSystemWebhooks: MiSystemWebhook[] = []; @@ -62,11 +61,9 @@ export class SystemWebhookService implements OnApplicationShutdown { private idService: IdService, private queueService: QueueService, private moderationLogService: ModerationLogService, - private loggerService: LoggerService, private globalEventService: GlobalEventService, ) { this.redisForSub.on('message', this.onMessage); - this.logger = this.loggerService.getLogger('webhook'); } @bindThis @@ -193,28 +190,24 @@ export class SystemWebhookService implements OnApplicationShutdown { /** * SystemWebhook をWebhook配送キューに追加する * @see QueueService.systemWebhookDeliver - * // TODO: contentの型を厳格化する */ @bindThis public async enqueueSystemWebhook<T extends SystemWebhookEventType>( - webhook: MiSystemWebhook | MiSystemWebhook['id'], type: T, content: SystemWebhookPayload<T>, + opts?: { + excludes?: MiSystemWebhook['id'][]; + }, ) { - const webhookEntity = typeof webhook === 'string' - ? (await this.fetchActiveSystemWebhooks()).find(a => a.id === webhook) - : webhook; - if (!webhookEntity || !webhookEntity.isActive) { - this.logger.info(`SystemWebhook is not active or not found : ${webhook}`); - return; - } - - if (!webhookEntity.on.includes(type)) { - this.logger.info(`SystemWebhook ${webhookEntity.id} is not listening to ${type}`); - return; - } - - return this.queueService.systemWebhookDeliver(webhookEntity, type, content); + const webhooks = await this.fetchActiveSystemWebhooks() + .then(webhooks => { + return webhooks.filter(webhook => !opts?.excludes?.includes(webhook.id) && webhook.on.includes(type)); + }); + return Promise.all( + webhooks.map(webhook => { + return this.queueService.systemWebhookDeliver(webhook, type, content); + }), + ); } @bindThis diff --git a/packages/backend/src/core/UserBlockingService.ts b/packages/backend/src/core/UserBlockingService.ts index 2f1310b8ef..8da1bb2092 100644 --- a/packages/backend/src/core/UserBlockingService.ts +++ b/packages/backend/src/core/UserBlockingService.ts @@ -118,13 +118,7 @@ export class UserBlockingService implements OnModuleInit { schema: 'UserDetailedNotMe', }).then(async packed => { this.globalEventService.publishMainStream(follower.id, 'unfollow', packed); - - const webhooks = (await this.webhookService.getActiveWebhooks()).filter(x => x.userId === follower.id && x.on.includes('unfollow')); - for (const webhook of webhooks) { - this.queueService.userWebhookDeliver(webhook, 'unfollow', { - user: packed, - }); - } + this.webhookService.enqueueUserWebhook(follower.id, 'unfollow', { user: packed }); }); } diff --git a/packages/backend/src/core/UserFollowingService.ts b/packages/backend/src/core/UserFollowingService.ts index 8963003057..b98ca97ec9 100644 --- a/packages/backend/src/core/UserFollowingService.ts +++ b/packages/backend/src/core/UserFollowingService.ts @@ -333,13 +333,7 @@ export class UserFollowingService implements OnModuleInit { schema: 'UserDetailedNotMe', }).then(async packed => { this.globalEventService.publishMainStream(follower.id, 'follow', packed); - - const webhooks = (await this.webhookService.getActiveWebhooks()).filter(x => x.userId === follower.id && x.on.includes('follow')); - for (const webhook of webhooks) { - this.queueService.userWebhookDeliver(webhook, 'follow', { - user: packed, - }); - } + this.webhookService.enqueueUserWebhook(follower.id, 'follow', { user: packed }); }); } @@ -347,13 +341,7 @@ export class UserFollowingService implements OnModuleInit { if (this.userEntityService.isLocalUser(followee)) { this.userEntityService.pack(follower.id, followee).then(async packed => { this.globalEventService.publishMainStream(followee.id, 'followed', packed); - - const webhooks = (await this.webhookService.getActiveWebhooks()).filter(x => x.userId === followee.id && x.on.includes('followed')); - for (const webhook of webhooks) { - this.queueService.userWebhookDeliver(webhook, 'followed', { - user: packed, - }); - } + this.webhookService.enqueueUserWebhook(followee.id, 'followed', { user: packed }); }); // 通知を作成 @@ -400,13 +388,7 @@ export class UserFollowingService implements OnModuleInit { schema: 'UserDetailedNotMe', }).then(async packed => { this.globalEventService.publishMainStream(follower.id, 'unfollow', packed); - - const webhooks = (await this.webhookService.getActiveWebhooks()).filter(x => x.userId === follower.id && x.on.includes('unfollow')); - for (const webhook of webhooks) { - this.queueService.userWebhookDeliver(webhook, 'unfollow', { - user: packed, - }); - } + this.webhookService.enqueueUserWebhook(follower.id, 'unfollow', { user: packed }); }); } @@ -744,13 +726,7 @@ export class UserFollowingService implements OnModuleInit { }); this.globalEventService.publishMainStream(follower.id, 'unfollow', packedFollowee); - - const webhooks = (await this.webhookService.getActiveWebhooks()).filter(x => x.userId === follower.id && x.on.includes('unfollow')); - for (const webhook of webhooks) { - this.queueService.userWebhookDeliver(webhook, 'unfollow', { - user: packedFollowee, - }); - } + this.webhookService.enqueueUserWebhook(follower.id, 'unfollow', { user: packedFollowee }); } @bindThis diff --git a/packages/backend/src/core/UserService.ts b/packages/backend/src/core/UserService.ts index 9b1961c631..1f471513f3 100644 --- a/packages/backend/src/core/UserService.ts +++ b/packages/backend/src/core/UserService.ts @@ -63,13 +63,6 @@ export class UserService { @bindThis public async notifySystemWebhook(user: MiUser, type: 'userCreated') { const packedUser = await this.userEntityService.pack(user, null, { schema: 'UserLite' }); - const recipientWebhookIds = await this.systemWebhookService.fetchSystemWebhooks({ isActive: true, on: [type] }); - for (const webhookId of recipientWebhookIds) { - await this.systemWebhookService.enqueueSystemWebhook( - webhookId, - type, - packedUser, - ); - } + return this.systemWebhookService.enqueueSystemWebhook(type, packedUser); } } diff --git a/packages/backend/src/core/UserWebhookService.ts b/packages/backend/src/core/UserWebhookService.ts index 911efdf768..08db4c9afc 100644 --- a/packages/backend/src/core/UserWebhookService.ts +++ b/packages/backend/src/core/UserWebhookService.ts @@ -5,13 +5,14 @@ import { Inject, Injectable } from '@nestjs/common'; import * as Redis from 'ioredis'; -import { type WebhooksRepository } from '@/models/_.js'; +import { MiUser, type WebhooksRepository } from '@/models/_.js'; import { MiWebhook, WebhookEventTypes } from '@/models/Webhook.js'; import { DI } from '@/di-symbols.js'; import { bindThis } from '@/decorators.js'; import { GlobalEvents } from '@/core/GlobalEventService.js'; -import type { OnApplicationShutdown } from '@nestjs/common'; import type { Packed } from '@/misc/json-schema.js'; +import { QueueService } from '@/core/QueueService.js'; +import type { OnApplicationShutdown } from '@nestjs/common'; export type UserWebhookPayload<T extends WebhookEventTypes> = T extends 'note' | 'reply' | 'renote' |'mention' | 'edited' ? { @@ -34,6 +35,7 @@ export class UserWebhookService implements OnApplicationShutdown { private redisForSub: Redis.Redis, @Inject(DI.webhooksRepository) private webhooksRepository: WebhooksRepository, + private queueService: QueueService, ) { this.redisForSub.on('message', this.onMessage); } @@ -75,6 +77,25 @@ export class UserWebhookService implements OnApplicationShutdown { return query.getMany(); } + /** + * UserWebhook をWebhook配送キューに追加する + * @see QueueService.userWebhookDeliver + */ + @bindThis + public async enqueueUserWebhook<T extends WebhookEventTypes>( + userId: MiUser['id'], + type: T, + content: UserWebhookPayload<T>, + ) { + const webhooks = await this.getActiveWebhooks() + .then(webhooks => webhooks.filter(webhook => webhook.userId === userId && webhook.on.includes(type))); + return Promise.all( + webhooks.map(webhook => { + return this.queueService.userWebhookDeliver(webhook, type, content); + }), + ); + } + @bindThis private async onMessage(_: string, data: string): Promise<void> { const obj = JSON.parse(data); diff --git a/packages/backend/src/core/UtilityService.ts b/packages/backend/src/core/UtilityService.ts index f905914022..81eaa5f95d 100644 --- a/packages/backend/src/core/UtilityService.ts +++ b/packages/backend/src/core/UtilityService.ts @@ -3,8 +3,7 @@ * SPDX-License-Identifier: AGPL-3.0-only */ -import { URL } from 'node:url'; -import punycode from 'punycode/punycode.js'; +import { URL, domainToASCII } from 'node:url'; import { Inject, Injectable } from '@nestjs/common'; import RE2 from 're2'; import psl from 'psl'; @@ -107,13 +106,13 @@ export class UtilityService { @bindThis public toPuny(host: string): string { - return punycode.toASCII(host.toLowerCase()); + return domainToASCII(host.toLowerCase()); } @bindThis public toPunyNullable(host: string | null | undefined): string | null { if (host == null) return null; - return punycode.toASCII(host.toLowerCase()); + return domainToASCII(host.toLowerCase()); } @bindThis diff --git a/packages/backend/src/core/WebAuthnService.ts b/packages/backend/src/core/WebAuthnService.ts index ad53192f18..ed75e4f467 100644 --- a/packages/backend/src/core/WebAuthnService.ts +++ b/packages/backend/src/core/WebAuthnService.ts @@ -189,14 +189,12 @@ export class WebAuthnService { */ @bindThis public async verifySignInWithPasskeyAuthentication(context: string, response: AuthenticationResponseJSON): Promise<MiUser['id'] | null> { - const challenge = await this.redisClient.get(`webauthn:challenge:${context}`); + const challenge = await this.redisClient.getdel(`webauthn:challenge:${context}`); if (!challenge) { throw new IdentifiableError('2d16e51c-007b-4edd-afd2-f7dd02c947f6', `challenge '${context}' not found`); } - await this.redisClient.del(`webauthn:challenge:${context}`); - const key = await this.userSecurityKeysRepository.findOneBy({ id: response.id, }); diff --git a/packages/backend/src/core/activitypub/ApRendererService.ts b/packages/backend/src/core/activitypub/ApRendererService.ts index fb706a775f..721cb77b2f 100644 --- a/packages/backend/src/core/activitypub/ApRendererService.ts +++ b/packages/backend/src/core/activitypub/ApRendererService.ts @@ -192,6 +192,9 @@ export class ApRendererService { // || emoji.originalUrl してるのは後方互換性のため（publicUrlはstringなので??はだめ） url: emoji.publicUrl || emoji.originalUrl, }, + _misskey_license: { + freeText: emoji.license, + }, }; } diff --git a/packages/backend/src/core/activitypub/ApRequestService.ts b/packages/backend/src/core/activitypub/ApRequestService.ts index aca322e745..6291768e8c 100644 --- a/packages/backend/src/core/activitypub/ApRequestService.ts +++ b/packages/backend/src/core/activitypub/ApRequestService.ts @@ -253,10 +253,8 @@ export class ApRequestService { if (alternate) { const href = alternate.getAttribute('href'); - if (href) { - if (this.utilityService.punyHostPSLDomain(url) === this.utilityService.punyHostPSLDomain(href)) { - return await this.signedGet(href, user, false); - } + if (href && this.utilityService.punyHostPSLDomain(url) === this.utilityService.punyHostPSLDomain(href)) { + return await this.signedGet(href, user, false); } } } catch (e) { @@ -268,7 +266,6 @@ export class ApRequestService { //#endregion validateContentTypeSetAsActivityPub(res); - const finalUrl = res.url; // redirects may have been involved const activity = await res.json() as IObject; diff --git a/packages/backend/src/core/activitypub/ApResolverService.ts b/packages/backend/src/core/activitypub/ApResolverService.ts index c82a9be3b1..a0c3a4846c 100644 --- a/packages/backend/src/core/activitypub/ApResolverService.ts +++ b/packages/backend/src/core/activitypub/ApResolverService.ts @@ -5,7 +5,6 @@ import { Inject, Injectable } from '@nestjs/common'; import { IsNull, Not } from 'typeorm'; -import { UnrecoverableError } from 'bullmq'; import type { MiLocalUser, MiRemoteUser } from '@/models/User.js'; import { InstanceActorService } from '@/core/InstanceActorService.js'; import type { NotesRepository, PollsRepository, NoteReactionsRepository, UsersRepository, FollowRequestsRepository, MiMeta } from '@/models/_.js'; @@ -17,6 +16,7 @@ import { bindThis } from '@/decorators.js'; import { LoggerService } from '@/core/LoggerService.js'; import type Logger from '@/logger.js'; import { fromTuple } from '@/misc/from-tuple.js'; +import { IdentifiableError } from '@/misc/identifiable-error.js'; import { isCollectionOrOrderedCollection } from './type.js'; import { ApDbResolverService } from './ApDbResolverService.js'; import { ApRendererService } from './ApRendererService.js'; @@ -68,7 +68,7 @@ export class Resolver { if (isCollectionOrOrderedCollection(collection)) { return collection; } else { - throw new UnrecoverableError(`unrecognized collection type: ${collection.type}`); + throw new IdentifiableError('f100eccf-f347-43fb-9b45-96a0831fb635', `unrecognized collection type: ${collection.type}`); } } @@ -85,15 +85,15 @@ export class Resolver { // URLs with fragment parts cannot be resolved correctly because // the fragment part does not get transmitted over HTTP(S). // Avoid strange behaviour by not trying to resolve these at all. - throw new UnrecoverableError(`cannot resolve URL with fragment: ${value}`); + throw new IdentifiableError('b94fd5b1-0e3b-4678-9df2-dad4cd515ab2', `cannot resolve URL with fragment: ${value}`); } if (this.history.has(value)) { - throw new Error(`cannot resolve already resolved URL: ${value}`); + throw new IdentifiableError('0dc86cf6-7cd6-4e56-b1e6-5903d62d7ea5', `cannot resolve already resolved URL: ${value}`); } if (this.history.size > this.recursionLimit) { - throw new Error(`hit recursion limit: ${value}`); + throw new IdentifiableError('d592da9f-822f-4d91-83d7-4ceefabcf3d2', `hit recursion limit: ${value}`); } this.history.add(value); @@ -104,7 +104,7 @@ export class Resolver { } if (!this.utilityService.isFederationAllowedHost(host)) { - throw new UnrecoverableError(`cannot fetch AP object ${value}: blocked instance ${host}`); + throw new IdentifiableError('09d79f9e-64f1-4316-9cfa-e75c4d091574', `cannot fetch AP object ${value}: blocked instance ${host}`); } if (this.config.signToActivityPubGet && !this.user) { @@ -120,13 +120,13 @@ export class Resolver { !(object['@context'] as unknown[]).includes('https://www.w3.org/ns/activitystreams') : object['@context'] !== 'https://www.w3.org/ns/activitystreams' ) { - throw new UnrecoverableError(`invalid AP object ${value}: does not have ActivityStreams context`); + throw new IdentifiableError('72180409-793c-4973-868e-5a118eb5519b', `invalid AP object ${value}: does not have ActivityStreams context`); } // Since redirects are allowed, we cannot safely validate an anonymous object. // Reject any responses without an ID, as all other checks depend on that value. if (object.id == null) { - throw new UnrecoverableError(`invalid AP object ${value}: missing id`); + throw new IdentifiableError('ad2dc287-75c1-44c4-839d-3d2e64576675', `invalid AP object ${value}: missing id`); } // We allow some limited cross-domain redirects, which means the host may have changed during fetch. @@ -135,12 +135,12 @@ export class Resolver { if (finalHost !== host) { // Make sure the redirect stayed within the same authority. if (this.utilityService.punyHostPSLDomain(object.id) !== this.utilityService.punyHostPSLDomain(value)) { - throw new UnrecoverableError(`invalid AP object ${value}: id ${object.id} has different host`); + throw new IdentifiableError('fd93c2fa-69a8-440f-880b-bf178e0ec877', `invalid AP object ${value}: id ${object.id} has different host`); } // Check if the redirect bounce from [allowed domain] to [blocked domain]. if (!this.utilityService.isFederationAllowedHost(finalHost)) { - throw new UnrecoverableError(`cannot fetch AP object ${value}: redirected to blocked instance ${finalHost}`); + throw new IdentifiableError('0a72bf24-2d9b-4f1d-886b-15aaa31adeda', `cannot fetch AP object ${value}: redirected to blocked instance ${finalHost}`); } } @@ -150,7 +150,7 @@ export class Resolver { @bindThis private resolveLocal(url: string): Promise<IObject> { const parsed = this.apDbResolverService.parseUri(url); - if (!parsed.local) throw new UnrecoverableError(`resolveLocal - not a local URL: ${url}`); + if (!parsed.local) throw new IdentifiableError('02b40cd0-fa92-4b0c-acc9-fb2ada952ab8', `resolveLocal - not a local URL: ${url}`); switch (parsed.type) { case 'notes': @@ -179,7 +179,7 @@ export class Resolver { case 'follows': return this.followRequestsRepository.findOneBy({ id: parsed.id }) .then(async followRequest => { - if (followRequest == null) throw new UnrecoverableError(`resolveLocal - invalid follow request ID ${parsed.id}: ${url}`); + if (followRequest == null) throw new IdentifiableError('a9d946e5-d276-47f8-95fb-f04230289bb0', `resolveLocal - invalid follow request ID ${parsed.id}: ${url}`); const [follower, followee] = await Promise.all([ this.usersRepository.findOneBy({ id: followRequest.followerId, @@ -191,12 +191,12 @@ export class Resolver { }), ]); if (follower == null || followee == null) { - throw new Error(`resolveLocal - follower or followee does not exist: ${url}`); + throw new IdentifiableError('06ae3170-1796-4d93-a697-2611ea6d83b6', `resolveLocal - follower or followee does not exist: ${url}`); } return this.apRendererService.addContext(this.apRendererService.renderFollow(follower as MiLocalUser | MiRemoteUser, followee as MiLocalUser | MiRemoteUser, url)); }); default: - throw new UnrecoverableError(`resolveLocal: type ${parsed.type} unhandled: ${url}`); + throw new IdentifiableError('7a5d2fc0-94bc-4db6-b8b8-1bf24a2e23d0', `resolveLocal: type ${parsed.type} unhandled: ${url}`); } } } diff --git a/packages/backend/src/core/activitypub/misc/contexts.ts b/packages/backend/src/core/activitypub/misc/contexts.ts index d7b6fc6589..5c0b8ffcbb 100644 --- a/packages/backend/src/core/activitypub/misc/contexts.ts +++ b/packages/backend/src/core/activitypub/misc/contexts.ts @@ -561,6 +561,11 @@ const extension_context_definition = { '_misskey_requireSigninToViewContents': 'misskey:_misskey_requireSigninToViewContents', '_misskey_makeNotesFollowersOnlyBefore': 'misskey:_misskey_makeNotesFollowersOnlyBefore', '_misskey_makeNotesHiddenBefore': 'misskey:_misskey_makeNotesHiddenBefore', + '_misskey_license': 'misskey:_misskey_license', + 'freeText': { + '@id': 'misskey:freeText', + '@type': 'schema:text', + }, 'isCat': 'misskey:isCat', // Firefish firefish: 'https://joinfirefish.org/ns#', diff --git a/packages/backend/src/core/activitypub/models/ApNoteService.ts b/packages/backend/src/core/activitypub/models/ApNoteService.ts index e4c4fe54b5..9fc6945edb 100644 --- a/packages/backend/src/core/activitypub/models/ApNoteService.ts +++ b/packages/backend/src/core/activitypub/models/ApNoteService.ts @@ -690,6 +690,8 @@ export class ApNoteService { originalUrl: tag.icon.url, publicUrl: tag.icon.url, updatedAt: new Date(), + // _misskey_license が存在しなければ `null` + license: (tag._misskey_license?.freeText ?? null), }); const emoji = await this.emojisRepository.findOneBy({ host, name }); @@ -711,6 +713,8 @@ export class ApNoteService { publicUrl: tag.icon.url, updatedAt: new Date(), aliases: [], + // _misskey_license が存在しなければ `null` + license: (tag._misskey_license?.freeText ?? null) }); })); } diff --git a/packages/backend/src/core/activitypub/type.ts b/packages/backend/src/core/activitypub/type.ts index 08bd224700..4e0f131647 100644 --- a/packages/backend/src/core/activitypub/type.ts +++ b/packages/backend/src/core/activitypub/type.ts @@ -277,6 +277,11 @@ export interface IApEmoji extends IObject { type: 'Emoji'; name: string; updated: string; + // Misskey拡張。後方互換性のためにoptional。 + // 将来の拡張性を考慮してobjectにしている + _misskey_license?: { + freeText: string | null; + }; } export const isEmoji = (object: IObject): object is IApEmoji => diff --git a/packages/backend/src/core/chart/ChartManagementService.ts b/packages/backend/src/core/chart/ChartManagementService.ts index ef65af2432..81495c8a6c 100644 --- a/packages/backend/src/core/chart/ChartManagementService.ts +++ b/packages/backend/src/core/chart/ChartManagementService.ts @@ -41,7 +41,7 @@ export class ChartManagementService implements OnApplicationShutdown { private perUserFollowingChart: PerUserFollowingChart, private perUserDriveChart: PerUserDriveChart, private apRequestChart: ApRequestChart, - private chartLoggerService: ChartLoggerService, + chartLoggerService: ChartLoggerService, ) { this.charts = [ this.federationChart, diff --git a/packages/backend/src/core/entities/EmojiEntityService.ts b/packages/backend/src/core/entities/EmojiEntityService.ts index 841bd731c0..490d3f2511 100644 --- a/packages/backend/src/core/entities/EmojiEntityService.ts +++ b/packages/backend/src/core/entities/EmojiEntityService.ts @@ -4,10 +4,10 @@ */ import { Inject, Injectable } from '@nestjs/common'; +import { In } from 'typeorm'; import { DI } from '@/di-symbols.js'; -import type { EmojisRepository } from '@/models/_.js'; +import type { EmojisRepository, MiRole, RolesRepository } from '@/models/_.js'; import type { Packed } from '@/misc/json-schema.js'; -import type { } from '@/models/Blocking.js'; import type { MiEmoji } from '@/models/Emoji.js'; import { bindThis } from '@/decorators.js'; @@ -16,6 +16,8 @@ export class EmojiEntityService { constructor( @Inject(DI.emojisRepository) private emojisRepository: EmojisRepository, + @Inject(DI.rolesRepository) + private rolesRepository: RolesRepository, ) { } @@ -68,8 +70,90 @@ export class EmojiEntityService { @bindThis public packDetailedMany( emojis: any[], - ) { + ): Promise<Packed<'EmojiDetailed'>[]> { return Promise.all(emojis.map(x => this.packDetailed(x))); } + + @bindThis + public async packDetailedAdmin( + src: MiEmoji['id'] | MiEmoji, + hint?: { + roles?: Map<MiRole['id'], MiRole> + }, + ): Promise<Packed<'EmojiDetailedAdmin'>> { + const emoji = typeof src === 'object' ? src : await this.emojisRepository.findOneByOrFail({ id: src }); + + const roles = Array.of<MiRole>(); + if (emoji.roleIdsThatCanBeUsedThisEmojiAsReaction.length > 0) { + if (hint?.roles) { + const hintRoles = hint.roles; + roles.push( + ...emoji.roleIdsThatCanBeUsedThisEmojiAsReaction + .filter(x => hintRoles.has(x)) + .map(x => hintRoles.get(x)!), + ); + } else { + roles.push( + ...await this.rolesRepository.findBy({ id: In(emoji.roleIdsThatCanBeUsedThisEmojiAsReaction) }), + ); + } + + roles.sort((a, b) => { + if (a.displayOrder !== b.displayOrder) { + return b.displayOrder - a.displayOrder; + } + + return a.id.localeCompare(b.id); + }); + } + + return { + id: emoji.id, + updatedAt: emoji.updatedAt?.toISOString() ?? null, + name: emoji.name, + host: emoji.host, + uri: emoji.uri, + type: emoji.type, + aliases: emoji.aliases, + category: emoji.category, + publicUrl: emoji.publicUrl, + originalUrl: emoji.originalUrl, + license: emoji.license, + localOnly: emoji.localOnly, + isSensitive: emoji.isSensitive, + roleIdsThatCanBeUsedThisEmojiAsReaction: roles.map(it => ({ id: it.id, name: it.name })), + }; + } + + @bindThis + public async packDetailedAdminMany( + emojis: MiEmoji['id'][] | MiEmoji[], + hint?: { + roles?: Map<MiRole['id'], MiRole> + }, + ): Promise<Packed<'EmojiDetailedAdmin'>[]> { + // IDのみの要素をピックアップし、DBからレコードを取り出して他の値を補完する + const emojiEntities = emojis.filter(x => typeof x === 'object') as MiEmoji[]; + const emojiIdOnlyList = emojis.filter(x => typeof x === 'string') as string[]; + if (emojiIdOnlyList.length > 0) { + emojiEntities.push(...await this.emojisRepository.findBy({ id: In(emojiIdOnlyList) })); + } + + // 特定ロール専用の絵文字である場合、そのロール情報をあらかじめまとめて取得しておく（pack側で都度取得も出来るが負荷が高いので） + let hintRoles: Map<MiRole['id'], MiRole>; + if (hint?.roles) { + hintRoles = hint.roles; + } else { + const roles = Array.of<MiRole>(); + const roleIds = [...new Set(emojiEntities.flatMap(x => x.roleIdsThatCanBeUsedThisEmojiAsReaction))]; + if (roleIds.length > 0) { + roles.push(...await this.rolesRepository.findBy({ id: In(roleIds) })); + } + + hintRoles = new Map(roles.map(x => [x.id, x])); + } + + return Promise.all(emojis.map(x => this.packDetailedAdmin(x, { roles: hintRoles }))); + } } diff --git a/packages/backend/src/core/entities/MetaEntityService.ts b/packages/backend/src/core/entities/MetaEntityService.ts index 857e8f5a7b..84d591ce7a 100644 --- a/packages/backend/src/core/entities/MetaEntityService.ts +++ b/packages/backend/src/core/entities/MetaEntityService.ts @@ -145,6 +145,7 @@ export class MetaEntityService { enableUrlPreview: instance.urlPreviewEnabled, noteSearchableScope: (this.config.meilisearch == null || this.config.meilisearch.scope !== 'local') ? 'global' : 'local', maxFileSize: this.config.maxFileSize, + federation: this.meta.federation, }; return packed; @@ -185,3 +186,4 @@ export class MetaEntityService { return packDetailed; } } + diff --git a/packages/backend/src/core/entities/NoteEntityService.ts b/packages/backend/src/core/entities/NoteEntityService.ts index b3cd191ad2..dca73567cc 100644 --- a/packages/backend/src/core/entities/NoteEntityService.ts +++ b/packages/backend/src/core/entities/NoteEntityService.ts @@ -23,7 +23,6 @@ import type { CustomEmojiService } from '../CustomEmojiService.js'; import type { ReactionService } from '../ReactionService.js'; import type { UserEntityService } from './UserEntityService.js'; import type { DriveFileEntityService } from './DriveFileEntityService.js'; -import type { Config } from '@/config.js'; // is-renote.tsとよしなにリンク function isPureRenote(note: MiNote): note is MiNote & { renoteId: MiNote['id']; renote: MiNote } { @@ -75,9 +74,6 @@ export class NoteEntityService implements OnModuleInit { @Inject(DI.usersRepository) private usersRepository: UsersRepository, - @Inject(DI.config) - private config: Config, - @Inject(DI.notesRepository) private notesRepository: NotesRepository, @@ -116,8 +112,7 @@ export class NoteEntityService implements OnModuleInit { } @bindThis - private async hideNote(packedNote: Packed<'Note'>, meId: MiUser['id'] | null): Promise<void> { - // FIXME: このvisibility変更処理が当関数にあるのは若干不自然かもしれない(関数名を treatVisibility とかに変える手もある) + private treatVisibility(packedNote: Packed<'Note'>): Packed<'Note'>['visibility'] { if (packedNote.visibility === 'public' || packedNote.visibility === 'home') { const followersOnlyBefore = packedNote.user.makeNotesFollowersOnlyBefore; if ((followersOnlyBefore != null) @@ -129,7 +124,11 @@ export class NoteEntityService implements OnModuleInit { packedNote.visibility = 'followers'; } } + return packedNote.visibility; + } + @bindThis + private async hideNote(packedNote: Packed<'Note'>, meId: MiUser['id'] | null): Promise<void> { if (meId === packedNote.userId) return; // TODO: isVisibleForMe を使うようにしても良さそう(型違うけど) @@ -508,6 +507,8 @@ export class NoteEntityService implements OnModuleInit { } : {}), }); + this.treatVisibility(packed); + if (!opts.skipHide) { await this.hideNote(packed, meId); } |