diff options
| author | Marie <Marie@kaifa.ch> | 2023-12-23 02:09:23 +0100 |
|---|---|---|
| committer | Marie <Marie@kaifa.ch> | 2023-12-23 02:09:23 +0100 |
| commit | 5db583a3eb61d50de14d875ebf7ecef20490e313 (patch) | |
| tree | 783dd43d2ac660c32e745a4485d499e9ddc43324 /packages/backend/src/core | |
| parent | add: Custom MOTDs (diff) | |
| parent | Update CHANGELOG.md (diff) | |
| download | sharkey-5db583a3eb61d50de14d875ebf7ecef20490e313.tar.gz sharkey-5db583a3eb61d50de14d875ebf7ecef20490e313.tar.bz2 sharkey-5db583a3eb61d50de14d875ebf7ecef20490e313.zip | |
merge: upstream
Diffstat (limited to 'packages/backend/src/core')
20 files changed, 637 insertions, 128 deletions
diff --git a/packages/backend/src/core/AntennaService.ts b/packages/backend/src/core/AntennaService.ts index 65be275548..2c27a02559 100644 --- a/packages/backend/src/core/AntennaService.ts +++ b/packages/backend/src/core/AntennaService.ts @@ -16,7 +16,7 @@ import type { AntennasRepository, UserListMembershipsRepository } from '@/models import { UtilityService } from '@/core/UtilityService.js'; import { bindThis } from '@/decorators.js'; import type { GlobalEvents } from '@/core/GlobalEventService.js'; -import { FunoutTimelineService } from '@/core/FunoutTimelineService.js'; +import { FanoutTimelineService } from '@/core/FanoutTimelineService.js'; import type { OnApplicationShutdown } from '@nestjs/common'; @Injectable() @@ -39,7 +39,7 @@ export class AntennaService implements OnApplicationShutdown { private utilityService: UtilityService, private globalEventService: GlobalEventService, - private funoutTimelineService: FunoutTimelineService, + private fanoutTimelineService: FanoutTimelineService, ) { this.antennasFetched = false; this.antennas = []; @@ -60,11 +60,21 @@ export class AntennaService implements OnApplicationShutdown { lastUsedAt: new Date(body.lastUsedAt), }); break; - case 'antennaUpdated': - this.antennas[this.antennas.findIndex(a => a.id === body.id)] = { - ...body, - lastUsedAt: new Date(body.lastUsedAt), - }; + case 'antennaUpdated': { + const idx = this.antennas.findIndex(a => a.id === body.id); + if (idx >= 0) { + this.antennas[idx] = { + ...body, + lastUsedAt: new Date(body.lastUsedAt), + }; + } else { + // サーバ起動時にactiveじゃなかった場合、リストに持っていないので追加する必要あり + this.antennas.push({ + ...body, + lastUsedAt: new Date(body.lastUsedAt), + }); + } + } break; case 'antennaDeleted': this.antennas = this.antennas.filter(a => a.id !== body.id); @@ -84,7 +94,7 @@ export class AntennaService implements OnApplicationShutdown { const redisPipeline = this.redisForTimelines.pipeline(); for (const antenna of matchedAntennas) { - this.funoutTimelineService.push(`antennaTimeline:${antenna.id}`, note.id, 200, redisPipeline); + this.fanoutTimelineService.push(`antennaTimeline:${antenna.id}`, note.id, 200, redisPipeline); this.globalEventService.publishAntennaStream(antenna.id, 'note', note); } diff --git a/packages/backend/src/core/CoreModule.ts b/packages/backend/src/core/CoreModule.ts index b3ab901fed..fa868ff8b0 100644 --- a/packages/backend/src/core/CoreModule.ts +++ b/packages/backend/src/core/CoreModule.ts @@ -4,6 +4,7 @@ */ import { Module } from '@nestjs/common'; +import { FanoutTimelineEndpointService } from '@/core/FanoutTimelineEndpointService.js'; import { AccountMoveService } from './AccountMoveService.js'; import { AccountUpdateService } from './AccountUpdateService.js'; import { AnnouncementService } from './AnnouncementService.js'; @@ -62,7 +63,7 @@ import { FileInfoService } from './FileInfoService.js'; import { SearchService } from './SearchService.js'; import { ClipService } from './ClipService.js'; import { FeaturedService } from './FeaturedService.js'; -import { FunoutTimelineService } from './FunoutTimelineService.js'; +import { FanoutTimelineService } from './FanoutTimelineService.js'; import { ChannelFollowingService } from './ChannelFollowingService.js'; import { RegistryApiService } from './RegistryApiService.js'; import { ChartLoggerService } from './chart/ChartLoggerService.js'; @@ -194,7 +195,8 @@ const $FileInfoService: Provider = { provide: 'FileInfoService', useExisting: Fi const $SearchService: Provider = { provide: 'SearchService', useExisting: SearchService }; const $ClipService: Provider = { provide: 'ClipService', useExisting: ClipService }; const $FeaturedService: Provider = { provide: 'FeaturedService', useExisting: FeaturedService }; -const $FunoutTimelineService: Provider = { provide: 'FunoutTimelineService', useExisting: FunoutTimelineService }; +const $FanoutTimelineService: Provider = { provide: 'FanoutTimelineService', useExisting: FanoutTimelineService }; +const $FanoutTimelineEndpointService: Provider = { provide: 'FanoutTimelineEndpointService', useExisting: FanoutTimelineEndpointService }; const $ChannelFollowingService: Provider = { provide: 'ChannelFollowingService', useExisting: ChannelFollowingService }; const $RegistryApiService: Provider = { provide: 'RegistryApiService', useExisting: RegistryApiService }; @@ -330,7 +332,8 @@ const $ApQuestionService: Provider = { provide: 'ApQuestionService', useExisting SearchService, ClipService, FeaturedService, - FunoutTimelineService, + FanoutTimelineService, + FanoutTimelineEndpointService, ChannelFollowingService, RegistryApiService, ChartLoggerService, @@ -459,7 +462,8 @@ const $ApQuestionService: Provider = { provide: 'ApQuestionService', useExisting $SearchService, $ClipService, $FeaturedService, - $FunoutTimelineService, + $FanoutTimelineService, + $FanoutTimelineEndpointService, $ChannelFollowingService, $RegistryApiService, $ChartLoggerService, @@ -589,7 +593,8 @@ const $ApQuestionService: Provider = { provide: 'ApQuestionService', useExisting SearchService, ClipService, FeaturedService, - FunoutTimelineService, + FanoutTimelineService, + FanoutTimelineEndpointService, ChannelFollowingService, RegistryApiService, FederationChart, @@ -717,7 +722,8 @@ const $ApQuestionService: Provider = { provide: 'ApQuestionService', useExisting $SearchService, $ClipService, $FeaturedService, - $FunoutTimelineService, + $FanoutTimelineService, + $FanoutTimelineEndpointService, $ChannelFollowingService, $RegistryApiService, $FederationChart, diff --git a/packages/backend/src/core/EmailService.ts b/packages/backend/src/core/EmailService.ts index c9da3f77c0..f31cec2b3a 100644 --- a/packages/backend/src/core/EmailService.ts +++ b/packages/backend/src/core/EmailService.ts @@ -3,9 +3,11 @@ * SPDX-License-Identifier: AGPL-3.0-only */ +import { URLSearchParams } from 'node:url'; import * as nodemailer from 'nodemailer'; import { Inject, Injectable } from '@nestjs/common'; import { validate as validateEmail } from 'deep-email-validator'; +import { SubOutputFormat } from 'deep-email-validator/dist/output/output.js'; import { MetaService } from '@/core/MetaService.js'; import { DI } from '@/di-symbols.js'; import type { Config } from '@/config.js'; @@ -13,6 +15,7 @@ import type Logger from '@/logger.js'; import type { UserProfilesRepository } from '@/models/_.js'; import { LoggerService } from '@/core/LoggerService.js'; import { bindThis } from '@/decorators.js'; +import { HttpRequestService } from '@/core/HttpRequestService.js'; @Injectable() export class EmailService { @@ -27,6 +30,7 @@ export class EmailService { private metaService: MetaService, private loggerService: LoggerService, + private httpRequestService: HttpRequestService, ) { this.logger = this.loggerService.getLogger('email'); } @@ -160,14 +164,25 @@ export class EmailService { email: emailAddress, }); - const validated = meta.enableActiveEmailValidation ? await validateEmail({ - email: emailAddress, - validateRegex: true, - validateMx: true, - validateTypo: false, // TLDを見ているみたいだけどclubとか弾かれるので - validateDisposable: true, // 捨てアドかどうかチェック - validateSMTP: false, // 日本だと25ポートが殆どのプロバイダーで塞がれていてタイムアウトになるので - }) : { valid: true, reason: null }; + const verifymailApi = meta.enableVerifymailApi && meta.verifymailAuthKey != null; + let validated; + + if (meta.enableActiveEmailValidation && meta.verifymailAuthKey) { + if (verifymailApi) { + validated = await this.verifyMail(emailAddress, meta.verifymailAuthKey); + } else { + validated = meta.enableActiveEmailValidation ? await validateEmail({ + email: emailAddress, + validateRegex: true, + validateMx: true, + validateTypo: false, // TLDを見ているみたいだけどclubとか弾かれるので + validateDisposable: true, // 捨てアドかどうかチェック + validateSMTP: false, // 日本だと25ポートが殆どのプロバイダーで塞がれていてタイムアウトになるので + }) : { valid: true, reason: null }; + } + } else { + validated = { valid: true, reason: null }; + } const available = exist === 0 && validated.valid; @@ -182,4 +197,65 @@ export class EmailService { null, }; } + + private async verifyMail(emailAddress: string, verifymailAuthKey: string): Promise<{ + valid: boolean; + reason: 'used' | 'format' | 'disposable' | 'mx' | 'smtp' | null; + }> { + const endpoint = 'https://verifymail.io/api/' + emailAddress + '?key=' + verifymailAuthKey; + const res = await this.httpRequestService.send(endpoint, { + method: 'GET', + headers: { + 'Content-Type': 'application/x-www-form-urlencoded', + Accept: 'application/json, */*', + }, + }); + + const json = (await res.json()) as { + block: boolean; + catch_all: boolean; + deliverable_email: boolean; + disposable: boolean; + domain: string; + email_address: string; + email_provider: string; + mx: boolean; + mx_fallback: boolean; + mx_host: string[]; + mx_ip: string[]; + mx_priority: { [key: string]: number }; + privacy: boolean; + related_domains: string[]; + }; + + if (json.email_address === undefined) { + return { + valid: false, + reason: 'format', + }; + } + if (json.deliverable_email !== undefined && !json.deliverable_email) { + return { + valid: false, + reason: 'smtp', + }; + } + if (json.disposable) { + return { + valid: false, + reason: 'disposable', + }; + } + if (json.mx !== undefined && !json.mx) { + return { + valid: false, + reason: 'mx', + }; + } + + return { + valid: true, + reason: null, + }; + } } diff --git a/packages/backend/src/core/FanoutTimelineEndpointService.ts b/packages/backend/src/core/FanoutTimelineEndpointService.ts new file mode 100644 index 0000000000..6d857d189e --- /dev/null +++ b/packages/backend/src/core/FanoutTimelineEndpointService.ts @@ -0,0 +1,192 @@ +/* + * SPDX-FileCopyrightText: syuilo and other misskey contributors + * SPDX-License-Identifier: AGPL-3.0-only + */ + +import { Inject, Injectable } from '@nestjs/common'; +import { DI } from '@/di-symbols.js'; +import { bindThis } from '@/decorators.js'; +import type { MiUser } from '@/models/User.js'; +import type { MiNote } from '@/models/Note.js'; +import { Packed } from '@/misc/json-schema.js'; +import type { NotesRepository } from '@/models/_.js'; +import { NoteEntityService } from '@/core/entities/NoteEntityService.js'; +import { FanoutTimelineName, FanoutTimelineService } from '@/core/FanoutTimelineService.js'; +import { isUserRelated } from '@/misc/is-user-related.js'; +import { isPureRenote } from '@/misc/is-pure-renote.js'; +import { CacheService } from '@/core/CacheService.js'; +import { isReply } from '@/misc/is-reply.js'; +import { isInstanceMuted } from '@/misc/is-instance-muted.js'; + +type TimelineOptions = { + untilId: string | null, + sinceId: string | null, + limit: number, + allowPartial: boolean, + me?: { id: MiUser['id'] } | undefined | null, + useDbFallback: boolean, + redisTimelines: FanoutTimelineName[], + noteFilter?: (note: MiNote) => boolean, + alwaysIncludeMyNotes?: boolean; + ignoreAuthorFromBlock?: boolean; + ignoreAuthorFromMute?: boolean; + excludeNoFiles?: boolean; + excludeReplies?: boolean; + excludeBots?: boolean; + excludePureRenotes: boolean; + dbFallback: (untilId: string | null, sinceId: string | null, limit: number) => Promise<MiNote[]>, +}; + +@Injectable() +export class FanoutTimelineEndpointService { + constructor( + @Inject(DI.notesRepository) + private notesRepository: NotesRepository, + + private noteEntityService: NoteEntityService, + private cacheService: CacheService, + private fanoutTimelineService: FanoutTimelineService, + ) { + } + + @bindThis + async timeline(ps: TimelineOptions): Promise<Packed<'Note'>[]> { + return await this.noteEntityService.packMany(await this.getMiNotes(ps), ps.me); + } + + @bindThis + private async getMiNotes(ps: TimelineOptions): Promise<MiNote[]> { + let noteIds: string[]; + let shouldFallbackToDb = false; + + // 呼び出し元と以下の処理をシンプルにするためにdbFallbackを置き換える + if (!ps.useDbFallback) ps.dbFallback = () => Promise.resolve([]); + + const shouldPrepend = ps.sinceId && !ps.untilId; + const idCompare: (a: string, b: string) => number = shouldPrepend ? (a, b) => a < b ? -1 : 1 : (a, b) => a > b ? -1 : 1; + + const redisResult = await this.fanoutTimelineService.getMulti(ps.redisTimelines, ps.untilId, ps.sinceId); + + // TODO: いい感じにgetMulti内でソート済だからuniqするときにredisResultが全てソート済なのを利用して再ソートを避けたい + const redisResultIds = Array.from(new Set(redisResult.flat(1))); + + redisResultIds.sort(idCompare); + noteIds = redisResultIds.slice(0, ps.limit); + + shouldFallbackToDb = shouldFallbackToDb || (noteIds.length === 0); + + if (!shouldFallbackToDb) { + let filter = ps.noteFilter ?? (_note => true); + + if (ps.alwaysIncludeMyNotes && ps.me) { + const me = ps.me; + const parentFilter = filter; + filter = (note) => note.userId === me.id || parentFilter(note); + } + + if (ps.excludeNoFiles) { + const parentFilter = filter; + filter = (note) => note.fileIds.length !== 0 && parentFilter(note); + } + + if (ps.excludeReplies) { + const parentFilter = filter; + filter = (note) => !isReply(note, ps.me?.id) && parentFilter(note); + } + + if (ps.excludeBots) { + const parentFilter = filter; + filter = (note) => !note.user?.isBot && parentFilter(note); + } + + if (ps.excludePureRenotes) { + const parentFilter = filter; + filter = (note) => !isPureRenote(note) && parentFilter(note); + } + + if (ps.me) { + const me = ps.me; + const [ + userIdsWhoMeMuting, + userIdsWhoMeMutingRenotes, + userIdsWhoBlockingMe, + userMutedInstances, + ] = await Promise.all([ + this.cacheService.userMutingsCache.fetch(ps.me.id), + this.cacheService.renoteMutingsCache.fetch(ps.me.id), + this.cacheService.userBlockedCache.fetch(ps.me.id), + this.cacheService.userProfileCache.fetch(me.id).then(p => new Set(p.mutedInstances)), + ]); + + const parentFilter = filter; + filter = (note) => { + if (isUserRelated(note, userIdsWhoBlockingMe, ps.ignoreAuthorFromBlock)) return false; + if (isUserRelated(note, userIdsWhoMeMuting, ps.ignoreAuthorFromMute)) return false; + if (isPureRenote(note) && isUserRelated(note, userIdsWhoMeMutingRenotes, ps.ignoreAuthorFromMute)) return false; + if (isInstanceMuted(note, userMutedInstances)) return false; + + return parentFilter(note); + }; + } + + const redisTimeline: MiNote[] = []; + let readFromRedis = 0; + let lastSuccessfulRate = 1; // rateをキャッシュする? + + while ((redisResultIds.length - readFromRedis) !== 0) { + const remainingToRead = ps.limit - redisTimeline.length; + + // DBからの取り直しを減らす初回と同じ割合以上で成功すると仮定するが、クエリの長さを考えて三倍まで + const countToGet = Math.ceil(remainingToRead * Math.min(1.1 / lastSuccessfulRate, 3)); + noteIds = redisResultIds.slice(readFromRedis, readFromRedis + countToGet); + + readFromRedis += noteIds.length; + + const gotFromDb = await this.getAndFilterFromDb(noteIds, filter, idCompare); + redisTimeline.push(...gotFromDb); + lastSuccessfulRate = gotFromDb.length / noteIds.length; + + if (ps.allowPartial ? redisTimeline.length !== 0 : redisTimeline.length >= ps.limit) { + // 十分Redisからとれた + const result = redisTimeline.slice(0, ps.limit); + if (shouldPrepend) result.reverse(); + return result; + } + } + + // まだ足りない分はDBにフォールバック + const remainingToRead = ps.limit - redisTimeline.length; + let dbUntil: string | null; + let dbSince: string | null; + if (shouldPrepend) { + redisTimeline.reverse(); + dbUntil = ps.untilId; + dbSince = noteIds[noteIds.length - 1]; + } else { + dbUntil = noteIds[noteIds.length - 1]; + dbSince = ps.sinceId; + } + const gotFromDb = await ps.dbFallback(dbUntil, dbSince, remainingToRead); + return shouldPrepend ? [...gotFromDb, ...redisTimeline] : [...redisTimeline, ...gotFromDb]; + } + + return await ps.dbFallback(ps.untilId, ps.sinceId, ps.limit); + } + + private async getAndFilterFromDb(noteIds: string[], noteFilter: (note: MiNote) => boolean, idCompare: (a: string, b: string) => number): Promise<MiNote[]> { + const query = this.notesRepository.createQueryBuilder('note') + .where('note.id IN (:...noteIds)', { noteIds: noteIds }) + .innerJoinAndSelect('note.user', 'user') + .leftJoinAndSelect('note.reply', 'reply') + .leftJoinAndSelect('note.renote', 'renote') + .leftJoinAndSelect('reply.user', 'replyUser') + .leftJoinAndSelect('renote.user', 'renoteUser') + .leftJoinAndSelect('note.channel', 'channel'); + + const notes = (await query.getMany()).filter(noteFilter); + + notes.sort((a, b) => idCompare(a.id, b.id)); + + return notes; + } +} diff --git a/packages/backend/src/core/FunoutTimelineService.ts b/packages/backend/src/core/FanoutTimelineService.ts index c633c329e5..9b2678fbcd 100644 --- a/packages/backend/src/core/FunoutTimelineService.ts +++ b/packages/backend/src/core/FanoutTimelineService.ts @@ -9,8 +9,37 @@ import { DI } from '@/di-symbols.js'; import { bindThis } from '@/decorators.js'; import { IdService } from '@/core/IdService.js'; +export type FanoutTimelineName = + // home timeline + | `homeTimeline:${string}` + | `homeTimelineWithFiles:${string}` // only notes with files are included + // local timeline + | `localTimeline` // replies are not included + | `localTimelineWithFiles` // only non-reply notes with files are included + | `localTimelineWithReplies` // only replies are included + | `localTimelineWithReplyTo:${string}` // Only replies to specific local user are included. Parameter is reply user id. + + // antenna + | `antennaTimeline:${string}` + + // user timeline + | `userTimeline:${string}` // replies are not included + | `userTimelineWithFiles:${string}` // only non-reply notes with files are included + | `userTimelineWithReplies:${string}` // only replies are included + | `userTimelineWithChannel:${string}` // only channel notes are included, replies are included + + // user list timelines + | `userListTimeline:${string}` + | `userListTimelineWithFiles:${string}` // only notes with files are included + + // channel timelines + | `channelTimeline:${string}` // replies are included + + // role timelines + | `roleTimeline:${string}` // any notes are included + @Injectable() -export class FunoutTimelineService { +export class FanoutTimelineService { constructor( @Inject(DI.redisForTimelines) private redisForTimelines: Redis.Redis, @@ -20,7 +49,7 @@ export class FunoutTimelineService { } @bindThis - public push(tl: string, id: string, maxlen: number, pipeline: Redis.ChainableCommander) { + public push(tl: FanoutTimelineName, id: string, maxlen: number, pipeline: Redis.ChainableCommander) { // リモートから遅れて届いた(もしくは後から追加された)投稿日時が古い投稿が追加されるとページネーション時に問題を引き起こすため、 // 3分以内に投稿されたものでない場合、Redisにある最古のIDより新しい場合のみ追加する if (this.idService.parse(id).date.getTime() > Date.now() - 1000 * 60 * 3) { @@ -41,7 +70,7 @@ export class FunoutTimelineService { } @bindThis - public get(name: string, untilId?: string | null, sinceId?: string | null) { + public get(name: FanoutTimelineName, untilId?: string | null, sinceId?: string | null) { if (untilId && sinceId) { return this.redisForTimelines.lrange('list:' + name, 0, -1) .then(ids => ids.filter(id => id < untilId && id > sinceId).sort((a, b) => a > b ? -1 : 1)); @@ -58,7 +87,7 @@ export class FunoutTimelineService { } @bindThis - public getMulti(name: string[], untilId?: string | null, sinceId?: string | null): Promise<string[][]> { + public getMulti(name: FanoutTimelineName[], untilId?: string | null, sinceId?: string | null): Promise<string[][]> { const pipeline = this.redisForTimelines.pipeline(); for (const n of name) { pipeline.lrange('list:' + n, 0, -1); @@ -79,7 +108,7 @@ export class FunoutTimelineService { } @bindThis - public purge(name: string) { + public purge(name: FanoutTimelineName) { return this.redisForTimelines.del('list:' + name); } } diff --git a/packages/backend/src/core/FeaturedService.ts b/packages/backend/src/core/FeaturedService.ts index 9617f83880..595383c82c 100644 --- a/packages/backend/src/core/FeaturedService.ts +++ b/packages/backend/src/core/FeaturedService.ts @@ -5,14 +5,17 @@ import { Inject, Injectable } from '@nestjs/common'; import * as Redis from 'ioredis'; -import type { MiNote, MiUser } from '@/models/_.js'; +import type { MiGalleryPost, MiNote, MiUser } from '@/models/_.js'; import { DI } from '@/di-symbols.js'; import { bindThis } from '@/decorators.js'; const GLOBAL_NOTES_RANKING_WINDOW = 1000 * 60 * 60 * 24 * 3; // 3日ごと +export const GALLERY_POSTS_RANKING_WINDOW = 1000 * 60 * 60 * 24 * 3; // 3日ごと const PER_USER_NOTES_RANKING_WINDOW = 1000 * 60 * 60 * 24 * 7; // 1週間ごと const HASHTAG_RANKING_WINDOW = 1000 * 60 * 60; // 1時間ごと +const featuredEpoc = new Date('2023-01-01T00:00:00Z').getTime(); + @Injectable() export class FeaturedService { constructor( @@ -23,7 +26,7 @@ export class FeaturedService { @bindThis private getCurrentWindow(windowRange: number): number { - const passed = new Date().getTime() - new Date(new Date().getFullYear(), 0, 1).getTime(); + const passed = new Date().getTime() - featuredEpoc; return Math.floor(passed / windowRange); } @@ -75,11 +78,27 @@ export class FeaturedService { } @bindThis + private async removeFromRanking(name: string, windowRange: number, element: string): Promise<void> { + const currentWindow = this.getCurrentWindow(windowRange); + const previousWindow = currentWindow - 1; + + const redisPipeline = this.redisClient.pipeline(); + redisPipeline.zrem(`${name}:${currentWindow}`, element); + redisPipeline.zrem(`${name}:${previousWindow}`, element); + await redisPipeline.exec(); + } + + @bindThis public updateGlobalNotesRanking(noteId: MiNote['id'], score = 1): Promise<void> { return this.updateRankingOf('featuredGlobalNotesRanking', GLOBAL_NOTES_RANKING_WINDOW, noteId, score); } @bindThis + public updateGalleryPostsRanking(galleryPostId: MiGalleryPost['id'], score = 1): Promise<void> { + return this.updateRankingOf('featuredGalleryPostsRanking', GALLERY_POSTS_RANKING_WINDOW, galleryPostId, score); + } + + @bindThis public updateInChannelNotesRanking(channelId: MiNote['channelId'], noteId: MiNote['id'], score = 1): Promise<void> { return this.updateRankingOf(`featuredInChannelNotesRanking:${channelId}`, GLOBAL_NOTES_RANKING_WINDOW, noteId, score); } @@ -100,6 +119,11 @@ export class FeaturedService { } @bindThis + public getGalleryPostsRanking(threshold: number): Promise<MiGalleryPost['id'][]> { + return this.getRankingOf('featuredGalleryPostsRanking', GALLERY_POSTS_RANKING_WINDOW, threshold); + } + + @bindThis public getInChannelNotesRanking(channelId: MiNote['channelId'], threshold: number): Promise<MiNote['id'][]> { return this.getRankingOf(`featuredInChannelNotesRanking:${channelId}`, GLOBAL_NOTES_RANKING_WINDOW, threshold); } @@ -113,4 +137,9 @@ export class FeaturedService { public getHashtagsRanking(threshold: number): Promise<string[]> { return this.getRankingOf('featuredHashtagsRanking', HASHTAG_RANKING_WINDOW, threshold); } + + @bindThis + public removeHashtagsFromRanking(hashtag: string): Promise<void> { + return this.removeFromRanking('featuredHashtagsRanking', HASHTAG_RANKING_WINDOW, hashtag); + } } diff --git a/packages/backend/src/core/IdService.ts b/packages/backend/src/core/IdService.ts index c98b8ea6fc..43e72d2d7b 100644 --- a/packages/backend/src/core/IdService.ts +++ b/packages/backend/src/core/IdService.ts @@ -7,11 +7,11 @@ import { Inject, Injectable } from '@nestjs/common'; import { ulid } from 'ulid'; import { DI } from '@/di-symbols.js'; import type { Config } from '@/config.js'; -import { genAid, parseAid } from '@/misc/id/aid.js'; -import { genAidx, parseAidx } from '@/misc/id/aidx.js'; -import { genMeid, parseMeid } from '@/misc/id/meid.js'; -import { genMeidg, parseMeidg } from '@/misc/id/meidg.js'; -import { genObjectId, parseObjectId } from '@/misc/id/object-id.js'; +import { genAid, isSafeAidT, parseAid } from '@/misc/id/aid.js'; +import { genAidx, isSafeAidxT, parseAidx } from '@/misc/id/aidx.js'; +import { genMeid, isSafeMeidT, parseMeid } from '@/misc/id/meid.js'; +import { genMeidg, isSafeMeidgT, parseMeidg } from '@/misc/id/meidg.js'; +import { genObjectId, isSafeObjectIdT, parseObjectId } from '@/misc/id/object-id.js'; import { bindThis } from '@/decorators.js'; import { parseUlid } from '@/misc/id/ulid.js'; @@ -26,6 +26,19 @@ export class IdService { this.method = config.id.toLowerCase(); } + @bindThis + public isSafeT(t: number): boolean { + switch (this.method) { + case 'aid': return isSafeAidT(t); + case 'aidx': return isSafeAidxT(t); + case 'meid': return isSafeMeidT(t); + case 'meidg': return isSafeMeidgT(t); + case 'ulid': return t > 0; + case 'objectid': return isSafeObjectIdT(t); + default: throw new Error('unrecognized id generation method'); + } + } + /** * 時間を元にIDを生成します(省略時は現在日時) * @param time 日時 diff --git a/packages/backend/src/core/MetaService.ts b/packages/backend/src/core/MetaService.ts index 508544dc07..80e8020961 100644 --- a/packages/backend/src/core/MetaService.ts +++ b/packages/backend/src/core/MetaService.ts @@ -11,6 +11,7 @@ import { MiMeta } from '@/models/Meta.js'; import { GlobalEventService } from '@/core/GlobalEventService.js'; import { bindThis } from '@/decorators.js'; import type { GlobalEvents } from '@/core/GlobalEventService.js'; +import { FeaturedService } from '@/core/FeaturedService.js'; import type { OnApplicationShutdown } from '@nestjs/common'; @Injectable() @@ -25,6 +26,7 @@ export class MetaService implements OnApplicationShutdown { @Inject(DI.db) private db: DataSource, + private featuredService: FeaturedService, private globalEventService: GlobalEventService, ) { //this.onMessage = this.onMessage.bind(this); @@ -95,6 +97,8 @@ export class MetaService implements OnApplicationShutdown { @bindThis public async update(data: Partial<MiMeta>): Promise<MiMeta> { + let before: MiMeta | undefined; + const updated = await this.db.transaction(async transactionalEntityManager => { const metas = await transactionalEntityManager.find(MiMeta, { order: { @@ -102,10 +106,10 @@ export class MetaService implements OnApplicationShutdown { }, }); - const meta = metas[0]; + before = metas[0]; - if (meta) { - await transactionalEntityManager.update(MiMeta, meta.id, data); + if (before) { + await transactionalEntityManager.update(MiMeta, before.id, data); const metas = await transactionalEntityManager.find(MiMeta, { order: { @@ -119,6 +123,21 @@ export class MetaService implements OnApplicationShutdown { } }); + if (data.hiddenTags) { + process.nextTick(() => { + const hiddenTags = new Set<string>(data.hiddenTags); + if (before) { + for (const previousHiddenTag of before.hiddenTags) { + hiddenTags.delete(previousHiddenTag); + } + } + + for (const hiddenTag of hiddenTags) { + this.featuredService.removeHashtagsFromRanking(hiddenTag); + } + }); + } + this.globalEventService.publishInternalEvent('metaUpdated', updated); return updated; diff --git a/packages/backend/src/core/MfmService.ts b/packages/backend/src/core/MfmService.ts index 12f4353ec4..60bf8b3c2a 100644 --- a/packages/backend/src/core/MfmService.ts +++ b/packages/backend/src/core/MfmService.ts @@ -250,6 +250,12 @@ export class MfmService { } } + function fnDefault(node: mfm.MfmFn) { + const el = doc.createElement('i'); + appendChildren(node.children, el); + return el; + } + const handlers: { [K in mfm.MfmNode['type']]: (node: mfm.NodeType<K>) => any } = { bold: (node) => { const el = doc.createElement('b'); @@ -276,17 +282,68 @@ export class MfmService { }, fn: (node) => { - if (node.props.name === 'unixtime') { - const text = node.children[0]!.type === 'text' ? node.children[0].props.text : ''; - const date = new Date(parseInt(text, 10) * 1000); - const el = doc.createElement('time'); - el.setAttribute('datetime', date.toISOString()); - el.textContent = date.toISOString(); - return el; - } else { - const el = doc.createElement('i'); - appendChildren(node.children, el); - return el; + switch (node.props.name) { + case 'unixtime': { + const text = node.children[0].type === 'text' ? node.children[0].props.text : ''; + try { + const date = new Date(parseInt(text, 10) * 1000); + const el = doc.createElement('time'); + el.setAttribute('datetime', date.toISOString()); + el.textContent = date.toISOString(); + return el; + } catch (err) { + return fnDefault(node); + } + } + + case 'ruby': { + if (node.children.length === 1) { + const child = node.children[0]; + const text = child.type === 'text' ? child.props.text : ''; + const rubyEl = doc.createElement('ruby'); + const rtEl = doc.createElement('rt'); + + // ruby未対応のHTMLサニタイザーを通したときにルビが「劉備(りゅうび)」となるようにする + const rpStartEl = doc.createElement('rp'); + rpStartEl.appendChild(doc.createTextNode('(')); + const rpEndEl = doc.createElement('rp'); + rpEndEl.appendChild(doc.createTextNode(')')); + + rubyEl.appendChild(doc.createTextNode(text.split(' ')[0])); + rtEl.appendChild(doc.createTextNode(text.split(' ')[1])); + rubyEl.appendChild(rpStartEl); + rubyEl.appendChild(rtEl); + rubyEl.appendChild(rpEndEl); + return rubyEl; + } else { + const rt = node.children.at(-1); + + if (!rt) { + return fnDefault(node); + } + + const text = rt.type === 'text' ? rt.props.text : ''; + const rubyEl = doc.createElement('ruby'); + const rtEl = doc.createElement('rt'); + + // ruby未対応のHTMLサニタイザーを通したときにルビが「劉備(りゅうび)」となるようにする + const rpStartEl = doc.createElement('rp'); + rpStartEl.appendChild(doc.createTextNode('(')); + const rpEndEl = doc.createElement('rp'); + rpEndEl.appendChild(doc.createTextNode(')')); + + appendChildren(node.children.slice(0, node.children.length - 1), rubyEl); + rtEl.appendChild(doc.createTextNode(text.trim())); + rubyEl.appendChild(rpStartEl); + rubyEl.appendChild(rtEl); + rubyEl.appendChild(rpEndEl); + return rubyEl; + } + } + + default: { + return fnDefault(node); + } } }, diff --git a/packages/backend/src/core/NoteCreateService.ts b/packages/backend/src/core/NoteCreateService.ts index 939662707f..270730c723 100644 --- a/packages/backend/src/core/NoteCreateService.ts +++ b/packages/backend/src/core/NoteCreateService.ts @@ -54,9 +54,10 @@ import { RoleService } from '@/core/RoleService.js'; import { MetaService } from '@/core/MetaService.js'; import { SearchService } from '@/core/SearchService.js'; import { FeaturedService } from '@/core/FeaturedService.js'; -import { FunoutTimelineService } from '@/core/FunoutTimelineService.js'; +import { FanoutTimelineService } from '@/core/FanoutTimelineService.js'; import { UtilityService } from '@/core/UtilityService.js'; import { UserBlockingService } from '@/core/UserBlockingService.js'; +import { isReply } from '@/misc/is-reply.js'; type NotificationType = 'reply' | 'renote' | 'quote' | 'mention'; @@ -194,7 +195,7 @@ export class NoteCreateService implements OnApplicationShutdown { private idService: IdService, private globalEventService: GlobalEventService, private queueService: QueueService, - private funoutTimelineService: FunoutTimelineService, + private fanoutTimelineService: FanoutTimelineService, private noteReadService: NoteReadService, private notificationService: NotificationService, private relayService: RelayService, @@ -461,7 +462,7 @@ export class NoteCreateService implements OnApplicationShutdown { } // Check blocking - if (data.renote && data.text == null && data.poll == null && (data.files == null || data.files.length === 0)) { + if (this.isQuote(data)) { if (data.renote.userHost === null) { if (data.renote.userId !== user.id) { const blocked = await this.userBlockingService.checkBlocked(data.renote.userId, user.id); @@ -804,7 +805,7 @@ export class NoteCreateService implements OnApplicationShutdown { // If it is renote if (data.renote) { - const type = data.text ? 'quote' : 'renote'; + const type = this.isQuote(data) ? 'quote' : 'renote'; // Notify if (data.renote.userHost === null) { @@ -1011,6 +1012,12 @@ export class NoteCreateService implements OnApplicationShutdown { } @bindThis + private isQuote(note: Option): note is Option & { renote: MiNote } { + // sync with misc/is-quote.ts + return !!note.renote && (!!note.text || !!note.cw || (!!note.files && !!note.files.length) || !!note.poll); + } + + @bindThis private incRenoteCount(renote: MiNote) { this.notesRepository.createQueryBuilder().update() .set({ @@ -1075,7 +1082,7 @@ export class NoteCreateService implements OnApplicationShutdown { private async renderNoteOrRenoteActivity(data: Option, note: MiNote) { if (data.localOnly) return null; - const content = data.renote && data.text == null && data.poll == null && (data.files == null || data.files.length === 0) + const content = data.renote && !this.isQuote(data) ? this.apRendererService.renderAnnounce(data.renote.uri ? data.renote.uri : `${this.config.url}/notes/${data.renote.id}`, note) : this.apRendererService.renderCreate(await this.apRendererService.renderNote(note, false), note); @@ -1125,9 +1132,9 @@ export class NoteCreateService implements OnApplicationShutdown { const r = this.redisForTimelines.pipeline(); if (note.channelId) { - this.funoutTimelineService.push(`channelTimeline:${note.channelId}`, note.id, this.config.perChannelMaxNoteCacheCount, r); + this.fanoutTimelineService.push(`channelTimeline:${note.channelId}`, note.id, this.config.perChannelMaxNoteCacheCount, r); - this.funoutTimelineService.push(`userTimelineWithChannel:${user.id}`, note.id, note.userHost == null ? meta.perLocalUserUserTimelineCacheMax : meta.perRemoteUserUserTimelineCacheMax, r); + this.fanoutTimelineService.push(`userTimelineWithChannel:${user.id}`, note.id, note.userHost == null ? meta.perLocalUserUserTimelineCacheMax : meta.perRemoteUserUserTimelineCacheMax, r); const channelFollowings = await this.channelFollowingsRepository.find({ where: { @@ -1137,9 +1144,9 @@ export class NoteCreateService implements OnApplicationShutdown { }); for (const channelFollowing of channelFollowings) { - this.funoutTimelineService.push(`homeTimeline:${channelFollowing.followerId}`, note.id, meta.perUserHomeTimelineCacheMax, r); + this.fanoutTimelineService.push(`homeTimeline:${channelFollowing.followerId}`, note.id, meta.perUserHomeTimelineCacheMax, r); if (note.fileIds.length > 0) { - this.funoutTimelineService.push(`homeTimelineWithFiles:${channelFollowing.followerId}`, note.id, meta.perUserHomeTimelineCacheMax / 2, r); + this.fanoutTimelineService.push(`homeTimelineWithFiles:${channelFollowing.followerId}`, note.id, meta.perUserHomeTimelineCacheMax / 2, r); } } } else { @@ -1173,13 +1180,13 @@ export class NoteCreateService implements OnApplicationShutdown { if (note.visibility === 'specified' && !note.visibleUserIds.some(v => v === following.followerId)) continue; // 「自分自身への返信 or そのフォロワーへの返信」のどちらでもない場合 - if (note.replyId && !(note.replyUserId === note.userId || note.replyUserId === following.followerId)) { + if (isReply(note, following.followerId)) { if (!following.withReplies) continue; } - this.funoutTimelineService.push(`homeTimeline:${following.followerId}`, note.id, meta.perUserHomeTimelineCacheMax, r); + this.fanoutTimelineService.push(`homeTimeline:${following.followerId}`, note.id, meta.perUserHomeTimelineCacheMax, r); if (note.fileIds.length > 0) { - this.funoutTimelineService.push(`homeTimelineWithFiles:${following.followerId}`, note.id, meta.perUserHomeTimelineCacheMax / 2, r); + this.fanoutTimelineService.push(`homeTimelineWithFiles:${following.followerId}`, note.id, meta.perUserHomeTimelineCacheMax / 2, r); } } @@ -1191,40 +1198,43 @@ export class NoteCreateService implements OnApplicationShutdown { ) continue; // 「自分自身への返信 or そのリストの作成者への返信」のどちらでもない場合 - if (note.replyId && !(note.replyUserId === note.userId || note.replyUserId === userListMembership.userListUserId)) { + if (isReply(note, userListMembership.userListUserId)) { if (!userListMembership.withReplies) continue; } - this.funoutTimelineService.push(`userListTimeline:${userListMembership.userListId}`, note.id, meta.perUserListTimelineCacheMax, r); + this.fanoutTimelineService.push(`userListTimeline:${userListMembership.userListId}`, note.id, meta.perUserListTimelineCacheMax, r); if (note.fileIds.length > 0) { - this.funoutTimelineService.push(`userListTimelineWithFiles:${userListMembership.userListId}`, note.id, meta.perUserListTimelineCacheMax / 2, r); + this.fanoutTimelineService.push(`userListTimelineWithFiles:${userListMembership.userListId}`, note.id, meta.perUserListTimelineCacheMax / 2, r); } } if (note.visibility !== 'specified' || !note.visibleUserIds.some(v => v === user.id)) { // 自分自身のHTL - this.funoutTimelineService.push(`homeTimeline:${user.id}`, note.id, meta.perUserHomeTimelineCacheMax, r); + this.fanoutTimelineService.push(`homeTimeline:${user.id}`, note.id, meta.perUserHomeTimelineCacheMax, r); if (note.fileIds.length > 0) { - this.funoutTimelineService.push(`homeTimelineWithFiles:${user.id}`, note.id, meta.perUserHomeTimelineCacheMax / 2, r); + this.fanoutTimelineService.push(`homeTimelineWithFiles:${user.id}`, note.id, meta.perUserHomeTimelineCacheMax / 2, r); } } // 自分自身以外への返信 - if (note.replyId && note.replyUserId !== note.userId) { - this.funoutTimelineService.push(`userTimelineWithReplies:${user.id}`, note.id, note.userHost == null ? meta.perLocalUserUserTimelineCacheMax : meta.perRemoteUserUserTimelineCacheMax, r); + if (isReply(note)) { + this.fanoutTimelineService.push(`userTimelineWithReplies:${user.id}`, note.id, note.userHost == null ? meta.perLocalUserUserTimelineCacheMax : meta.perRemoteUserUserTimelineCacheMax, r); if (note.visibility === 'public' && note.userHost == null) { - this.funoutTimelineService.push('localTimelineWithReplies', note.id, 300, r); + this.fanoutTimelineService.push('localTimelineWithReplies', note.id, 300, r); + if (note.replyUserHost == null) { + this.fanoutTimelineService.push(`localTimelineWithReplyTo:${note.replyUserId}`, note.id, 300 / 10, r); + } } } else { - this.funoutTimelineService.push(`userTimeline:${user.id}`, note.id, note.userHost == null ? meta.perLocalUserUserTimelineCacheMax : meta.perRemoteUserUserTimelineCacheMax, r); + this.fanoutTimelineService.push(`userTimeline:${user.id}`, note.id, note.userHost == null ? meta.perLocalUserUserTimelineCacheMax : meta.perRemoteUserUserTimelineCacheMax, r); if (note.fileIds.length > 0) { - this.funoutTimelineService.push(`userTimelineWithFiles:${user.id}`, note.id, note.userHost == null ? meta.perLocalUserUserTimelineCacheMax / 2 : meta.perRemoteUserUserTimelineCacheMax / 2, r); + this.fanoutTimelineService.push(`userTimelineWithFiles:${user.id}`, note.id, note.userHost == null ? meta.perLocalUserUserTimelineCacheMax / 2 : meta.perRemoteUserUserTimelineCacheMax / 2, r); } if (note.visibility === 'public' && note.userHost == null) { - this.funoutTimelineService.push('localTimeline', note.id, 1000, r); + this.fanoutTimelineService.push('localTimeline', note.id, 1000, r); if (note.fileIds.length > 0) { - this.funoutTimelineService.push('localTimelineWithFiles', note.id, 500, r); + this.fanoutTimelineService.push('localTimelineWithFiles', note.id, 500, r); } } } diff --git a/packages/backend/src/core/NoteEditService.ts b/packages/backend/src/core/NoteEditService.ts index c4314c65ac..dae3f485ff 100644 --- a/packages/backend/src/core/NoteEditService.ts +++ b/packages/backend/src/core/NoteEditService.ts @@ -44,7 +44,7 @@ import { DB_MAX_NOTE_TEXT_LENGTH } from '@/const.js'; import { RoleService } from '@/core/RoleService.js'; import { MetaService } from '@/core/MetaService.js'; import { SearchService } from '@/core/SearchService.js'; -import { FunoutTimelineService } from '@/core/FunoutTimelineService.js'; +import { FanoutTimelineService } from '@/core/FanoutTimelineService.js'; import { UtilityService } from '@/core/UtilityService.js'; type NotificationType = 'reply' | 'renote' | 'quote' | 'mention'; @@ -191,7 +191,7 @@ export class NoteEditService implements OnApplicationShutdown { private idService: IdService, private globalEventService: GlobalEventService, private queueService: QueueService, - private funoutTimelineService: FunoutTimelineService, + private fanoutTimelineService: FanoutTimelineService, private noteReadService: NoteReadService, private notificationService: NotificationService, private relayService: RelayService, @@ -786,9 +786,9 @@ export class NoteEditService implements OnApplicationShutdown { const r = this.redisForTimelines.pipeline(); if (note.channelId) { - this.funoutTimelineService.push(`channelTimeline:${note.channelId}`, note.id, this.config.perChannelMaxNoteCacheCount, r); + this.fanoutTimelineService.push(`channelTimeline:${note.channelId}`, note.id, this.config.perChannelMaxNoteCacheCount, r); - this.funoutTimelineService.push(`userTimelineWithChannel:${user.id}`, note.id, note.userHost == null ? meta.perLocalUserUserTimelineCacheMax : meta.perRemoteUserUserTimelineCacheMax, r); + this.fanoutTimelineService.push(`userTimelineWithChannel:${user.id}`, note.id, note.userHost == null ? meta.perLocalUserUserTimelineCacheMax : meta.perRemoteUserUserTimelineCacheMax, r); const channelFollowings = await this.channelFollowingsRepository.find({ where: { @@ -798,9 +798,9 @@ export class NoteEditService implements OnApplicationShutdown { }); for (const channelFollowing of channelFollowings) { - this.funoutTimelineService.push(`homeTimeline:${channelFollowing.followerId}`, note.id, meta.perUserHomeTimelineCacheMax, r); + this.fanoutTimelineService.push(`homeTimeline:${channelFollowing.followerId}`, note.id, meta.perUserHomeTimelineCacheMax, r); if (note.fileIds.length > 0) { - this.funoutTimelineService.push(`homeTimelineWithFiles:${channelFollowing.followerId}`, note.id, meta.perUserHomeTimelineCacheMax / 2, r); + this.fanoutTimelineService.push(`homeTimelineWithFiles:${channelFollowing.followerId}`, note.id, meta.perUserHomeTimelineCacheMax / 2, r); } } } else { @@ -838,9 +838,9 @@ export class NoteEditService implements OnApplicationShutdown { if (!following.withReplies) continue; } - this.funoutTimelineService.push(`homeTimeline:${following.followerId}`, note.id, meta.perUserHomeTimelineCacheMax, r); + this.fanoutTimelineService.push(`homeTimeline:${following.followerId}`, note.id, meta.perUserHomeTimelineCacheMax, r); if (note.fileIds.length > 0) { - this.funoutTimelineService.push(`homeTimelineWithFiles:${following.followerId}`, note.id, meta.perUserHomeTimelineCacheMax / 2, r); + this.fanoutTimelineService.push(`homeTimelineWithFiles:${following.followerId}`, note.id, meta.perUserHomeTimelineCacheMax / 2, r); } } @@ -856,36 +856,36 @@ export class NoteEditService implements OnApplicationShutdown { if (!userListMembership.withReplies) continue; } - this.funoutTimelineService.push(`userListTimeline:${userListMembership.userListId}`, note.id, meta.perUserListTimelineCacheMax, r); + this.fanoutTimelineService.push(`userListTimeline:${userListMembership.userListId}`, note.id, meta.perUserListTimelineCacheMax, r); if (note.fileIds.length > 0) { - this.funoutTimelineService.push(`userListTimelineWithFiles:${userListMembership.userListId}`, note.id, meta.perUserListTimelineCacheMax / 2, r); + this.fanoutTimelineService.push(`userListTimelineWithFiles:${userListMembership.userListId}`, note.id, meta.perUserListTimelineCacheMax / 2, r); } } if (note.visibility !== 'specified' || !note.visibleUserIds.some(v => v === user.id)) { // 自分自身のHTL - this.funoutTimelineService.push(`homeTimeline:${user.id}`, note.id, meta.perUserHomeTimelineCacheMax, r); + this.fanoutTimelineService.push(`homeTimeline:${user.id}`, note.id, meta.perUserHomeTimelineCacheMax, r); if (note.fileIds.length > 0) { - this.funoutTimelineService.push(`homeTimelineWithFiles:${user.id}`, note.id, meta.perUserHomeTimelineCacheMax / 2, r); + this.fanoutTimelineService.push(`homeTimelineWithFiles:${user.id}`, note.id, meta.perUserHomeTimelineCacheMax / 2, r); } } // 自分自身以外への返信 if (note.replyId && note.replyUserId !== note.userId) { - this.funoutTimelineService.push(`userTimelineWithReplies:${user.id}`, note.id, note.userHost == null ? meta.perLocalUserUserTimelineCacheMax : meta.perRemoteUserUserTimelineCacheMax, r); + this.fanoutTimelineService.push(`userTimelineWithReplies:${user.id}`, note.id, note.userHost == null ? meta.perLocalUserUserTimelineCacheMax : meta.perRemoteUserUserTimelineCacheMax, r); if (note.visibility === 'public' && note.userHost == null) { - this.funoutTimelineService.push('localTimelineWithReplies', note.id, 300, r); + this.fanoutTimelineService.push('localTimelineWithReplies', note.id, 300, r); } } else { - this.funoutTimelineService.push(`userTimeline:${user.id}`, note.id, note.userHost == null ? meta.perLocalUserUserTimelineCacheMax : meta.perRemoteUserUserTimelineCacheMax, r); + this.fanoutTimelineService.push(`userTimeline:${user.id}`, note.id, note.userHost == null ? meta.perLocalUserUserTimelineCacheMax : meta.perRemoteUserUserTimelineCacheMax, r); if (note.fileIds.length > 0) { - this.funoutTimelineService.push(`userTimelineWithFiles:${user.id}`, note.id, note.userHost == null ? meta.perLocalUserUserTimelineCacheMax / 2 : meta.perRemoteUserUserTimelineCacheMax / 2, r); + this.fanoutTimelineService.push(`userTimelineWithFiles:${user.id}`, note.id, note.userHost == null ? meta.perLocalUserUserTimelineCacheMax / 2 : meta.perRemoteUserUserTimelineCacheMax / 2, r); } if (note.visibility === 'public' && note.userHost == null) { - this.funoutTimelineService.push('localTimeline', note.id, 1000, r); + this.fanoutTimelineService.push('localTimeline', note.id, 1000, r); if (note.fileIds.length > 0) { - this.funoutTimelineService.push('localTimelineWithFiles', note.id, 500, r); + this.fanoutTimelineService.push('localTimelineWithFiles', note.id, 500, r); } } } diff --git a/packages/backend/src/core/NotePiningService.ts b/packages/backend/src/core/NotePiningService.ts index 52abb4c2a1..74e53c5c46 100644 --- a/packages/backend/src/core/NotePiningService.ts +++ b/packages/backend/src/core/NotePiningService.ts @@ -77,7 +77,7 @@ export class NotePiningService { } as MiUserNotePining); // Deliver to remote followers - if (this.userEntityService.isLocalUser(user)) { + if (this.userEntityService.isLocalUser(user) && !note.localOnly && ['public', 'home'].includes(note.visibility)) { this.deliverPinnedChange(user.id, note.id, true); } } @@ -105,7 +105,7 @@ export class NotePiningService { }); // Deliver to remote followers - if (this.userEntityService.isLocalUser(user)) { + if (this.userEntityService.isLocalUser(user) && !note.localOnly && ['public', 'home'].includes(note.visibility)) { this.deliverPinnedChange(user.id, noteId, false); } } diff --git a/packages/backend/src/core/RoleService.ts b/packages/backend/src/core/RoleService.ts index 79c1ecc76f..dcd9d7399f 100644 --- a/packages/backend/src/core/RoleService.ts +++ b/packages/backend/src/core/RoleService.ts @@ -6,7 +6,14 @@ import { Inject, Injectable } from '@nestjs/common'; import * as Redis from 'ioredis'; import { In } from 'typeorm'; -import type { MiRole, MiRoleAssignment, RoleAssignmentsRepository, RolesRepository, UsersRepository } from '@/models/_.js'; +import { ModuleRef } from '@nestjs/core'; +import type { + MiRole, + MiRoleAssignment, + RoleAssignmentsRepository, + RolesRepository, + UsersRepository, +} from '@/models/_.js'; import { MemoryKVCache, MemorySingleCache } from '@/misc/cache.js'; import type { MiUser } from '@/models/User.js'; import { DI } from '@/di-symbols.js'; @@ -16,12 +23,13 @@ import { CacheService } from '@/core/CacheService.js'; import type { RoleCondFormulaValue } from '@/models/Role.js'; import { UserEntityService } from '@/core/entities/UserEntityService.js'; import type { GlobalEvents } from '@/core/GlobalEventService.js'; -import { IdService } from '@/core/IdService.js'; import { GlobalEventService } from '@/core/GlobalEventService.js'; +import { IdService } from '@/core/IdService.js'; import { ModerationLogService } from '@/core/ModerationLogService.js'; import type { Packed } from '@/misc/json-schema.js'; -import { FunoutTimelineService } from '@/core/FunoutTimelineService.js'; -import type { OnApplicationShutdown } from '@nestjs/common'; +import { FanoutTimelineService } from '@/core/FanoutTimelineService.js'; +import { NotificationService } from '@/core/NotificationService.js'; +import type { OnApplicationShutdown, OnModuleInit } from '@nestjs/common'; export type RolePolicies = { gtlAvailable: boolean; @@ -49,6 +57,7 @@ export type RolePolicies = { userEachUserListsLimit: number; rateLimitFactor: number; canImportNotes: boolean; + avatarDecorationLimit: number; }; export const DEFAULT_POLICIES: RolePolicies = { @@ -77,20 +86,27 @@ export const DEFAULT_POLICIES: RolePolicies = { userEachUserListsLimit: 50, rateLimitFactor: 1, canImportNotes: true, + avatarDecorationLimit: 1, }; @Injectable() -export class RoleService implements OnApplicationShutdown { +export class RoleService implements OnApplicationShutdown, OnModuleInit { private rolesCache: MemorySingleCache<MiRole[]>; private roleAssignmentByUserIdCache: MemoryKVCache<MiRoleAssignment[]>; + private notificationService: NotificationService; public static AlreadyAssignedError = class extends Error {}; public static NotAssignedError = class extends Error {}; constructor( + private moduleRef: ModuleRef, + @Inject(DI.redis) private redisClient: Redis.Redis, + @Inject(DI.redisForTimelines) + private redisForTimelines: Redis.Redis, + @Inject(DI.redisForSub) private redisForSub: Redis.Redis, @@ -109,7 +125,7 @@ export class RoleService implements OnApplicationShutdown { private globalEventService: GlobalEventService, private idService: IdService, private moderationLogService: ModerationLogService, - private funoutTimelineService: FunoutTimelineService, + private fanoutTimelineService: FanoutTimelineService, ) { //this.onMessage = this.onMessage.bind(this); @@ -119,6 +135,10 @@ export class RoleService implements OnApplicationShutdown { this.redisForSub.on('message', this.onMessage); } + async onModuleInit() { + this.notificationService = this.moduleRef.get(NotificationService.name); + } + @bindThis private async onMessage(_: string, data: string): Promise<void> { const obj = JSON.parse(data); @@ -329,6 +349,7 @@ export class RoleService implements OnApplicationShutdown { userEachUserListsLimit: calc('userEachUserListsLimit', vs => Math.max(...vs)), rateLimitFactor: calc('rateLimitFactor', vs => Math.max(...vs)), canImportNotes: calc('canImportNotes', vs => vs.some(v => v === true)), + avatarDecorationLimit: calc('avatarDecorationLimit', vs => Math.max(...vs)), }; } @@ -427,6 +448,12 @@ export class RoleService implements OnApplicationShutdown { this.globalEventService.publishInternalEvent('userRoleAssigned', created); + if (role.isPublic) { + this.notificationService.createNotification(userId, 'roleAssigned', { + roleId: roleId, + }); + } + if (moderator) { const user = await this.usersRepository.findOneByOrFail({ id: userId }); this.moderationLogService.log(moderator, 'assignRole', { @@ -482,10 +509,10 @@ export class RoleService implements OnApplicationShutdown { public async addNoteToRoleTimeline(note: Packed<'Note'>): Promise<void> { const roles = await this.getUserRoles(note.userId); - const redisPipeline = this.redisClient.pipeline(); + const redisPipeline = this.redisForTimelines.pipeline(); for (const role of roles) { - this.funoutTimelineService.push(`roleTimeline:${role.id}`, note.id, 1000, redisPipeline); + this.fanoutTimelineService.push(`roleTimeline:${role.id}`, note.id, 1000, redisPipeline); this.globalEventService.publishRoleTimelineStream(role.id, 'note', note); } diff --git a/packages/backend/src/core/SearchService.ts b/packages/backend/src/core/SearchService.ts index b399ace0ae..57c7e4baba 100644 --- a/packages/backend/src/core/SearchService.ts +++ b/packages/backend/src/core/SearchService.ts @@ -12,6 +12,8 @@ import { MiNote } from '@/models/Note.js'; import { MiUser } from '@/models/_.js'; import type { NotesRepository } from '@/models/_.js'; import { sqlLikeEscape } from '@/misc/sql-like-escape.js'; +import { isUserRelated } from '@/misc/is-user-related.js'; +import { CacheService } from '@/core/CacheService.js'; import { QueryService } from '@/core/QueryService.js'; import { IdService } from '@/core/IdService.js'; import type { Index, MeiliSearch } from 'meilisearch'; @@ -74,6 +76,7 @@ export class SearchService { @Inject(DI.notesRepository) private notesRepository: NotesRepository, + private cacheService: CacheService, private queryService: QueryService, private idService: IdService, ) { @@ -230,8 +233,19 @@ export class SearchService { limit: pagination.limit, }); if (res.hits.length === 0) return []; - const notes = await this.notesRepository.findBy({ + const [ + userIdsWhoMeMuting, + userIdsWhoBlockingMe, + ] = me ? await Promise.all([ + this.cacheService.userMutingsCache.fetch(me.id), + this.cacheService.userBlockedCache.fetch(me.id), + ]) : [new Set<string>(), new Set<string>()]; + const notes = (await this.notesRepository.findBy({ id: In(res.hits.map(x => x.id)), + })).filter(note => { + if (me && isUserRelated(note, userIdsWhoBlockingMe)) return false; + if (me && isUserRelated(note, userIdsWhoMeMuting)) return false; + return true; }); return notes.sort((a, b) => a.id > b.id ? -1 : 1); } else { diff --git a/packages/backend/src/core/UserFollowingService.ts b/packages/backend/src/core/UserFollowingService.ts index bd7f298021..d600ffb9d9 100644 --- a/packages/backend/src/core/UserFollowingService.ts +++ b/packages/backend/src/core/UserFollowingService.ts @@ -29,7 +29,7 @@ import { CacheService } from '@/core/CacheService.js'; import type { Config } from '@/config.js'; import { AccountMoveService } from '@/core/AccountMoveService.js'; import { UtilityService } from '@/core/UtilityService.js'; -import { FunoutTimelineService } from '@/core/FunoutTimelineService.js'; +import { FanoutTimelineService } from '@/core/FanoutTimelineService.js'; import Logger from '../logger.js'; const logger = new Logger('following/create'); @@ -84,7 +84,7 @@ export class UserFollowingService implements OnModuleInit { private webhookService: WebhookService, private apRendererService: ApRendererService, private accountMoveService: AccountMoveService, - private funoutTimelineService: FunoutTimelineService, + private fanoutTimelineService: FanoutTimelineService, private perUserFollowingChart: PerUserFollowingChart, private instanceChart: InstanceChart, ) { @@ -304,8 +304,6 @@ export class UserFollowingService implements OnModuleInit { }); } }); - - this.funoutTimelineService.purge(`homeTimeline:${follower.id}`); } // Publish followed event @@ -373,8 +371,6 @@ export class UserFollowingService implements OnModuleInit { }); } }); - - this.funoutTimelineService.purge(`homeTimeline:${follower.id}`); } if (this.userEntityService.isLocalUser(follower) && this.userEntityService.isRemoteUser(followee)) { diff --git a/packages/backend/src/core/UserListService.ts b/packages/backend/src/core/UserListService.ts index 702c731fc3..b6e4e1e884 100644 --- a/packages/backend/src/core/UserListService.ts +++ b/packages/backend/src/core/UserListService.ts @@ -3,30 +3,34 @@ * SPDX-License-Identifier: AGPL-3.0-only */ -import { Inject, Injectable, OnApplicationShutdown } from '@nestjs/common'; +import { Inject, Injectable, OnApplicationShutdown, OnModuleInit } from '@nestjs/common'; import * as Redis from 'ioredis'; +import { ModuleRef } from '@nestjs/core'; import type { UserListMembershipsRepository } from '@/models/_.js'; import type { MiUser } from '@/models/User.js'; import type { MiUserList } from '@/models/UserList.js'; import type { MiUserListMembership } from '@/models/UserListMembership.js'; import { IdService } from '@/core/IdService.js'; +import type { GlobalEvents } from '@/core/GlobalEventService.js'; import { GlobalEventService } from '@/core/GlobalEventService.js'; import { DI } from '@/di-symbols.js'; import { UserEntityService } from '@/core/entities/UserEntityService.js'; import { ProxyAccountService } from '@/core/ProxyAccountService.js'; import { bindThis } from '@/decorators.js'; -import { RoleService } from '@/core/RoleService.js'; import { QueueService } from '@/core/QueueService.js'; import { RedisKVCache } from '@/misc/cache.js'; -import type { GlobalEvents } from '@/core/GlobalEventService.js'; +import { RoleService } from '@/core/RoleService.js'; @Injectable() -export class UserListService implements OnApplicationShutdown { +export class UserListService implements OnApplicationShutdown, OnModuleInit { public static TooManyUsersError = class extends Error {}; public membersCache: RedisKVCache<Set<string>>; + private roleService: RoleService; constructor( + private moduleRef: ModuleRef, + @Inject(DI.redis) private redisClient: Redis.Redis, @@ -38,7 +42,6 @@ export class UserListService implements OnApplicationShutdown { private userEntityService: UserEntityService, private idService: IdService, - private roleService: RoleService, private globalEventService: GlobalEventService, private proxyAccountService: ProxyAccountService, private queueService: QueueService, @@ -54,6 +57,10 @@ export class UserListService implements OnApplicationShutdown { this.redisForSub.on('message', this.onMessage); } + async onModuleInit() { + this.roleService = this.moduleRef.get(RoleService.name); + } + @bindThis private async onMessage(_: string, data: string): Promise<void> { const obj = JSON.parse(data); diff --git a/packages/backend/src/core/activitypub/ApInboxService.ts b/packages/backend/src/core/activitypub/ApInboxService.ts index 5a7b07e666..d8616d293d 100644 --- a/packages/backend/src/core/activitypub/ApInboxService.ts +++ b/packages/backend/src/core/activitypub/ApInboxService.ts @@ -306,9 +306,15 @@ export class ApInboxService { this.logger.info(`Creating the (Re)Note: ${uri}`); const activityAudience = await this.apAudienceService.parseAudience(actor, activity.to, activity.cc); + const createdAt = activity.published ? new Date(activity.published) : null; + + if (createdAt && createdAt < this.idService.parse(renote.id).date) { + this.logger.warn('skip: malformed createdAt'); + return; + } await this.noteCreateService.create(actor, { - createdAt: activity.published ? new Date(activity.published) : null, + createdAt, renote, visibility: activityAudience.visibility, visibleUsers: activityAudience.visibleUsers, diff --git a/packages/backend/src/core/activitypub/models/ApNoteService.ts b/packages/backend/src/core/activitypub/models/ApNoteService.ts index 72488b1c37..2595783e04 100644 --- a/packages/backend/src/core/activitypub/models/ApNoteService.ts +++ b/packages/backend/src/core/activitypub/models/ApNoteService.ts @@ -97,6 +97,10 @@ export class ApNoteService { return new Error(`invalid Note: attributedTo has different host. expected: ${expectHost}, actual: ${actualHost}`); } + if (object.published && !this.idService.isSafeT(new Date(object.published).valueOf())) { + return new Error('invalid Note: published timestamp is malformed'); + } + return null; } diff --git a/packages/backend/src/core/entities/NotificationEntityService.ts b/packages/backend/src/core/entities/NotificationEntityService.ts index f74594ff0c..704081ed00 100644 --- a/packages/backend/src/core/entities/NotificationEntityService.ts +++ b/packages/backend/src/core/entities/NotificationEntityService.ts @@ -15,8 +15,8 @@ import type { Packed } from '@/misc/json-schema.js'; import { bindThis } from '@/decorators.js'; import { isNotNull } from '@/misc/is-not-null.js'; import { FilterUnionByProperty, notificationTypes } from '@/types.js'; +import { RoleEntityService } from './RoleEntityService.js'; import type { OnModuleInit } from '@nestjs/common'; -import type { CustomEmojiService } from '../CustomEmojiService.js'; import type { UserEntityService } from './UserEntityService.js'; import type { NoteEntityService } from './NoteEntityService.js'; @@ -27,7 +27,7 @@ const NOTE_REQUIRED_GROUPED_NOTIFICATION_TYPES = new Set(['note', 'mention', 're export class NotificationEntityService implements OnModuleInit { private userEntityService: UserEntityService; private noteEntityService: NoteEntityService; - private customEmojiService: CustomEmojiService; + private roleEntityService: RoleEntityService; constructor( private moduleRef: ModuleRef, @@ -43,14 +43,13 @@ export class NotificationEntityService implements OnModuleInit { //private userEntityService: UserEntityService, //private noteEntityService: NoteEntityService, - //private customEmojiService: CustomEmojiService, ) { } onModuleInit() { this.userEntityService = this.moduleRef.get('UserEntityService'); this.noteEntityService = this.moduleRef.get('NoteEntityService'); - this.customEmojiService = this.moduleRef.get('CustomEmojiService'); + this.roleEntityService = this.moduleRef.get('RoleEntityService'); } @bindThis @@ -81,6 +80,7 @@ export class NotificationEntityService implements OnModuleInit { detail: false, }) ) : undefined; + const role = notification.type === 'roleAssigned' ? await this.roleEntityService.pack(notification.roleId) : undefined; return await awaitAll({ id: notification.id, @@ -92,6 +92,9 @@ export class NotificationEntityService implements OnModuleInit { ...(notification.type === 'reaction' ? { reaction: notification.reaction, } : {}), + ...(notification.type === 'roleAssigned' ? { + role: role, + } : {}), ...(notification.type === 'achievementEarned' ? { achievement: notification.achievement, } : {}), @@ -198,12 +201,14 @@ export class NotificationEntityService implements OnModuleInit { }); } else if (notification.type === 'renote:grouped') { const users = await Promise.all(notification.userIds.map(userId => { - const user = hint?.packedUsers != null - ? hint.packedUsers.get(userId) - : this.userEntityService.pack(userId!, { id: meId }, { - detail: false, - }); - return user; + const packedUser = hint?.packedUsers != null ? hint.packedUsers.get(userId) : null; + if (packedUser) { + return packedUser; + } + + return this.userEntityService.pack(userId, { id: meId }, { + detail: false, + }); })); return await awaitAll({ id: notification.id, @@ -214,6 +219,8 @@ export class NotificationEntityService implements OnModuleInit { }); } + const role = notification.type === 'roleAssigned' ? await this.roleEntityService.pack(notification.roleId) : undefined; + return await awaitAll({ id: notification.id, createdAt: new Date(notification.createdAt).toISOString(), @@ -224,6 +231,9 @@ export class NotificationEntityService implements OnModuleInit { ...(notification.type === 'reaction' ? { reaction: notification.reaction, } : {}), + ...(notification.type === 'roleAssigned' ? { + role: role, + } : {}), ...(notification.type === 'achievementEarned' ? { achievement: notification.achievement, } : {}), diff --git a/packages/backend/src/core/entities/UserEntityService.ts b/packages/backend/src/core/entities/UserEntityService.ts index b9d5f27d54..adb7dfbf86 100644 --- a/packages/backend/src/core/entities/UserEntityService.ts +++ b/packages/backend/src/core/entities/UserEntityService.ts @@ -361,13 +361,13 @@ export class UserEntityService implements OnModuleInit { const mastoapi = !opts.detail ? opts.userProfile ?? await this.userProfilesRepository.findOneByOrFail({ userId: user.id }) : null; const followingCount = profile == null ? null : - (profile.ffVisibility === 'public') || isMe ? user.followingCount : - (profile.ffVisibility === 'followers') && (relation && relation.isFollowing) ? user.followingCount : + (profile.followingVisibility === 'public') || isMe ? user.followingCount : + (profile.followingVisibility === 'followers') && (relation && relation.isFollowing) ? user.followingCount : null; const followersCount = profile == null ? null : - (profile.ffVisibility === 'public') || isMe ? user.followersCount : - (profile.ffVisibility === 'followers') && (relation && relation.isFollowing) ? user.followersCount : + (profile.followersVisibility === 'public') || isMe ? user.followersCount : + (profile.followersVisibility === 'followers') && (relation && relation.isFollowing) ? user.followersCount : null; const isModerator = isMe && opts.detail ? this.roleService.isModerator(user) : null; @@ -395,6 +395,8 @@ export class UserEntityService implements OnModuleInit { id: ud.id, angle: ud.angle || undefined, flipH: ud.flipH || undefined, + offsetX: ud.offsetX || undefined, + offsetY: ud.offsetY || undefined, url: decorations.find(d => d.id === ud.id)!.url, }))) : [], isBot: user.isBot, @@ -452,7 +454,8 @@ export class UserEntityService implements OnModuleInit { pinnedPageId: profile!.pinnedPageId, pinnedPage: profile!.pinnedPageId ? this.pageEntityService.pack(profile!.pinnedPageId, me) : null, publicReactions: profile!.publicReactions, - ffVisibility: profile!.ffVisibility, + followersVisibility: profile!.followersVisibility, + followingVisibility: profile!.followingVisibility, twoFactorEnabled: profile!.twoFactorEnabled, usePasswordLessLogin: profile!.usePasswordLessLogin, securityKeys: profile!.twoFactorEnabled @@ -511,6 +514,7 @@ export class UserEntityService implements OnModuleInit { hasPendingReceivedFollowRequest: this.getHasPendingReceivedFollowRequest(user.id), unreadNotificationsCount: notificationsInfo?.unreadCount, mutedWords: profile!.mutedWords, + hardMutedWords: profile!.hardMutedWords, mutedInstances: profile!.mutedInstances, mutingNotificationTypes: [], // 後方互換性のため notificationRecieveConfig: profile!.notificationRecieveConfig, |