From a35c2f214b1b1054229f31569f6df4090a7375a5 Mon Sep 17 00:00:00 2001 From: Hazelnoot Date: Fri, 21 Feb 2025 22:04:36 -0500 Subject: convert Authorized Fetch to a setting and add support for hybrid mode (essential metadata only) --- packages/backend/src/core/CacheService.ts | 8 ++++++++ 1 file changed, 8 insertions(+) (limited to 'packages/backend/src/core/CacheService.ts') diff --git a/packages/backend/src/core/CacheService.ts b/packages/backend/src/core/CacheService.ts index 6725ebe75b..e9900373b4 100644 --- a/packages/backend/src/core/CacheService.ts +++ b/packages/backend/src/core/CacheService.ts @@ -5,6 +5,7 @@ import { Inject, Injectable } from '@nestjs/common'; import * as Redis from 'ioredis'; +import { IsNull } from 'typeorm'; import type { BlockingsRepository, FollowingsRepository, MutingsRepository, RenoteMutingsRepository, MiUserProfile, UserProfilesRepository, UsersRepository, MiFollowing } from '@/models/_.js'; import { MemoryKVCache, RedisKVCache } from '@/misc/cache.js'; import type { MiLocalUser, MiUser } from '@/models/User.js'; @@ -179,6 +180,13 @@ export class CacheService implements OnApplicationShutdown { return this.userByIdCache.fetch(userId, () => this.usersRepository.findOneByOrFail({ id: userId })); } + @bindThis + public async findLocalUserById(userId: MiUser['id']): Promise { + return await this.localUserByIdCache.fetchMaybe(userId, async () => { + return await this.usersRepository.findOneBy({ id: userId, host: IsNull() }) as MiLocalUser | null ?? undefined; + }) ?? null; + } + @bindThis public dispose(): void { this.redisForSub.off('message', this.onMessage); -- cgit v1.2.3-freya From 40a73bfcbe083d5a2aa4be57880e389def4757c4 Mon Sep 17 00:00:00 2001 From: Hazelnoot Date: Fri, 9 May 2025 11:53:29 -0400 Subject: add new role conditions for local/remote followers/followees --- locales/index.d.ts | 32 ++++++++++ packages/backend/src/core/CacheService.ts | 66 ++++++++++++++++++++ packages/backend/src/core/RoleService.ts | 39 ++++++++++-- packages/backend/src/models/Role.ts | 72 ++++++++++++++++++++++ .../src/pages/admin/RolesEditorFormula.vue | 29 ++++++++- sharkey-locales/en-US.yml | 8 +++ 6 files changed, 239 insertions(+), 7 deletions(-) (limited to 'packages/backend/src/core/CacheService.ts') diff --git a/locales/index.d.ts b/locales/index.d.ts index 9679cc3acf..8f032cd518 100644 --- a/locales/index.d.ts +++ b/locales/index.d.ts @@ -7689,6 +7689,38 @@ export interface Locale extends ILocale { * Match subdomains */ "isFromInstanceSubdomains": string; + /** + * Has X or fewer local followers + */ + "localFollowersLessThanOrEq": string; + /** + * Has X or more local followers + */ + "localFollowersMoreThanOrEq": string; + /** + * Follows X or fewer local accounts + */ + "localFollowingLessThanOrEq": string; + /** + * Follows X or more local accounts + */ + "localFollowingMoreThanOrEq": string; + /** + * Has X or fewer remote followers + */ + "remoteFollowersLessThanOrEq": string; + /** + * Has X or more remote followers + */ + "remoteFollowersMoreThanOrEq": string; + /** + * Follows X or fewer remote accounts + */ + "remoteFollowingLessThanOrEq": string; + /** + * Follows X or more remote accounts + */ + "remoteFollowingMoreThanOrEq": string; }; }; "_sensitiveMediaDetection": { diff --git a/packages/backend/src/core/CacheService.ts b/packages/backend/src/core/CacheService.ts index e9900373b4..822bb9d42c 100644 --- a/packages/backend/src/core/CacheService.ts +++ b/packages/backend/src/core/CacheService.ts @@ -15,6 +15,13 @@ import { bindThis } from '@/decorators.js'; import type { GlobalEvents } from '@/core/GlobalEventService.js'; import type { OnApplicationShutdown } from '@nestjs/common'; +export interface FollowStats { + localFollowing: number; + localFollowers: number; + remoteFollowing: number; + remoteFollowers: number; +} + @Injectable() export class CacheService implements OnApplicationShutdown { public userByIdCache: MemoryKVCache; @@ -27,6 +34,7 @@ export class CacheService implements OnApplicationShutdown { public userBlockedCache: RedisKVCache>; // NOTE: 「被」Blockキャッシュ public renoteMutingsCache: RedisKVCache>; public userFollowingsCache: RedisKVCache | undefined>>; + private readonly userFollowStatsCache = new MemoryKVCache(1000 * 60 * 10); // 10 minutes constructor( @Inject(DI.redis) @@ -167,6 +175,18 @@ export class CacheService implements OnApplicationShutdown { const followee = this.userByIdCache.get(body.followeeId); if (followee) followee.followersCount++; this.userFollowingsCache.delete(body.followerId); + this.userFollowStatsCache.delete(body.followerId); + this.userFollowStatsCache.delete(body.followeeId); + break; + } + case 'unfollow': { + const follower = this.userByIdCache.get(body.followerId); + if (follower) follower.followingCount--; + const followee = this.userByIdCache.get(body.followeeId); + if (followee) followee.followersCount--; + this.userFollowingsCache.delete(body.followerId); + this.userFollowStatsCache.delete(body.followerId); + this.userFollowStatsCache.delete(body.followeeId); break; } default: @@ -187,6 +207,52 @@ export class CacheService implements OnApplicationShutdown { }) ?? null; } + @bindThis + public async getFollowStats(userId: MiUser['id']): Promise { + return await this.userFollowStatsCache.fetch(userId, async () => { + const stats = { + localFollowing: 0, + localFollowers: 0, + remoteFollowing: 0, + remoteFollowers: 0, + }; + + const followings = await this.followingsRepository.findBy([ + { followerId: userId }, + { followeeId: userId }, + ]); + + for (const following of followings) { + if (following.followerId === userId) { + // increment following; user is a follower of someone else + if (following.followeeHost == null) { + stats.localFollowing++; + } else { + stats.remoteFollowing++; + } + } else if (following.followeeId === userId) { + // increment followers; user is followed by someone else + if (following.followerHost == null) { + stats.localFollowers++; + } else { + stats.remoteFollowers++; + } + } else { + // Should never happen + } + } + + // Infer remote-remote followers heuristically, since we don't track that info directly. + const user = await this.findUserById(userId); + if (user.host !== null) { + stats.remoteFollowing = Math.max(0, user.followingCount - stats.localFollowing); + stats.remoteFollowers = Math.max(0, user.followersCount - stats.localFollowers); + } + + return stats; + }); + } + @bindThis public dispose(): void { this.redisForSub.off('message', this.onMessage); diff --git a/packages/backend/src/core/RoleService.ts b/packages/backend/src/core/RoleService.ts index d948325503..3fdac4c580 100644 --- a/packages/backend/src/core/RoleService.ts +++ b/packages/backend/src/core/RoleService.ts @@ -20,6 +20,7 @@ import type { MiUser } from '@/models/User.js'; import { DI } from '@/di-symbols.js'; import { bindThis } from '@/decorators.js'; import { CacheService } from '@/core/CacheService.js'; +import type { FollowStats } from '@/core/CacheService.js'; import type { RoleCondFormulaValue } from '@/models/Role.js'; import { UserEntityService } from '@/core/entities/UserEntityService.js'; import type { GlobalEvents } from '@/core/GlobalEventService.js'; @@ -221,20 +222,20 @@ export class RoleService implements OnApplicationShutdown, OnModuleInit { } @bindThis - private evalCond(user: MiUser, roles: MiRole[], value: RoleCondFormulaValue): boolean { + private evalCond(user: MiUser, roles: MiRole[], value: RoleCondFormulaValue, followStats: FollowStats): boolean { try { switch (value.type) { // ～かつ～ case 'and': { - return value.values.every(v => this.evalCond(user, roles, v)); + return value.values.every(v => this.evalCond(user, roles, v, followStats)); } // ～または～ case 'or': { - return value.values.some(v => this.evalCond(user, roles, v)); + return value.values.some(v => this.evalCond(user, roles, v, followStats)); } // ～ではない case 'not': { - return !this.evalCond(user, roles, value.value); + return !this.evalCond(user, roles, value.value, followStats); } // マニュアルロールがアサインされている case 'roleAssignedTo': { @@ -305,6 +306,30 @@ export class RoleService implements OnApplicationShutdown, OnModuleInit { case 'followingMoreThanOrEq': { return user.followingCount >= value.value; } + case 'localFollowersLessThanOrEq': { + return followStats.localFollowers <= value.value; + } + case 'localFollowersMoreThanOrEq': { + return followStats.localFollowers >= value.value; + } + case 'localFollowingLessThanOrEq': { + return followStats.localFollowing <= value.value; + } + case 'localFollowingMoreThanOrEq': { + return followStats.localFollowing >= value.value; + } + case 'remoteFollowersLessThanOrEq': { + return followStats.remoteFollowers <= value.value; + } + case 'remoteFollowersMoreThanOrEq': { + return followStats.remoteFollowers >= value.value; + } + case 'remoteFollowingLessThanOrEq': { + return followStats.remoteFollowing <= value.value; + } + case 'remoteFollowingMoreThanOrEq': { + return followStats.remoteFollowing >= value.value; + } // ノート数が指定値以下 case 'notesLessThanOrEq': { return user.notesCount <= value.value; @@ -340,10 +365,11 @@ export class RoleService implements OnApplicationShutdown, OnModuleInit { @bindThis public async getUserRoles(userId: MiUser['id']) { const roles = await this.rolesCache.fetch(() => this.rolesRepository.findBy({})); + const followStats = await this.cacheService.getFollowStats(userId); const assigns = await this.getUserAssigns(userId); const assignedRoles = roles.filter(r => assigns.map(x => x.roleId).includes(r.id)); const user = roles.some(r => r.target === 'conditional') ? await this.cacheService.findUserById(userId) : null; - const matchedCondRoles = roles.filter(r => r.target === 'conditional' && this.evalCond(user!, assignedRoles, r.condFormula)); + const matchedCondRoles = roles.filter(r => r.target === 'conditional' && this.evalCond(user!, assignedRoles, r.condFormula, followStats)); return [...assignedRoles, ...matchedCondRoles]; } @@ -357,12 +383,13 @@ export class RoleService implements OnApplicationShutdown, OnModuleInit { // 期限切れのロールを除外 assigns = assigns.filter(a => a.expiresAt == null || (a.expiresAt.getTime() > now)); const roles = await this.rolesCache.fetch(() => this.rolesRepository.findBy({})); + const followStats = await this.cacheService.getFollowStats(userId); const assignedRoles = roles.filter(r => assigns.map(x => x.roleId).includes(r.id)); const assignedBadgeRoles = assignedRoles.filter(r => r.asBadge); const badgeCondRoles = roles.filter(r => r.asBadge && (r.target === 'conditional')); if (badgeCondRoles.length > 0) { const user = roles.some(r => r.target === 'conditional') ? await this.cacheService.findUserById(userId) : null; - const matchedBadgeCondRoles = badgeCondRoles.filter(r => this.evalCond(user!, assignedRoles, r.condFormula)); + const matchedBadgeCondRoles = badgeCondRoles.filter(r => this.evalCond(user!, assignedRoles, r.condFormula, followStats)); return [...assignedBadgeRoles, ...matchedBadgeCondRoles]; } else { return assignedBadgeRoles; diff --git a/packages/backend/src/models/Role.ts b/packages/backend/src/models/Role.ts index d7ae8ed38c..2caf3e0bd3 100644 --- a/packages/backend/src/models/Role.ts +++ b/packages/backend/src/models/Role.ts @@ -147,6 +147,70 @@ type CondFormulaValueFollowingMoreThanOrEq = { value: number; }; +/** + * Is followed by at most N local users + */ +type CondFormulaValueLocalFollowersLessThanOrEq = { + type: 'localFollowersLessThanOrEq'; + value: number; +}; + +/** + * Is followed by at least N local users + */ +type CondFormulaValueLocalFollowersMoreThanOrEq = { + type: 'localFollowersMoreThanOrEq'; + value: number; +}; + +/** + * Is following at most N local users + */ +type CondFormulaValueLocalFollowingLessThanOrEq = { + type: 'localFollowingLessThanOrEq'; + value: number; +}; + +/** + * Is following at least N local users + */ +type CondFormulaValueLocalFollowingMoreThanOrEq = { + type: 'localFollowingMoreThanOrEq'; + value: number; +}; + +/** + * Is followed by at most N remote users + */ +type CondFormulaValueRemoteFollowersLessThanOrEq = { + type: 'remoteFollowersLessThanOrEq'; + value: number; +}; + +/** + * Is followed by at least N remote users + */ +type CondFormulaValueRemoteFollowersMoreThanOrEq = { + type: 'remoteFollowersMoreThanOrEq'; + value: number; +}; + +/** + * Is following at most N remote users + */ +type CondFormulaValueRemoteFollowingLessThanOrEq = { + type: 'remoteFollowingLessThanOrEq'; + value: number; +}; + +/** + * Is following at least N remote users + */ +type CondFormulaValueRemoteFollowingMoreThanOrEq = { + type: 'remoteFollowingMoreThanOrEq'; + value: number; +}; + /** * 投稿数が指定値以下の場合のみ成立とする */ @@ -182,6 +246,14 @@ export type RoleCondFormulaValue = { id: string } & ( CondFormulaValueFollowersMoreThanOrEq | CondFormulaValueFollowingLessThanOrEq | CondFormulaValueFollowingMoreThanOrEq | + CondFormulaValueLocalFollowersLessThanOrEq | + CondFormulaValueLocalFollowersMoreThanOrEq | + CondFormulaValueLocalFollowingLessThanOrEq | + CondFormulaValueLocalFollowingMoreThanOrEq | + CondFormulaValueRemoteFollowersLessThanOrEq | + CondFormulaValueRemoteFollowersMoreThanOrEq | + CondFormulaValueRemoteFollowingLessThanOrEq | + CondFormulaValueRemoteFollowingMoreThanOrEq | CondFormulaValueNotesLessThanOrEq | CondFormulaValueNotesMoreThanOrEq ); diff --git a/packages/frontend/src/pages/admin/RolesEditorFormula.vue b/packages/frontend/src/pages/admin/RolesEditorFormula.vue index 53a4836caa..c937f3be71 100644 --- a/packages/frontend/src/pages/admin/RolesEditorFormula.vue +++ b/packages/frontend/src/pages/admin/RolesEditorFormula.vue @@ -22,6 +22,14 @@ SPDX-License-Identifier: AGPL-3.0-only + + + + + + + + @@ -56,7 +64,26 @@ SPDX-License-Identifier: AGPL-3.0-only - + diff --git a/sharkey-locales/en-US.yml b/sharkey-locales/en-US.yml index 034d995b01..5922fb63ca 100644 --- a/sharkey-locales/en-US.yml +++ b/sharkey-locales/en-US.yml @@ -245,6 +245,14 @@ _role: isFromInstance: "Is from a specific instance" isFromInstanceHost: "Hostname (case-insensitive)" isFromInstanceSubdomains: "Match subdomains" + localFollowersLessThanOrEq: "Has X or fewer local followers" + localFollowersMoreThanOrEq: "Has X or more local followers" + localFollowingLessThanOrEq: "Follows X or fewer local accounts" + localFollowingMoreThanOrEq: "Follows X or more local accounts" + remoteFollowersLessThanOrEq: "Has X or fewer remote followers" + remoteFollowersMoreThanOrEq: "Has X or more remote followers" + remoteFollowingLessThanOrEq: "Follows X or fewer remote accounts" + remoteFollowingMoreThanOrEq: "Follows X or more remote accounts" _emailUnavailable: banned: "This email address is banned" _signup: -- cgit v1.2.3-freya From 7db48ffa8d27c4be37f87ea12e8d65942d5f9cdc Mon Sep 17 00:00:00 2001 From: Hazelnoot Date: Mon, 12 May 2025 13:16:01 -0400 Subject: add redis cache for note translations * Partitioned by target language * Invalidated if the note is edited --- packages/backend/src/core/CacheService.ts | 47 ++++++++++++++++++++++++++++++- 1 file changed, 46 insertions(+), 1 deletion(-) (limited to 'packages/backend/src/core/CacheService.ts') diff --git a/packages/backend/src/core/CacheService.ts b/packages/backend/src/core/CacheService.ts index 822bb9d42c..1cf63221f9 100644 --- a/packages/backend/src/core/CacheService.ts +++ b/packages/backend/src/core/CacheService.ts @@ -6,7 +6,7 @@ import { Inject, Injectable } from '@nestjs/common'; import * as Redis from 'ioredis'; import { IsNull } from 'typeorm'; -import type { BlockingsRepository, FollowingsRepository, MutingsRepository, RenoteMutingsRepository, MiUserProfile, UserProfilesRepository, UsersRepository, MiFollowing } from '@/models/_.js'; +import type { BlockingsRepository, FollowingsRepository, MutingsRepository, RenoteMutingsRepository, MiUserProfile, UserProfilesRepository, UsersRepository, MiFollowing, MiNote } from '@/models/_.js'; import { MemoryKVCache, RedisKVCache } from '@/misc/cache.js'; import type { MiLocalUser, MiUser } from '@/models/User.js'; import { DI } from '@/di-symbols.js'; @@ -22,6 +22,17 @@ export interface FollowStats { remoteFollowers: number; } +export interface CachedTranslation { + sourceLang: string | undefined; + text: string | undefined; +} + +interface CachedTranslationEntity { + l?: string; + t?: string; + u?: number; +} + @Injectable() export class CacheService implements OnApplicationShutdown { public userByIdCache: MemoryKVCache; @@ -35,6 +46,7 @@ export class CacheService implements OnApplicationShutdown { public renoteMutingsCache: RedisKVCache>; public userFollowingsCache: RedisKVCache | undefined>>; private readonly userFollowStatsCache = new MemoryKVCache(1000 * 60 * 10); // 10 minutes + private readonly translationsCache: RedisKVCache; constructor( @Inject(DI.redis) @@ -124,6 +136,11 @@ export class CacheService implements OnApplicationShutdown { fromRedisConverter: (value) => JSON.parse(value), }); + this.translationsCache = new RedisKVCache(this.redisClient, 'translations', { + lifetime: 1000 * 60 * 60 * 24 * 7, // 1 week, + memoryCacheLifetime: 1000 * 60, // 1 minute + }); + // NOTE: チャンネルのフォロー状況キャッシュはChannelFollowingServiceで行っている this.redisForSub.on('message', this.onMessage); @@ -253,6 +270,34 @@ export class CacheService implements OnApplicationShutdown { }); } + @bindThis + public async getCachedTranslation(note: MiNote, targetLang: string): Promise { + const cacheKey = `${note.id}@${targetLang}`; + + // Use cached translation, if present and up-to-date + const cached = await this.translationsCache.get(cacheKey); + if (cached && cached.u === note.updatedAt?.valueOf()) { + return { + sourceLang: cached.l, + text: cached.t, + }; + } + + // No cache entry :( + return null; + } + + @bindThis + public async setCachedTranslation(note: MiNote, targetLang: string, translation: CachedTranslation): Promise { + const cacheKey = `${note.id}@${targetLang}`; + + await this.translationsCache.set(cacheKey, { + l: translation.sourceLang, + t: translation.text, + u: note.updatedAt?.valueOf(), + }); + } + @bindThis public dispose(): void { this.redisForSub.off('message', this.onMessage); -- cgit v1.2.3-freya