From 305250d0738b1fc7332ee9556b8d9e6aed5a8237 Mon Sep 17 00:00:00 2001 From: Hazelnoot Date: Sat, 24 May 2025 18:29:10 -0400 Subject: persist changes to meta host lists to instance table --- packages/backend/src/core/MetaService.ts | 55 ++++++++++++++++++++++++++++++-- 1 file changed, 53 insertions(+), 2 deletions(-) (limited to 'packages/backend/src/core/MetaService.ts') diff --git a/packages/backend/src/core/MetaService.ts b/packages/backend/src/core/MetaService.ts index 40e7439f5f..16be22ac19 100644 --- a/packages/backend/src/core/MetaService.ts +++ b/packages/backend/src/core/MetaService.ts @@ -4,7 +4,7 @@ */ import { Inject, Injectable } from '@nestjs/common'; -import { DataSource } from 'typeorm'; +import { DataSource, EntityManager } from 'typeorm'; import * as Redis from 'ioredis'; import { DI } from '@/di-symbols.js'; import { MiMeta } from '@/models/Meta.js'; @@ -12,6 +12,8 @@ import { GlobalEventService } from '@/core/GlobalEventService.js'; import { bindThis } from '@/decorators.js'; import type { GlobalEvents } from '@/core/GlobalEventService.js'; import { FeaturedService } from '@/core/FeaturedService.js'; +import { MiInstance } from '@/models/Instance.js'; +import { diffArrays } from '@/misc/diff-arrays.js'; import type { OnApplicationShutdown } from '@nestjs/common'; @Injectable() @@ -103,7 +105,7 @@ export class MetaService implements OnApplicationShutdown { let before: MiMeta | undefined; const updated = await this.db.transaction(async transactionalEntityManager => { - const metas = await transactionalEntityManager.find(MiMeta, { + const metas: (MiMeta | undefined)[] = await transactionalEntityManager.find(MiMeta, { order: { id: 'DESC', }, @@ -126,6 +128,10 @@ export class MetaService implements OnApplicationShutdown { }, }); + // Propagate changes to blockedHosts, silencedHosts, mediaSilencedHosts, federationInstances, and bubbleInstances to the relevant instance rows + // Do this inside the transaction to avoid potential race condition (when an instance gets registered while we're updating). + await this.persistBlocks(transactionalEntityManager, before ?? {}, afters[0]); + return afters[0]; }); @@ -159,4 +165,49 @@ export class MetaService implements OnApplicationShutdown { public onApplicationShutdown(signal?: string | undefined): void { this.dispose(); } + + private async persistBlocks(tem: EntityManager, before: Partial, after: Partial): Promise { + await this.persistBlock(tem, before.blockedHosts, after.blockedHosts, 'isBlocked'); + await this.persistBlock(tem, before.silencedHosts, after.silencedHosts, 'isSilenced'); + await this.persistBlock(tem, before.mediaSilencedHosts, after.mediaSilencedHosts, 'isMediaSilenced'); + await this.persistBlock(tem, before.federationHosts, after.federationHosts, 'isAllowListed'); + await this.persistBlock(tem, before.bubbleInstances, after.bubbleInstances, 'isBubbled'); + } + + private async persistBlock(tem: EntityManager, before: string[] | undefined, after: string[] | undefined, field: keyof MiInstance): Promise { + const { added, removed } = diffArrays(before, after); + + if (removed.length > 0) { + await this.updateInstancesByHost(tem, field, false, removed); + } + + if (added.length > 0) { + await this.updateInstancesByHost(tem, field, true, added); + } + } + + private async updateInstancesByHost(tem: EntityManager, field: keyof MiInstance, value: boolean, hosts: string[]): Promise { + // Use non-array queries when possible, as they are indexed and can be much faster. + if (hosts.length === 1) { + const pattern = genHostPattern(hosts[0]); + await tem + .createQueryBuilder(MiInstance, 'instance') + .update() + .set({ [field]: value }) + .where('(lower(reverse("host")) || \'.\') LIKE :pattern', { pattern }) + .execute(); + } else if (hosts.length > 1) { + const patterns = hosts.map(host => genHostPattern(host)); + await tem + .createQueryBuilder(MiInstance, 'instance') + .update() + .set({ [field]: value }) + .where('(lower(reverse("host")) || \'.\') LIKE ANY (:patterns)', {patterns}) + .execute(); + } + } +} + +function genHostPattern(host: string): string { + return host.toLowerCase().split('').reverse().join('') + '.%'; } -- cgit v1.2.3-freya From fad82000f09ec4a9843e5ffb14924ab1288f2e0c Mon Sep 17 00:00:00 2001 From: Hazelnoot Date: Sat, 24 May 2025 23:12:52 -0400 Subject: fix lint error in MetaService --- packages/backend/src/core/MetaService.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'packages/backend/src/core/MetaService.ts') diff --git a/packages/backend/src/core/MetaService.ts b/packages/backend/src/core/MetaService.ts index 16be22ac19..5b6ee8920e 100644 --- a/packages/backend/src/core/MetaService.ts +++ b/packages/backend/src/core/MetaService.ts @@ -202,7 +202,7 @@ export class MetaService implements OnApplicationShutdown { .createQueryBuilder(MiInstance, 'instance') .update() .set({ [field]: value }) - .where('(lower(reverse("host")) || \'.\') LIKE ANY (:patterns)', {patterns}) + .where('(lower(reverse("host")) || \'.\') LIKE ANY (:patterns)', { patterns }) .execute(); } } -- cgit v1.2.3-freya From 3e7ab07b3caa91c9caff7aeb072cf26f2e13ffc9 Mon Sep 17 00:00:00 2001 From: Hazelnoot Date: Sun, 25 May 2025 08:44:45 -0400 Subject: avoid race conditions in meta / instance insert --- .../backend/src/core/FederatedInstanceService.ts | 30 ++++++-------- packages/backend/src/core/MetaService.ts | 48 +++++++++++----------- 2 files changed, 37 insertions(+), 41 deletions(-) (limited to 'packages/backend/src/core/MetaService.ts') diff --git a/packages/backend/src/core/FederatedInstanceService.ts b/packages/backend/src/core/FederatedInstanceService.ts index e31d802e8c..662c7f7f2d 100644 --- a/packages/backend/src/core/FederatedInstanceService.ts +++ b/packages/backend/src/core/FederatedInstanceService.ts @@ -5,15 +5,13 @@ import { Inject, Injectable, OnApplicationShutdown } from '@nestjs/common'; import * as Redis from 'ioredis'; -import { DataSource, QueryFailedError } from 'typeorm'; import type { InstancesRepository } from '@/models/_.js'; -import { MiInstance } from '@/models/Instance.js'; +import type { MiInstance } from '@/models/Instance.js'; import { MemoryKVCache, RedisKVCache } from '@/misc/cache.js'; import { IdService } from '@/core/IdService.js'; import { DI } from '@/di-symbols.js'; import { UtilityService } from '@/core/UtilityService.js'; import { bindThis } from '@/decorators.js'; -import { isDuplicateKeyValueError } from '@/misc/is-duplicate-key-value-error.js'; @Injectable() export class FederatedInstanceService implements OnApplicationShutdown { @@ -26,9 +24,6 @@ export class FederatedInstanceService implements OnApplicationShutdown { @Inject(DI.instancesRepository) private instancesRepository: InstancesRepository, - @Inject(DI.db) - private readonly db: DataSource, - private utilityService: UtilityService, private idService: IdService, ) { @@ -58,11 +53,11 @@ export class FederatedInstanceService implements OnApplicationShutdown { const cached = await this.federatedInstanceCache.get(host); if (cached) return cached; - return await this.db.transaction(async tem => { - let index = await tem.findOneBy(MiInstance, { host }); - - if (index == null) { - await tem.insert(MiInstance, { + let index = await this.instancesRepository.findOneBy({ host }); + if (index == null) { + await this.instancesRepository.createQueryBuilder('instance') + .insert() + .values({ id: this.idService.gen(), host, firstRetrievedAt: new Date(), @@ -71,14 +66,15 @@ export class FederatedInstanceService implements OnApplicationShutdown { isMediaSilenced: this.utilityService.isMediaSilencedHost(host), isAllowListed: this.utilityService.isAllowListedHost(host), isBubbled: this.utilityService.isBubbledHost(host), - }); + }) + .orIgnore() + .execute(); - index = await tem.findOneByOrFail(MiInstance, { host }); - } + index = await this.instancesRepository.findOneByOrFail({ host }); + } - await this.federatedInstanceCache.set(host, index); - return index; - }); + await this.federatedInstanceCache.set(host, index); + return index; } @bindThis diff --git a/packages/backend/src/core/MetaService.ts b/packages/backend/src/core/MetaService.ts index 5b6ee8920e..b4ccfec4cc 100644 --- a/packages/backend/src/core/MetaService.ts +++ b/packages/backend/src/core/MetaService.ts @@ -14,6 +14,7 @@ import type { GlobalEvents } from '@/core/GlobalEventService.js'; import { FeaturedService } from '@/core/FeaturedService.js'; import { MiInstance } from '@/models/Instance.js'; import { diffArrays } from '@/misc/diff-arrays.js'; +import type { MetasRepository } from '@/models/_.js'; import type { OnApplicationShutdown } from '@nestjs/common'; @Injectable() @@ -28,6 +29,9 @@ export class MetaService implements OnApplicationShutdown { @Inject(DI.db) private db: DataSource, + @Inject(DI.metasRepository) + private readonly metasRepository: MetasRepository, + private featuredService: FeaturedService, private globalEventService: GlobalEventService, ) { @@ -69,35 +73,31 @@ export class MetaService implements OnApplicationShutdown { public async fetch(noCache = false): Promise { if (!noCache && this.cache) return this.cache; - return await this.db.transaction(async transactionalEntityManager => { - // 過去のバグでレコードが複数出来てしまっている可能性があるので新しいIDを優先する - const metas = await transactionalEntityManager.find(MiMeta, { + // 過去のバグでレコードが複数出来てしまっている可能性があるので新しいIDを優先する + let meta = await this.metasRepository.findOne({ + order: { + id: 'DESC', + }, + }); + + if (!meta) { + await this.metasRepository.createQueryBuilder('meta') + .insert() + .values({ + id: 'x', + }) + .orIgnore() + .execute(); + + meta = await this.metasRepository.findOneOrFail({ order: { id: 'DESC', }, }); + } - const meta = metas[0]; - - if (meta) { - this.cache = meta; - return meta; - } else { - // metaが空のときfetchMetaが同時に呼ばれるとここが同時に呼ばれてしまうことがあるのでフェイルセーフなupsertを使う - const saved = await transactionalEntityManager - .upsert( - MiMeta, - { - id: 'x', - }, - ['id'], - ) - .then((x) => transactionalEntityManager.findOneByOrFail(MiMeta, x.identifiers[0])); - - this.cache = saved; - return saved; - } - }); + this.cache = meta; + return meta; } @bindThis -- cgit v1.2.3-freya From 4738b14d1ca76c29f71f3665c7a1b7658835404a Mon Sep 17 00:00:00 2001 From: Hazelnoot Date: Sun, 25 May 2025 18:42:47 -0400 Subject: fix TypeORM error from MetaService.fetch --- packages/backend/src/core/MetaService.ts | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) (limited to 'packages/backend/src/core/MetaService.ts') diff --git a/packages/backend/src/core/MetaService.ts b/packages/backend/src/core/MetaService.ts index b4ccfec4cc..07f82dc23e 100644 --- a/packages/backend/src/core/MetaService.ts +++ b/packages/backend/src/core/MetaService.ts @@ -74,11 +74,13 @@ export class MetaService implements OnApplicationShutdown { if (!noCache && this.cache) return this.cache; // 過去のバグでレコードが複数出来てしまっている可能性があるので新しいIDを優先する - let meta = await this.metasRepository.findOne({ - order: { + let meta = await this.metasRepository.createQueryBuilder('meta') + .select() + .orderBy({ id: 'DESC', - }, - }); + }) + .limit(1) + .getOne(); if (!meta) { await this.metasRepository.createQueryBuilder('meta') @@ -89,11 +91,13 @@ export class MetaService implements OnApplicationShutdown { .orIgnore() .execute(); - meta = await this.metasRepository.findOneOrFail({ - order: { + meta = await this.metasRepository.createQueryBuilder('meta') + .select() + .orderBy({ id: 'DESC', - }, - }); + }) + .limit(1) + .getOneOrFail(); } this.cache = meta; -- cgit v1.2.3-freya