feat: suspend instance improvements (#13861)

* feat(backend): dead instance detection

* feat(backend): suspend type detection

* feat(frontend): show suspend reason on frontend

* feat(backend): resume federation automatically if the server is automatically suspended

* docs(changelog): 配信停止まわりの改善

* lint: fix lint errors

* Update packages/frontend/src/pages/instance-info.vue

* lint: fix lint error

* chore: suspendedState => suspensionState

---------

Co-authored-by: syuilo <4439005+syuilo@users.noreply.github.com>

7 files changed, 44 insertions, 10 deletions

diff --git a/packages/backend/src/core/entities/InstanceEntityService.ts b/packages/backend/src/core/entities/InstanceEntityService.ts
index e46bd8b963..9117b13914 100644
--- a/packages/backend/src/core/entities/InstanceEntityService.ts
+++ b/packages/backend/src/core/entities/InstanceEntityService.ts
@@ -39,7 +39,8 @@ export class InstanceEntityService {
 			followingCount: instance.followingCount,
 			followersCount: instance.followersCount,
 			isNotResponding: instance.isNotResponding,
-			isSuspended: instance.isSuspended,
+			isSuspended: instance.suspensionState !== 'none',
+			suspensionState: instance.suspensionState,
 			isBlocked: this.utilityService.isBlockedHost(meta.blockedHosts, instance.host),
 			softwareName: instance.softwareName,
 			softwareVersion: instance.softwareVersion,
diff --git a/packages/backend/src/models/Instance.ts b/packages/backend/src/models/Instance.ts
index 9863c9d75d..17cd5c6665 100644
--- a/packages/backend/src/models/Instance.ts
+++ b/packages/backend/src/models/Instance.ts
@@ -81,13 +81,22 @@ export class MiInstance {
 	public isNotResponding: boolean;
 
 	/**
-	 * このインスタンスへの配信を停止するか
+	 * このインスタンスと不通になった日時
+	 */
+	@Column('timestamp with time zone', {
+		nullable: true,
+	})
+	public notRespondingSince: Date | null;
+
+	/**
+	 * このインスタンスへの配信状態
 	 */
 	@Index()
-	@Column('boolean', {
-		default: false,
+	@Column('enum', {
+		default: 'none',
+		enum: ['none', 'manuallySuspended', 'goneSuspended', 'autoSuspendedForNotResponding'],
 	})
-	public isSuspended: boolean;
+	public suspensionState: 'none' | 'manuallySuspended' | 'goneSuspended' | 'autoSuspendedForNotResponding';
 
 	@Column('varchar', {
 		length: 64, nullable: true,
diff --git a/packages/backend/src/models/json-schema/federation-instance.ts b/packages/backend/src/models/json-schema/federation-instance.ts
index 42d98fe523..ed40d405c6 100644
--- a/packages/backend/src/models/json-schema/federation-instance.ts
+++ b/packages/backend/src/models/json-schema/federation-instance.ts
@@ -45,6 +45,11 @@ export const packedFederationInstanceSchema = {
 			type: 'boolean',
 			optional: false, nullable: false,
 		},
+		suspensionState: {
+			type: 'string',
+			nullable: false, optional: false,
+			enum: ['none', 'manuallySuspended', 'goneSuspended', 'autoSuspendedForNotResponding'],
+		},
 		isBlocked: {
 			type: 'boolean',
 			optional: false, nullable: false,
diff --git a/packages/backend/src/queue/processors/DeliverProcessorService.ts b/packages/backend/src/queue/processors/DeliverProcessorService.ts
index 5fed070929..b73195afc3 100644
--- a/packages/backend/src/queue/processors/DeliverProcessorService.ts
+++ b/packages/backend/src/queue/processors/DeliverProcessorService.ts
@@ -5,6 +5,7 @@
 
 import { Inject, Injectable } from '@nestjs/common';
 import * as Bull from 'bullmq';
+import { Not } from 'typeorm';
 import { DI } from '@/di-symbols.js';
 import type { InstancesRepository } from '@/models/_.js';
 import type Logger from '@/logger.js';
@@ -62,7 +63,7 @@ export class DeliverProcessorService {
 		if (suspendedHosts == null) {
 			suspendedHosts = await this.instancesRepository.find({
 				where: {
-					isSuspended: true,
+					suspensionState: Not('none'),
 				},
 			});
 			this.suspendedHostsCache.set(suspendedHosts);
@@ -79,6 +80,7 @@ export class DeliverProcessorService {
 				if (i.isNotResponding) {
 					this.federatedInstanceService.update(i.id, {
 						isNotResponding: false,
+						notRespondingSince: null,
 					});
 				}
 
@@ -98,7 +100,15 @@ export class DeliverProcessorService {
 				if (!i.isNotResponding) {
 					this.federatedInstanceService.update(i.id, {
 						isNotResponding: true,
+						notRespondingSince: new Date(),
 					});
+				} else if (i.notRespondingSince) {
+					// 1週間以上不通ならサスペンド
+					if (i.suspensionState === 'none' && i.notRespondingSince.getTime() <= Date.now() - 1000 * 60 * 60 * 24 * 7) {
+						this.federatedInstanceService.update(i.id, {
+							suspensionState: 'autoSuspendedForNotResponding',
+						});
+					}
 				}
 
 				this.apRequestChart.deliverFail();
@@ -116,7 +126,7 @@ export class DeliverProcessorService {
 					if (job.data.isSharedInbox && res.statusCode === 410) {
 						this.federatedInstanceService.fetch(host).then(i => {
 							this.federatedInstanceService.update(i.id, {
-								isSuspended: true,
+								suspensionState: 'goneSuspended',
 							});
 						});
 						throw new Bull.UnrecoverableError(`${host} is gone`);
diff --git a/packages/backend/src/queue/processors/InboxProcessorService.ts b/packages/backend/src/queue/processors/InboxProcessorService.ts
index 1d05f4ade1..f465339075 100644
--- a/packages/backend/src/queue/processors/InboxProcessorService.ts
+++ b/packages/backend/src/queue/processors/InboxProcessorService.ts
@@ -188,6 +188,8 @@ export class InboxProcessorService {
 			this.federatedInstanceService.update(i.id, {
 				latestRequestReceivedAt: new Date(),
 				isNotResponding: false,
+				// もしサーバーが死んでるために配信が止まっていた場合には自動的に復活させてあげる
+				suspensionState: i.suspensionState === 'autoSuspendedForNotResponding' ? 'none' : undefined,
 			});
 
 			this.fetchInstanceMetadataService.fetchInstanceMetadata(i);
diff --git a/packages/backend/src/server/api/ApiServerService.ts b/packages/backend/src/server/api/ApiServerService.ts
index e99244cdd0..4a5935f930 100644
--- a/packages/backend/src/server/api/ApiServerService.ts
+++ b/packages/backend/src/server/api/ApiServerService.ts
@@ -137,7 +137,7 @@ export class ApiServerService {
 			const instances = await this.instancesRepository.find({
 				select: ['host'],
 				where: {
-					isSuspended: false,
+					suspensionState: 'none',
 				},
 			});
 
diff --git a/packages/backend/src/server/api/endpoints/admin/federation/update-instance.ts b/packages/backend/src/server/api/endpoints/admin/federation/update-instance.ts
index 0bcdc2a4b8..fed7bfbbde 100644
--- a/packages/backend/src/server/api/endpoints/admin/federation/update-instance.ts
+++ b/packages/backend/src/server/api/endpoints/admin/federation/update-instance.ts
@@ -46,12 +46,19 @@ export default class extends Endpoint<typeof meta, typeof paramDef> { // eslint-
 				throw new Error('instance not found');
 			}
 
+			const isSuspendedBefore = instance.suspensionState !== 'none';
+			let suspensionState: undefined | 'manuallySuspended' | 'none';
+
+			if (ps.isSuspended != null && isSuspendedBefore !== ps.isSuspended) {
+				suspensionState = ps.isSuspended ? 'manuallySuspended' : 'none';
+			}
+
 			await this.federatedInstanceService.update(instance.id, {
-				isSuspended: ps.isSuspended,
+				suspensionState,
 				moderationNote: ps.moderationNote,
 			});
 
-			if (ps.isSuspended != null && instance.isSuspended !== ps.isSuspended) {
+			if (ps.isSuspended != null && isSuspendedBefore !== ps.isSuspended) {
 				if (ps.isSuspended) {
 					this.moderationLogService.log(me, 'suspendRemoteInstance', {
 						id: instance.id,