Store nodeinfo per federated instances (#5578)

* Store nodeinfo per federated instances

* Update fetch-nodeinfo.ts

* Update fetch-nodeinfo.ts

* update

7 files changed, 144 insertions, 10 deletions

diff --git a/src/misc/app-lock.ts b/src/misc/app-lock.ts
index 30579ed934..3d5ff91882 100644
--- a/src/misc/app-lock.ts
+++ b/src/misc/app-lock.ts
@@ -20,3 +20,7 @@ const lock: (key: string, timeout?: number) => Promise<() => void>
 export function getApLock(uri: string, timeout = 30 * 1000) {
 	return lock(`ap-object:${uri}`, timeout);
 }
+
+export function getNodeinfoLock(host: string, timeout = 30 * 1000) {
+	return lock(`nodeinfo:${host}`, timeout);
+}
diff --git a/src/models/entities/instance.ts b/src/models/entities/instance.ts
index 52c5215f14..dd0de100d6 100644
--- a/src/models/entities/instance.ts
+++ b/src/models/entities/instance.ts
@@ -26,15 +26,6 @@ export class Instance {
 	public host: string;
 
 	/**
-	 * インスタンスのシステム (MastodonとかMisskeyとかPleromaとか)
-	 */
-	@Column('varchar', {
-		length: 64, nullable: true,
-		comment: 'The system of the Instance.'
-	})
-	public system: string | null;
-
-	/**
 	 * インスタンスのユーザー数
 	 */
 	@Column('integer', {
@@ -129,4 +120,45 @@ export class Instance {
 		default: false
 	})
 	public isMarkedAsClosed: boolean;
+
+	@Column('varchar', {
+		length: 64, nullable: true, default: null,
+		comment: 'The software of the Instance.'
+	})
+	public softwareName: string | null;
+
+	@Column('varchar', {
+		length: 64, nullable: true, default: null,
+	})
+	public softwareVersion: string | null;
+
+	@Column('boolean', {
+		nullable: true, default: null,
+	})
+	public openRegistrations: boolean | null;
+
+	@Column('varchar', {
+		length: 256, nullable: true, default: null,
+	})
+	public name: string | null;
+
+	@Column('varchar', {
+		length: 4096, nullable: true, default: null,
+	})
+	public description: string | null;
+
+	@Column('varchar', {
+		length: 128, nullable: true, default: null,
+	})
+	public maintainerName: string | null;
+
+	@Column('varchar', {
+		length: 256, nullable: true, default: null,
+	})
+	public maintainerEmail: string | null;
+
+	@Column('timestamp with time zone', {
+		nullable: true,
+	})
+	public infoUpdatedAt: Date | null;
 }
diff --git a/src/queue/processors/deliver.ts b/src/queue/processors/deliver.ts
index 8837c80d87..b252c20163 100644
--- a/src/queue/processors/deliver.ts
+++ b/src/queue/processors/deliver.ts
@@ -4,6 +4,7 @@ import { registerOrFetchInstanceDoc } from '../../services/register-or-fetch-ins
 import Logger from '../../services/logger';
 import { Instances } from '../../models';
 import { instanceChart } from '../../services/chart';
+import { fetchNodeinfo } from '../../services/fetch-nodeinfo';
 
 const logger = new Logger('deliver');
 
@@ -28,6 +29,8 @@ export default async (job: Bull.Job) => {
 				isNotResponding: false
 			});
 
+			fetchNodeinfo(i);
+
 			instanceChart.requestSent(i.host, true);
 		});
 
diff --git a/src/queue/processors/inbox.ts b/src/queue/processors/inbox.ts
index e71181ee73..1a583ec865 100644
--- a/src/queue/processors/inbox.ts
+++ b/src/queue/processors/inbox.ts
@@ -13,6 +13,7 @@ import { fetchMeta } from '../../misc/fetch-meta';
 import { toPuny } from '../../misc/convert-host';
 import { validActor } from '../../remote/activitypub/type';
 import { ensure } from '../../prelude/ensure';
+import { fetchNodeinfo } from '../../services/fetch-nodeinfo';
 
 const logger = new Logger('inbox');
 
@@ -105,6 +106,8 @@ export default async (job: Bull.Job): Promise<void> => {
 			isNotResponding: false
 		});
 
+		fetchNodeinfo(i);
+
 		instanceChart.requestReceived(i.host);
 	});
 
diff --git a/src/remote/activitypub/models/person.ts b/src/remote/activitypub/models/person.ts
index 198fd78bd5..c7a6d56638 100644
--- a/src/remote/activitypub/models/person.ts
+++ b/src/remote/activitypub/models/person.ts
@@ -27,6 +27,7 @@ import { validActor } from '../../../remote/activitypub/type';
 import { getConnection } from 'typeorm';
 import { ensure } from '../../../prelude/ensure';
 import { toArray } from '../../../prelude/array';
+import { fetchNodeinfo } from '../../../services/fetch-nodeinfo';
 
 const logger = apLogger;
 
@@ -191,6 +192,7 @@ export async function createPerson(uri: string, resolver?: Resolver): Promise<Us
 	registerOrFetchInstanceDoc(host).then(i => {
 		Instances.increment({ id: i.id }, 'usersCount', 1);
 		instanceChart.newUser(i.host);
+		fetchNodeinfo(i);
 	});
 
 	usersChart.update(user!, true);
diff --git a/src/services/fetch-nodeinfo.ts b/src/services/fetch-nodeinfo.ts
new file mode 100644
index 0000000000..e5d652a6b3
--- /dev/null
+++ b/src/services/fetch-nodeinfo.ts
@@ -0,0 +1,91 @@
+import * as request from 'request-promise-native';
+import { Instance } from '../models/entities/instance';
+import { Instances } from '../models';
+import config from '../config';
+import { getNodeinfoLock } from '../misc/app-lock';
+import Logger from '../services/logger';
+
+export const logger = new Logger('nodeinfo', 'cyan');
+
+export async function fetchNodeinfo(instance: Instance) {
+	const unlock = await getNodeinfoLock(instance.host);
+
+	const _instance = await Instances.findOne({ host: instance.host });
+	const now = Date.now();
+	if (_instance && _instance.infoUpdatedAt && (now - _instance.infoUpdatedAt.getTime() < 1000 * 60 * 60 * 24)) {
+		unlock();
+		return;
+	}
+
+	logger.info(`Fetching nodeinfo of ${instance.host} ...`);
+
+	try {
+		const wellknown = await request({
+			url: 'https://' + instance.host + '/.well-known/nodeinfo',
+			proxy: config.proxy,
+			timeout: 1000 * 10,
+			forever: true,
+			headers: {
+				'User-Agent': config.userAgent,
+				Accept: 'application/json, */*'
+			},
+			json: true
+		}).catch(e => {
+			if (e.statusCode === 404) {
+				throw 'No nodeinfo provided';
+			} else {
+				throw e.statusCode || e.message;
+			}
+		});
+
+		if (wellknown.links == null || !Array.isArray(wellknown.links)) {
+			throw 'No wellknown links';
+		}
+
+		const links = wellknown.links as any[];
+
+		const lnik1_0 = links.find(link => link.rel === 'http://nodeinfo.diaspora.software/ns/schema/1.0');
+		const lnik2_0 = links.find(link => link.rel === 'http://nodeinfo.diaspora.software/ns/schema/2.0');
+		const lnik2_1 = links.find(link => link.rel === 'http://nodeinfo.diaspora.software/ns/schema/2.1');
+		const link = lnik2_1 || lnik2_0 || lnik1_0;
+
+		if (link == null) {
+			throw 'No nodeinfo link provided';
+		}
+
+		const info = await request({
+			url: link.href,
+			proxy: config.proxy,
+			timeout: 1000 * 10,
+			forever: true,
+			headers: {
+				'User-Agent': config.userAgent,
+				Accept: 'application/json, */*'
+			},
+			json: true
+		}).catch(e => {
+			throw e.statusCode || e.message;
+		});
+
+		await Instances.update(instance.id, {
+			infoUpdatedAt: new Date(),
+			softwareName: info.software.name.toLowerCase(),
+			softwareVersion: info.software.version,
+			openRegistrations: info.openRegistrations,
+			name: info.metadata ? (info.metadata.nodeName || info.metadata.name || null) : null,
+			description: info.metadata ? (info.metadata.nodeDescription || info.metadata.description || null) : null,
+			maintainerName: info.metadata ? info.metadata.maintainer ? (info.metadata.maintainer.name || null) : null : null,
+			maintainerEmail: info.metadata ? info.metadata.maintainer ? (info.metadata.maintainer.email || null) : null : null,
+		});
+
+		logger.succ(`Successfuly fetched nodeinfo of ${instance.host}`);
+	} catch (e) {
+		logger.error(`Failed to fetch nodeinfo of ${instance.host}: ${e}`);
+
+		await Instances.update(instance.id, {
+			infoUpdatedAt: new Date(),
+		});
+	} finally {
+		unlock();
+	}
+}
diff --git a/src/services/register-or-fetch-instance-doc.ts b/src/services/register-or-fetch-instance-doc.ts
index 9957edd3db..3501e20de1 100644
--- a/src/services/register-or-fetch-instance-doc.ts
+++ b/src/services/register-or-fetch-instance-doc.ts
@@ -15,7 +15,6 @@ export async function registerOrFetchInstanceDoc(host: string): Promise<Instance
 			host,
 			caughtAt: new Date(),
 			lastCommunicatedAt: new Date(),
-			system: null // TODO
 		});
 
 		federationChart.update(true);