From 0e4a111f81cceed275d9bec2695f6e401fb654d8 Mon Sep 17 00:00:00 2001 From: syuilo Date: Fri, 12 Nov 2021 02:02:25 +0900 Subject: refactoring Resolve #7779 --- src/services/fetch-instance-metadata.ts | 265 -------------------------------- 1 file changed, 265 deletions(-) delete mode 100644 src/services/fetch-instance-metadata.ts (limited to 'src/services/fetch-instance-metadata.ts') diff --git a/src/services/fetch-instance-metadata.ts b/src/services/fetch-instance-metadata.ts deleted file mode 100644 index 2c401508a9..0000000000 --- a/src/services/fetch-instance-metadata.ts +++ /dev/null @@ -1,265 +0,0 @@ -import { DOMWindow, JSDOM } from 'jsdom'; -import fetch from 'node-fetch'; -import { getJson, getHtml, getAgentByUrl } from '@/misc/fetch'; -import { Instance } from '@/models/entities/instance'; -import { Instances } from '@/models/index'; -import { getFetchInstanceMetadataLock } from '@/misc/app-lock'; -import Logger from './logger'; -import { URL } from 'url'; - -const logger = new Logger('metadata', 'cyan'); - -export async function fetchInstanceMetadata(instance: Instance, force = false): Promise { - const unlock = await getFetchInstanceMetadataLock(instance.host); - - if (!force) { - const _instance = await Instances.findOne({ host: instance.host }); - const now = Date.now(); - if (_instance && _instance.infoUpdatedAt && (now - _instance.infoUpdatedAt.getTime() < 1000 * 60 * 60 * 24)) { - unlock(); - return; - } - } - - logger.info(`Fetching metadata of ${instance.host} ...`); - - try { - const [info, dom, manifest] = await Promise.all([ - fetchNodeinfo(instance).catch(() => null), - fetchDom(instance).catch(() => null), - fetchManifest(instance).catch(() => null), - ]); - - const [favicon, icon, themeColor, name, description] = await Promise.all([ - fetchFaviconUrl(instance, dom).catch(() => null), - fetchIconUrl(instance, dom, manifest).catch(() => null), - getThemeColor(dom, manifest).catch(() => null), - getSiteName(info, dom, manifest).catch(() => null), - getDescription(info, dom, manifest).catch(() => null), - ]); - - logger.succ(`Successfuly fetched metadata of ${instance.host}`); - - const updates = { - infoUpdatedAt: new Date(), - } as Record; - - if (info) { - updates.softwareName = info.software?.name.toLowerCase(); - updates.softwareVersion = info.software?.version; - updates.openRegistrations = info.openRegistrations; - updates.maintainerName = info.metadata ? info.metadata.maintainer ? (info.metadata.maintainer.name || null) : null : null; - updates.maintainerEmail = info.metadata ? info.metadata.maintainer ? (info.metadata.maintainer.email || null) : null : null; - } - - if (name) updates.name = name; - if (description) updates.description = description; - if (icon || favicon) updates.iconUrl = icon || favicon; - if (favicon) updates.faviconUrl = favicon; - if (themeColor) updates.themeColor = themeColor; - - await Instances.update(instance.id, updates); - - logger.succ(`Successfuly updated metadata of ${instance.host}`); - } catch (e) { - logger.error(`Failed to update metadata of ${instance.host}: ${e}`); - } finally { - unlock(); - } -} - -type NodeInfo = { - openRegistrations?: any; - software?: { - name?: any; - version?: any; - }; - metadata?: { - name?: any; - nodeName?: any; - nodeDescription?: any; - description?: any; - maintainer?: { - name?: any; - email?: any; - }; - }; -}; - -async function fetchNodeinfo(instance: Instance): Promise { - logger.info(`Fetching nodeinfo of ${instance.host} ...`); - - try { - const wellknown = await getJson('https://' + instance.host + '/.well-known/nodeinfo') - .catch(e => { - if (e.statusCode === 404) { - throw 'No nodeinfo provided'; - } else { - throw e.statusCode || e.message; - } - }); - - if (wellknown.links == null || !Array.isArray(wellknown.links)) { - throw 'No wellknown links'; - } - - const links = wellknown.links as any[]; - - const lnik1_0 = links.find(link => link.rel === 'http://nodeinfo.diaspora.software/ns/schema/1.0'); - const lnik2_0 = links.find(link => link.rel === 'http://nodeinfo.diaspora.software/ns/schema/2.0'); - const lnik2_1 = links.find(link => link.rel === 'http://nodeinfo.diaspora.software/ns/schema/2.1'); - const link = lnik2_1 || lnik2_0 || lnik1_0; - - if (link == null) { - throw 'No nodeinfo link provided'; - } - - const info = await getJson(link.href) - .catch(e => { - throw e.statusCode || e.message; - }); - - logger.succ(`Successfuly fetched nodeinfo of ${instance.host}`); - - return info; - } catch (e) { - logger.error(`Failed to fetch nodeinfo of ${instance.host}: ${e}`); - - throw e; - } -} - -async function fetchDom(instance: Instance): Promise { - logger.info(`Fetching HTML of ${instance.host} ...`); - - const url = 'https://' + instance.host; - - const html = await getHtml(url); - - const { window } = new JSDOM(html); - const doc = window.document; - - return doc; -} - -async function fetchManifest(instance: Instance): Promise | null> { - const url = 'https://' + instance.host; - - const manifestUrl = url + '/manifest.json'; - - const manifest = await getJson(manifestUrl); - - return manifest; -} - -async function fetchFaviconUrl(instance: Instance, doc: DOMWindow['document'] | null): Promise { - const url = 'https://' + instance.host; - - if (doc) { - const href = doc.querySelector('link[rel="icon"]')?.getAttribute('href'); - - if (href) { - return (new URL(href, url)).href; - } - } - - const faviconUrl = url + '/favicon.ico'; - - const favicon = await fetch(faviconUrl, { - timeout: 10000, - agent: getAgentByUrl, - }); - - if (favicon.ok) { - return faviconUrl; - } - - return null; -} - -async function fetchIconUrl(instance: Instance, doc: DOMWindow['document'] | null, manifest: Record | null): Promise { - if (manifest && manifest.icons && manifest.icons.length > 0 && manifest.icons[0].src) { - const url = 'https://' + instance.host; - return (new URL(manifest.icons[0].src, url)).href; - } - - if (doc) { - const url = 'https://' + instance.host; - - const hrefAppleTouchIconPrecomposed = doc.querySelector('link[rel="apple-touch-icon-precomposed"]')?.getAttribute('href'); - const hrefAppleTouchIcon = doc.querySelector('link[rel="apple-touch-icon"]')?.getAttribute('href'); - const hrefIcon = doc.querySelector('link[rel="icon"]')?.getAttribute('href'); - - const href = hrefAppleTouchIconPrecomposed || hrefAppleTouchIcon || hrefIcon; - - if (href) { - return (new URL(href, url)).href; - } - } - - return null; -} - -async function getThemeColor(doc: DOMWindow['document'] | null, manifest: Record | null): Promise { - if (doc) { - const themeColor = doc.querySelector('meta[name="theme-color"]')?.getAttribute('content'); - - if (themeColor) { - return themeColor; - } - } - - if (manifest) { - return manifest.theme_color; - } - - return null; -} - -async function getSiteName(info: NodeInfo | null, doc: DOMWindow['document'] | null, manifest: Record | null): Promise { - if (info && info.metadata) { - if (info.metadata.nodeName || info.metadata.name) { - return info.metadata.nodeName || info.metadata.name; - } - } - - if (doc) { - const og = doc.querySelector('meta[property="og:title"]')?.getAttribute('content'); - - if (og) { - return og; - } - } - - if (manifest) { - return manifest?.name || manifest?.short_name; - } - - return null; -} - -async function getDescription(info: NodeInfo | null, doc: DOMWindow['document'] | null, manifest: Record | null): Promise { - if (info && info.metadata) { - if (info.metadata.nodeDescription || info.metadata.description) { - return info.metadata.nodeDescription || info.metadata.description; - } - } - - if (doc) { - const meta = doc.querySelector('meta[name="description"]')?.getAttribute('content'); - if (meta) { - return meta; - } - - const og = doc.querySelector('meta[property="og:description"]')?.getAttribute('content'); - if (og) { - return og; - } - } - - if (manifest) { - return manifest?.name || manifest?.short_name; - } - - return null; -} -- cgit v1.2.3-freya