From def32107af9e34f162cfe586e68c4379d61c4a22 Mon Sep 17 00:00:00 2001
From: MeiMei <30769358+mei23@users.noreply.github.com>
Date: Mon, 16 Aug 2021 17:44:43 +0900
Subject: perf: Improve network request performance (#7636)

* perf: Improve fetch

* CHANGELOG

* lifo
---
 src/misc/fetch.ts | 33 +++++++++++++++++++++++++++------
 1 file changed, 27 insertions(+), 6 deletions(-)

(limited to 'src/misc')

diff --git a/src/misc/fetch.ts b/src/misc/fetch.ts
index 37f8203543..f7756f9256 100644
--- a/src/misc/fetch.ts
+++ b/src/misc/fetch.ts
@@ -1,9 +1,8 @@
 import * as http from 'http';
 import * as https from 'https';
-import * as cache from 'lookup-dns-cache';
+import CacheableLookup from 'cacheable-lookup';
 import fetch, { HeadersInit } from 'node-fetch';
-import { HttpProxyAgent } from 'http-proxy-agent';
-import { HttpsProxyAgent } from 'https-proxy-agent';
+import { HttpProxyAgent, HttpsProxyAgent } from 'hpagent';
 import config from '@/config';
 import { URL } from 'url';
 
@@ -49,6 +48,12 @@ export async function getHtml(url: string, accept = 'text/html, */*', timeout =
 	return await res.text();
 }
 
+const cache = new CacheableLookup({
+	maxTtl: 3600,	// 1hours
+	errorTtl: 30,	// 30secs
+	lookup: false,	// nativeのdns.lookupにfallbackしない
+});
+
 /**
  * Get http non-proxy agent
  */
@@ -65,20 +70,36 @@ const _https = new https.Agent({
 	keepAlive: true,
 	keepAliveMsecs: 30 * 1000,
 	lookup: cache.lookup,
-});
+} as https.AgentOptions);
+
+const maxSockets = Math.max(256, config.deliverJobConcurrency || 128);
 
 /**
  * Get http proxy or non-proxy agent
  */
 export const httpAgent = config.proxy
-	? new HttpProxyAgent(config.proxy)
+	? new HttpProxyAgent({
+		keepAlive: true,
+		keepAliveMsecs: 30 * 1000,
+		maxSockets,
+		maxFreeSockets: 256,
+		scheduling: 'lifo',
+		proxy: config.proxy
+	})
 	: _http;
 
 /**
  * Get https proxy or non-proxy agent
  */
 export const httpsAgent = config.proxy
-	? new HttpsProxyAgent(config.proxy)
+	? new HttpsProxyAgent({
+		keepAlive: true,
+		keepAliveMsecs: 30 * 1000,
+		maxSockets,
+		maxFreeSockets: 256,
+		scheduling: 'lifo',
+		proxy: config.proxy
+	})
 	: _https;
 
 /**
-- 
cgit v1.2.3-freya