1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
|
import * as http from 'http';
import * as https from 'https';
import CacheableLookup from 'cacheable-lookup';
import fetch, { HeadersInit } from 'node-fetch';
import { HttpProxyAgent, HttpsProxyAgent } from 'hpagent';
import config from '@/config';
import { URL } from 'url';
export async function getJson(url: string, accept = 'application/json, */*', timeout = 10000, headers?: HeadersInit) {
const res = await fetch(url, {
headers: Object.assign({
'User-Agent': config.userAgent,
Accept: accept
}, headers || {}),
timeout,
agent: getAgentByUrl,
});
if (!res.ok) {
throw {
name: `StatusError`,
statusCode: res.status,
message: `${res.status} ${res.statusText}`,
};
}
return await res.json();
}
export async function getHtml(url: string, accept = 'text/html, */*', timeout = 10000, headers?: HeadersInit) {
const res = await fetch(url, {
headers: Object.assign({
'User-Agent': config.userAgent,
Accept: accept
}, headers || {}),
timeout,
agent: getAgentByUrl,
});
if (!res.ok) {
throw {
name: `StatusError`,
statusCode: res.status,
message: `${res.status} ${res.statusText}`,
};
}
return await res.text();
}
const cache = new CacheableLookup({
maxTtl: 3600, // 1hours
errorTtl: 30, // 30secs
lookup: false, // nativeのdns.lookupにfallbackしない
});
/**
* Get http non-proxy agent
*/
const _http = new http.Agent({
keepAlive: true,
keepAliveMsecs: 30 * 1000,
lookup: cache.lookup,
} as http.AgentOptions);
/**
* Get https non-proxy agent
*/
const _https = new https.Agent({
keepAlive: true,
keepAliveMsecs: 30 * 1000,
lookup: cache.lookup,
} as https.AgentOptions);
const maxSockets = Math.max(256, config.deliverJobConcurrency || 128);
/**
* Get http proxy or non-proxy agent
*/
export const httpAgent = config.proxy
? new HttpProxyAgent({
keepAlive: true,
keepAliveMsecs: 30 * 1000,
maxSockets,
maxFreeSockets: 256,
scheduling: 'lifo',
proxy: config.proxy
})
: _http;
/**
* Get https proxy or non-proxy agent
*/
export const httpsAgent = config.proxy
? new HttpsProxyAgent({
keepAlive: true,
keepAliveMsecs: 30 * 1000,
maxSockets,
maxFreeSockets: 256,
scheduling: 'lifo',
proxy: config.proxy
})
: _https;
/**
* Get agent by URL
* @param url URL
* @param bypassProxy Allways bypass proxy
*/
export function getAgentByUrl(url: URL, bypassProxy = false) {
if (bypassProxy || (config.proxyBypassHosts || []).includes(url.hostname)) {
return url.protocol == 'http:' ? _http : _https;
} else {
return url.protocol == 'http:' ? httpAgent : httpsAgent;
}
}
|
