diff options
Diffstat (limited to 'src/misc/fetch.ts')
| -rw-r--r-- | src/misc/fetch.ts | 141 |
1 files changed, 0 insertions, 141 deletions
diff --git a/src/misc/fetch.ts b/src/misc/fetch.ts deleted file mode 100644 index f4f16a27e2..0000000000 --- a/src/misc/fetch.ts +++ /dev/null @@ -1,141 +0,0 @@ -import * as http from 'http'; -import * as https from 'https'; -import CacheableLookup from 'cacheable-lookup'; -import fetch from 'node-fetch'; -import { HttpProxyAgent, HttpsProxyAgent } from 'hpagent'; -import config from '@/config/index'; -import { URL } from 'url'; - -export async function getJson(url: string, accept = 'application/json, */*', timeout = 10000, headers?: Record<string, string>) { - const res = await getResponse({ - url, - method: 'GET', - headers: Object.assign({ - 'User-Agent': config.userAgent, - Accept: accept - }, headers || {}), - timeout - }); - - return await res.json(); -} - -export async function getHtml(url: string, accept = 'text/html, */*', timeout = 10000, headers?: Record<string, string>) { - const res = await getResponse({ - url, - method: 'GET', - headers: Object.assign({ - 'User-Agent': config.userAgent, - Accept: accept - }, headers || {}), - timeout - }); - - return await res.text(); -} - -export async function getResponse(args: { url: string, method: string, body?: string, headers: Record<string, string>, timeout?: number, size?: number }) { - const timeout = args?.timeout || 10 * 1000; - - const controller = new AbortController(); - setTimeout(() => { - controller.abort(); - }, timeout * 6); - - const res = await fetch(args.url, { - method: args.method, - headers: args.headers, - body: args.body, - timeout, - size: args?.size || 10 * 1024 * 1024, - agent: getAgentByUrl, - signal: controller.signal, - }); - - if (!res.ok) { - throw new StatusError(`${res.status} ${res.statusText}`, res.status, res.statusText); - } - - return res; -} - -const cache = new CacheableLookup({ - maxTtl: 3600, // 1hours - errorTtl: 30, // 30secs - lookup: false, // nativeのdns.lookupにfallbackしない -}); - -/** - * Get http non-proxy agent - */ -const _http = new http.Agent({ - keepAlive: true, - keepAliveMsecs: 30 * 1000, - lookup: cache.lookup, -} as http.AgentOptions); - -/** - * Get https non-proxy agent - */ -const _https = new https.Agent({ - keepAlive: true, - keepAliveMsecs: 30 * 1000, - lookup: cache.lookup, -} as https.AgentOptions); - -const maxSockets = Math.max(256, config.deliverJobConcurrency || 128); - -/** - * Get http proxy or non-proxy agent - */ -export const httpAgent = config.proxy - ? new HttpProxyAgent({ - keepAlive: true, - keepAliveMsecs: 30 * 1000, - maxSockets, - maxFreeSockets: 256, - scheduling: 'lifo', - proxy: config.proxy - }) - : _http; - -/** - * Get https proxy or non-proxy agent - */ -export const httpsAgent = config.proxy - ? new HttpsProxyAgent({ - keepAlive: true, - keepAliveMsecs: 30 * 1000, - maxSockets, - maxFreeSockets: 256, - scheduling: 'lifo', - proxy: config.proxy - }) - : _https; - -/** - * Get agent by URL - * @param url URL - * @param bypassProxy Allways bypass proxy - */ -export function getAgentByUrl(url: URL, bypassProxy = false) { - if (bypassProxy || (config.proxyBypassHosts || []).includes(url.hostname)) { - return url.protocol == 'http:' ? _http : _https; - } else { - return url.protocol == 'http:' ? httpAgent : httpsAgent; - } -} - -export class StatusError extends Error { - public statusCode: number; - public statusMessage?: string; - public isClientError: boolean; - - constructor(message: string, statusCode: number, statusMessage?: string) { - super(message); - this.name = 'StatusError'; - this.statusCode = statusCode; - this.statusMessage = statusMessage; - this.isClientError = typeof this.statusCode === 'number' && this.statusCode >= 400 && this.statusCode < 500; - } -} |