fix(server): node-fetchおよびgotを使う以前の実装に戻す

see #9710

1 files changed, 82 insertions, 295 deletions

diff --git a/packages/backend/src/core/HttpRequestService.ts b/packages/backend/src/core/HttpRequestService.ts
index cd859d0023..baf74acfa6 100644
--- a/packages/backend/src/core/HttpRequestService.ts
+++ b/packages/backend/src/core/HttpRequestService.ts
@@ -1,284 +1,67 @@
 import * as http from 'node:http';
 import * as https from 'node:https';
-import { LookupFunction } from 'node:net';
 import CacheableLookup from 'cacheable-lookup';
+import fetch from 'node-fetch';
 import { HttpProxyAgent, HttpsProxyAgent } from 'hpagent';
 import { Inject, Injectable } from '@nestjs/common';
-import * as undici from 'undici';
 import { DI } from '@/di-symbols.js';
 import type { Config } from '@/config.js';
 import { StatusError } from '@/misc/status-error.js';
 import { bindThis } from '@/decorators.js';
-import { LoggerService } from '@/core/LoggerService.js';
-import type Logger from '@/logger.js';
+import type { Response } from 'node-fetch';
+import type { URL } from 'node:url';
 
-// true to allow, false to deny
-export type IpChecker = (ip: string) => boolean;
-
-/* 
- *  Child class to create and save Agent for fetch.
- *  You should construct this when you want
- *  to change timeout, size limit, socket connect function, etc.
- */
-export class UndiciFetcher {
+@Injectable()
+export class HttpRequestService {
 	/**
-	 * Get http non-proxy agent (undici)
+	 * Get http non-proxy agent
 	 */
-	public nonProxiedAgent: undici.Agent;
+	private http: http.Agent;
 
 	/**
-	 * Get http proxy or non-proxy agent (undici)
+	 * Get https non-proxy agent
 	 */
-	public agent: undici.ProxyAgent | undici.Agent;
-
-	private proxyBypassHosts: string[];
-	private userAgent: string | undefined;
-
-	private logger: Logger | undefined;
-
-	constructor(
-		args: {
-			agentOptions: undici.Agent.Options;
-			proxy?: {
-				uri: string;
-				options?: undici.Agent.Options; // Override of agentOptions
-			},
-			proxyBypassHosts?: string[];
-			userAgent?: string;
-		},
-		logger?: Logger,
-	) {
-		this.logger = logger;
-		this.logger?.debug('UndiciFetcher constructor', args);
-
-		this.proxyBypassHosts = args.proxyBypassHosts ?? [];
-		this.userAgent = args.userAgent;
-
-		this.nonProxiedAgent = new undici.Agent({
-			...args.agentOptions,
-			connect: (process.env.NODE_ENV !== 'production' && typeof args.agentOptions.connect !== 'function')
-				? (options, cb) => {
-					// Custom connector for debug
-					undici.buildConnector(args.agentOptions.connect as undici.buildConnector.BuildOptions)(options, (err, socket) => {
-						this.logger?.debug('Socket connector called', socket);
-						if (err) {
-							this.logger?.debug('Socket error', err);
-							cb(new Error(`Error while socket connecting\n${err}`), null);
-							return;
-						}
-						this.logger?.debug(`Socket connected: port ${socket.localPort} => remote ${socket.remoteAddress}`);
-						cb(null, socket);
-					});
-				} : args.agentOptions.connect,
-		});
-
-		this.agent = args.proxy
-			? new undici.ProxyAgent({
-				...args.agentOptions,
-				...args.proxy.options,
-
-				uri: args.proxy.uri,
-
-				connect: (process.env.NODE_ENV !== 'production' && typeof (args.proxy.options?.connect ?? args.agentOptions.connect) !== 'function')
-					? (options, cb) => {
-						// Custom connector for debug
-						undici.buildConnector((args.proxy?.options?.connect ?? args.agentOptions.connect) as undici.buildConnector.BuildOptions)(options, (err, socket) => {
-							this.logger?.debug('Socket connector called (secure)', socket);
-							if (err) {
-								this.logger?.debug('Socket error', err);
-								cb(new Error(`Error while socket connecting\n${err}`), null);
-								return;
-							}
-							this.logger?.debug(`Socket connected (secure): port ${socket.localPort} => remote ${socket.remoteAddress}`);
-							cb(null, socket);
-						});
-					} : (args.proxy.options?.connect ?? args.agentOptions.connect),
-			})
-			: this.nonProxiedAgent;
-	}
+	private https: https.Agent;
 
 	/**
-	 * Get agent by URL
-	 * @param url URL
-	 * @param bypassProxy Allways bypass proxy
+	 * Get http proxy or non-proxy agent
 	 */
-	@bindThis
-	public getAgentByUrl(url: URL, bypassProxy = false): undici.Agent | undici.ProxyAgent {
-		if (bypassProxy || this.proxyBypassHosts.includes(url.hostname)) {
-			return this.nonProxiedAgent;
-		} else {
-			return this.agent;
-		}
-	}
-
-	@bindThis
-	public async fetch(
-		url: string | URL,
-		options: undici.RequestInit = {},
-		privateOptions: { noOkError?: boolean; bypassProxy?: boolean; } = { noOkError: false, bypassProxy: false },
-	): Promise<undici.Response> {
-		const res = await undici.fetch(url, {
-			dispatcher: this.getAgentByUrl(new URL(url), privateOptions.bypassProxy),
-			...options,
-			headers: {
-				'User-Agent': this.userAgent ?? '',
-				...(options.headers ?? {}),
-			},
-		}).catch((err) => {
-			this.logger?.error(`fetch error to ${typeof url === 'string' ? url : url.href}`, err);
-			throw new StatusError('Resource Unreachable', 500, 'Resource Unreachable');
-		});
-		if (!res.ok && !privateOptions.noOkError) {
-			throw new StatusError(`${res.status} ${res.statusText}`, res.status, res.statusText);
-		}
-		return res;
-	}
-
-	@bindThis
-	public async request(
-		url: string | URL,
-		options: { dispatcher?: undici.Dispatcher } & Omit<undici.Dispatcher.RequestOptions, 'origin' | 'path' | 'method'> & Partial<Pick<undici.Dispatcher.RequestOptions, 'method'>> = {},
-		privateOptions: { noOkError?: boolean; bypassProxy?: boolean; } = { noOkError: false, bypassProxy: false },
-	): Promise<undici.Dispatcher.ResponseData> {
-		const res = await undici.request(url, {
-			dispatcher: this.getAgentByUrl(new URL(url), privateOptions.bypassProxy),
-			...options,
-			headers: {
-				'user-agent': this.userAgent ?? '',
-				...(options.headers ?? {}),
-			},
-		}).catch((err) => {
-			this.logger?.error(`fetch error to ${typeof url === 'string' ? url : url.href}`, err);
-			throw new StatusError('Resource Unreachable', 500, 'Resource Unreachable');
-		});
-
-		if (res.statusCode >= 400) {
-			throw new StatusError(`${res.statusCode}`, res.statusCode, '');
-		}
-
-		return res;
-	}
-
-	@bindThis
-	public async getJson<T extends unknown>(url: string, accept = 'application/json, */*', headers?: Record<string, string>): Promise<T> {
-		const { body } = await this.request( 
-			url,
-			{
-				headers: Object.assign({
-					Accept: accept,
-				}, headers ?? {}),
-			},
-		);
-
-		return await body.json() as T;
-	}
-
-	@bindThis
-	public async getHtml(url: string, accept = 'text/html, */*', headers?: Record<string, string>): Promise<string> {
-		const { body } = await this.request(
-			url,
-			{
-				headers: Object.assign({
-					Accept: accept,
-				}, headers ?? {}),
-			},
-		);
-
-		return await body.text();
-	}
-}
-
-@Injectable()
-export class HttpRequestService {
-	public defaultFetcher: UndiciFetcher;
-	public fetch: UndiciFetcher['fetch'];
-	public request: UndiciFetcher['request'];
-	public getHtml: UndiciFetcher['getHtml'];
-	public defaultJsonFetcher: UndiciFetcher;
-	public getJson: UndiciFetcher['getJson'];
-
-	//#region for old http/https, only used in S3Service
-	// http non-proxy agent
-	private http: http.Agent;
-
-	// https non-proxy agent
-	private https: https.Agent;
-
-	// http proxy or non-proxy agent
 	public httpAgent: http.Agent;
 
-	// https proxy or non-proxy agent
+	/**
+	 * Get https proxy or non-proxy agent
+	 */
 	public httpsAgent: https.Agent;
-	//#endregion
-
-	public readonly dnsCache: CacheableLookup;
-	public readonly clientDefaults: undici.Agent.Options;
-	private maxSockets: number;
-
-	private logger: Logger;
 
 	constructor(
 		@Inject(DI.config)
 		private config: Config,
-		private loggerService: LoggerService,
 	) {
-		this.logger = this.loggerService.getLogger('http-request');
-
-		this.dnsCache = new CacheableLookup({
+		const cache = new CacheableLookup({
 			maxTtl: 3600,	// 1hours
 			errorTtl: 30,	// 30secs
 			lookup: false,	// nativeのdns.lookupにfallbackしない
 		});
-
-		this.clientDefaults = {
-			keepAliveTimeout: 30 * 1000,
-			keepAliveMaxTimeout: 10 * 60 * 1000,
-			keepAliveTimeoutThreshold: 1 * 1000,
-			strictContentLength: true,
-			headersTimeout: 10 * 1000,
-			bodyTimeout: 10 * 1000,
-			maxHeaderSize: 16364, // default
-			maxResponseSize: 10 * 1024 * 1024,
-			maxRedirections: 3,
-			connect: {
-				timeout: 10 * 1000, // コネクションが確立するまでのタイムアウト
-				maxCachedSessions: 300, // TLSセッションのキャッシュ数 https://github.com/nodejs/undici/blob/v5.14.0/lib/core/connect.js#L80
-				lookup: this.dnsCache.lookup as LookupFunction, // https://github.com/nodejs/undici/blob/v5.14.0/lib/core/connect.js#L98
-			},
-		};
-
-		this.maxSockets = Math.max(64, ((this.config.deliverJobConcurrency ?? 128) / (this.config.clusterLimit ?? 1)));
-
-		this.defaultFetcher = this.createFetcher({}, {}, this.logger);
-
-		this.fetch = this.defaultFetcher.fetch;
-		this.request = this.defaultFetcher.request;
-		this.getHtml = this.defaultFetcher.getHtml;
-
-		this.defaultJsonFetcher = this.createFetcher({
-			maxResponseSize: 1024 * 256,
-		}, {}, this.logger);
-
-		this.getJson = this.defaultJsonFetcher.getJson;
-
-		//#region for old http/https, only used in S3Service
+		
 		this.http = new http.Agent({
 			keepAlive: true,
 			keepAliveMsecs: 30 * 1000,
-			lookup: this.dnsCache.lookup,
+			lookup: cache.lookup,
 		} as http.AgentOptions);
 		
 		this.https = new https.Agent({
 			keepAlive: true,
 			keepAliveMsecs: 30 * 1000,
-			lookup: this.dnsCache.lookup,
+			lookup: cache.lookup,
 		} as https.AgentOptions);
-
+		
+		const maxSockets = Math.max(256, config.deliverJobConcurrency ?? 128);
+		
 		this.httpAgent = config.proxy
 			? new HttpProxyAgent({
 				keepAlive: true,
 				keepAliveMsecs: 30 * 1000,
-				maxSockets: this.maxSockets,
+				maxSockets,
 				maxFreeSockets: 256,
 				scheduling: 'lifo',
 				proxy: config.proxy,
@@ -289,47 +72,21 @@ export class HttpRequestService {
 			? new HttpsProxyAgent({
 				keepAlive: true,
 				keepAliveMsecs: 30 * 1000,
-				maxSockets: this.maxSockets,
+				maxSockets,
 				maxFreeSockets: 256,
 				scheduling: 'lifo',
 				proxy: config.proxy,
 			})
 			: this.https;
-		//#endregion
-	}
-
-	@bindThis
-	private getStandardUndiciFetcherOption(opts: undici.Agent.Options = {}, proxyOpts: undici.Agent.Options = {}) {
-		return {
-			agentOptions: {
-				...this.clientDefaults,
-				...opts,
-			},
-			...(this.config.proxy ? {
-				proxy: {
-					uri: this.config.proxy,
-					options: {
-						connections: this.maxSockets,
-						...proxyOpts,
-					},
-				},
-			} : {}),
-			userAgent: this.config.userAgent,
-		};
-	}
-
-	@bindThis
-	public createFetcher(opts: undici.Agent.Options = {}, proxyOpts: undici.Agent.Options = {}, logger: Logger) {
-		return new UndiciFetcher(this.getStandardUndiciFetcherOption(opts, proxyOpts), logger);
 	}
 
 	/**
-	 * Get http agent by URL
+	 * Get agent by URL
 	 * @param url URL
 	 * @param bypassProxy Allways bypass proxy
 	 */
 	@bindThis
-	public getHttpAgentByUrl(url: URL, bypassProxy = false): http.Agent | https.Agent {
+	public getAgentByUrl(url: URL, bypassProxy = false): http.Agent | https.Agent {
 		if (bypassProxy || (this.config.proxyBypassHosts || []).includes(url.hostname)) {
 			return url.protocol === 'http:' ? this.http : this.https;
 		} else {
@@ -337,37 +94,67 @@ export class HttpRequestService {
 		}
 	}
 
-	/**
-	 * check ip
-	 */
 	@bindThis
-	public getConnectorWithIpCheck(connector: undici.buildConnector.connector, checkIp: IpChecker): undici.buildConnector.connectorAsync {
-		return (options, cb) => {
-			connector(options, (err, socket) => {
-				this.logger.debug('Socket connector (with ip checker) called', socket);
-				if (err) {
-					this.logger.error('Socket error', err);
-					cb(new Error(`Error while socket connecting\n${err}`), null);
-					return;
-				}
+	public async getJson(url: string, accept = 'application/json, */*', headers?: Record<string, string>): Promise<unknown> {
+		const res = await this.send(url, {
+			method: 'GET',
+			headers: Object.assign({
+				'User-Agent': this.config.userAgent,
+				Accept: accept,
+			}, headers ?? {}),
+			timeout: 5000,
+			size: 1024 * 256,
+		});
+
+		return await res.json();
+	}
+
+	@bindThis
+	public async getHtml(url: string, accept = 'text/html, */*', headers?: Record<string, string>): Promise<string> {
+		const res = await this.send(url, {
+			method: 'GET',
+			headers: Object.assign({
+				'User-Agent': this.config.userAgent,
+				Accept: accept,
+			}, headers ?? {}),
+			timeout: 5000,
+		});
+
+		return await res.text();
+	}
+
+	@bindThis
+	public async send(url: string, args: {
+		method?: string,
+		body?: string,
+		headers?: Record<string, string>,
+		timeout?: number,
+		size?: number,
+	} = {}, extra: {
+		throwErrorWhenResponseNotOk: boolean;
+	} = {
+		throwErrorWhenResponseNotOk: true,
+	}): Promise<Response> {
+		const timeout = args.timeout ?? 5000;
+
+		const controller = new AbortController();
+		setTimeout(() => {
+			controller.abort();
+		}, timeout);
 
-				if (socket.remoteAddress == undefined) {
-					this.logger.error('Socket error: remoteAddress is undefined');
-					cb(new Error('remoteAddress is undefined (maybe socket destroyed)'), null);
-					return;
-				}
+		const res = await fetch(url, {
+			method: args.method ?? 'GET',
+			headers: args.headers,
+			body: args.body,
+			size: args.size ?? 10 * 1024 * 1024,
+			agent: (url) => this.getAgentByUrl(url),
+			signal: controller.signal,
+		});
 
-				// allow
-				if (checkIp(socket.remoteAddress)) {
-					this.logger.debug(`Socket connected (ip ok): ${socket.localPort} => ${socket.remoteAddress}`);
-					cb(null, socket);
-					return;
-				}
+		if (!res.ok && extra.throwErrorWhenResponseNotOk) {
+			throw new StatusError(`${res.status} ${res.statusText}`, res.status, res.statusText);
+		}
 
-				this.logger.error('IP is not allowed', socket);
-				cb(new StatusError('IP is not allowed', 403, 'IP is not allowed'), null);
-				socket.destroy();
-			});
-		};
+		return res;
 	}
 }