From 978a9bbb3beafb7322aaf81d097430de6dbf527e Mon Sep 17 00:00:00 2001
From: tamaina <tamaina@hotmail.co.jp>
Date: Thu, 12 Jan 2023 21:03:02 +0900
Subject: perf(backend): Use undici instead of node-fetch and got (#9459)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

* Implement? HttpFetchService

* :v:

* remove node-fetch

* fix

* refactor

* fix

* gateway timeout

* UndiciFetcherクラスを追加 (仮コミット, ビルドもstartもさせていない)

* fix

* add logger and fix url preview

* fix ip check

* enhance logger and error handling

* fix

* fix

* clean up

* Use custom fetcher for ApRequest / ApResolver

* bypassProxyはproxyBypassHostsに判断を委譲するように

* set maxRedirections (default 3, ApRequest/ApResolver: 0)

* fix comment

* handle error s3 upload

* add debug message

* no return await

* Revert "no return await"

This reverts commit b5b0dc58a342393d260492e3a6f58304372f53b2.

* reduce maxSockets

* apResolverのUndiciFetcherを廃止しapRequestのものを使う、 add ap logger

* Revert "apResolverのUndiciFetcherを廃止しapRequestのものを使う、 add ap logger"

This reverts commit 997243915c8e1f8472da64f607f88c36cb1d5cb4.

* add logger

* fix

* change logger name

* safe

* デフォルトでUser-Agentを設定
---
 packages/backend/src/core/DownloadService.ts | 90 ++++++++++------------------
 1 file changed, 32 insertions(+), 58 deletions(-)

(limited to 'packages/backend/src/core/DownloadService.ts')

diff --git a/packages/backend/src/core/DownloadService.ts b/packages/backend/src/core/DownloadService.ts
index 62123246a7..a3078bff45 100644
--- a/packages/backend/src/core/DownloadService.ts
+++ b/packages/backend/src/core/DownloadService.ts
@@ -8,11 +8,12 @@ import got, * as Got from 'got';
 import chalk from 'chalk';
 import { DI } from '@/di-symbols.js';
 import type { Config } from '@/config.js';
-import { HttpRequestService } from '@/core/HttpRequestService.js';
+import { HttpRequestService, UndiciFetcher } from '@/core/HttpRequestService.js';
 import { createTemp } from '@/misc/create-temp.js';
 import { StatusError } from '@/misc/status-error.js';
 import { LoggerService } from '@/core/LoggerService.js';
 import type Logger from '@/logger.js';
+import { buildConnector } from 'undici';
 
 const pipeline = util.promisify(stream.pipeline);
 import { bindThis } from '@/decorators.js';
@@ -20,6 +21,7 @@ import { bindThis } from '@/decorators.js';
 @Injectable()
 export class DownloadService {
 	private logger: Logger;
+	private undiciFetcher: UndiciFetcher;
 
 	constructor(
 		@Inject(DI.config)
@@ -29,70 +31,42 @@ export class DownloadService {
 		private loggerService: LoggerService,
 	) {
 		this.logger = this.loggerService.getLogger('download');
+
+		this.undiciFetcher = new UndiciFetcher(this.httpRequestService.getStandardUndiciFetcherOption(
+			{
+				connect: process.env.NODE_ENV === 'development' ?
+					this.httpRequestService.clientDefaults.connect
+					:
+					this.httpRequestService.getConnectorWithIpCheck(
+						buildConnector({
+							...this.httpRequestService.clientDefaults.connect,
+						}),
+						(ip) => !this.isPrivateIp(ip)
+					),
+				bodyTimeout: 30 * 1000,
+			},
+			{
+				connect: this.httpRequestService.clientDefaults.connect,
+			}
+		), this.logger);
 	}
 
 	@bindThis
 	public async downloadUrl(url: string, path: string): Promise<void> {
 		this.logger.info(`Downloading ${chalk.cyan(url)} to ${chalk.cyanBright(path)} ...`);
-	
+
 		const timeout = 30 * 1000;
 		const operationTimeout = 60 * 1000;
 		const maxSize = this.config.maxFileSize ?? 262144000;
-	
-		const req = got.stream(url, {
-			headers: {
-				'User-Agent': this.config.userAgent,
-			},
-			timeout: {
-				lookup: timeout,
-				connect: timeout,
-				secureConnect: timeout,
-				socket: timeout,	// read timeout
-				response: timeout,
-				send: timeout,
-				request: operationTimeout,	// whole operation timeout
-			},
-			agent: {
-				http: this.httpRequestService.httpAgent,
-				https: this.httpRequestService.httpsAgent,
-			},
-			http2: false,	// default
-			retry: {
-				limit: 0,
-			},
-		}).on('response', (res: Got.Response) => {
-			if ((process.env.NODE_ENV === 'production' || process.env.NODE_ENV === 'test') && !this.config.proxy && res.ip) {
-				if (this.isPrivateIp(res.ip)) {
-					this.logger.warn(`Blocked address: ${res.ip}`);
-					req.destroy();
-				}
-			}
-	
-			const contentLength = res.headers['content-length'];
-			if (contentLength != null) {
-				const size = Number(contentLength);
-				if (size > maxSize) {
-					this.logger.warn(`maxSize exceeded (${size} > ${maxSize}) on response`);
-					req.destroy();
-				}
-			}
-		}).on('downloadProgress', (progress: Got.Progress) => {
-			if (progress.transferred > maxSize) {
-				this.logger.warn(`maxSize exceeded (${progress.transferred} > ${maxSize}) on downloadProgress`);
-				req.destroy();
-			}
-		});
-	
-		try {
-			await pipeline(req, fs.createWriteStream(path));
-		} catch (e) {
-			if (e instanceof Got.HTTPError) {
-				throw new StatusError(`${e.response.statusCode} ${e.response.statusMessage}`, e.response.statusCode, e.response.statusMessage);
-			} else {
-				throw e;
-			}
+
+		const response = await this.undiciFetcher.fetch(url);
+
+		if (response.body === null) {
+			throw new StatusError('No body', 400, 'No body');
 		}
-	
+
+		await pipeline(stream.Readable.fromWeb(response.body), fs.createWriteStream(path));
+
 		this.logger.succ(`Download finished: ${chalk.cyan(url)}`);
 	}
 
@@ -114,7 +88,7 @@ export class DownloadService {
 			cleanup();
 		}
 	}
-	
+
 	@bindThis
 	private isPrivateIp(ip: string): boolean {
 		for (const net of this.config.allowedPrivateNetworks ?? []) {
@@ -124,6 +98,6 @@ export class DownloadService {
 			}
 		}
 
-		return PrivateIp(ip);
+		return PrivateIp(ip) ?? false;
 	}
 }
-- 
cgit v1.2.3-freya