From b75184ec8e3436200bacdcd832e3324702553d20 Mon Sep 17 00:00:00 2001
From: syuilo <Syuilotan@yahoo.co.jp>
Date: Sun, 18 Sep 2022 03:27:08 +0900
Subject: なんかもうめっちゃ変えた
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 packages/backend/src/core/DownloadService.ts | 123 +++++++++++++++++++++++++++
 1 file changed, 123 insertions(+)
 create mode 100644 packages/backend/src/core/DownloadService.ts

(limited to 'packages/backend/src/core/DownloadService.ts')

diff --git a/packages/backend/src/core/DownloadService.ts b/packages/backend/src/core/DownloadService.ts
new file mode 100644
index 0000000000..84d5ca2e8b
--- /dev/null
+++ b/packages/backend/src/core/DownloadService.ts
@@ -0,0 +1,123 @@
+import * as fs from 'node:fs';
+import * as stream from 'node:stream';
+import * as util from 'node:util';
+import { Inject, Injectable } from '@nestjs/common';
+import IPCIDR from 'ip-cidr';
+import PrivateIp from 'private-ip';
+import got, * as Got from 'got';
+import chalk from 'chalk';
+import { DI } from '@/di-symbols.js';
+import { Config } from '@/config.js';
+import Logger from '@/logger.js';
+import { HttpRequestService } from '@/core/HttpRequestService.js';
+import { createTemp } from '@/misc/create-temp.js';
+import { StatusError } from '@/misc/status-error.js';
+
+const pipeline = util.promisify(stream.pipeline);
+
+@Injectable()
+export class DownloadService {
+	#logger: Logger;
+
+	constructor(
+		@Inject(DI.config)
+		private config: Config,
+
+		private httpRequestService: HttpRequestService,
+	) {
+		this.#logger = new Logger('download');
+	}
+
+	public async downloadUrl(url: string, path: string): Promise<void> {
+		this.#logger.info(`Downloading ${chalk.cyan(url)} ...`);
+	
+		const timeout = 30 * 1000;
+		const operationTimeout = 60 * 1000;
+		const maxSize = this.config.maxFileSize ?? 262144000;
+	
+		const req = got.stream(url, {
+			headers: {
+				'User-Agent': this.config.userAgent,
+			},
+			timeout: {
+				lookup: timeout,
+				connect: timeout,
+				secureConnect: timeout,
+				socket: timeout,	// read timeout
+				response: timeout,
+				send: timeout,
+				request: operationTimeout,	// whole operation timeout
+			},
+			agent: {
+				http: this.httpRequestService.httpAgent,
+				https: this.httpRequestService.httpsAgent,
+			},
+			http2: false,	// default
+			retry: {
+				limit: 0,
+			},
+		}).on('response', (res: Got.Response) => {
+			if ((process.env.NODE_ENV === 'production' || process.env.NODE_ENV === 'test') && !this.config.proxy && res.ip) {
+				if (this.#isPrivateIp(res.ip)) {
+					this.#logger.warn(`Blocked address: ${res.ip}`);
+					req.destroy();
+				}
+			}
+	
+			const contentLength = res.headers['content-length'];
+			if (contentLength != null) {
+				const size = Number(contentLength);
+				if (size > maxSize) {
+					this.#logger.warn(`maxSize exceeded (${size} > ${maxSize}) on response`);
+					req.destroy();
+				}
+			}
+		}).on('downloadProgress', (progress: Got.Progress) => {
+			if (progress.transferred > maxSize) {
+				this.#logger.warn(`maxSize exceeded (${progress.transferred} > ${maxSize}) on downloadProgress`);
+				req.destroy();
+			}
+		});
+	
+		try {
+			await pipeline(req, fs.createWriteStream(path));
+		} catch (e) {
+			if (e instanceof Got.HTTPError) {
+				throw new StatusError(`${e.response.statusCode} ${e.response.statusMessage}`, e.response.statusCode, e.response.statusMessage);
+			} else {
+				throw e;
+			}
+		}
+	
+		this.#logger.succ(`Download finished: ${chalk.cyan(url)}`);
+	}
+
+	public async downloadTextFile(url: string): Promise<string> {
+		// Create temp file
+		const [path, cleanup] = await createTemp();
+	
+		this.#logger.info(`text file: Temp file is ${path}`);
+	
+		try {
+			// write content at URL to temp file
+			await this.downloadUrl(url, path);
+	
+			const text = await util.promisify(fs.readFile)(path, 'utf8');
+	
+			return text;
+		} finally {
+			cleanup();
+		}
+	}
+	
+	#isPrivateIp(ip: string): boolean {
+		for (const net of this.config.allowedPrivateNetworks ?? []) {
+			const cidr = new IPCIDR(net);
+			if (cidr.contains(ip)) {
+				return false;
+			}
+		}
+
+		return PrivateIp(ip);
+	}
+}
-- 
cgit v1.2.3-freya