Merge branch 'develop'

4 files changed, 74 insertions, 31 deletions

diff --git a/src/misc/download-url.ts b/src/misc/download-url.ts
index 8a8640a8cd..c96b4fd1d6 100644
--- a/src/misc/download-url.ts
+++ b/src/misc/download-url.ts
@@ -2,7 +2,7 @@ import * as fs from 'fs';
 import * as stream from 'stream';
 import * as util from 'util';
 import got, * as Got from 'got';
-import { httpAgent, httpsAgent } from './fetch';
+import { httpAgent, httpsAgent, StatusError } from './fetch';
 import config from '@/config/index';
 import * as chalk from 'chalk';
 import Logger from '@/services/logger';
@@ -37,6 +37,7 @@ export async function downloadUrl(url: string, path: string) {
 			http: httpAgent,
 			https: httpsAgent,
 		},
+		http2: false,	// default
 		retry: 0,
 	}).on('response', (res: Got.Response) => {
 		if ((process.env.NODE_ENV === 'production' || process.env.NODE_ENV === 'test') && !config.proxy && res.ip) {
@@ -59,17 +60,17 @@ export async function downloadUrl(url: string, path: string) {
 			logger.warn(`maxSize exceeded (${progress.transferred} > ${maxSize}) on downloadProgress`);
 			req.destroy();
 		}
-	}).on('error', (e: any) => {
-		if (e.name === 'HTTPError') {
-			const statusCode = e.response?.statusCode;
-			const statusMessage = e.response?.statusMessage;
-			e.name = `StatusError`;
-			e.statusCode = statusCode;
-			e.message = `${statusCode} ${statusMessage}`;
-		}
 	});
 
-	await pipeline(req, fs.createWriteStream(path));
+	try {
+		await pipeline(req, fs.createWriteStream(path));
+	} catch (e) {
+		if (e instanceof Got.HTTPError) {
+			throw new StatusError(`${e.response.statusCode} ${e.response.statusMessage}`, e.response.statusCode, e.response.statusMessage);
+		} else {
+			throw e;
+		}
+	}
 
 	logger.succ(`Download finished: ${chalk.cyan(url)}`);
 }
diff --git a/src/misc/fetch.ts b/src/misc/fetch.ts
index 82db2f2f8c..f4f16a27e2 100644
--- a/src/misc/fetch.ts
+++ b/src/misc/fetch.ts
@@ -1,51 +1,62 @@
 import * as http from 'http';
 import * as https from 'https';
 import CacheableLookup from 'cacheable-lookup';
-import fetch, { HeadersInit } from 'node-fetch';
+import fetch from 'node-fetch';
 import { HttpProxyAgent, HttpsProxyAgent } from 'hpagent';
 import config from '@/config/index';
 import { URL } from 'url';
 
-export async function getJson(url: string, accept = 'application/json, */*', timeout = 10000, headers?: HeadersInit) {
-	const res = await fetch(url, {
+export async function getJson(url: string, accept = 'application/json, */*', timeout = 10000, headers?: Record<string, string>) {
+	const res = await getResponse({
+		url,
+		method: 'GET',
 		headers: Object.assign({
 			'User-Agent': config.userAgent,
 			Accept: accept
 		}, headers || {}),
-		timeout,
-		agent: getAgentByUrl,
+		timeout
 	});
 
-	if (!res.ok) {
-		throw {
-			name: `StatusError`,
-			statusCode: res.status,
-			message: `${res.status} ${res.statusText}`,
-		};
-	}
-
 	return await res.json();
 }
 
-export async function getHtml(url: string, accept = 'text/html, */*', timeout = 10000, headers?: HeadersInit) {
-	const res = await fetch(url, {
+export async function getHtml(url: string, accept = 'text/html, */*', timeout = 10000, headers?: Record<string, string>) {
+	const res = await getResponse({
+		url,
+		method: 'GET',
 		headers: Object.assign({
 			'User-Agent': config.userAgent,
 			Accept: accept
 		}, headers || {}),
+		timeout
+	});
+
+	return await res.text();
+}
+
+export async function getResponse(args: { url: string, method: string, body?: string, headers: Record<string, string>, timeout?: number, size?: number }) {
+	const timeout = args?.timeout || 10 * 1000;
+
+	const controller = new AbortController();
+	setTimeout(() => {
+		controller.abort();
+	}, timeout * 6);
+
+	const res = await fetch(args.url, {
+		method: args.method,
+		headers: args.headers,
+		body: args.body,
 		timeout,
+		size: args?.size || 10 * 1024 * 1024,
 		agent: getAgentByUrl,
+		signal: controller.signal,
 	});
 
 	if (!res.ok) {
-		throw {
-			name: `StatusError`,
-			statusCode: res.status,
-			message: `${res.status} ${res.statusText}`,
-		};
+		throw new StatusError(`${res.status} ${res.statusText}`, res.status, res.statusText);
 	}
 
-	return await res.text();
+	return res;
 }
 
 const cache = new CacheableLookup({
@@ -114,3 +125,17 @@ export function getAgentByUrl(url: URL, bypassProxy = false) {
 		return url.protocol == 'http:' ? httpAgent : httpsAgent;
 	}
 }
+
+export class StatusError extends Error {
+	public statusCode: number;
+	public statusMessage?: string;
+	public isClientError: boolean;
+
+	constructor(message: string, statusCode: number, statusMessage?: string) {
+		super(message);
+		this.name = 'StatusError';
+		this.statusCode = statusCode;
+		this.statusMessage = statusMessage;
+		this.isClientError = typeof this.statusCode === 'number' && this.statusCode >= 400 && this.statusCode < 500;
+	}
+}
diff --git a/src/misc/hard-limits.ts b/src/misc/hard-limits.ts
index 2a61cb321b..1039f7335a 100644
--- a/src/misc/hard-limits.ts
+++ b/src/misc/hard-limits.ts
@@ -6,3 +6,9 @@
  * Surrogate pairs count as one
  */
 export const DB_MAX_NOTE_TEXT_LENGTH = 8192;
+
+/**
+ * Maximum image description length that can be stored in DB.
+ * Surrogate pairs count as one
+ */
+export const DB_MAX_IMAGE_COMMENT_LENGTH = 512;
diff --git a/src/misc/truncate.ts b/src/misc/truncate.ts
new file mode 100644
index 0000000000..cb120331a1
--- /dev/null
+++ b/src/misc/truncate.ts
@@ -0,0 +1,11 @@
+import { substring } from 'stringz';
+
+export function truncate(input: string, size: number): string;
+export function truncate(input: string | undefined, size: number): string | undefined;
+export function truncate(input: string | undefined, size: number): string | undefined {
+	if (!input) {
+		return input;
+	} else {
+		return substring(input, 0, size);
+	}
+}