From e1a8b158e04ad567d92d8daf3cc0898ee18f1a2e Mon Sep 17 00:00:00 2001 From: MeiMei <30769358+mei23@users.noreply.github.com> Date: Fri, 3 Sep 2021 21:00:44 +0900 Subject: Tune download (#2) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * s2-2 * allowedPrivateNetworks * style * Proxyの間にあると誤解しそうなのでconfigの記述順を変更 * Fix error handler --- src/misc/download-url.ts | 67 +++++++++++++++++++++++++++++++++++------------- 1 file changed, 49 insertions(+), 18 deletions(-) (limited to 'src/misc') diff --git a/src/misc/download-url.ts b/src/misc/download-url.ts index 43e061c715..463fb555bb 100644 --- a/src/misc/download-url.ts +++ b/src/misc/download-url.ts @@ -1,13 +1,13 @@ import * as fs from 'fs'; import * as stream from 'stream'; import * as util from 'util'; -import { URL } from 'url'; -import fetch from 'node-fetch'; -import { getAgentByUrl } from './fetch'; -import { AbortController } from 'abort-controller'; +import got, * as Got from 'got'; +import { httpAgent, httpsAgent } from './fetch'; import config from '@/config/index'; import * as chalk from 'chalk'; import Logger from '@/services/logger'; +import * as IPCIDR from 'ip-cidr'; +const PrivateIp = require('private-ip'); const pipeline = util.promisify(stream.pipeline); @@ -15,26 +15,57 @@ export async function downloadUrl(url: string, path: string) { const logger = new Logger('download'); logger.info(`Downloading ${chalk.cyan(url)} ...`); - const controller = new AbortController(); - setTimeout(() => { - controller.abort(); - }, 60 * 1000); - const response = await fetch(new URL(url).href, { + const timeout = 30 * 1000; + const operationTimeout = 60 * 1000; + + const req = got.stream(url, { headers: { 'User-Agent': config.userAgent }, - timeout: 10 * 1000, - signal: controller.signal, - agent: getAgentByUrl, + timeout: { + lookup: timeout, + connect: timeout, + secureConnect: timeout, + socket: timeout, // read timeout + response: timeout, + send: timeout, + request: operationTimeout, // whole operation timeout + }, + agent: { + http: httpAgent, + https: httpsAgent, + }, + retry: 0, + }).on('response', (res: Got.Response) => { + if ((process.env.NODE_ENV === 'production' || process.env.NODE_ENV === 'test') && !config.proxy && res.ip) { + if (isPrivateIp(res.ip)) { + logger.warn(`Blocked address: ${res.ip}`); + req.destroy(); + } + } + }).on('error', (e: any) => { + if (e.name === 'HTTPError') { + const statusCode = e.response?.statusCode; + const statusMessage = e.response?.statusMessage; + e.name = `StatusError`; + e.statusCode = statusCode; + e.message = `${statusCode} ${statusMessage}`; + } }); - if (!response.ok) { - logger.error(`Got ${response.status} (${url})`); - throw response.status; - } - - await pipeline(response.body, fs.createWriteStream(path)); + await pipeline(req, fs.createWriteStream(path)); logger.succ(`Download finished: ${chalk.cyan(url)}`); } + +function isPrivateIp(ip: string) { + for (const net of config.allowedPrivateNetworks || []) { + const cidr = new IPCIDR(net); + if (cidr.contains(ip)) { + return false; + } + } + + return PrivateIp(ip); +} -- cgit v1.2.3-freya From e21ff916b09258fb989a9eaa651d1b9d99266b22 Mon Sep 17 00:00:00 2001 From: MeiMei <30769358+mei23@users.noreply.github.com> Date: Sat, 4 Sep 2021 20:33:14 +0900 Subject: ファイルサイズのハードリミット (#7760) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * maxFileSize * CHANGELOG --- .config/example.yml | 3 +++ CHANGELOG.md | 2 ++ src/config/types.ts | 2 ++ src/misc/download-url.ts | 15 +++++++++++++++ src/server/api/index.ts | 7 ++++++- 5 files changed, 28 insertions(+), 1 deletion(-) (limited to 'src/misc') diff --git a/.config/example.yml b/.config/example.yml index 2e13b63977..38d5e59ea7 100644 --- a/.config/example.yml +++ b/.config/example.yml @@ -161,3 +161,6 @@ id: 'aid' #allowedPrivateNetworks: [ # '127.0.0.1/32' #] + +# Upload or download file size limits (bytes) +#maxFileSize: 262144000 diff --git a/CHANGELOG.md b/CHANGELOG.md index d02f177e93..7ee4486eb9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,6 +17,8 @@ - Proxy使用時には、この制限は適用されません。 Proxy使用時に同等の制限を行いたい場合は、Proxy側で設定を行う必要があります。 - `default.yml`にて`allowedPrivateNetworks`にCIDRを追加することにより、宛先ネットワークを指定してこの制限から除外することが出来ます。 +- アップロード, ダウンロード出来るファイルサイズにハードリミットが適用されるようになりました。(約250MB) + - `default.yml`にて`maxFileSize`を変更することにより、制限値を変更することが出来ます。 ### Bugfixes - 管理者が最初にサインアップするページでログインされないのを修正 diff --git a/src/config/types.ts b/src/config/types.ts index 55beac6f55..e3ca6c1ab6 100644 --- a/src/config/types.ts +++ b/src/config/types.ts @@ -39,6 +39,8 @@ export type Source = { allowedPrivateNetworks?: string[]; + maxFileSize?: number; + accesslog?: string; clusterLimit?: number; diff --git a/src/misc/download-url.ts b/src/misc/download-url.ts index 463fb555bb..8a8640a8cd 100644 --- a/src/misc/download-url.ts +++ b/src/misc/download-url.ts @@ -18,6 +18,7 @@ export async function downloadUrl(url: string, path: string) { const timeout = 30 * 1000; const operationTimeout = 60 * 1000; + const maxSize = config.maxFileSize || 262144000; const req = got.stream(url, { headers: { @@ -44,6 +45,20 @@ export async function downloadUrl(url: string, path: string) { req.destroy(); } } + + const contentLength = res.headers['content-length']; + if (contentLength != null) { + const size = Number(contentLength); + if (size > maxSize) { + logger.warn(`maxSize exceeded (${size} > ${maxSize}) on response`); + req.destroy(); + } + } + }).on('downloadProgress', (progress: Got.Progress) => { + if (progress.transferred > maxSize) { + logger.warn(`maxSize exceeded (${progress.transferred} > ${maxSize}) on downloadProgress`); + req.destroy(); + } }).on('error', (e: any) => { if (e.name === 'HTTPError') { const statusCode = e.response?.statusCode; diff --git a/src/server/api/index.ts b/src/server/api/index.ts index 55083261ee..db35fdf9e0 100644 --- a/src/server/api/index.ts +++ b/src/server/api/index.ts @@ -16,6 +16,7 @@ import discord from './service/discord'; import github from './service/github'; import twitter from './service/twitter'; import { Instances, AccessTokens, Users } from '@/models/index'; +import config from '@/config'; // Init app const app = new Koa(); @@ -37,7 +38,11 @@ app.use(bodyParser({ // Init multer instance const upload = multer({ - storage: multer.diskStorage({}) + storage: multer.diskStorage({}), + limits: { + fileSize: config.maxFileSize || 262144000, + files: 1, + } }); // Init router -- cgit v1.2.3-freya