1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
|
import * as fs from 'node:fs';
import * as stream from 'node:stream';
import * as util from 'node:util';
import { Inject, Injectable } from '@nestjs/common';
import IPCIDR from 'ip-cidr';
import PrivateIp from 'private-ip';
import chalk from 'chalk';
import got, * as Got from 'got';
import { DI } from '@/di-symbols.js';
import type { Config } from '@/config.js';
import { HttpRequestService } from '@/core/HttpRequestService.js';
import { createTemp } from '@/misc/create-temp.js';
import { StatusError } from '@/misc/status-error.js';
import { LoggerService } from '@/core/LoggerService.js';
import type Logger from '@/logger.js';
const pipeline = util.promisify(stream.pipeline);
import { bindThis } from '@/decorators.js';
@Injectable()
export class DownloadService {
private logger: Logger;
constructor(
@Inject(DI.config)
private config: Config,
private httpRequestService: HttpRequestService,
private loggerService: LoggerService,
) {
this.logger = this.loggerService.getLogger('download');
}
@bindThis
public async downloadUrl(url: string, path: string): Promise<void> {
this.logger.info(`Downloading ${chalk.cyan(url)} to ${chalk.cyanBright(path)} ...`);
const timeout = 30 * 1000;
const operationTimeout = 60 * 1000;
const maxSize = this.config.maxFileSize ?? 262144000;
const req = got.stream(url, {
headers: {
'User-Agent': this.config.userAgent,
},
timeout: {
lookup: timeout,
connect: timeout,
secureConnect: timeout,
socket: timeout, // read timeout
response: timeout,
send: timeout,
request: operationTimeout, // whole operation timeout
},
agent: {
http: this.httpRequestService.httpAgent,
https: this.httpRequestService.httpsAgent,
},
http2: false, // default
retry: {
limit: 0,
},
enableUnixSockets: false,
}).on('response', (res: Got.Response) => {
if ((process.env.NODE_ENV === 'production' || process.env.NODE_ENV === 'test') && !this.config.proxy && res.ip) {
if (this.isPrivateIp(res.ip)) {
this.logger.warn(`Blocked address: ${res.ip}`);
req.destroy();
}
}
const contentLength = res.headers['content-length'];
if (contentLength != null) {
const size = Number(contentLength);
if (size > maxSize) {
this.logger.warn(`maxSize exceeded (${size} > ${maxSize}) on response`);
req.destroy();
}
}
}).on('downloadProgress', (progress: Got.Progress) => {
if (progress.transferred > maxSize) {
this.logger.warn(`maxSize exceeded (${progress.transferred} > ${maxSize}) on downloadProgress`);
req.destroy();
}
});
try {
await pipeline(req, fs.createWriteStream(path));
} catch (e) {
if (e instanceof Got.HTTPError) {
throw new StatusError(`${e.response.statusCode} ${e.response.statusMessage}`, e.response.statusCode, e.response.statusMessage);
} else {
throw e;
}
}
this.logger.succ(`Download finished: ${chalk.cyan(url)}`);
}
@bindThis
public async downloadTextFile(url: string): Promise<string> {
// Create temp file
const [path, cleanup] = await createTemp();
this.logger.info(`text file: Temp file is ${path}`);
try {
// write content at URL to temp file
await this.downloadUrl(url, path);
const text = await util.promisify(fs.readFile)(path, 'utf8');
return text;
} finally {
cleanup();
}
}
@bindThis
private isPrivateIp(ip: string): boolean {
for (const net of this.config.allowedPrivateNetworks ?? []) {
const cidr = new IPCIDR(net);
if (cidr.contains(ip)) {
return false;
}
}
return PrivateIp(ip) ?? false;
}
}
|
