refactoring

Resolve #7779

packages/backend/src/misc/download-url.ts

@@ -0,0 +1,87 @@
+import * as fs from 'fs';
+import * as stream from 'stream';
+import * as util from 'util';
+import got, * as Got from 'got';
+import { httpAgent, httpsAgent, StatusError } from './fetch';
+import config from '@/config/index';
+import * as chalk from 'chalk';
+import Logger from '@/services/logger';
+import * as IPCIDR from 'ip-cidr';
+const PrivateIp = require('private-ip');
+
+const pipeline = util.promisify(stream.pipeline);
+
+export async function downloadUrl(url: string, path: string) {
+	const logger = new Logger('download');
+
+	logger.info(`Downloading ${chalk.cyan(url)} ...`);
+
+	const timeout = 30 * 1000;
+	const operationTimeout = 60 * 1000;
+	const maxSize = config.maxFileSize || 262144000;
+
+	const req = got.stream(url, {
+		headers: {
+			'User-Agent': config.userAgent
+		},
+		timeout: {
+			lookup: timeout,
+			connect: timeout,
+			secureConnect: timeout,
+			socket: timeout,	// read timeout
+			response: timeout,
+			send: timeout,
+			request: operationTimeout,	// whole operation timeout
+		},
+		agent: {
+			http: httpAgent,
+			https: httpsAgent,
+		},
+		http2: false,	// default
+		retry: 0,
+	}).on('response', (res: Got.Response) => {
+		if ((process.env.NODE_ENV === 'production' || process.env.NODE_ENV === 'test') && !config.proxy && res.ip) {
+			if (isPrivateIp(res.ip)) {
+				logger.warn(`Blocked address: ${res.ip}`);
+				req.destroy();
+			}
+		}
+
+		const contentLength = res.headers['content-length'];
+		if (contentLength != null) {
+			const size = Number(contentLength);
+			if (size > maxSize) {
+				logger.warn(`maxSize exceeded (${size} > ${maxSize}) on response`);
+				req.destroy();
+			}
+		}
+	}).on('downloadProgress', (progress: Got.Progress) => {
+		if (progress.transferred > maxSize) {
+			logger.warn(`maxSize exceeded (${progress.transferred} > ${maxSize}) on downloadProgress`);
+			req.destroy();
+		}
+	});
+
+	try {
+		await pipeline(req, fs.createWriteStream(path));
+	} catch (e) {
+		if (e instanceof Got.HTTPError) {
+			throw new StatusError(`${e.response.statusCode} ${e.response.statusMessage}`, e.response.statusCode, e.response.statusMessage);
+		} else {
+			throw e;
+		}
+	}
+
+	logger.succ(`Download finished: ${chalk.cyan(url)}`);
+}
+
+function isPrivateIp(ip: string) {
+	for (const net of config.allowedPrivateNetworks || []) {
+		const cidr = new IPCIDR(net);
+		if (cidr.contains(ip)) {
+			return false;
+		}
+	}
+
+	return PrivateIp(ip);
+}