forked from mirrors/firefish
104 lines
2.5 KiB
TypeScript
104 lines
2.5 KiB
TypeScript
import * as fs from "node:fs";
|
|
import * as stream from "node:stream";
|
|
import * as util from "node:util";
|
|
import got, * as Got from "got";
|
|
import { httpAgent, httpsAgent, StatusError } from "./fetch.js";
|
|
import config from "@/config/index.js";
|
|
import chalk from "chalk";
|
|
import Logger from "@/services/logger.js";
|
|
import IPCIDR from "ip-cidr";
|
|
import PrivateIp from "private-ip";
|
|
|
|
const pipeline = util.promisify(stream.pipeline);
|
|
|
|
export async function downloadUrl(url: string, path: string): Promise<void> {
|
|
const logger = new Logger("download");
|
|
|
|
logger.info(`Downloading ${chalk.cyan(url)} ...`);
|
|
|
|
const timeout = 30 * 1000;
|
|
const operationTimeout = 60 * 1000;
|
|
const maxSize = config.maxFileSize || 262144000;
|
|
|
|
const req = got
|
|
.stream(url, {
|
|
headers: {
|
|
"User-Agent": config.userAgent,
|
|
},
|
|
timeout: {
|
|
lookup: timeout,
|
|
connect: timeout,
|
|
secureConnect: timeout,
|
|
socket: timeout, // read timeout
|
|
response: timeout,
|
|
send: timeout,
|
|
request: operationTimeout, // whole operation timeout
|
|
},
|
|
agent: {
|
|
http: httpAgent,
|
|
https: httpsAgent,
|
|
},
|
|
http2: false, // default
|
|
retry: {
|
|
limit: 0,
|
|
},
|
|
})
|
|
.on("response", (res: Got.Response) => {
|
|
if (
|
|
(process.env.NODE_ENV === "production" ||
|
|
process.env.NODE_ENV === "test") &&
|
|
!config.proxy &&
|
|
res.ip
|
|
) {
|
|
if (isPrivateIp(res.ip)) {
|
|
logger.warn(`Blocked address: ${res.ip}`);
|
|
req.destroy();
|
|
}
|
|
}
|
|
|
|
const contentLength = res.headers["content-length"];
|
|
if (contentLength != null) {
|
|
const size = Number(contentLength);
|
|
if (size > maxSize) {
|
|
logger.warn(`maxSize exceeded (${size} > ${maxSize}) on response`);
|
|
req.destroy();
|
|
}
|
|
}
|
|
})
|
|
.on("downloadProgress", (progress: Got.Progress) => {
|
|
if (progress.transferred > maxSize) {
|
|
logger.warn(
|
|
`maxSize exceeded (${progress.transferred} > ${maxSize}) on downloadProgress`,
|
|
);
|
|
req.destroy();
|
|
}
|
|
});
|
|
|
|
try {
|
|
await pipeline(req, fs.createWriteStream(path));
|
|
} catch (e) {
|
|
if (e instanceof Got.HTTPError) {
|
|
throw new StatusError(
|
|
`${e.response.statusCode} ${e.response.statusMessage}`,
|
|
e.response.statusCode,
|
|
e.response.statusMessage,
|
|
);
|
|
} else {
|
|
throw e;
|
|
}
|
|
}
|
|
|
|
logger.succ(`Download finished: ${chalk.cyan(url)}`);
|
|
}
|
|
|
|
function isPrivateIp(ip: string): boolean {
|
|
for (const net of config.allowedPrivateNetworks || []) {
|
|
const cidr = new IPCIDR(net);
|
|
if (cidr.contains(ip)) {
|
|
return false;
|
|
}
|
|
}
|
|
|
|
return PrivateIp(ip);
|
|
}
|