117 lines
2.5 KiB
TypeScript
117 lines
2.5 KiB
TypeScript
import * as http from 'http';
|
|
import * as https from 'https';
|
|
import CacheableLookup from 'cacheable-lookup';
|
|
import fetch, { HeadersInit } from 'node-fetch';
|
|
import { HttpProxyAgent, HttpsProxyAgent } from 'hpagent';
|
|
import config from '@/config/index.js';
|
|
import { URL } from 'url';
|
|
|
|
export async function getJson(url: string, accept = 'application/json, */*', timeout = 10000, headers?: HeadersInit) {
|
|
const res = await fetch(url, {
|
|
headers: Object.assign({
|
|
'User-Agent': config.userAgent,
|
|
Accept: accept
|
|
}, headers || {}),
|
|
timeout,
|
|
agent: getAgentByUrl,
|
|
});
|
|
|
|
if (!res.ok) {
|
|
throw {
|
|
name: `StatusError`,
|
|
statusCode: res.status,
|
|
message: `${res.status} ${res.statusText}`,
|
|
};
|
|
}
|
|
|
|
return await res.json();
|
|
}
|
|
|
|
export async function getHtml(url: string, accept = 'text/html, */*', timeout = 10000, headers?: HeadersInit) {
|
|
const res = await fetch(url, {
|
|
headers: Object.assign({
|
|
'User-Agent': config.userAgent,
|
|
Accept: accept
|
|
}, headers || {}),
|
|
timeout,
|
|
agent: getAgentByUrl,
|
|
});
|
|
|
|
if (!res.ok) {
|
|
throw {
|
|
name: `StatusError`,
|
|
statusCode: res.status,
|
|
message: `${res.status} ${res.statusText}`,
|
|
};
|
|
}
|
|
|
|
return await res.text();
|
|
}
|
|
|
|
const cache = new CacheableLookup({
|
|
maxTtl: 3600, // 1hours
|
|
errorTtl: 30, // 30secs
|
|
lookup: false, // nativeのdns.lookupにfallbackしない
|
|
});
|
|
|
|
/**
|
|
* Get http non-proxy agent
|
|
*/
|
|
const _http = new http.Agent({
|
|
keepAlive: true,
|
|
keepAliveMsecs: 30 * 1000,
|
|
lookup: cache.lookup,
|
|
} as http.AgentOptions);
|
|
|
|
/**
|
|
* Get https non-proxy agent
|
|
*/
|
|
const _https = new https.Agent({
|
|
keepAlive: true,
|
|
keepAliveMsecs: 30 * 1000,
|
|
lookup: cache.lookup,
|
|
} as https.AgentOptions);
|
|
|
|
const maxSockets = Math.max(256, config.deliverJobConcurrency || 128);
|
|
|
|
/**
|
|
* Get http proxy or non-proxy agent
|
|
*/
|
|
export const httpAgent = config.proxy
|
|
? new HttpProxyAgent({
|
|
keepAlive: true,
|
|
keepAliveMsecs: 30 * 1000,
|
|
maxSockets,
|
|
maxFreeSockets: 256,
|
|
scheduling: 'lifo',
|
|
proxy: config.proxy
|
|
})
|
|
: _http;
|
|
|
|
/**
|
|
* Get https proxy or non-proxy agent
|
|
*/
|
|
export const httpsAgent = config.proxy
|
|
? new HttpsProxyAgent({
|
|
keepAlive: true,
|
|
keepAliveMsecs: 30 * 1000,
|
|
maxSockets,
|
|
maxFreeSockets: 256,
|
|
scheduling: 'lifo',
|
|
proxy: config.proxy
|
|
})
|
|
: _https;
|
|
|
|
/**
|
|
* Get agent by URL
|
|
* @param url URL
|
|
* @param bypassProxy Allways bypass proxy
|
|
*/
|
|
export function getAgentByUrl(url: URL, bypassProxy = false) {
|
|
if (bypassProxy || (config.proxyBypassHosts || []).includes(url.hostname)) {
|
|
return url.protocol == 'http:' ? _http : _https;
|
|
} else {
|
|
return url.protocol == 'http:' ? httpAgent : httpsAgent;
|
|
}
|
|
}
|