Spaces:
Build error
Build error
File size: 3,364 Bytes
0bfe2e3 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 |
import { Cache } from './cache';
import { HEADERS_FOR_IP_FORWARDING } from './constants';
import { Env } from './env';
import { createLogger, maskSensitiveInfo } from './logger';
import { fetch, ProxyAgent } from 'undici';
const logger = createLogger('http');
const urlCount = Cache.getInstance<string, number>('url-count');
export class PossibleRecursiveRequestError extends Error {
constructor(message: string) {
super(message);
this.name = 'PossibleRecursiveRequestError';
}
}
export function makeUrlLogSafe(url: string) {
// for each component of the path, if it is longer than 10 characters, mask it
// and replace the query params of key 'password' with '****'
return url
.split('/')
.map((component) => {
if (component.length > 10 && !component.includes('.')) {
return maskSensitiveInfo(component);
}
return component;
})
.join('/')
.replace(/(?<![^?&])(password=[^&]+)/g, 'password=****');
}
export function makeRequest(
url: string,
timeout: number,
headers: HeadersInit = {},
forwardIp?: string,
ignoreRecursion?: boolean
) {
const useProxy = shouldProxy(url);
headers = new Headers(headers);
if (forwardIp) {
for (const header of HEADERS_FOR_IP_FORWARDING) {
headers.set(header, forwardIp);
}
}
if (headers.get('User-Agent') === 'none') {
headers.delete('User-Agent');
}
// block recursive requests
const key = `${url}-${forwardIp}`;
const currentCount = urlCount.get(key, false) ?? 0;
if (currentCount > Env.RECURSION_THRESHOLD_LIMIT && !ignoreRecursion) {
logger.warn(
`Detected possible recursive requests to ${url}. Current count: ${currentCount}. Blocking request.`
);
throw new PossibleRecursiveRequestError(
`Possible recursive request to ${url}`
);
}
if (currentCount > 0) {
urlCount.update(key, currentCount + 1);
} else {
urlCount.set(key, 1, Env.RECURSION_THRESHOLD_WINDOW);
}
logger.debug(
`Making a ${useProxy ? 'proxied' : 'direct'} request to ${makeUrlLogSafe(
url
)} with forwarded ip ${maskSensitiveInfo(forwardIp ?? 'none')} and headers ${maskSensitiveInfo(JSON.stringify(headers))}`
);
let response = fetch(url, {
dispatcher: useProxy ? new ProxyAgent(Env.ADDON_PROXY!) : undefined,
method: 'GET',
headers: headers,
signal: AbortSignal.timeout(timeout),
});
return response;
}
function shouldProxy(url: string) {
let shouldProxy = false;
let hostname: string;
try {
hostname = new URL(url).hostname;
} catch (error) {
return false;
}
if (!Env.ADDON_PROXY) {
return false;
}
shouldProxy = true;
if (Env.ADDON_PROXY_CONFIG) {
for (const rule of Env.ADDON_PROXY_CONFIG.split(',')) {
const [ruleHostname, ruleShouldProxy] = rule.split(':');
if (['true', 'false'].includes(ruleShouldProxy) === false) {
logger.error(`Invalid proxy config: ${rule}`);
continue;
}
if (ruleHostname === '*') {
shouldProxy = !(ruleShouldProxy === 'false');
} else if (ruleHostname.startsWith('*')) {
if (hostname.endsWith(ruleHostname.slice(1))) {
shouldProxy = !(ruleShouldProxy === 'false');
}
}
if (hostname === ruleHostname) {
shouldProxy = !(ruleShouldProxy === 'false');
}
}
}
return shouldProxy;
}
|