Spaces:
Sleeping
Sleeping
/** | |
* Index.js | |
* | |
* a request API compatible with window.fetch | |
* | |
* All spec algorithm step numbers are based on https://fetch.spec.whatwg.org/commit-snapshots/ae716822cb3a61843226cd090eefc6589446c1d2/. | |
*/ | |
import http from 'node:http'; | |
import https from 'node:https'; | |
import zlib from 'node:zlib'; | |
import Stream, {PassThrough, pipeline as pump} from 'node:stream'; | |
import {Buffer} from 'node:buffer'; | |
import dataUriToBuffer from 'data-uri-to-buffer'; | |
import {writeToStream, clone} from './body.js'; | |
import Response from './response.js'; | |
import Headers, {fromRawHeaders} from './headers.js'; | |
import Request, {getNodeRequestOptions} from './request.js'; | |
import {FetchError} from './errors/fetch-error.js'; | |
import {AbortError} from './errors/abort-error.js'; | |
import {isRedirect} from './utils/is-redirect.js'; | |
import {FormData} from 'formdata-polyfill/esm.min.js'; | |
import {isDomainOrSubdomain, isSameProtocol} from './utils/is.js'; | |
import {parseReferrerPolicyFromHeader} from './utils/referrer.js'; | |
import { | |
Blob, | |
File, | |
fileFromSync, | |
fileFrom, | |
blobFromSync, | |
blobFrom | |
} from 'fetch-blob/from.js'; | |
export {FormData, Headers, Request, Response, FetchError, AbortError, isRedirect}; | |
export {Blob, File, fileFromSync, fileFrom, blobFromSync, blobFrom}; | |
const supportedSchemas = new Set(['data:', 'http:', 'https:']); | |
/** | |
* Fetch function | |
* | |
* @param {string | URL | import('./request').default} url - Absolute url or Request instance | |
* @param {*} [options_] - Fetch options | |
* @return {Promise<import('./response').default>} | |
*/ | |
export default async function fetch(url, options_) { | |
return new Promise((resolve, reject) => { | |
// Build request object | |
const request = new Request(url, options_); | |
const {parsedURL, options} = getNodeRequestOptions(request); | |
if (!supportedSchemas.has(parsedURL.protocol)) { | |
throw new TypeError(`node-fetch cannot load ${url}. URL scheme "${parsedURL.protocol.replace(/:$/, '')}" is not supported.`); | |
} | |
if (parsedURL.protocol === 'data:') { | |
const data = dataUriToBuffer(request.url); | |
const response = new Response(data, {headers: {'Content-Type': data.typeFull}}); | |
resolve(response); | |
return; | |
} | |
// Wrap http.request into fetch | |
const send = (parsedURL.protocol === 'https:' ? https : http).request; | |
const {signal} = request; | |
let response = null; | |
const abort = () => { | |
const error = new AbortError('The operation was aborted.'); | |
reject(error); | |
if (request.body && request.body instanceof Stream.Readable) { | |
request.body.destroy(error); | |
} | |
if (!response || !response.body) { | |
return; | |
} | |
response.body.emit('error', error); | |
}; | |
if (signal && signal.aborted) { | |
abort(); | |
return; | |
} | |
const abortAndFinalize = () => { | |
abort(); | |
finalize(); | |
}; | |
// Send request | |
const request_ = send(parsedURL.toString(), options); | |
if (signal) { | |
signal.addEventListener('abort', abortAndFinalize); | |
} | |
const finalize = () => { | |
request_.abort(); | |
if (signal) { | |
signal.removeEventListener('abort', abortAndFinalize); | |
} | |
}; | |
request_.on('error', error => { | |
reject(new FetchError(`request to ${request.url} failed, reason: ${error.message}`, 'system', error)); | |
finalize(); | |
}); | |
fixResponseChunkedTransferBadEnding(request_, error => { | |
if (response && response.body) { | |
response.body.destroy(error); | |
} | |
}); | |
/* c8 ignore next 18 */ | |
if (process.version < 'v14') { | |
// Before Node.js 14, pipeline() does not fully support async iterators and does not always | |
// properly handle when the socket close/end events are out of order. | |
request_.on('socket', s => { | |
let endedWithEventsCount; | |
s.prependListener('end', () => { | |
endedWithEventsCount = s._eventsCount; | |
}); | |
s.prependListener('close', hadError => { | |
// if end happened before close but the socket didn't emit an error, do it now | |
if (response && endedWithEventsCount < s._eventsCount && !hadError) { | |
const error = new Error('Premature close'); | |
error.code = 'ERR_STREAM_PREMATURE_CLOSE'; | |
response.body.emit('error', error); | |
} | |
}); | |
}); | |
} | |
request_.on('response', response_ => { | |
request_.setTimeout(0); | |
const headers = fromRawHeaders(response_.rawHeaders); | |
// HTTP fetch step 5 | |
if (isRedirect(response_.statusCode)) { | |
// HTTP fetch step 5.2 | |
const location = headers.get('Location'); | |
// HTTP fetch step 5.3 | |
let locationURL = null; | |
try { | |
locationURL = location === null ? null : new URL(location, request.url); | |
} catch { | |
// error here can only be invalid URL in Location: header | |
// do not throw when options.redirect == manual | |
// let the user extract the errorneous redirect URL | |
if (request.redirect !== 'manual') { | |
reject(new FetchError(`uri requested responds with an invalid redirect URL: ${location}`, 'invalid-redirect')); | |
finalize(); | |
return; | |
} | |
} | |
// HTTP fetch step 5.5 | |
switch (request.redirect) { | |
case 'error': | |
reject(new FetchError(`uri requested responds with a redirect, redirect mode is set to error: ${request.url}`, 'no-redirect')); | |
finalize(); | |
return; | |
case 'manual': | |
// Nothing to do | |
break; | |
case 'follow': { | |
// HTTP-redirect fetch step 2 | |
if (locationURL === null) { | |
break; | |
} | |
// HTTP-redirect fetch step 5 | |
if (request.counter >= request.follow) { | |
reject(new FetchError(`maximum redirect reached at: ${request.url}`, 'max-redirect')); | |
finalize(); | |
return; | |
} | |
// HTTP-redirect fetch step 6 (counter increment) | |
// Create a new Request object. | |
const requestOptions = { | |
headers: new Headers(request.headers), | |
follow: request.follow, | |
counter: request.counter + 1, | |
agent: request.agent, | |
compress: request.compress, | |
method: request.method, | |
body: clone(request), | |
signal: request.signal, | |
size: request.size, | |
referrer: request.referrer, | |
referrerPolicy: request.referrerPolicy | |
}; | |
// when forwarding sensitive headers like "Authorization", | |
// "WWW-Authenticate", and "Cookie" to untrusted targets, | |
// headers will be ignored when following a redirect to a domain | |
// that is not a subdomain match or exact match of the initial domain. | |
// For example, a redirect from "foo.com" to either "foo.com" or "sub.foo.com" | |
// will forward the sensitive headers, but a redirect to "bar.com" will not. | |
// headers will also be ignored when following a redirect to a domain using | |
// a different protocol. For example, a redirect from "https://foo.com" to "http://foo.com" | |
// will not forward the sensitive headers | |
if (!isDomainOrSubdomain(request.url, locationURL) || !isSameProtocol(request.url, locationURL)) { | |
for (const name of ['authorization', 'www-authenticate', 'cookie', 'cookie2']) { | |
requestOptions.headers.delete(name); | |
} | |
} | |
// HTTP-redirect fetch step 9 | |
if (response_.statusCode !== 303 && request.body && options_.body instanceof Stream.Readable) { | |
reject(new FetchError('Cannot follow redirect with body being a readable stream', 'unsupported-redirect')); | |
finalize(); | |
return; | |
} | |
// HTTP-redirect fetch step 11 | |
if (response_.statusCode === 303 || ((response_.statusCode === 301 || response_.statusCode === 302) && request.method === 'POST')) { | |
requestOptions.method = 'GET'; | |
requestOptions.body = undefined; | |
requestOptions.headers.delete('content-length'); | |
} | |
// HTTP-redirect fetch step 14 | |
const responseReferrerPolicy = parseReferrerPolicyFromHeader(headers); | |
if (responseReferrerPolicy) { | |
requestOptions.referrerPolicy = responseReferrerPolicy; | |
} | |
// HTTP-redirect fetch step 15 | |
resolve(fetch(new Request(locationURL, requestOptions))); | |
finalize(); | |
return; | |
} | |
default: | |
return reject(new TypeError(`Redirect option '${request.redirect}' is not a valid value of RequestRedirect`)); | |
} | |
} | |
// Prepare response | |
if (signal) { | |
response_.once('end', () => { | |
signal.removeEventListener('abort', abortAndFinalize); | |
}); | |
} | |
let body = pump(response_, new PassThrough(), error => { | |
if (error) { | |
reject(error); | |
} | |
}); | |
// see https://github.com/nodejs/node/pull/29376 | |
/* c8 ignore next 3 */ | |
if (process.version < 'v12.10') { | |
response_.on('aborted', abortAndFinalize); | |
} | |
const responseOptions = { | |
url: request.url, | |
status: response_.statusCode, | |
statusText: response_.statusMessage, | |
headers, | |
size: request.size, | |
counter: request.counter, | |
highWaterMark: request.highWaterMark | |
}; | |
// HTTP-network fetch step 12.1.1.3 | |
const codings = headers.get('Content-Encoding'); | |
// HTTP-network fetch step 12.1.1.4: handle content codings | |
// in following scenarios we ignore compression support | |
// 1. compression support is disabled | |
// 2. HEAD request | |
// 3. no Content-Encoding header | |
// 4. no content response (204) | |
// 5. content not modified response (304) | |
if (!request.compress || request.method === 'HEAD' || codings === null || response_.statusCode === 204 || response_.statusCode === 304) { | |
response = new Response(body, responseOptions); | |
resolve(response); | |
return; | |
} | |
// For Node v6+ | |
// Be less strict when decoding compressed responses, since sometimes | |
// servers send slightly invalid responses that are still accepted | |
// by common browsers. | |
// Always using Z_SYNC_FLUSH is what cURL does. | |
const zlibOptions = { | |
flush: zlib.Z_SYNC_FLUSH, | |
finishFlush: zlib.Z_SYNC_FLUSH | |
}; | |
// For gzip | |
if (codings === 'gzip' || codings === 'x-gzip') { | |
body = pump(body, zlib.createGunzip(zlibOptions), error => { | |
if (error) { | |
reject(error); | |
} | |
}); | |
response = new Response(body, responseOptions); | |
resolve(response); | |
return; | |
} | |
// For deflate | |
if (codings === 'deflate' || codings === 'x-deflate') { | |
// Handle the infamous raw deflate response from old servers | |
// a hack for old IIS and Apache servers | |
const raw = pump(response_, new PassThrough(), error => { | |
if (error) { | |
reject(error); | |
} | |
}); | |
raw.once('data', chunk => { | |
// See http://stackoverflow.com/questions/37519828 | |
if ((chunk[0] & 0x0F) === 0x08) { | |
body = pump(body, zlib.createInflate(), error => { | |
if (error) { | |
reject(error); | |
} | |
}); | |
} else { | |
body = pump(body, zlib.createInflateRaw(), error => { | |
if (error) { | |
reject(error); | |
} | |
}); | |
} | |
response = new Response(body, responseOptions); | |
resolve(response); | |
}); | |
raw.once('end', () => { | |
// Some old IIS servers return zero-length OK deflate responses, so | |
// 'data' is never emitted. See https://github.com/node-fetch/node-fetch/pull/903 | |
if (!response) { | |
response = new Response(body, responseOptions); | |
resolve(response); | |
} | |
}); | |
return; | |
} | |
// For br | |
if (codings === 'br') { | |
body = pump(body, zlib.createBrotliDecompress(), error => { | |
if (error) { | |
reject(error); | |
} | |
}); | |
response = new Response(body, responseOptions); | |
resolve(response); | |
return; | |
} | |
// Otherwise, use response as-is | |
response = new Response(body, responseOptions); | |
resolve(response); | |
}); | |
// eslint-disable-next-line promise/prefer-await-to-then | |
writeToStream(request_, request).catch(reject); | |
}); | |
} | |
function fixResponseChunkedTransferBadEnding(request, errorCallback) { | |
const LAST_CHUNK = Buffer.from('0\r\n\r\n'); | |
let isChunkedTransfer = false; | |
let properLastChunkReceived = false; | |
let previousChunk; | |
request.on('response', response => { | |
const {headers} = response; | |
isChunkedTransfer = headers['transfer-encoding'] === 'chunked' && !headers['content-length']; | |
}); | |
request.on('socket', socket => { | |
const onSocketClose = () => { | |
if (isChunkedTransfer && !properLastChunkReceived) { | |
const error = new Error('Premature close'); | |
error.code = 'ERR_STREAM_PREMATURE_CLOSE'; | |
errorCallback(error); | |
} | |
}; | |
const onData = buf => { | |
properLastChunkReceived = Buffer.compare(buf.slice(-5), LAST_CHUNK) === 0; | |
// Sometimes final 0-length chunk and end of message code are in separate packets | |
if (!properLastChunkReceived && previousChunk) { | |
properLastChunkReceived = ( | |
Buffer.compare(previousChunk.slice(-3), LAST_CHUNK.slice(0, 3)) === 0 && | |
Buffer.compare(buf.slice(-2), LAST_CHUNK.slice(3)) === 0 | |
); | |
} | |
previousChunk = buf; | |
}; | |
socket.prependListener('close', onSocketClose); | |
socket.on('data', onData); | |
request.on('close', () => { | |
socket.removeListener('close', onSocketClose); | |
socket.removeListener('data', onData); | |
}); | |
}); | |
} | |