Spaces:
Build error
Build error
fix: curl redirection location
Browse files- src/services/curl.ts +3 -3
src/services/curl.ts
CHANGED
|
@@ -10,7 +10,7 @@ import { AssertionFailureError, FancyFile } from 'civkit';
|
|
| 10 |
import { ServiceBadAttemptError, TempFileManager } from '../shared';
|
| 11 |
import { createBrotliDecompress, createInflate, createGunzip } from 'zlib';
|
| 12 |
import { ZSTDDecompress } from 'simple-zstd';
|
| 13 |
-
import _
|
| 14 |
import { Readable } from 'stream';
|
| 15 |
import { AsyncLocalContext } from './async-context';
|
| 16 |
|
|
@@ -277,7 +277,7 @@ export class CurlControl extends AsyncService {
|
|
| 277 |
|
| 278 |
if ([301, 302, 307, 308].includes(r.statusCode)) {
|
| 279 |
const headers = r.headers[r.headers.length - 1];
|
| 280 |
-
const location = headers.Location || headers.location;
|
| 281 |
|
| 282 |
const setCookieHeader = headers['Set-Cookie'] || headers['set-cookie'];
|
| 283 |
if (setCookieHeader) {
|
|
@@ -292,7 +292,7 @@ export class CurlControl extends AsyncService {
|
|
| 292 |
throw new AssertionFailureError(`Failed to access ${urlToCrawl}: Bad redirection from ${nextHopUrl}`);
|
| 293 |
}
|
| 294 |
|
| 295 |
-
nextHopUrl = new URL(location, nextHopUrl);
|
| 296 |
fakeHeaderInfos.push(...r.headers);
|
| 297 |
leftRedirection -= 1;
|
| 298 |
continue;
|
|
|
|
| 10 |
import { ServiceBadAttemptError, TempFileManager } from '../shared';
|
| 11 |
import { createBrotliDecompress, createInflate, createGunzip } from 'zlib';
|
| 12 |
import { ZSTDDecompress } from 'simple-zstd';
|
| 13 |
+
import _ from 'lodash';
|
| 14 |
import { Readable } from 'stream';
|
| 15 |
import { AsyncLocalContext } from './async-context';
|
| 16 |
|
|
|
|
| 277 |
|
| 278 |
if ([301, 302, 307, 308].includes(r.statusCode)) {
|
| 279 |
const headers = r.headers[r.headers.length - 1];
|
| 280 |
+
const location: string | undefined = headers.Location || headers.location;
|
| 281 |
|
| 282 |
const setCookieHeader = headers['Set-Cookie'] || headers['set-cookie'];
|
| 283 |
if (setCookieHeader) {
|
|
|
|
| 292 |
throw new AssertionFailureError(`Failed to access ${urlToCrawl}: Bad redirection from ${nextHopUrl}`);
|
| 293 |
}
|
| 294 |
|
| 295 |
+
nextHopUrl = new URL(location || '', nextHopUrl);
|
| 296 |
fakeHeaderInfos.push(...r.headers);
|
| 297 |
leftRedirection -= 1;
|
| 298 |
continue;
|