Spaces:
Build error
Build error
puppeteer: ditch puppeteer-stealth and use the real stable chrome
Browse files
backend/functions/Dockerfile
CHANGED
|
@@ -29,6 +29,7 @@ COPY licensed ./licensed
|
|
| 29 |
|
| 30 |
RUN rm -rf ~/.config/chromium && mkdir -p ~/.config/chromium
|
| 31 |
|
|
|
|
| 32 |
ENV LD_PRELOAD=/usr/local/lib/libcurl-impersonate.so CURL_IMPERSONATE=chrome116 CURL_IMPERSONATE_HEADERS=no
|
| 33 |
ENV PORT=8080
|
| 34 |
|
|
|
|
| 29 |
|
| 30 |
RUN rm -rf ~/.config/chromium && mkdir -p ~/.config/chromium
|
| 31 |
|
| 32 |
+
ENV OVERRIDE_CHROME_EXECUTABLE_PATH=/usr/bin/google-chrome-stable
|
| 33 |
ENV LD_PRELOAD=/usr/local/lib/libcurl-impersonate.so CURL_IMPERSONATE=chrome116 CURL_IMPERSONATE_HEADERS=no
|
| 34 |
ENV PORT=8080
|
| 35 |
|
backend/functions/src/services/puppeteer.ts
CHANGED
|
@@ -85,8 +85,8 @@ export interface ScrappingOptions {
|
|
| 85 |
}
|
| 86 |
|
| 87 |
|
| 88 |
-
const puppeteerStealth = require('puppeteer-extra-plugin-stealth');
|
| 89 |
-
puppeteer.use(puppeteerStealth());
|
| 90 |
// const puppeteerUAOverride = require('puppeteer-extra-plugin-stealth/evasions/user-agent-override');
|
| 91 |
// puppeteer.use(puppeteerUAOverride({
|
| 92 |
// userAgent: `Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; GPTBot/1.0; +https://openai.com/gptbot)`,
|
|
@@ -224,7 +224,16 @@ const MUTATION_IDLE_WATCH = `
|
|
| 224 |
`;
|
| 225 |
|
| 226 |
const DONT_MESS_WITH_THE_FUNDAMENTALS = `
|
| 227 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 228 |
delete Function.prototype.bind.apply;
|
| 229 |
delete Function.prototype.bind.call;
|
| 230 |
`;
|
|
@@ -500,6 +509,8 @@ export class PuppeteerControl extends AsyncService {
|
|
| 500 |
}
|
| 501 |
this.browser = await puppeteer.launch({
|
| 502 |
timeout: 10_000,
|
|
|
|
|
|
|
| 503 |
args: ['--disable-dev-shm-usage']
|
| 504 |
}).catch((err: any) => {
|
| 505 |
this.logger.error(`Unknown firebase issue, just die fast.`, { err });
|
|
@@ -886,7 +897,7 @@ export class PuppeteerControl extends AsyncService {
|
|
| 886 |
}
|
| 887 |
|
| 888 |
let nextSnapshotDeferred = Defer();
|
| 889 |
-
nextSnapshotDeferred.promise.catch(()=> 'just dont crash anything');
|
| 890 |
const crippleListener = () => nextSnapshotDeferred.reject(new ServiceCrashedError({ message: `Browser crashed, try again` }));
|
| 891 |
this.once('crippled', crippleListener);
|
| 892 |
nextSnapshotDeferred.promise.finally(() => {
|
|
|
|
| 85 |
}
|
| 86 |
|
| 87 |
|
| 88 |
+
// const puppeteerStealth = require('puppeteer-extra-plugin-stealth');
|
| 89 |
+
// puppeteer.use(puppeteerStealth());
|
| 90 |
// const puppeteerUAOverride = require('puppeteer-extra-plugin-stealth/evasions/user-agent-override');
|
| 91 |
// puppeteer.use(puppeteerUAOverride({
|
| 92 |
// userAgent: `Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; GPTBot/1.0; +https://openai.com/gptbot)`,
|
|
|
|
| 224 |
`;
|
| 225 |
|
| 226 |
const DONT_MESS_WITH_THE_FUNDAMENTALS = `
|
| 227 |
+
(()=> {
|
| 228 |
+
const fnToStringDesc = Object.getOwnPropertyDescriptor(Function.prototype, 'toString');
|
| 229 |
+
Object.defineProperty(Function.prototype, 'toString', {
|
| 230 |
+
get: ()=> fnToStringDesc.value,
|
| 231 |
+
set: ()=> 'Dont mess with this',
|
| 232 |
+
writeable: true,
|
| 233 |
+
configurable: false,
|
| 234 |
+
enumerable: false,
|
| 235 |
+
});
|
| 236 |
+
})();
|
| 237 |
delete Function.prototype.bind.apply;
|
| 238 |
delete Function.prototype.bind.call;
|
| 239 |
`;
|
|
|
|
| 509 |
}
|
| 510 |
this.browser = await puppeteer.launch({
|
| 511 |
timeout: 10_000,
|
| 512 |
+
headless: true,
|
| 513 |
+
executablePath: process.env.OVERRIDE_CHROME_EXECUTABLE_PATH,
|
| 514 |
args: ['--disable-dev-shm-usage']
|
| 515 |
}).catch((err: any) => {
|
| 516 |
this.logger.error(`Unknown firebase issue, just die fast.`, { err });
|
|
|
|
| 897 |
}
|
| 898 |
|
| 899 |
let nextSnapshotDeferred = Defer();
|
| 900 |
+
nextSnapshotDeferred.promise.catch(() => 'just dont crash anything');
|
| 901 |
const crippleListener = () => nextSnapshotDeferred.reject(new ServiceCrashedError({ message: `Browser crashed, try again` }));
|
| 902 |
this.once('crippled', crippleListener);
|
| 903 |
nextSnapshotDeferred.promise.finally(() => {
|