Spaces:
Build error
Build error
fix: req cap issues
Browse files
backend/functions/src/services/puppeteer.ts
CHANGED
|
@@ -1,7 +1,7 @@
|
|
| 1 |
import os from 'os';
|
| 2 |
import fs from 'fs';
|
| 3 |
import { container, singleton } from 'tsyringe';
|
| 4 |
-
import { AsyncService, Defer, marshalErrorLike, AssertionFailureError, delay, maxConcurrency, Deferred } from 'civkit';
|
| 5 |
import { Logger } from '../shared/services/logger';
|
| 6 |
|
| 7 |
import type { Browser, CookieParam, GoToOptions, Page } from 'puppeteer';
|
|
@@ -310,11 +310,14 @@ export class PuppeteerControl extends AsyncService {
|
|
| 310 |
this.logger.warn(`Browser killed`);
|
| 311 |
}
|
| 312 |
|
|
|
|
| 313 |
reqCapRoutine() {
|
| 314 |
const now = Date.now();
|
| 315 |
const numToPass = Math.round((now - this.lastReqSentAt) / 1000 * this.rpsCap);
|
| 316 |
this.requestDeferredQueue.splice(0, numToPass).forEach((x) => x.resolve(true));
|
| 317 |
-
|
|
|
|
|
|
|
| 318 |
if (!this.requestDeferredQueue.length) {
|
| 319 |
if (this.__reqCapInterval) {
|
| 320 |
clearInterval(this.__reqCapInterval);
|
|
@@ -403,10 +406,12 @@ export class PuppeteerControl extends AsyncService {
|
|
| 403 |
return req.abort('blockedbyclient', 1000);
|
| 404 |
}
|
| 405 |
|
| 406 |
-
|
| 407 |
-
|
| 408 |
-
|
| 409 |
-
|
|
|
|
|
|
|
| 410 |
|
| 411 |
if (req.isInterceptResolutionHandled()) {
|
| 412 |
return;
|
|
|
|
| 1 |
import os from 'os';
|
| 2 |
import fs from 'fs';
|
| 3 |
import { container, singleton } from 'tsyringe';
|
| 4 |
+
import { AsyncService, Defer, marshalErrorLike, AssertionFailureError, delay, maxConcurrency, Deferred, perNextTick } from 'civkit';
|
| 5 |
import { Logger } from '../shared/services/logger';
|
| 6 |
|
| 7 |
import type { Browser, CookieParam, GoToOptions, Page } from 'puppeteer';
|
|
|
|
| 310 |
this.logger.warn(`Browser killed`);
|
| 311 |
}
|
| 312 |
|
| 313 |
+
@perNextTick()
|
| 314 |
reqCapRoutine() {
|
| 315 |
const now = Date.now();
|
| 316 |
const numToPass = Math.round((now - this.lastReqSentAt) / 1000 * this.rpsCap);
|
| 317 |
this.requestDeferredQueue.splice(0, numToPass).forEach((x) => x.resolve(true));
|
| 318 |
+
if (numToPass) {
|
| 319 |
+
this.lastReqSentAt = now;
|
| 320 |
+
}
|
| 321 |
if (!this.requestDeferredQueue.length) {
|
| 322 |
if (this.__reqCapInterval) {
|
| 323 |
clearInterval(this.__reqCapInterval);
|
|
|
|
| 406 |
return req.abort('blockedbyclient', 1000);
|
| 407 |
}
|
| 408 |
|
| 409 |
+
if (requestUrl.startsWith('http')) {
|
| 410 |
+
const d = Defer();
|
| 411 |
+
this.requestDeferredQueue.push(d);
|
| 412 |
+
this.reqCapRoutine();
|
| 413 |
+
await d.promise;
|
| 414 |
+
}
|
| 415 |
|
| 416 |
if (req.isInterceptResolutionHandled()) {
|
| 417 |
return;
|