nomagick commited on
Commit
5171e5f
·
unverified ·
1 Parent(s): 405fe63

fix: req cap issues

Browse files
backend/functions/src/services/puppeteer.ts CHANGED
@@ -1,7 +1,7 @@
1
  import os from 'os';
2
  import fs from 'fs';
3
  import { container, singleton } from 'tsyringe';
4
- import { AsyncService, Defer, marshalErrorLike, AssertionFailureError, delay, maxConcurrency, Deferred } from 'civkit';
5
  import { Logger } from '../shared/services/logger';
6
 
7
  import type { Browser, CookieParam, GoToOptions, Page } from 'puppeteer';
@@ -310,11 +310,14 @@ export class PuppeteerControl extends AsyncService {
310
  this.logger.warn(`Browser killed`);
311
  }
312
 
 
313
  reqCapRoutine() {
314
  const now = Date.now();
315
  const numToPass = Math.round((now - this.lastReqSentAt) / 1000 * this.rpsCap);
316
  this.requestDeferredQueue.splice(0, numToPass).forEach((x) => x.resolve(true));
317
- this.lastReqSentAt = now;
 
 
318
  if (!this.requestDeferredQueue.length) {
319
  if (this.__reqCapInterval) {
320
  clearInterval(this.__reqCapInterval);
@@ -403,10 +406,12 @@ export class PuppeteerControl extends AsyncService {
403
  return req.abort('blockedbyclient', 1000);
404
  }
405
 
406
- const d = Defer();
407
- this.requestDeferredQueue.push(d);
408
- process.nextTick(() => this.reqCapRoutine());
409
- await d.promise;
 
 
410
 
411
  if (req.isInterceptResolutionHandled()) {
412
  return;
 
1
  import os from 'os';
2
  import fs from 'fs';
3
  import { container, singleton } from 'tsyringe';
4
+ import { AsyncService, Defer, marshalErrorLike, AssertionFailureError, delay, maxConcurrency, Deferred, perNextTick } from 'civkit';
5
  import { Logger } from '../shared/services/logger';
6
 
7
  import type { Browser, CookieParam, GoToOptions, Page } from 'puppeteer';
 
310
  this.logger.warn(`Browser killed`);
311
  }
312
 
313
+ @perNextTick()
314
  reqCapRoutine() {
315
  const now = Date.now();
316
  const numToPass = Math.round((now - this.lastReqSentAt) / 1000 * this.rpsCap);
317
  this.requestDeferredQueue.splice(0, numToPass).forEach((x) => x.resolve(true));
318
+ if (numToPass) {
319
+ this.lastReqSentAt = now;
320
+ }
321
  if (!this.requestDeferredQueue.length) {
322
  if (this.__reqCapInterval) {
323
  clearInterval(this.__reqCapInterval);
 
406
  return req.abort('blockedbyclient', 1000);
407
  }
408
 
409
+ if (requestUrl.startsWith('http')) {
410
+ const d = Defer();
411
+ this.requestDeferredQueue.push(d);
412
+ this.reqCapRoutine();
413
+ await d.promise;
414
+ }
415
 
416
  if (req.isInterceptResolutionHandled()) {
417
  return;