Repository: codemanki/cloudscraper (js)

Issue:
Post request fails after initial cloudflare bypass.
Hi!

I'm using Cloudscraper version `4.14` on Node version `12.10.0`.

I'm attempting to access [this website](https://nakedcph.com), which has a cloudflare protection page with a captcha.

I can bypass the cloudflare and access the site's homepage/any page, however, after bypassing, I am unable to successfully send a post request. The weirdest part about this is that my code works using the Python version of the module recreating the exact same requests.

When sending the post request, the console sometimes prints:

`request received invalid json` when debug is on.

In my second request I need a csrf token that gets returned with the first request's (the bypass request) response. Essentially I am trying to create an account on this website by first retrieving the csrf after the initial bypass (which I can do successfully) and then sending a post request with the account information.

Like I said, I can do this successfully in Python which leads me to believe the issue is related to the module's way of handling post requests, but of course I'm probably wrong. This is my code when sending both requests.

```js
const captchaAPI = require("imagetyperz-api")
const cloudscraper = require("cloudscraper").defaults({ onCaptcha })

cloudscraper.debug = true

let captchaRes
// CALLED IF SCRAPER RUNS INTO CAPTCHA
async function onCaptcha(options, response, body) {
  const captchaData = response.captcha
  captchaRes = await solveCaptcha(
    response.request.uri.href,
    captchaData.siteKey
  )
  captchaData.form["g-recaptcha-response"] = captchaRes
  captchaData.submit()
}

// CALLED WHEN SOLVING A CAPTCHA IS NECESSARY
async function solveCaptcha(uri, sitekey) {
  captchaAPI.set_access_key("KEY")
  const params = {
    page_url: uri,
    sitekey: sitekey
  }
  console.log("Solving captcha...")
  const id = await captchaAPI.submit_recaptcha(params)
  const token = await captchaAPI.retrieve_recaptcha(id)
  return token
}

// GENERATE AN ACCOUNT
async function genAccount(req) {
  const Csrf = req.body.match(/setRequestHeader\('X-AntiCsrfToken', '(.+)'/)[1]
  console.log(`CSRF: ${Csrf}`)
  cloudscraper.defaultParams.headers = {}
  console.log(`CAPTCHA: ${captchaRes}`)
  const headers = {
    authority: "www.nakedcph.com",
    path: "/auth/submit",
    scheme: "https",
    accept: "application/json, text/javascript, */*; q=0.01",
    "accept-encoding": "gzip, deflate, br",
    "accept-language": "en-US,en;q=0.9,pt;q=0.8",
    "Content-Type": "application/x-www-form-urlencoded; charset=UTF-8",
    origin: "https://www.nakedcph.com",
    referer: "https://www.nakedcph.com/auth/view?op=register",
    "sec-fetch-mode": "cors",
    "sec-fetch-site": "same-origin",
    "user-agent":
      "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/76.0.3809.87 Safari/537.36",
    "X-AntiCsrfToken": Csrf,
    "X-Requested-With": "XMLHttpRequest"
  }
  const gen = await cloudscraper({
    url: "https://nakedcph.com/auth/submit",
    method: "POST",
    resolveWithFullResponse: true,
    // followOriginalHttpMethod: true,
    json: true,
    simple: false,
    headers: headers,
    formData: {
      _AntiCsrfToken: Csrf,
      firstName: "CLOUDSCRAPER TEST",
      email: "cloudscraper+123456@gmail.com",
      password: "MyPassword123",
      "g-recaptcha-response": String(captchaRes),
      action: "register"
    }
  })
  console.log("RESULT:")
  console.log(gen.statusCode)
  console.log(gen.body)
}

const headers = {
    "User-Agent":
      "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.81 Safari/537.36",
    Accept: "application/json, text/javascript, */*; q=0.01",
    "Accept-Language": "en-US,en;q=0.5",
    "Accept-Encoding": "gzip, deflate, br",
    "X-Requested-With": "XMLHttpRequest"
  }
// FIRST REQUEST GETS SENT HERE
  const req = await cloudscraper({
    url: "https://www.nakedcph.com",
    method: "GET",
    resolveWithFullResponse: true,
    // json: true,
    simple: false,
    headers: headers
  })
  if (req.statusCode === 200) {
// IF FIRST REQUEST IS GOOD AND CLOUDFLARE IS BYPASSED, WE TRY SENDING THE POST
    await genAccount(req)
  }
  console.log(req.statusCode)
```

I get a 200 on the first request, and a 403 on the 2nd. This is what the server returns on the 403:

`{ Response: null, StatusCode: 500, Status: '' }`

Hopefully I'm being really stupid and there is a super simple solution to this. And like I said, my Python version is 100% functional doing the exact same request with the same headers and everything.

Thanks and sorry for any confusions and the long code. I've been looking at this for way longer than I should have and haven't been able to find the solution. Any help is much appreciated.

Pull request:
Update reCaptcha handling
Fixes #259 

Patch:
diff --git a/docs/examples/solve-recaptcha-v2.js b/docs/examples/solve-recaptcha-v2.js
index d31b6dd..83a95f7 100755
--- a/docs/examples/solve-recaptcha-v2.js
+++ b/docs/examples/solve-recaptcha-v2.js
@@ -8,7 +8,7 @@ const cloudscraper = require('../..').defaults({ onCaptcha: handler, headers });
 function handler (options, { captcha }) {
   return new Promise((resolve, reject) => {
     // Here you do some magic with the siteKey provided by cloudscraper
-    console.error('The url is "' + captcha.url + '"');
+    console.error('The url is "' + captcha.uri.href + '"');
     console.error('The site key is "' + captcha.siteKey + '"');
     // captcha.form['g-recaptcha-response'] = /* Obtain from your service */
     reject(new Error('This is a dummy function.'));
@@ -16,9 +16,9 @@ function handler (options, { captcha }) {
 }
 
 // An example handler with destructuring arguments
-function alternative (options, { captcha: { url, siteKey } }) {
+function alternative (options, { captcha: { uri, siteKey } }) {
   // Here you do some magic with the siteKey provided by cloudscraper
-  console.error('The url is "' + url + '"');
+  console.error('The url is "' + uri.href + '"');
   console.error('The site key is "' + siteKey + '"');
   return Promise.reject(new Error('This is a dummy function'));
 }
diff --git a/docs/examples/solve-recaptcha.js b/docs/examples/solve-recaptcha.js
index 10ea7a7..5962092 100755
--- a/docs/examples/solve-recaptcha.js
+++ b/docs/examples/solve-recaptcha.js
@@ -7,7 +7,7 @@ function solveReCAPTCHA (url, sitekey, callback) {
 function onCaptcha (options, response, body) {
   const captcha = response.captcha;
   // solveReCAPTCHA is a method that you should come up with and pass it href and sitekey, in return it will return you a reponse
-  solveReCAPTCHA(response.request.uri.href, captcha.siteKey, (error, gRes) => {
+  solveReCAPTCHA(captcha.uri.href, captcha.siteKey, (error, gRes) => {
     // eslint-disable-next-line no-void
     if (error) return void captcha.submit(error);
     captcha.form['g-recaptcha-response'] = gRes;
diff --git a/index.js b/index.js
index 4e9dabb..55134f3 100644
--- a/index.js
+++ b/index.js
@@ -6,6 +6,7 @@ const decodeEmails = require('./lib/email-decode.js');
 const { getDefaultHeaders, caseless } = require('./lib/headers');
 const brotli = require('./lib/brotli');
 const crypto = require('crypto');
+const { deprecate } = require('util');
 
 const {
   RequestError,
@@ -14,6 +15,8 @@ const {
   ParserError
 } = require('./errors');
 
+let debugging = false;
+
 const HOST = Symbol('host');
 
 module.exports = defaults.call(requestModule);
@@ -61,9 +64,18 @@ function defaults (params) {
   if (isRequestModule) {
     cloudscraper.defaults = defaults;
   }
+
   // Expose the debug option
-  Object.defineProperty(cloudscraper, 'debug',
-    Object.getOwnPropertyDescriptor(this, 'debug'));
+  Object.defineProperty(cloudscraper, 'debug', {
+    configurable: true,
+    enumerable: true,
+    set (value) {
+      requestModule.debug = debugging = true;
+    },
+    get () {
+      return debugging;
+    }
+  });
 
   return cloudscraper;
 }
@@ -295,7 +307,7 @@ function onChallenge (options, response, body) {
       timeout = parseInt(match[2]);
 
       if (timeout > options.cloudflareMaxTimeout) {
-        if (requestModule.debug) {
+        if (debugging) {
           console.warn('Cloudflare\'s timeout is excessive: ' + (timeout / 1000) + 's');
         }
 
@@ -358,26 +370,53 @@ function onCaptcha (options, response, body) {
     return callback(new ParserError(cause, options, response));
   }
 
-  // Defining response.challengeForm for debugging purposes
-  const form = response.challengeForm = match[1];
+  const form = match[1];
+  let siteKey;
 
-  match = form.match(/\/recaptcha\/api\/fallback\?k=([^\s"'<>]*)/);
-  if (!match) {
-    // The site key wasn't inside the form so search the entire document
-    match = body.match(/data-sitekey=["']?([^\s"'<>]*)/);
-    if (!match) {
+  match = body.match(/\sdata-sitekey=["']?([^\s"'<>&]+)/);
+  if (match) {
+    siteKey = match[1];
+  } else {
+    const keys = [];
+    const re = /\/recaptcha\/api2?\/(?:fallback|anchor|bframe)\?(?:[^\s<>]+&(?:amp;)?)?[Kk]=["']?([^\s"'<>&]+)/g;
+
+    while ((match = re.exec(body)) !== null) {
+      // Prioritize the explicit fallback siteKey over other matches
+      if (match[0].indexOf('fallback') !== -1) {
+        keys.unshift(match[1]);
+        if (!debugging) break;
+      } else {
+        keys.push(match[1]);
+      }
+    }
+
+    siteKey = keys[0];
+
+    if (!siteKey) {
       cause = 'Unable to find the reCAPTCHA site key';
       return callback(new ParserError(cause, options, response));
     }
+
+    if (debugging) {
+      console.warn('Failed to find data-sitekey, using a fallback:', keys);
+    }
   }
 
   // Everything that is needed to solve the reCAPTCHA
   response.captcha = {
-    url: response.request.uri.href,
-    siteKey: match[1],
+    siteKey,
+    uri: response.request.uri,
     form: payload
   };
 
+  Object.defineProperty(response.captcha, 'url', {
+    configurable: true,
+    enumerable: false,
+    get: deprecate(function () {
+      return response.request.uri.href;
+    }, 'captcha.url is deprecated. Please use captcha.uri instead.')
+  });
+
   // Adding formData
   match = form.match(/<input(?: [^<>]*)? name=[^<>]+>/g);
   if (!match) {
@@ -403,6 +442,10 @@ function onCaptcha (options, response, body) {
     return callback(new ParserError(cause, options, response));
   }
 
+  if (debugging) {
+    console.warn('Captcha:', response.captcha);
+  }
+
   // The callback used to green light form submission
   const submit = function (error) {
     if (error) {


Test patch:
diff --git a/test/test-captcha.js b/test/test-captcha.js
index 74f41eb..b537232 100644
--- a/test/test-captcha.js
+++ b/test/test-captcha.js
@@ -96,12 +96,12 @@ describe('Cloudscraper', function () {
           isCloudflare: true,
           isHTML: true,
           isCaptcha: true,
-          challengeForm: sinon.match.string,
           captcha: sinon.match.object
         });
 
         sinon.assert.match(response.captcha, {
-          url: uri,
+          url: uri, // <-- Deprecated
+          uri: sinon.match.same(response.request.uri),
           form: { s: secret },
           siteKey: siteKey,
           submit: sinon.match.func