Repository: codemanki/cloudscraper
Pull request #140 (2019-03-13 20:46:40)

Update for new challenge
https://github.com/Anorov/cloudflare-scrape/issues/193


Patch:
diff --git a/index.js b/index.js
index 17b66b1..7675618 100644
--- a/index.js
+++ b/index.js
@@ -211,8 +211,6 @@ function solveChallenge (options, response, body) {
   var uri = response.request.uri;
   // The JS challenge to be evaluated for answer/response.
   var challenge;
-  // The result of challenge being evaluated in sandbox
-  var answer;
   // The query string to send back to Cloudflare
   // var payload = { jschl_vc, jschl_answer, pass };
   var payload = {};
@@ -229,7 +227,7 @@ function solveChallenge (options, response, body) {
 
   payload.jschl_vc = match[1];
 
-  match = body.match(/getElementById\('cf-content'\)[\s\S]+?setTimeout.+?\r?\n([\s\S]+?a\.value =.+?)\r?\n/i);
+  match = body.match(/getElementById\('cf-content'\)[\s\S]+?setTimeout.+?\r?\n([\s\S]+?a\.value\s*=.+?)\r?\n/);
 
   if (!match) {
     cause = 'setTimeout callback extraction failed';
@@ -237,18 +235,24 @@ function solveChallenge (options, response, body) {
   }
 
   challenge = match[1]
-    .replace(/a\.value =(.+?) \+ .+?;/i, '$1')
-    .replace(/\s{3,}[a-z](?: = |\.).+/g, '')
+    .replace(/a\.value\s*=\s*(.*)/, function (_, value) {
+      return value.replace(/[A-Za-z0-9_$]+\.length/, uri.hostname.length);
+    })
+    .replace(/^\s*[a-z0-9_$]+\s*[=.].+/gm, '')
     .replace(/'; \d+'/g, '');
 
   try {
-    answer = vm.runInNewContext(challenge, undefined, VM_OPTIONS);
-    payload.jschl_answer = answer + uri.hostname.length;
+    payload.jschl_answer = vm.runInNewContext(challenge, undefined, VM_OPTIONS);
   } catch (error) {
     error.message = 'Challenge evaluation failed: ' + error.message;
     return callback(new errors.ParserError(error, options, response));
   }
 
+  if (isNaN(payload.jschl_answer)) {
+    cause = 'Challenge answer is not a number';
+    return callback(new errors.ParserError(cause, options, response));
+  }
+
   match = body.match(/name="pass" value="(.+?)"/);
 
   if (!match) {


Test patch:
diff --git a/test/fixtures/js_challenge_13_03_2019.html b/test/fixtures/js_challenge_13_03_2019.html
new file mode 100644
index 0000000..f28af47
--- /dev/null
+++ b/test/fixtures/js_challenge_13_03_2019.html
@@ -0,0 +1,83 @@
+<!DOCTYPE HTML>
+<html lang="en-US">
+<head>
+  <meta charset="UTF-8" />
+  <meta http-equiv="Content-Type" content="text/html; charset=UTF-8" />
+  <meta http-equiv="X-UA-Compatible" content="IE=Edge,chrome=1" />
+  <meta name="robots" content="noindex, nofollow" />
+  <meta name="viewport" content="width=device-width, initial-scale=1, maximum-scale=1" />
+  <title>Just a moment...</title>
+  <style type="text/css">
+    html, body {width: 100%; height: 100%; margin: 0; padding: 0;}
+    body {background-color: #ffffff; font-family: Helvetica, Arial, sans-serif; font-size: 100%;}
+    h1 {font-size: 1.5em; color: #404040; text-align: center;}
+    p {font-size: 1em; color: #404040; text-align: center; margin: 10px 0 0 0;}
+    #spinner {margin: 0 auto 30px auto; display: block;}
+    .attribution {margin-top: 20px;}
+    @-webkit-keyframes bubbles { 33%: { -webkit-transform: translateY(10px); transform: translateY(10px); } 66% { -webkit-transform: translateY(-10px); transform: translateY(-10px); } 100% { -webkit-transform: translateY(0); transform: translateY(0); } }
+    @keyframes bubbles { 33%: { -webkit-transform: translateY(10px); transform: translateY(10px); } 66% { -webkit-transform: translateY(-10px); transform: translateY(-10px); } 100% { -webkit-transform: translateY(0); transform: translateY(0); } }
+    .bubbles { background-color: #404040; width:15px; height: 15px; margin:2px; border-radius:100%; -webkit-animation:bubbles 0.6s 0.07s infinite ease-in-out; animation:bubbles 0.6s 0.07s infinite ease-in-out; -webkit-animation-fill-mode:both; animation-fill-mode:both; display:inline-block; }
+  </style>
+
+    <script type="text/javascript">
+  //<![CDATA[
+  (function(){
+    var a = function() {try{return !!window.addEventListener} catch(e) {return !1} },
+    b = function(b, c) {a() ? document.addEventListener("DOMContentLoaded", b, c) : document.attachEvent("onreadystatechange", b)};
+    b(function(){
+      var a = document.getElementById('cf-content');a.style.display = 'block';
+      setTimeout(function(){
+        var s,t,o,p,b,r,e,a,k,i,n,g,f, PkaItmy={"qmAZKRxjdVI":+((!+[]+!![]+!![]+!![]+!![]+!![]+!![]+!![]+!![]+[])+(!+[]+!![]+!![]+!![])+(+!![])+(+[])+(!+[]+!![]+!![]+!![]+!![]+!![]+!![]+!![])+(!+[]+!![]+!![])+(!+[]+!![]+!![]+!![]+!![]+!![]+!![]+!![]+!![])+(!+[]+!![]+!![]+!![]+!![]+!![]+!![]+!![])+(+!![]))/+((+!![]+[])+(!+[]+!![]+!![]+!![]+!![]+!![])+(!+[]+!![])+(!+[]+!![]+!![]+!![]+!![])+(!+[]+!![]+!![]+!![]+!![]+!![]+!![]+!![])+(!+[]+!![]+!![]+!![])+(!+[]+!![])+(!+[]+!![]+!![]+!![]+!![]+!![]+!![]+!![])+(+!![]))};
+        t = document.createElement('div');
+        t.innerHTML="<a href='/'>x</a>";
+        t = t.firstChild.href;r = t.match(/https?:\/\//)[0];
+        t = t.substr(r.length); t = t.substr(0,t.length-1);
+        a = document.getElementById('jschl-answer');
+        f = document.getElementById('challenge-form');
+        ;PkaItmy.qmAZKRxjdVI*=+((!+[]+!![]+!![]+!![]+!![]+!![]+!![]+!![]+[])+(+!![])+(!+[]+!![]+!![]+!![]+!![]+!![]+!![])+(!+[]+!![]+!![]+!![]+!![])+(+[])+(!+[]+!![]+!![]+!![])+(!+[]+!![])+(!+[]+!![]+!![]+!![]+!![])+(!+[]+!![]+!![]))/+((!+[]+!![]+!![]+!![]+!![]+!![]+[])+(!+[]+!![]+!![]+!![])+(!+[]+!![]+!![]+!![]+!![]+!![]+!![]+!![])+(!+[]+!![]+!![]+!![]+!![])+(!+[]+!![]+!![]+!![]+!![]+!![])+(!+[]+!![]+!![]+!![]+!![]+!![])+(!+[]+!![]+!![]+!![]+!![]+!![]+!![]+!![])+(!+[]+!![]+!![]+!![]+!![])+(+!![]));PkaItmy.qmAZKRxjdVI+=+((!+[]+!![]+!![]+!![]+!![]+!![]+!![]+!![]+[])+(+!![])+(!+[]+!![]+!![]+!![]+!![]+!![]+!![])+(!+[]+!![]+!![]+!![]+!![])+(+[])+(!+[]+!![]+!![]+!![])+(!+[]+!![])+(!+[]+!![]+!![]+!![]+!![])+(!+[]+!![]+!![]))/+((!+[]+!![]+!![]+[])+(!+[]+!![]+!![])+(!+[]+!![]+!![]+!![]+!![]+!![]+!![])+(!+[]+!![]+!![]+!![]+!![]+!![]+!![]+!![])+(+[])+(!+[]+!![]+!![]+!![])+(!+[]+!![]+!![]+!![]+!![]+!![]+!![]+!![])+(!+[]+!![]+!![]+!![]+!![]+!![]+!![])+(!+[]+!![]+!![]+!![]));PkaItmy.qmAZKRxjdVI*=+((!+[]+!![]+!![]+!![]+!![]+!![]+!![]+!![]+[])+(+!![])+(!+[]+!![]+!![]+!![]+!![]+!![]+!![])+(!+[]+!![]+!![]+!![]+!![])+(+[])+(!+[]+!![]+!![]+!![])+(!+[]+!![])+(!+[]+!![]+!![]+!![]+!![])+(!+[]+!![]+!![]))/+((!+[]+!![]+!![]+!![]+!![]+[])+(!+[]+!![]+!![]+!![]+!![])+(!+[]+!![]+!![]+!![])+(+[])+(!+[]+!![]+!![]+!![])+(!+[]+!![]+!![]+!![]+!![]+!![]+!![])+(+!![])+(!+[]+!![]+!![]+!![]+!![])+(+[]));PkaItmy.qmAZKRxjdVI+=+((!+[]+!![]+!![]+!![]+!![]+!![]+!![]+!![]+[])+(+!![])+(!+[]+!![]+!![]+!![]+!![]+!![]+!![])+(!+[]+!![]+!![]+!![]+!![])+(+[])+(!+[]+!![]+!![]+!![])+(!+[]+!![])+(!+[]+!![]+!![]+!![]+!![])+(!+[]+!![]+!![]))/+((!+[]+!![]+!![]+!![]+!![]+!![]+!![]+!![]+[])+(+[])+(!+[]+!![]+!![]+!![]+!![]+!![])+(!+[]+!![]+!![]+!![])+(!+[]+!![]+!![]+!![]+!![])+(!+[]+!![])+(+!![])+(+[])+(+!![]));PkaItmy.qmAZKRxjdVI-=+((!+[]+!![]+!![]+!![]+!![]+!![]+!![]+!![]+[])+(!+[]+!![]+!![]+!![]+!![])+(!+[]+!![]+!![]+!![]+!![]+!![]+!![]+!![])+(!+[]+!![]+!![]+!![]+!![])+(!+[]+!![]+!![]+!![]+!![]+!![]+!![]+!![]+!![])+(!+[]+!![]+!![]+!![]+!![]+!![]+!![]+!![]+!![])+(!+[]+!![]+!![]+!![]+!![])+(+[])+(!+[]+!![]+!![]))/+((!+[]+!![]+!![]+!![]+!![]+[])+(!+[]+!![]+!![])+(!+[]+!![]+!![]+!![]+!![])+(!+[]+!![]+!![])+(!+[]+!![]+!![]+!![]+!![]+!![]+!![]+!![])+(!+[]+!![]+!![]+!![]+!![])+(!+[]+!![]+!![]+!![])+(!+[]+!![]+!![]+!![]+!![]+!![]+!![]+!![])+(!+[]+!![]+!![]+!![]+!![]+!![]+!![]+!![]));PkaItmy.qmAZKRxjdVI*=+((!+[]+!![]+!![]+!![]+!![]+!![]+!![]+!![]+[])+(!+[]+!![]+!![])+(!+[]+!![]+!![]+!![]+!![]+!![]+!![]+!![])+(+[])+(!+[]+!![]+!![]+!![])+(+!![])+(!+[]+!![]+!![]+!![]+!![]+!![])+(!+[]+!![]+!![]+!![])+(+!![]))/+((!+[]+!![]+!![]+!![]+[])+(+!![])+(!+[]+!![]+!![]+!![]+!![])+(!+[]+!![]+!![])+(+[])+(!+[]+!![]+!![]+!![]+!![]+!![]+!![]+!![]+!![])+(!+[]+!![]+!![]+!![]+!![]+!![])+(!+[]+!![]+!![]+!![]+!![]+!![]+!![]+!![]+!![])+(+[]));PkaItmy.qmAZKRxjdVI-=+((!+[]+!![]+!![]+!![]+!![]+!![]+!![]+!![]+!![]+[])+(!+[]+!![]+!![]+!![])+(+!![])+(+[])+(!+[]+!![]+!![]+!![]+!![]+!![]+!![]+!![])+(!+[]+!![]+!![])+(!+[]+!![]+!![]+!![]+!![]+!![]+!![]+!![]+!![])+(!+[]+!![]+!![]+!![]+!![]+!![]+!![]+!![])+(+!![]))/+((!+[]+!![]+!![]+!![]+!![]+!![]+!![]+[])+(+[])+(+[])+(!+[]+!![]+!![])+(+!![])+(!+[]+!![]+!![]+!![]+!![]+!![])+(!+[]+!![]+!![]+!![])+(!+[]+!![]+!![]+!![]+!![]+!![]+!![]+!![])+(!+[]+!![]+!![]+!![]+!![]));PkaItmy.qmAZKRxjdVI-=+((!+[]+!![]+!![]+!![]+!![]+!![]+!![]+!![]+[])+(+!![])+(!+[]+!![]+!![]+!![]+!![]+!![]+!![])+(!+[]+!![]+!![]+!![]+!![])+(+[])+(!+[]+!![]+!![]+!![])+(!+[]+!![])+(!+[]+!![]+!![]+!![]+!![])+(!+[]+!![]+!![]))/+((!+[]+!![]+[])+(!+[]+!![]+!![]+!![]+!![]+!![]+!![]+!![]+!![])+(+[])+(+[])+(!+[]+!![]+!![]+!![]+!![]+!![]+!![]+!![])+(!+[]+!![]+!![]+!![]+!![]+!![]+!![])+(!+[]+!![]+!![]+!![]+!![])+(!+[]+!![]+!![])+(!+[]+!![]+!![]+!![]));PkaItmy.qmAZKRxjdVI-=+((!+[]+!![]+!![]+!![]+!![]+!![]+!![]+[])+(!+[]+!![]+!![]+!![]+!![]+!![]+!![]+!![]+!![])+(!+[]+!![]+!![]+!![]+!![]+!![]+!![])+(+[])+(+[])+(!+[]+!![]+!![])+(!+[]+!![]+!![]+!![])+(!+[]+!![]+!![])+(!+[]+!![]+!![]+!![]+!![]+!![]+!![]))/+((!+[]+!![]+!![]+!![]+!![]+!![]+!![]+!![]+[])+(+[])+(!+[]+!![]+!![]+!![]+!![]+!![])+(!+[]+!![]+!![]+!![]+!![])+(!+[]+!![]+!![]+!![]+!![]+!![]+!![]+!![])+(+[])+(!+[]+!![]+!![]+!![])+(!+[]+!![])+(!+[]+!![]));a.value = (+PkaItmy.qmAZKRxjdVI + t.length).toFixed(10); '; 121'
+        f.action += location.hash;
+        f.submit();
+      }, 4000);
+    }, false);
+  })();
+  //]]>
+</script>
+
+
+</head>
+<body>
+  <table width="100%" height="100%" cellpadding="20">
+    <tr>
+      <td align="center" valign="middle">
+          <div class="cf-browser-verification cf-im-under-attack">
+  <noscript><h1 data-translate="turn_on_js" style="color:#bd2426;">Please turn JavaScript on and reload the page.</h1></noscript>
+  <div id="cf-content" style="display:none">
+
+    <div>
+      <div class="bubbles"></div>
+      <div class="bubbles"></div>
+      <div class="bubbles"></div>
+    </div>
+    <h1><span data-translate="checking_browser">Checking your browser before accessing</span> website.com.</h1>
+
+    <p data-translate="process_is_automatic">This process is automatic. Your browser will redirect to your requested content shortly.</p>
+    <p data-translate="allow_5_secs">Please allow up to 5 seconds&hellip;</p>
+  </div>
+  <form id="challenge-form" action="/cdn-cgi/l/chk_jschl" method="get">
+    <input type="hidden" name="jschl_vc" value="18e0eb4e7cc844880cd9822df9d8546e"/>
+    <input type="hidden" name="pass" value="1552499230.142-MOc6blXorq"/>
+    <input type="hidden" id="jschl-answer" name="jschl_answer"/>
+  </form>
+</div>
+
+
+          <div class="attribution">
+            <a href="https://www.cloudflare.com/5xx-error-landing?utm_source=iuam" target="_blank" style="font-size: 12px;">DDoS protection by Cloudflare</a>
+            <br>
+          </div>
+      </td>
+
+    </tr>
+  </table>
+</body>
+</html>
diff --git a/test/test-errors.js b/test/test-errors.js
index c6e7d9c..44da28b 100644
--- a/test/test-errors.js
+++ b/test/test-errors.js
@@ -341,6 +341,31 @@ describe('Cloudscraper', function () {
     this.clock.tick(7000); // tick the timeout
   });
 
+  it('should return error if challenge answer is not a number', function (done) {
+    var html = helper.getFixture('js_challenge_03_12_2018_1.html').toString('utf-8');
+
+    var expectedResponse = helper.cloudflareResponse({
+      body: Buffer.from(html.replace(/a.value.*/, 'a.value="abc"'), 'utf8')
+    });
+
+    Request.callsFake(helper.fakeRequest({ response: expectedResponse }));
+
+    var promise = cloudscraper.get(uri, function (error) {
+      expect(error).to.be.instanceOf(errors.ParserError);
+      expect(error).to.have.property('error', 'Challenge answer is not a number');
+      expect(error).to.have.property('errorType', 3);
+
+      expect(Request).to.be.calledOnceWithExactly(helper.defaultParams);
+
+      expect(error.response).to.be.equal(expectedResponse);
+      expect(error.response.body).to.be.equal(expectedResponse.body);
+    });
+
+    expect(promise).to.be.rejectedWith(errors.ParserError).and.notify(done);
+
+    this.clock.tick(7000); // tick the timeout
+  });
+
   it('should return error if it was thrown by request when solving challenge', function (done) {
     var expectedResponse = helper.cloudflareResponse({
       body: helper.getFixture('js_challenge_21_05_2015.html')
diff --git a/test/test-index.js b/test/test-index.js
index 69a20b3..886badc 100644
--- a/test/test-index.js
+++ b/test/test-index.js
@@ -248,6 +248,53 @@ describe('Cloudscraper', function () {
     this.clock.tick(7000); // tick the timeout
   });
 
+  it('should resolve challenge (version as on 13.03.2019) and then return page', function (done) {
+    // Cloudflare is enabled for site. It returns a page with js challenge
+    var firstResponse = helper.cloudflareResponse({
+      body: helper.getFixture('js_challenge_13_03_2019.html')
+    });
+
+    Request.onFirstCall()
+      .callsFake(helper.fakeRequest({ response: firstResponse }));
+
+    var secondParams = helper.extendParams({
+      uri: 'http://example-site.dev/cdn-cgi/l/chk_jschl',
+      qs: {
+        'jschl_vc': '18e0eb4e7cc844880cd9822df9d8546e',
+        // 6632 is a answer to cloudflares js challenge in this particular case
+        'jschl_answer': String(22.587957833300003 + 'example-site.dev'.length),
+        'pass': '1552499230.142-MOc6blXorq'
+      },
+      headers: {
+        'Referer': 'http://example-site.dev/path/'
+      },
+      challengesToSolve: 2
+    });
+
+    // Second call to Request will have challenge solution
+    // It should contain uri, answer, headers with Referer
+    var secondResponse = helper.fakeResponse({ body: requestedPage });
+    var expectedBody = secondResponse.body.toString('utf8');
+
+    Request.onSecondCall()
+      .callsFake(helper.fakeRequest({ response: secondResponse }));
+
+    var promise = cloudscraper.get(uri, function (error, response, body) {
+      expect(error).to.be.null;
+
+      expect(Request).to.be.calledTwice;
+      expect(Request.firstCall).to.be.calledWithExactly(helper.defaultParams);
+      expect(Request.secondCall).to.be.calledWithExactly(secondParams);
+
+      expect(response).to.be.equal(secondResponse);
+      expect(body).to.be.equal(expectedBody);
+    });
+
+    expect(promise).to.eventually.equal(expectedBody).and.notify(done);
+
+    this.clock.tick(7000); // tick the timeout
+  });
+
   it('should resolve 2 consequent challenges', function (done) {
     var firstParams = helper.extendParams({ resolveWithFullResponse: true });
     // First call and CF returns a challenge