Repository: codemanki/cloudscraper (js)

Issue:
Email protection is not parsed by cloudscraper
On Crunchyroll, they have activated the "email protection" from cloudflare and when I try to scrape some of their webpages I get "[email protected]" instead of the expected text.

Example: http://www.crunchyroll.com/the-idolmster-cinderella-girls-theater#

On that webpage the "DOLM@STER" is seen as an email by cloudflare (that mean their regexp is fabulously wrong) and it screw my scraping because I'm not expecting ton of the javascript they insert for that "protection". 

It would be nice if cloudscraper could detect these insertion and parse them the same as it does for the browser detection.

Pull request:
Add option to decode emails
Fixes #41 

I've removed the method that was in #146, added a flag + additional test, and fixture.

Patch:
diff --git a/index.js b/index.js
index 02a5bce..9b164d8 100644
--- a/index.js
+++ b/index.js
@@ -3,6 +3,7 @@
 var vm = require('vm');
 var requestModule = require('request-promise');
 var errors = require('./errors');
+var decodeEmails = require('./lib/email-decode.js');
 
 var VM_OPTIONS = {
   contextOrigin: 'cloudflare:challenge.js',
@@ -30,7 +31,9 @@ function defaults (params) {
     // followAllRedirects - follow non-GET HTTP 3xx responses as redirects
     followAllRedirects: true,
     // Support only this max challenges in row. If CF returns more, throw an error
-    challengesToSolve: 3
+    challengesToSolve: 3,
+    // Remove Cloudflare's email protection
+    decodeEmails: false
   };
 
   // Object.assign requires at least nodejs v4, request only test/supports v6+
@@ -339,6 +342,10 @@ function processResponseBody (options, response, body) {
         return callback(error);
       }
     }
+
+    if (options.decodeEmails) {
+      response.body = body = decodeEmails(body);
+    }
   }
 
   callback(null, response, body);
diff --git a/lib/email-decode.js b/lib/email-decode.js
new file mode 100644
index 0000000..87c2e9c
--- /dev/null
+++ b/lib/email-decode.js
@@ -0,0 +1,49 @@
+var pattern = (
+  // Opening tag
+  // $1 = TAG_NAME
+  '<([a-z]+)(?: [^>]*)?' + '(?:' +
+  // href attribute
+  // $2 = /cdn-cgi/l/email-protection#HEX_STRING
+  // $3 = HEX_STRING
+  ' href=[\'"]?(\\/cdn-cgi\\/l\\/email-protection#([a-f0-9]{4,}))' + '|' +
+  // data attribute
+  // $4 = HEX_STRING
+  ' data-cfemail=["\']?([a-f0-9]{4,})' +
+  // Self-closing or innerHTML(disallow child nodes) followed by closing tag
+  // \1 backreference to $1
+  '(?:[^<]*\\/>|[^<]*?<\\/\\1>)' + ')'
+);
+
+var re = new RegExp(pattern, 'gi');
+
+module.exports = function (html) {
+  var match, result;
+
+  re.lastIndex = 0;
+
+  while ((match = re.exec(html)) !== null) {
+    if (match[2] !== undefined) {
+      result = match[0].replace(match[2], 'mailto:' + decode(match[3]));
+    } else {
+      result = decode(match[4]);
+    }
+
+    html = html.substr(0, match.index) + result + html.substr(re.lastIndex);
+    re.lastIndex = match.index + result.length - 1;
+  }
+
+  return html;
+};
+
+function decode (hexStr) {
+  var key = parseInt(hexStr.substr(0, 2), 16);
+  var email = '';
+
+  for (var codePoint, i = 2; i < hexStr.length; i += 2) {
+    codePoint = parseInt(hexStr.substr(i, 2), 16) ^ key;
+    email += String.fromCharCode(codePoint);
+  }
+
+  // noinspection JSDeprecatedSymbols
+  return decodeURIComponent(escape(email));
+}


Test patch:
diff --git a/test/fixtures/page_with_emails.html b/test/fixtures/page_with_emails.html
new file mode 100644
index 0000000..dbd0f2d
--- /dev/null
+++ b/test/fixtures/page_with_emails.html
@@ -0,0 +1,16 @@
+<!DOCTYPE html>
+<html lang="en">
+<head>
+    <title>Cloudscraper</title>
+    <meta charset="utf-8">
+    <meta name="description" content="Site with protected emails">
+    <meta name="author" content="cloudscraper">
+
+    <link rel="stylesheet" href="styles/page.css?timestamp=1552884935864">
+</head>
+<body>
+<p> The email is <a href="/cdn-cgi/l/email-protection" class="__cf_email__"
+                    data-cfemail="6506090a10011606170415001725001d040815090048160c11004b010013">[email&#160;protected]</a>
+</p>
+</body>
+</html>
diff --git a/test/helper.js b/test/helper.js
index 87779ec..ab61d95 100644
--- a/test/helper.js
+++ b/test/helper.js
@@ -29,7 +29,8 @@ var helper = {
       realEncoding: 'utf8',
       followAllRedirects: true,
       cloudflareTimeout: 1,
-      challengesToSolve: 3
+      challengesToSolve: 3,
+      decodeEmails: false
     };
   },
   getFixture: function (fileName) {
diff --git a/test/test-emails.js b/test/test-emails.js
new file mode 100644
index 0000000..860bfe4
--- /dev/null
+++ b/test/test-emails.js
@@ -0,0 +1,97 @@
+/* eslint-disable no-unused-expressions */
+/* eslint-env node, mocha */
+'use strict';
+
+var decode = require('../lib/email-decode');
+var expect = require('chai').expect;
+
+var EMAIL = 'cloudscraper@example-site.dev';
+var HEX_STRING = '6506090a10011606170415001725001d040815090048160c11004b010013';
+
+function genHTML (body) {
+  return '<!DOCTYPE html>\n' +
+  '<html lang="en">\n' +
+  '<head>\n' +
+    '  <title>Cloudscraper</title>\n' +
+    '  <meta charset="utf-8">\n' +
+    '  <meta name="description" content="Site with protected emails">\n' +
+    '  <meta name="author" content="cloudscraper">\n\n' +
+    '  <link rel="stylesheet" href="styles/page.css?timestamp=1552884935864">' +
+  '</head>' +
+  '<body>\n' + body + '</body>\n' +
+  '</html>\n';
+}
+
+describe('Cloudscraper', function () {
+  it('should not modify unprotected html', function () {
+    var raw = genHTML('');
+
+    expect(decode(raw)).to.equal(raw);
+  });
+
+  it('should remove email protection', function () {
+    var protection = '<a data-cfemail="' + HEX_STRING + '">!@#&*9^%()[]/\\</a>';
+
+    expect(decode(protection)).to.equal(EMAIL);
+  });
+
+  it('should replace anchors that have a data-cfemail attribute', function () {
+    var protection = '<a href="/cdn-cgi/l/email-protection" class="__cf_email__" data-cfemail="' +
+      HEX_STRING + '">[email&#160;protected]</a>';
+
+    var raw = genHTML('<p> The email is ' + EMAIL + '</p>');
+    var enc = genHTML('<p> The email is ' + protection + '</p>');
+
+    expect(decode(enc)).to.equal(raw);
+  });
+
+  it('should replace spans that have a data-cfemail attribute', function () {
+    var protection = '<span href="/cdn-cgi/l/email-protection" data-cfemail="' +
+      HEX_STRING + '">[email&#160;protected]</span>';
+
+    var raw = genHTML('<p> The email is ' + EMAIL + '</p>');
+    var enc = genHTML('<p> The email is ' + protection + '</p>');
+
+    expect(decode(enc)).to.equal(raw);
+  });
+
+  it('should be space agnostic', function () {
+    var protection = '<NODE    href="/cdn-cgi/l/email-protection"    data-cfemail="' +
+      HEX_STRING + '"  \r\n>\n[email&#160;protected]\r\n</NODE>';
+
+    var raw = genHTML('\r\n<p>\n The email <br/>is ' + EMAIL + '\r\n</p>\n');
+    var enc = genHTML('\r\n<p>\n The email <br/>is ' + protection + '\r\n</p>\n');
+
+    expect(decode(enc)).to.equal(raw);
+  });
+
+  it('should not replace nodes if they have children', function () {
+    var protection = '<a href="/cdn-cgi/l/email-protection" class="__cf_email__" data-cfemail="' +
+      HEX_STRING + '"><span>[email&#160;protected]</span></a>';
+    var enc = genHTML('<p> The email is ' + protection + '</p>');
+
+    expect(decode(enc)).to.equal(enc);
+  });
+
+  it('should not replace malformed html', function () {
+    var protection = '<a data-cfemail="' + HEX_STRING + '">\n<\n</a>';
+    var enc = genHTML('<p> The email is ' + protection + '</p>');
+
+    expect(decode(enc)).to.equal(enc);
+  });
+
+  it('should account for self-closing nodes', function () {
+    var protection = '<a data-cfemail="' + HEX_STRING + '"/><span>test</span>';
+
+    expect(decode(protection)).to.equal(EMAIL + '<span>test</span>');
+  });
+
+  it('should update href attribute values', function () {
+    var protection = '<a b="c" href="/cdn-cgi/l/email-protection#' + HEX_STRING + '"></a>';
+
+    var raw = genHTML('<a b="c" href="mailto:' + EMAIL + '"></a>');
+    var enc = genHTML(protection);
+
+    expect(decode(enc)).to.equal(raw);
+  });
+});
diff --git a/test/test-index.js b/test/test-index.js
index 9ca9682..6e166fd 100644
--- a/test/test-index.js
+++ b/test/test-index.js
@@ -499,6 +499,31 @@ describe('Cloudscraper', function () {
     done();
   });
 
+  it('should decode emails', function (done) {
+    helper.router
+      .get('/test', function (req, res) {
+        res.sendChallenge('js_challenge_13_03_2019.html');
+      })
+      .get('/cdn-cgi/l/chk_jschl', function (req, res) {
+        res.sendFixture('page_with_emails.html');
+      });
+
+    var cf = cloudscraper.defaults({ decodeEmails: true });
+
+    var firstParams = helper.extendParams({ decodeEmails: true });
+
+    var promise = cf.get(uri, function (error, response, body) {
+      expect(error).to.be.null;
+
+      expect(Request).to.be.calledTwice;
+      expect(Request.firstCall).to.be.calledWithExactly(firstParams);
+
+      expect(body).to.include('cloudscraper@example-site.dev');
+    });
+
+    expect(promise).to.eventually.include('cloudscraper@example-site.dev').and.notify(done);
+  });
+
   it('should not error when using the baseUrl option', function (done) {
     helper.router
       .get('/test', function (req, res) {