Repository: codemanki/cloudscraper (js)

Issue:
Update docs
The `README.md` still contains the old message about not having any support for solving CAPTCHA but we have that now. The TODO list hasn't been updated.

The migration guide could be moved to it's own document and just mentioned as the `README.md` is already long. The link can be relative: `[Migration Guide](migration-v2-to-v3.md)` or to make room for other docs, make a docs directory: `[Migration Guide](docs/migration-v2-to-v3.md)`

Might as well move the examples directory into the docs directory if taking that approach:
```sh
mkdir docs
git mv examples/ docs/
```

Pull request description:
Add captcha.url and an example
Adds a `url` property to captcha which may be used instead of `response.request.uri.href`.
Adds an example of using this property and promises.
Sets the executable bit on newer files in the example directory.

I went ahead and updated the docs. I'm sure it can be improved further.
Fixes #206 

Patch:
diff --git a/README.md b/README.md
index 4cfd468..1059fc9 100644
--- a/README.md
+++ b/README.md
@@ -19,71 +19,32 @@ You can use cloudscraper even if you are not sure if Cloudflare protection is tu
 In general, Cloudflare has 4 types of _common_ anti-bot pages:
   - Simple html+javascript page with challenge
   - Page which redirects to original site
-  - Page with recaptcha
+  - Page with reCAPTCHA
   - Page with error ( your ip was banned, etc)
 
-__Unfortunately, there is no support for handling a CAPTCHA, if the response contains it, but some convenience methods may be added in the future.__
+If you notice that for some reason cloudscraper stops working, do not hesitate and get in touch with me ( by creating an issue [here](https://github.com/codemanki/cloudscraper/issues), for example), so i can update it.
 
-If you notice that for some reason cloudscraper stopped to work, do not hesitate and get in touch with me ( by creating an issue here, for example), so i can update it.
-
-Migration from v2 to v3
+Install
 ============
-- Replace `cloudscraper.request(options)` with `cloudscraper(options)`
-- `cloudscraper.get()` and `cloudscraper.post()` method signatures are aligned with corresponding methods from [request](https://github.com/request/request#requestmethod):
-```
-var options = {
-  uri: 'https://website.com/',
-  headers: {/*...*/}
-};
-
-cloudscraper.get(options, function(error, response, body) {
-  console.log(body);
-});
-```
-or for **POST**
-```
-var options = {
-  uri: 'https://website.com/',
-  headers: {/*...*/},
-  formData: { field1: 'value', field2: 2 }
-};
-
-cloudscraper.post(options, function(error, response, body) {
-  console.log(body);
-});
-```
-- If you are using custom promise support workarounds please remove them as cloudscraper now uses [request-promise](https://github.com/request/request-promise):
-
+```sh
+npm install cloudscraper
 ```
-var cloudscraper = require('cloudscraper');
-var options = {
-  uri: 'https://website.com/',
-  method: 'GET'
-};
 
-cloudscraper(options).then(function(body) {
-  console.log(body);
-});
-```
+Saving the `request` module as a dependency is compulsory.
 
-Install
-============
-```javascript
-npm install cloudscraper
+```sh
+# Pin the request version
+npm install --save request
 ```
 
 Usage
 ============
+Cloudscraper uses `request-promise` by default since v3. You can find the migration guide [here.](docs/migration-guide.md)
+
 ```javascript
 var cloudscraper = require('cloudscraper');
 
-cloudscraper.get('https://website.com/', function(error, response, body) {
-  if (error) {
-    console.log('Error occurred');
-  } else {
-    console.log(body, response);
-  }
-});
+cloudscraper.get('https://website.com/').then(console.log, console.error);
 ```
 
 or for `POST` action:
@@ -94,12 +55,12 @@ var options = {
   formData: { field1: 'value', field2: 2 }
 };
 
-cloudscraper.post(options, function(error, response, body) {
-  console.log(body);
-});
+cloudscraper.post(options).then(console.log).catch(console.error);
 ```
 
-A generic request can be made with `cloudscraper(options, callback)`. The options object should follow [request's options](https://www.npmjs.com/package/request#request-options-callback). Not everything is supported however, for example http methods other than GET and POST. If you wanted to request an image in binary data you could use the encoding option:
+*Examples live in the docs directory of the Github repo and can be found [here.](docs/examples)*
+
+A generic request can be made with `cloudscraper(options)`. The options object should follow [request's options](https://www.npmjs.com/package/request#request-options-callback). Not everything is supported however, for example http methods other than GET and POST. If you wanted to request an image in binary data you could use the encoding option:
 
 ```javascript
 var options = {
@@ -107,19 +68,17 @@ var options = {
   url:'http://website.com/',
 };
 
-cloudscraper(options, function(err, response, body) {
-  console.log(response)
-});
+cloudscraper(options).then(console.log);
 ```
 
 ## Advanced usage
-Cloudscraper wraps request and request-promise, so using cloudscraper is pretty much like using those two libraries.
- - Cloudscraper exposes [the same request methods as request](https://github.com/request/request#requestmethod):
- `cloudscraper.get(options, callback)`
- `cloudscraper.post(options, callback)`
- `cloudscraper(uri)`
- Please refer to request's documentation for further instructions
- - Cloudscraper uses request-promise, promise chaining is done exactly the same as described in [docs](https://github.com/request/request-promise#cheat-sheet):
+Cloudscraper allows you to specify your own requester, one of either `request` or `request-promise`.
+Cloudscraper wraps the requester and accepts the same options, so using cloudscraper is pretty much like using those two libraries.
+ - Cloudscraper exposes [the same HTTP verb methods as request](https://github.com/request/request#requestmethod):
+   * `cloudscraper.get(options, callback)`
+   * `cloudscraper.post(options, callback)`
+   * `cloudscraper(uri)`
+ - Cloudscraper uses request-promise by default, promise chaining is done exactly the same as described in [docs](https://github.com/request/request-promise#cheat-sheet):
  ```
   cloudscraper(options)
     .then(function (htmlString) {
@@ -127,29 +86,28 @@ Cloudscraper wraps request and request-promise, so using cloudscraper is pretty
     .catch(function (err) {
     });
   ```
+Please refer to the requester's documentation for further instructions.
 
-## Recaptcha
-Cloudscraper may help you with the recaptcha page. Take a look at [this example](https://github.com/codemanki/cloudscraper/blob/master/examples/solve-recaptcha.js).
+## ReCAPTCHA
+Cloudscraper may help you with the reCAPTCHA page. Take a look at [this example](docs/examples/solve-recaptcha.js) and an [example using promises](docs/examples/solve-recaptcha-v2.js).
 
 ## Defaults method
 
 `cloudscraper.defaults` is a very convenient way of extending the cloudscraper requests with any of your settings.
 
-```
+```javascript
 var cloudscraper = require('cloudscraper').defaults({ 'proxy': 'http://localproxy.com' });
-// Override headers
+// Overriding headers to remove them or using uncommon headers will cause reCAPTCHA responses
 var headers = { /* ... */ };
 var cloudscraper = require('cloudscraper').defaults({ headers: headers });
 
-cloudscraper(options, function(error, response, body) {
-  console.log(body)
-});
+cloudscraper(options).then(console.log);
 ```
 
 ## Configuration
-Cloudscraper exposes following options that are required by default but might be changed. Please note that the default values increase chances of correct work.
+Cloudscraper exposes the following options that are required by default but might be changed. *Please note that the default values eliminate the chance of getting sent a CAPTCHA.*
 
-```
+```javascript
 var options = {
   uri: 'https://website',
   jar: requestModule.jar(), // Custom cookie jar
@@ -173,50 +131,50 @@ var options = {
   decodeEmails: false
 };
 
-cloudscraper(options, function(error, response, body) {
-  console.log(body)
-});
+cloudscraper(options).then(console.log);
 
 ```
-You can access default default configuration with `cloudscraper.defaultParams`
+You can access the default configuration with `cloudscraper.defaultParams`
 
 ## Error object
 Cloudscraper error object inherits from `Error` has following fields:
-  * `name` - `RequestError`/`CaptchaError`/`CloudflareError`/`ParserError`
+  * `name` - `RequestError`/`CAPTCHAError`/`CloudflareError`/`ParserError`
   * `options` - The request options
   * `cause` - An alias for `error`
   * `response` - The request response
   * `errorType` - Custom error code
 Where `errorType` can be following:
  - `0` if request to page failed due to some native reason as bad url, http connection or so. `error` in this case will be error [event](http://nodejs.org/api/http.html#http_class_http_server)
- - `1` Cloudflare returned captcha. Nothing to do here. Bad luck
+ - `1` Cloudflare returned CAPTCHA. Nothing to do here. Bad luck
  - `2` Cloudflare returned page with some inner error. `error` will be `Number` within this range `1012, 1011, 1002, 1000, 1004, 1010, 1006, 1007, 1008`. See more [here](https://support.cloudflare.com/hc/en-us/sections/200820298-Error-Pages)
  - `3` this error is returned when library failed to parse and solve js challenge. `error` will be `String` with some details. :warning: :warning: __Most likely it means that Cloudflare have changed their js challenge.__
  - `4` CF went into a loop and started to return challenge after challenge. If number of solved challenges is greater than `3` and another challenge is returned, throw an error
 
-Do not always rely on `error.cause` to be an error, it can be a string
+Errors are descriptive. You can find a list of all known errors [here.](errors.js)
+
+
+Do not always rely on `error.cause` to be an error, it can be a string.
 
 Running tests
 ============
 Clone this repo, do `npm install` and then just `npm test`
 
 ### Unknown error? Library stopped working? ###
-Let me know, by opening [issue](https://github.com/codemanki/cloudscraper/issues) in this repo and i will update library asap. Please, provide url and body of page where cloudscraper failed.
+Let me know, by opening an [issue](https://github.com/codemanki/cloudscraper/issues) in this repo and I will update library asap. Please, provide url and body of page where cloudscraper failed.
 
 WAT
 ===========
 Current Cloudflare implementation requires browser to respect the timeout of 5 seconds and cloudscraper mimics this behaviour. So everytime you call `cloudscraper.get/post` you should expect it to return result after minimum 6 seconds. If you want to change this behaviour, you would need to make a generic request as described in above and pass `cloudflareTimeout` options with your value. But be aware that Cloudflare might track this timeout and use it against you ;)
 
 ## TODO
- - [x] Check for recaptcha
+ - [x] Check for reCAPTCHA
  - [x] Support cookies, so challenge can be solved once per session
  - [x] Support page with simple redirects
  - [x] Add proper testing
  - [x] Remove manual 302 processing, replace with `followAllRedirects` param
- - [ ] Parse out the timeout from challenge page
- - [x] Reoder the arguments in get/post/request methods and allow custom options to be passed in
- - [ ] Expose solve methods to use them independently
- - [ ] Support recaptcha solving
+ - [x] Parse out the timeout from challenge page
+ - [x] Reorder the arguments in get/post/request methods and allow custom options to be passed in
+ - [x] Support reCAPTCHA solving
  - [x] Promisification
 
 ## Kudos to contributors
@@ -231,7 +189,6 @@ Current Cloudflare implementation requires browser to respect the timeout of 5 s
  - In the beginning cloudscraper was a port of python module [cloudflare-scrape](https://github.com/Anorov/cloudflare-scrape). Thank you [Anorov](https://github.com/Anorov) for an inspiration.
 
 ## Dependencies
-* [request](https://github.com/request/request)
 * [request-promise](https://github.com/request/request-promise)
 
 
diff --git a/examples/banned.js b/docs/examples/banned.js
old mode 100644
new mode 100755
similarity index 100%
rename from examples/banned.js
rename to docs/examples/banned.js
diff --git a/examples/captcha-html.js b/docs/examples/captcha-html.js
similarity index 100%
rename from examples/captcha-html.js
rename to docs/examples/captcha-html.js
diff --git a/examples/custom-headers-v2.js b/docs/examples/custom-headers-v2.js
similarity index 100%
rename from examples/custom-headers-v2.js
rename to docs/examples/custom-headers-v2.js
diff --git a/examples/custom-headers.js b/docs/examples/custom-headers.js
similarity index 100%
rename from examples/custom-headers.js
rename to docs/examples/custom-headers.js
diff --git a/examples/custom-requester-v2.js b/docs/examples/custom-requester-v2.js
similarity index 100%
rename from examples/custom-requester-v2.js
rename to docs/examples/custom-requester-v2.js
diff --git a/examples/custom-requester.js b/docs/examples/custom-requester.js
similarity index 100%
rename from examples/custom-requester.js
rename to docs/examples/custom-requester.js
diff --git a/examples/debugging.js b/docs/examples/debugging.js
similarity index 100%
rename from examples/debugging.js
rename to docs/examples/debugging.js
diff --git a/examples/download-v2.js b/docs/examples/download-v2.js
similarity index 100%
rename from examples/download-v2.js
rename to docs/examples/download-v2.js
diff --git a/examples/download.js b/docs/examples/download.js
similarity index 100%
rename from examples/download.js
rename to docs/examples/download.js
diff --git a/examples/ignore-error.js b/docs/examples/ignore-error.js
similarity index 100%
rename from examples/ignore-error.js
rename to docs/examples/ignore-error.js
diff --git a/examples/redirects.js b/docs/examples/redirects.js
similarity index 100%
rename from examples/redirects.js
rename to docs/examples/redirects.js
diff --git a/docs/examples/solve-recaptcha-v2.js b/docs/examples/solve-recaptcha-v2.js
new file mode 100755
index 0000000..a5a0916
--- /dev/null
+++ b/docs/examples/solve-recaptcha-v2.js
@@ -0,0 +1,27 @@
+#!/usr/bin/env node
+
+// Force a CAPTCHA response by sending bogus headers
+const headers = { /* headers without user-agent, etc. */ };
+const cloudscraper = require('..').defaults({ onCaptcha: handler, headers });
+
+// Pseudo function that returns a promise instead of calling captcha.submit()
+function handler (options, { captcha }) {
+  return new Promise((resolve, reject) => {
+    // Here you do some magic with the siteKey provided by cloudscraper
+    console.error('The url is "' + captcha.url + '"');
+    console.error('The site key is "' + captcha.siteKey + '"');
+    // captcha.form['g-recaptcha-response'] = /* Obtain from your service */
+    reject(new Error('This is a dummy function.'));
+  });
+}
+
+// An example handler with destructuring arguments
+function alternative (options, { captcha: { url, siteKey } }) {
+  // Here you do some magic with the siteKey provided by cloudscraper
+  console.error('The url is "' + url + '"');
+  console.error('The site key is "' + siteKey + '"');
+  return Promise.reject(new Error('This is a dummy function'));
+}
+
+const uri = process.argv[2];
+cloudscraper.get({ uri, onCaptcha: alternative }).then(console.log).catch(console.warn);
diff --git a/examples/solve-recaptcha.js b/docs/examples/solve-recaptcha.js
old mode 100644
new mode 100755
similarity index 100%
rename from examples/solve-recaptcha.js
rename to docs/examples/solve-recaptcha.js
diff --git a/docs/migration-guide.md b/docs/migration-guide.md
new file mode 100644
index 0000000..d7abdd6
--- /dev/null
+++ b/docs/migration-guide.md
@@ -0,0 +1,39 @@
+Migration from v2 to v3
+============
+- Replace `cloudscraper.request(options)` with `cloudscraper(options)`
+- `cloudscraper.get()` and `cloudscraper.post()` method signatures are aligned with corresponding methods from [request](https://github.com/request/request#requestmethod):
+```
+var options = {
+  uri: 'https://website.com/',
+  headers: {/*...*/}
+};
+
+cloudscraper.get(options, function(error, response, body) {
+  console.log(body);
+});
+```
+or for **POST**
+```
+var options = {
+  uri: 'https://website.com/',
+  headers: {/*...*/},
+  formData: { field1: 'value', field2: 2 }
+};
+
+cloudscraper.post(options, function(error, response, body) {
+  console.log(body);
+});
+```
+- If you are using custom promise support workarounds please remove them as cloudscraper now uses [request-promise](https://github.com/request/request-promise):
+
+```
+var cloudscraper = require('cloudscraper');
+var options = {
+  uri: 'https://website.com/',
+  method: 'GET'
+};
+
+cloudscraper(options).then(function(body) {
+  console.log(body);
+});
+```
diff --git a/index.js b/index.js
index ab7bb5c..25be0f4 100644
--- a/index.js
+++ b/index.js
@@ -371,7 +371,11 @@ function onCaptcha (options, response, body) {
   }
 
   // Everything that is needed to solve the reCAPTCHA
-  response.captcha = { siteKey: match[1], form: payload };
+  response.captcha = {
+    url: response.request.uri.href,
+    siteKey: match[1],
+    form: payload
+  };
 
   // Adding formData
   match = form.match(/<input(?: [^<>]*)? name=[^<>]+>/g);


Test patch:
diff --git a/test/test-captcha.js b/test/test-captcha.js
index d223231..76f6a25 100644
--- a/test/test-captcha.js
+++ b/test/test-captcha.js
@@ -102,6 +102,7 @@ describe('Cloudscraper', function () {
         });
 
         sinon.assert.match(response.captcha, {
+          url: uri,
           form: { s: secret },
           siteKey: siteKey,
           submit: sinon.match.func