Repository: sindresorhus/got
Pull request #405 (2017-11-03 20:45:36)

Use HTTPS for protocol-less URLs
Previously it used HTTP. Better to have safe defaults.


Patch:
diff --git a/index.js b/index.js
index d1668f0..e7d80a9 100644
--- a/index.js
+++ b/index.js
@@ -68,6 +68,7 @@ function requestAsEventEmitter(opts) {
 	const ee = new EventEmitter();
 	const requestUrl = opts.href || urlLib.resolve(urlLib.format(opts), opts.path);
 	const redirects = [];
+	const agents = typeof opts.agent === 'object' ? opts.agent : null;
 	let retryCount = 0;
 	let redirectUrl;
 	let uploadBodySize;
@@ -81,6 +82,11 @@ function requestAsEventEmitter(opts) {
 
 		let fn = opts.protocol === 'https:' ? https : http;
 
+		if (agents) {
+			const protocolName = opts.protocol === 'https:' ? 'https' : 'http';
+			opts.agent = agents[protocolName] || opts.agent;
+		}
+
 		if (opts.useElectronNet && process.versions.electron) {
 			const electron = require('electron');
 			fn = electron.net || electron.remote.net;
@@ -324,7 +330,7 @@ function asPromise(opts) {
 					}
 
 					if (statusCode !== 304 && (statusCode < 200 || statusCode > limitStatusCode)) {
-						throw new got.HTTPError(statusCode, res.headers, opts);
+						throw new got.HTTPError(statusCode, res.statusMessage, res.headers, opts);
 					}
 
 					resolve(res);
@@ -338,6 +344,7 @@ function asPromise(opts) {
 		ee.on('error', reject);
 		ee.on('uploadProgress', proxy.emit.bind(proxy, 'uploadProgress'));
 		ee.on('downloadProgress', proxy.emit.bind(proxy, 'downloadProgress'));
+		ee.on('redirect', proxy.emit.bind(proxy, 'redirect'));
 	});
 
 	const promise = timeoutFn(cancelable);
@@ -407,7 +414,7 @@ function asStream(opts) {
 		res.pipe(output);
 
 		if (statusCode !== 304 && (statusCode < 200 || statusCode > 299)) {
-			proxy.emit('error', new got.HTTPError(statusCode, res.headers, opts), null, res);
+			proxy.emit('error', new got.HTTPError(statusCode, res.statusMessage, res.headers, opts), null, res);
 			return;
 		}
 
@@ -627,8 +634,12 @@ got.ParseError = class extends StdError {
 };
 
 got.HTTPError = class extends StdError {
-	constructor(statusCode, headers, opts) {
-		const statusMessage = http.STATUS_CODES[statusCode];
+	constructor(statusCode, statusMessage, headers, opts) {
+		if (statusMessage) {
+			statusMessage = statusMessage.replace(/\r?\n/g, ' ').trim();
+		} else {
+			statusMessage = http.STATUS_CODES[statusCode];
+		}
 		super(`Response code ${statusCode} (${statusMessage})`, {}, opts);
 		this.name = 'HTTPError';
 		this.statusCode = statusCode;
diff --git a/package.json b/package.json
index 2ef78ca..6119377 100644
--- a/package.json
+++ b/package.json
@@ -66,7 +66,7 @@
 		"pify": "^3.0.0",
 		"safe-buffer": "^5.0.1",
 		"timed-out": "^4.0.0",
-		"url-parse-lax": "^1.0.0",
+		"url-parse-lax": "^3.0.0",
 		"url-to-options": "^1.0.1"
 	},
 	"devDependencies": {
diff --git a/readme.md b/readme.md
index 31c5d11..7e070a4 100644
--- a/readme.md
+++ b/readme.md
@@ -44,7 +44,7 @@ $ npm install got
 const fs = require('fs');
 const got = require('got');
 
-got('todomvc.com')
+got('sindresorhus.com')
 	.then(response => {
 		console.log(response.body);
 		//=> '<!doctype html> ...'
@@ -55,10 +55,10 @@ got('todomvc.com')
 	});
 
 // Streams
-got.stream('todomvc.com').pipe(fs.createWriteStream('index.html'));
+got.stream('sindresorhus.com').pipe(fs.createWriteStream('index.html'));
 
 // For POST, PUT and PATCH methods got.stream returns a WritableStream
-fs.createReadStream('index.html').pipe(got.stream.post('todomvc.com'));
+fs.createReadStream('index.html').pipe(got.stream.post('sindresorhus.com'));
 ```
 
 
@@ -82,6 +82,8 @@ The URL to request as simple string, a [`http.request` options](https://nodejs.o
 
 Properties from `options` will override properties in the parsed `url`.
 
+If no protocol is specified, it will default to `https`.
+
 ##### options
 
 Type: `Object`
@@ -233,7 +235,7 @@ If it's not possible to retrieve the body size (can happen when streaming), `tot
 **Note**: Progress events can also be used with promises.
 
 ```js
-got('todomvc.com')
+got('sindresorhus.com')
 	.on('downloadProgress', progress => {
 		// Report download progress
 	})
@@ -343,11 +345,11 @@ const got = require('got');
 const map = new Map();
 
 (async () => {
-		let response = await got('todomvc.com', {cache: map});
+		let response = await got('sindresorhus.com', {cache: map});
 		console.log(response.fromCache);
 		//=> false
 
-		response = await got('todomvc.com', {cache: map});
+		response = await got('sindresorhus.com', {cache: map});
 		console.log(response.fromCache);
 		//=> true
 })();
@@ -365,7 +367,7 @@ const KeyvRedis = require('@keyv/redis');
 
 const redis = new KeyvRedis('redis://user:pass@localhost:6379');
 
-got('todomvc.com', {cache: redis});
+got('sindresorhus.com', {cache: redis});
 ```
 
 Got supports anything that follows the Map API, so it's easy to write your own storage adapter or use a third-party solution.
@@ -380,7 +382,7 @@ const storageAdapter = require('./my-storage-adapter');
 const QuickLRU = require('quick-lru');
 const storageAdapter = new QuickLRU({maxSize: 1000});
 
-got('todomvc.com', {cache: storageAdapter});
+got('sindresorhus.com', {cache: storageAdapter});
 ```
 
 View the [Keyv docs](https://github.com/lukechilds/keyv) for more information on how to use storage adapters.
@@ -394,7 +396,7 @@ You can use the [`tunnel`](https://github.com/koichik/node-tunnel) module with t
 const got = require('got');
 const tunnel = require('tunnel');
 
-got('todomvc.com', {
+got('sindresorhus.com', {
 	agent: tunnel.httpOverHttp({
 		proxy: {
 			host: 'localhost'
@@ -403,6 +405,21 @@ got('todomvc.com', {
 });
 ```
 
+If you require different agents for different protocols, you can pass a map of agents to the `agent` option. This is necessary because a request to one protocol might redirect to another. In such a scenario, `got` will switch over to the right protocol agent for you.
+
+```js
+const got = require('got');
+const HttpAgent = require('agentkeepalive');
+const HttpsAgent = HttpAgent.HttpsAgent;
+
+got('sindresorhus.com', {
+	agent: {
+		http: new HttpAgent(),
+		https: new HttpsAgent()
+	}
+});
+```
+
 
 ## Cookies
 
@@ -535,7 +552,7 @@ It's a good idea to set the `'user-agent'` header so the provider can more easil
 const got = require('got');
 const pkg = require('./package.json');
 
-got('todomvc.com', {
+got('sindresorhus.com', {
 	headers: {
 		'user-agent': `my-module/${pkg.version} (https://github.com/username/my-module)`
 	}


Test patch:
diff --git a/test/error.js b/test/error.js
index b9d7ab0..f9a116a 100644
--- a/test/error.js
+++ b/test/error.js
@@ -14,6 +14,17 @@ test.before('setup', async () => {
 		res.end('not');
 	});
 
+	s.on('/default-status-message', (req, res) => {
+		res.statusCode = 400;
+		res.end('body');
+	});
+
+	s.on('/custom-status-message', (req, res) => {
+		res.statusCode = 400;
+		res.statusMessage = 'Something Exploded';
+		res.end('body');
+	});
+
 	await s.listen(s.port);
 });
 
@@ -45,6 +56,18 @@ test('options.body error message', async t => {
 	t.regex(err.message, /options\.body must be a ReadableStream, string, Buffer or plain Object/);
 });
 
+test('default status message', async t => {
+	const err = await t.throws(got(`${s.url}/default-status-message`));
+	t.is(err.statusCode, 400);
+	t.is(err.statusMessage, 'Bad Request');
+});
+
+test('custom status message', async t => {
+	const err = await t.throws(got(`${s.url}/custom-status-message`));
+	t.is(err.statusCode, 400);
+	t.is(err.statusMessage, 'Something Exploded');
+});
+
 test.serial('http.request error', async t => {
 	const stub = sinon.stub(http, 'request').callsFake(() => {
 		throw new TypeError('The header content contains invalid characters');
diff --git a/test/http.js b/test/http.js
index dd7fbfa..34bc566 100644
--- a/test/http.js
+++ b/test/http.js
@@ -36,10 +36,6 @@ test('simple request', async t => {
 	t.is((await got(s.url)).body, 'ok');
 });
 
-test('protocol-less URLs', async t => {
-	t.is((await got(s.url.replace(/^http:\/\//, ''))).body, 'ok');
-});
-
 test('empty response', async t => {
 	t.is((await got(`${s.url}/empty`)).body, '');
 });
diff --git a/test/https.js b/test/https.js
index e1a0d3a..d0f773e 100644
--- a/test/https.js
+++ b/test/https.js
@@ -53,6 +53,15 @@ test('make request to https server with ca', async t => {
 	t.is(body, 'ok');
 });
 
+test('protocol-less URLs default to HTTPS', async t => {
+	const {body, requestUrl} = await got(s.url.replace(/^https:\/\//, ''), {
+		ca: caRootCert,
+		headers: {host: 'sindresorhus.com'}
+	});
+	t.is(body, 'ok');
+	t.true(requestUrl.startsWith('https://'));
+});
+
 test.after('cleanup', async () => {
 	await s.close();
 });
diff --git a/test/redirects.js b/test/redirects.js
index 5306573..bf05b64 100644
--- a/test/redirects.js
+++ b/test/redirects.js
@@ -1,6 +1,9 @@
+import {Agent as HttpAgent} from 'http';
+import {Agent as HttpsAgent} from 'https';
 import test from 'ava';
 import pem from 'pem';
 import pify from 'pify';
+import sinon from 'sinon';
 import got from '..';
 import {createServer, createSSLServer} from './helpers/server';
 
@@ -35,12 +38,22 @@ test.before('setup', async () => {
 	const cert = keys.certificate;
 
 	https = await createSSLServer({key, cert}); // eslint-disable-line object-property-newline
+	http = await createServer();
+
+	// HTTPS Handlers
 
 	https.on('/', (req, res) => {
 		res.end('https');
 	});
 
-	http = await createServer();
+	https.on('/httpsToHttp', (req, res) => {
+		res.writeHead(302, {
+			location: http.url
+		});
+		res.end();
+	});
+
+	// HTTP Handlers
 
 	http.on('/', (req, res) => {
 		res.end('reached');
@@ -173,6 +186,29 @@ test('redirects from http to https works', async t => {
 	t.truthy((await got(`${http.url}/httpToHttps`, {rejectUnauthorized: false})).body);
 });
 
+test('redirects from https to http works', async t => {
+	t.truthy((await got(`${https.url}/httpsToHttp`, {rejectUnauthorized: false})).body);
+});
+
+test('redirects from https to http works with an agent object', async t => {
+	const httpAgent = new HttpAgent({keepAlive: true});
+	const httpsAgent = new HttpsAgent({keepAlive: true});
+	const httpSpy = sinon.spy(httpAgent, 'addRequest');
+	const httpsSpy = sinon.spy(httpsAgent, 'addRequest');
+	t.truthy((await got(`${https.url}/httpsToHttp`, {
+		rejectUnauthorized: false,
+		agent: {
+			http: httpAgent,
+			https: httpsAgent
+		}
+	})).body);
+	t.true(httpSpy.calledOnce);
+	t.true(httpsSpy.calledOnce);
+	// Make sure to close all open sockets
+	httpAgent.destroy();
+	httpsAgent.destroy();
+});
+
 test('redirects works with lowercase method', async t => {
 	const body = (await got(`${http.url}/relative`, {method: 'head'})).body;
 	t.is(body, '');