Repository: luin/ioredis
Pull request #293 (2016-05-03 02:09:31)

feat: add dropBufferSupport option to improve the performance
https://github.com/luin/ioredis/wiki/Improve-Performance


Patch:
diff --git a/API.md b/API.md
index ab0898e..de42c18 100644
--- a/API.md
+++ b/API.md
@@ -59,6 +59,8 @@ Creates a Redis instance
 | [options.connectionName] | <code>string</code> | <code>null</code> | Connection name. |
 | [options.db] | <code>number</code> | <code>0</code> | Database index to use. |
 | [options.password] | <code>string</code> | <code>null</code> | If set, client will send AUTH command with the value of this option when connected. |
+| [options.parser] | <code>string</code> | <code>null</code> | Either "hiredis" or "javascript". If not set, "hiredis" parser will be used if it's installed (`npm install hiredis`), otherwise "javascript" parser will be used. |
+| [options.dropBufferSupport] | <code>boolean</code> | <code>false</code> | Drop the buffer support for better performance. This option is recommanded to be enabled when "hiredis" parser is used. Refer to https://github.com/luin/ioredis/wiki/Improve-Performance for more details. |
 | [options.enableReadyCheck] | <code>boolean</code> | <code>true</code> | When a connection is established to the Redis server, the server might still be loading the database from disk. While loading, the server not respond to any commands. To work around this, when this option is `true`, ioredis will check the status of the Redis server, and when the Redis server is able to process commands, a `ready` event will be emitted. |
 | [options.enableOfflineQueue] | <code>boolean</code> | <code>true</code> | By default, if there is no active connection to the Redis server, commands are added to a queue and are executed once the connection is "ready" (when `enableReadyCheck` is `true`, "ready" means the Redis server has loaded the database from disk, otherwise means the connection to the Redis server has been established). If this option is false, when execute the command when the connection isn't ready, an error will be returned. |
 | [options.connectTimeout] | <code>number</code> | <code>10000</code> | The milliseconds before a timeout occurs during the initial connection to the Redis server. |
diff --git a/README.md b/README.md
index f777ad5..f0a8ecd 100644
--- a/README.md
+++ b/README.md
@@ -807,10 +807,9 @@ var cluster = new Redis.Cluster([
 });
 ```
 
-## Native Parser
-If [hiredis](https://github.com/redis/hiredis-node) is installed (by `npm install hiredis`),
-ioredis will use it by default. Otherwise, a pure JavaScript parser will be used.
-Typically, there's not much difference between them in terms of performance.
+## Improve Performance
+ioredis supports two parsers, "hiredis" and "javascript". Refer to https://github.com/luin/ioredis/wiki/Improve-Performance
+for details about the differences between them in terms of performance.
 
 <hr>
 
diff --git a/benchmarks/single_node.js b/benchmarks/single_node.js
index baf8b8c..fe1c4f8 100644
--- a/benchmarks/single_node.js
+++ b/benchmarks/single_node.js
@@ -1,13 +1,10 @@
 'use strict';
 
 var childProcess = require('child_process');
-var nodeRedis = require('redis');
-var IORedis = require('../');
-var ndredis, ioredis;
+var Redis = require('../');
 
 console.log('==========================');
-console.log('ioredis: ' + require('../package.json').version);
-console.log('node_redis: ' + require('redis/package.json').version);
+console.log('redis: ' + require('../package.json').version);
 var os = require('os');
 console.log('CPU: ' + os.cpus().length);
 console.log('OS: ' + os.platform() + ' ' + os.arch());
@@ -15,207 +12,87 @@ console.log('node version: ' + process.version);
 console.log('current commit: ' + childProcess.execSync('git rev-parse --short HEAD'));
 console.log('==========================');
 
+var redisJD, redisJ, redisBD, redisB;
 var waitReady = function (next) {
-  var pending = 2;
-  ndredis.on('ready', function () {
+  var pending = 4;
+  function check() {
     if (!--pending) {
       next();
     }
-  });
+  }
+  redisJD = new Redis({ parser: 'javascript', dropBufferSupport: true });
+  redisJ = new Redis({ parser: 'javascript', dropBufferSupport: false });
+  redisBD = new Redis({ parser: 'hiredis', dropBufferSupport: true });
+  redisB = new Redis({ parser: 'hiredis', dropBufferSupport: false });
+  redisJD.on('ready', check);
+  redisJ.on('ready', check);
+  redisBD.on('ready', check);
+  redisB.on('ready', check);
+};
 
-  ioredis.on('ready', function () {
-    if (!--pending) {
-      next();
-    }
-  });
+var quit = function () {
+  redisJD.quit();
+  redisJ.quit();
+  redisBD.quit();
+  redisB.quit();
 };
 
-suite('simple set', function () {
+suite('SET foo bar', function () {
   set('mintime', 5000);
   set('concurrency', 300);
   before(function (start) {
-    ndredis = nodeRedis.createClient();
-    ioredis = new IORedis();
     waitReady(start);
   });
 
-  bench('ioredis', function (next) {
-    ioredis.set('foo', 'bar', next);
-  });
-
-  bench('node_redis', function (next) {
-    ndredis.set('foo', 'bar', next);
+  bench('javascript parser + dropBufferSupport: true', function (next) {
+    redisJD.set('foo', 'bar', next);
   });
 
-  after(function () {
-    ndredis.quit();
-    ioredis.quit();
-  });
-});
-
-suite('simple get', function () {
-  set('mintime', 5000);
-  set('concurrency', 300);
-  before(function (start) {
-    ndredis = nodeRedis.createClient();
-    ioredis = new IORedis();
-    waitReady(function () {
-      ndredis.set('foo', 'bar', start);
-    });
+  bench('javascript parser', function (next) {
+    redisJ.setBuffer('foo', 'bar', next);
   });
 
-  bench('ioredis', function (next) {
-    ioredis.get('foo', next);
+  bench('hiredis parser + dropBufferSupport: true', function (next) {
+    redisBD.set('foo', 'bar', next);
   });
 
-  bench('node_redis', function (next) {
-    ndredis.get('foo', next);
+  bench('hiredis parser', function (next) {
+    redisB.setBuffer('foo', 'bar', next);
   });
 
-  after(function () {
-    ndredis.quit();
-    ioredis.quit();
-  });
+  after(quit);
 });
 
-suite('simple get with pipeline', function () {
+suite('LRANGE foo 0 99', function () {
   set('mintime', 5000);
   set('concurrency', 300);
   before(function (start) {
-    ndredis = nodeRedis.createClient();
-    ioredis = new IORedis();
-    waitReady(function () {
-      ndredis.set('foo', 'bar', start);
-    });
-  });
-
-  bench('ioredis', function (next) {
-    var pipeline = ioredis.pipeline();
-    for (var i = 0; i < 10; ++i) {
-      pipeline.get('foo');
-    }
-    pipeline.exec(next);
-  });
-
-  bench('node_redis', function (next) {
-    var pending = 0;
-    for (var i = 0; i < 10; ++i) {
-      pending += 1;
-      ndredis.get('foo', check);
+    var redis = new Redis();
+    var item = [];
+    for (var i = 0; i < 100; ++i) {
+      item.push((Math.random() * 100000 | 0) + 'str');
     }
-    function check() {
-      if (!--pending) {
-        next();
-      }
-    }
-  });
-
-  after(function () {
-    ndredis.quit();
-    ioredis.quit();
-  });
-});
-
-suite('lrange 100', function () {
-  set('mintime', 5000);
-  set('concurrency', 300);
-  before(function (start) {
-    ndredis = nodeRedis.createClient();
-    ioredis = new IORedis();
-    waitReady(function () {
-      var item = [];
-      for (var i = 0; i < 100; ++i) {
-        item.push((Math.random() * 100000 | 0) + 'str');
-      }
-      ndredis.del('foo');
-      ndredis.lpush('foo', item, start);
-    });
-  });
-
-  bench('ioredis', function (next) {
-    ioredis.lrange('foo', 0, 99, next);
-  });
-
-  bench('node_redis', function (next) {
-    ndredis.lrange('foo', 0, 99, next);
-  });
-
-  after(function () {
-    ndredis.quit();
-    ioredis.quit();
-  });
-});
-
-suite('publish', function () {
-  set('mintime', 5000);
-  set('concurrency', 300);
-
-  before(function (start) {
-    ndredis = nodeRedis.createClient();
-    ioredis = new IORedis();
-    waitReady(function () {
-      start();
+    redis.del('foo');
+    redis.lpush('foo', item, function () {
+      waitReady(start);
     });
   });
 
-  bench('ioredis', function (next) {
-    ioredis.publish('foo', 'bar', next);
-  });
-
-  bench('node_redis', function (next) {
-    ndredis.publish('foo', 'bar', next);
+  bench('javascript parser + dropBufferSupport: true', function (next) {
+    redisJD.lrange('foo', 0, 99, next);
   });
 
-  after(function () {
-    ndredis.quit();
-    ioredis.quit();
-  });
-});
-
-suite('subscribe', function () {
-  set('mintime', 5000);
-  set('concurrency', 300);
-
-  var ndpublisher = null;
-  var iopublisher = null;
-  var ndsubscriber = null;
-  var iosubscriber = null;
-
-  before(function (start) {
-    ndredis = nodeRedis.createClient();
-    ioredis = new IORedis();
-    waitReady(function () {
-      ndsubscriber = ndredis;
-      ndsubscriber.subscribe('foo');
-      iosubscriber = ioredis;
-      iosubscriber.subscribe('foo');
-
-      ndredis = nodeRedis.createClient();
-      ioredis = new IORedis();
-      waitReady(function () {
-        ndpublisher = ndredis;
-        iopublisher = ioredis;
-        start();
-      });
-    });
+  bench('javascript parser', function (next) {
+    redisJ.lrangeBuffer('foo', 0, 99, next);
   });
 
-  bench('ioredis', function (next) {
-    iosubscriber.removeAllListeners('message');
-    ndsubscriber.removeAllListeners('message');
-    iosubscriber.on('message', next);
-    iopublisher.publish('foo', 'bar');
+  bench('hiredis parser + dropBufferSupport: true', function (next) {
+    redisBD.lrange('foo', 0, 99, next);
   });
 
-  bench('node_redis', function (next) {
-    iosubscriber.removeAllListeners('message');
-    ndsubscriber.removeAllListeners('message');
-    ndsubscriber.on('message', next);
-    ndpublisher.publish('foo', 'bar');
+  bench('hiredis parser', function (next) {
+    redisB.lrangeBuffer('foo', 0, 99, next);
   });
 
-  after(function () {
-    ndredis.quit();
-    ioredis.quit();
-  });
+  after(quit);
 });
diff --git a/lib/commander.js b/lib/commander.js
index e5f9c1c..701ab84 100644
--- a/lib/commander.js
+++ b/lib/commander.js
@@ -3,6 +3,11 @@
 var _ = require('lodash');
 var Command = require('./command');
 var Script = require('./script');
+var Promise = require('bluebird');
+
+var DROP_BUFFER_SUPPORT_ERROR = '*Buffer methods are not available ' +
+  'because "dropBufferSupport" option is enabled.' +
+  'Refer to https://github.com/luin/ioredis/wiki/Improve-Performance for more details.';
 
 /**
  * Commander
@@ -106,7 +111,16 @@ function generateFunction(_commandName, _encoding) {
       args[i - firstArgIndex] = arguments[i];
     }
 
-    var options = { replyEncoding: _encoding };
+    var options;
+    if (this.options.dropBufferSupport) {
+      if (!_encoding) {
+        return Promise.reject(new Error(DROP_BUFFER_SUPPORT_ERROR)).nodeify(callback);
+      }
+      options = { replyEncoding: null };
+    } else {
+      options = { replyEncoding: _encoding };
+    }
+
     if (this.options.showFriendlyErrorStack) {
       options.errorStack = new Error().stack;
     }
@@ -133,7 +147,16 @@ function generateScriptingFunction(_script, _encoding) {
       args[i] = arguments[i];
     }
 
-    var options = { replyEncoding: _encoding };
+    var options;
+    if (this.options.dropBufferSupport) {
+      if (!_encoding) {
+        return Promise.reject(new Error(DROP_BUFFER_SUPPORT_ERROR)).nodeify(callback);
+      }
+      options = { replyEncoding: null };
+    } else {
+      options = { replyEncoding: _encoding };
+    }
+
     if (this.options.showFriendlyErrorStack) {
       options.errorStack = new Error().stack;
     }
diff --git a/lib/redis.js b/lib/redis.js
index b810807..d119760 100644
--- a/lib/redis.js
+++ b/lib/redis.js
@@ -36,6 +36,11 @@ var ScanStream = require('./scan_stream');
  * @param {number} [options.db=0] - Database index to use.
  * @param {string} [options.password=null] - If set, client will send AUTH command
  * with the value of this option when connected.
+ * @param {string} [options.parser=null] - Either "hiredis" or "javascript". If not set, "hiredis" parser
+ * will be used if it's installed (`npm install hiredis`), otherwise "javascript" parser will be used.
+ * @param {boolean} [options.dropBufferSupport=false] - Drop the buffer support for better performance.
+ * This option is recommanded to be enabled when "hiredis" parser is used.
+ * Refer to https://github.com/luin/ioredis/wiki/Improve-Performance for more details.
  * @param {boolean} [options.enableReadyCheck=true] - When a connection is established to
  * the Redis server, the server might still be loading the database from disk.
  * While loading, the server not respond to any commands.
@@ -166,6 +171,7 @@ Redis.defaultOptions = {
   db: 0,
   // Others
   parser: null,
+  dropBufferSupport: false,
   enableOfflineQueue: true,
   enableReadyCheck: true,
   autoResubscribe: true,
diff --git a/lib/redis/parser.js b/lib/redis/parser.js
index 6137cc4..2068c42 100644
--- a/lib/redis/parser.js
+++ b/lib/redis/parser.js
@@ -20,7 +20,7 @@ exports.initParser = function () {
   this.replyParser = new Parser({
     name: this.options.parser,
     stringNumbers: this.options.stringNumbers,
-    returnBuffers: true,
+    returnBuffers: !this.options.dropBufferSupport,
     returnError: function (err) {
       _this.returnError(new ReplyError(err.message));
     },
@@ -33,6 +33,12 @@ exports.initParser = function () {
       _this.disconnect(true);
     }
   });
+
+  if (this.replyParser.name === 'hiredis' && !this.options.dropBufferSupport) {
+    console.warn('[WARN] ioredis is using hiredis parser, however "dropBufferSupport" is disabled. ' +
+     'It\'s highly recommanded to enable this option. ' +
+     'Refer to https://github.com/luin/ioredis/wiki/Improve-Performance for more details.');
+  }
 };
 
 exports.returnError = function (err) {


Test patch:
diff --git a/test/functional/drop_buffer_support.js b/test/functional/drop_buffer_support.js
new file mode 100644
index 0000000..9f16fd0
--- /dev/null
+++ b/test/functional/drop_buffer_support.js
@@ -0,0 +1,91 @@
+'use strict';
+
+describe('dropBufferSupport', function () {
+  it('should be disabled by default', function () {
+    var redis = new Redis({ lazyConnect: true });
+    expect(redis.options).to.have.property('dropBufferSupport', false);
+  });
+
+  it('should return strings correctly', function (done) {
+    var redis = new Redis({ dropBufferSupport: false });
+    redis.set('foo', new Buffer('bar'), function (err, res) {
+      expect(err).to.eql(null);
+      expect(res).to.eql('OK');
+      redis.get('foo', function (err, res) {
+        expect(err).to.eql(null);
+        expect(res).to.eql('bar');
+        redis.disconnect();
+        done();
+      });
+    });
+  });
+
+  context('enabled', function () {
+    it('should reject the buffer commands', function (done) {
+      var redis = new Redis({ dropBufferSupport: true });
+      redis.getBuffer('foo', function (err) {
+        expect(err.message).to.match(/Buffer methods are not available/);
+
+        redis.callBuffer('get', 'foo', function (err) {
+          expect(err.message).to.match(/Buffer methods are not available/);
+          redis.disconnect();
+          done();
+        });
+      });
+    });
+
+    it('should reject the custom buffer commands', function (done) {
+      var redis = new Redis({ dropBufferSupport: true });
+      redis.defineCommand('geteval', {
+        numberOfKeys: 0,
+        lua: 'return "string"'
+      });
+      redis.getevalBuffer(function (err) {
+        expect(err.message).to.match(/Buffer methods are not available/);
+        redis.disconnect();
+        done();
+      });
+    });
+
+    it('should return strings correctly', function (done) {
+      var redis = new Redis({ dropBufferSupport: true });
+      redis.set('foo', new Buffer('bar'), function (err, res) {
+        expect(err).to.eql(null);
+        expect(res).to.eql('OK');
+        redis.get('foo', function (err, res) {
+          expect(err).to.eql(null);
+          expect(res).to.eql('bar');
+          redis.disconnect();
+          done();
+        });
+      });
+    });
+
+    it('should return strings for custom commands', function (done) {
+      var redis = new Redis({ dropBufferSupport: true });
+      redis.defineCommand('geteval', {
+        numberOfKeys: 0,
+        lua: 'return "string"'
+      });
+      redis.geteval(function (err, res) {
+        expect(err).to.eql(null);
+        expect(res).to.eql('string');
+        redis.disconnect();
+        done();
+      });
+    });
+
+    it('should work with pipeline', function (done) {
+      var redis = new Redis({ dropBufferSupport: true });
+      var pipeline = redis.pipeline();
+      pipeline.set('foo', 'bar');
+      pipeline.get(new Buffer('foo'));
+      pipeline.exec(function (err, res) {
+        expect(err).to.eql(null);
+        expect(res[0][1]).to.eql('OK');
+        expect(res[1][1]).to.eql('bar');
+        done();
+      });
+    });
+  });
+});