Repository: animir/node-rate-limiter-flexible
Pull request #293: feature(sqlite): atomic upsert support
Author: login: muco-rolle
Created: 2025-03-10T16:06:05Z | Merged: 2025-03-13T11:57:25Z

@animir Hi, 

This extends the work that @no-on3 began in #291 to add SQLite support. I've implemented atomic upsert functionality and I've tested this implementation

Let me know if you'd like any adjustments to the implementation.


Diff:
diff --git a/index.js b/index.js
index ba47098..4a2254d 100644
--- a/index.js
+++ b/index.js
@@ -13,6 +13,7 @@ const RateLimiterRes = require('./lib/RateLimiterRes');
 const RateLimiterDynamo = require('./lib/RateLimiterDynamo');
 const RateLimiterPrisma = require('./lib/RateLimiterPrisma');
 const RateLimiterValkey = require('./lib/RateLimiterValkey');
+const RateLimiterSQLite = require('./lib/RateLimiterSQLite');
 
 module.exports = {
   RateLimiterRedis,
@@ -32,4 +33,5 @@ module.exports = {
   RateLimiterDynamo,
   RateLimiterPrisma,
   RateLimiterValkey,
+  RateLimiterSQLite,
 };
diff --git a/lib/RateLimiterSQLite.js b/lib/RateLimiterSQLite.js
new file mode 100644
index 0000000..488ae87
--- /dev/null
+++ b/lib/RateLimiterSQLite.js
@@ -0,0 +1,207 @@
+const RateLimiterStoreAbstract = require("./RateLimiterStoreAbstract");
+const RateLimiterRes = require("./RateLimiterRes");
+
+class RateLimiterSQLite extends RateLimiterStoreAbstract {
+  /**
+   * @callback callback
+   * @param {Object} err
+   *
+   * @param {Object} opts
+   * @param {callback} cb
+   * Defaults {
+   *   ... see other in RateLimiterStoreAbstract
+   *
+   *   storeClient: sqliteClient, // SQLite database instance
+   *   tableName: 'string',
+   * }
+   */
+  constructor(opts, cb = null) {
+    super(opts);
+
+    this.client = opts.storeClient;
+    this.tableName = opts.tableName;
+    this.tableCreated = opts.tableCreated || false;
+    this.clearExpiredByTimeout = opts.clearExpiredByTimeout;
+
+    if (!/^[A-Za-z0-9_]*$/.test(this.tableName)) {
+      const err = new Error("Table name must contain only letters and numbers");
+      if (typeof cb === "function") {
+        return cb(err);
+      }
+      throw err;
+    }
+
+    if (!this.tableCreated) {
+      this._createDbAndTable()
+        .then(() => {
+          this.tableCreated = true;
+
+          if (this.clearExpiredByTimeout) {
+            this._clearExpiredHourAgo();
+          }
+          if (typeof cb === "function") {
+            cb();
+          }
+        })
+        .catch((err) => {
+          if (typeof cb === "function") {
+            cb(err);
+          } else {
+            throw err;
+          }
+        });
+    } else {
+      if (this.clearExpiredByTimeout) {
+        this._clearExpiredHourAgo();
+      }
+      if (typeof cb === "function") {
+        cb();
+      }
+    }
+  }
+  async _createDbAndTable() {
+    return new Promise((resolve, reject) => {
+      const createTableSQL = this._getCreateTableSQL();
+      this.client.run(createTableSQL, (err) => {
+        if (err) {
+          reject(err);
+        } else {
+          resolve();
+        }
+      });
+    });
+  }
+
+  _getCreateTableSQL() {
+    return `CREATE TABLE IF NOT EXISTS ${this.tableName} (
+      key TEXT PRIMARY KEY,
+      points INTEGER NOT NULL DEFAULT 0,
+      expire INTEGER
+    )`;
+  }
+
+  _clearExpiredHourAgo() {
+    if (this._clearExpiredTimeoutId) {
+      clearTimeout(this._clearExpiredTimeoutId);
+    }
+    this._clearExpiredTimeoutId = setTimeout(() => {
+      this.clearExpired(Date.now() - 3600000) // Never rejected
+        .then(() => {
+          this._clearExpiredHourAgo();
+        });
+    }, 300000);
+    this._clearExpiredTimeoutId.unref();
+  }
+
+  clearExpired(nowMs) {
+    return new Promise((resolve) => {
+      this.client.run(
+        `DELETE FROM ${this.tableName} WHERE expire < ?`,
+        [nowMs],
+        () => resolve()
+      );
+    });
+  }
+
+  _getRateLimiterRes(rlKey, changedPoints, result) {
+    const res = new RateLimiterRes();
+    res.isFirstInDuration = changedPoints === result.points;
+    res.consumedPoints = res.isFirstInDuration ? changedPoints : result.points;
+    res.remainingPoints = Math.max(this.points - res.consumedPoints, 0);
+    res.msBeforeNext = result.expire
+      ? Math.max(result.expire - Date.now(), 0)
+      : -1;
+
+    return res;
+  }
+
+  async _upsertTransaction(rlKey, points, msDuration, forceExpire) {
+    return new Promise((resolve, reject) => {
+      const dateNow = Date.now();
+      const newExpire = msDuration > 0 ? dateNow + msDuration : null;
+
+      const upsertQuery = forceExpire
+        ? `INSERT OR REPLACE INTO ${this.tableName} (key, points, expire) VALUES (?, ?, ?) RETURNING points, expire;`
+        : `
+          INSERT INTO ${this.tableName} (key, points, expire)
+          VALUES (?, ?, ?)
+          ON CONFLICT(key) DO UPDATE SET
+            points = CASE
+              WHEN expire IS NULL OR expire > ? THEN points + excluded.points
+              ELSE excluded.points
+            END,
+            expire = CASE
+              WHEN expire IS NULL OR expire > ? THEN expire
+              ELSE excluded.expire
+            END
+          RETURNING points, expire;
+        `;
+
+      const upsertParams = forceExpire
+        ? [rlKey, points, newExpire]
+        : [rlKey, points, newExpire, dateNow, dateNow];
+
+      this.client.serialize(() => {
+        this.client.run("SAVEPOINT rate_limiter_trx;", (savepointErr) => {
+          if (savepointErr) return reject(savepointErr);
+
+          this.client.get(upsertQuery, upsertParams, (queryErr, row) => {
+            if (queryErr) {
+              return this.client.run(
+                "ROLLBACK TO SAVEPOINT rate_limiter_trx;",
+                () => reject(queryErr)
+              );
+            }
+
+            this.client.run("RELEASE SAVEPOINT rate_limiter_trx;", () =>
+              resolve(row)
+            );
+          });
+        });
+      });
+    });
+  }
+  _upsert(rlKey, points, msDuration, forceExpire = false) {
+    if (!this.tableCreated) {
+      return Promise.reject(Error("Table is not created yet"));
+    }
+    return this._upsertTransaction(rlKey, points, msDuration, forceExpire);
+  }
+  _get(rlKey) {
+    return new Promise((resolve, reject) => {
+      this.client.get(
+        `SELECT points, expire FROM ${this.tableName} WHERE key = ? AND (expire > ? OR expire IS NULL)`,
+        [rlKey, Date.now()],
+        (err, row) => {
+          if (err) {
+            reject(err);
+          } else {
+            resolve(row || null);
+          }
+        }
+      );
+    });
+  }
+
+  _delete(rlKey) {
+    if (!this.tableCreated) {
+      return Promise.reject(Error("Table is not created yet"));
+    }
+
+    return new Promise((resolve, reject) => {
+      this.client.run(
+        `DELETE FROM ${this.tableName} WHERE key = ?`,
+        [rlKey],
+        function (err) {
+          if (err) {
+            reject(err);
+          } else {
+            resolve(this.changes > 0);
+          }
+        }
+      );
+    });
+  }
+}
+
+module.exports = RateLimiterSQLite;
diff --git a/lib/constants.js b/lib/constants.js
index e24313c..50bcdf9 100644
--- a/lib/constants.js
+++ b/lib/constants.js
@@ -8,6 +8,7 @@ const LIMITER_TYPES = {
   POSTGRES: 'postgres',
   DYNAMO: 'dynamo',
   PRISMA: 'prisma',
+  SQLITE: 'sqlite',
 };
 
 const ERR_UNKNOWN_LIMITER_TYPE_MESSAGE = 'Unknown limiter type. Use one of LIMITER_TYPES constants.';
diff --git a/package.json b/package.json
index dd6f6fe..6df961f 100644
--- a/package.json
+++ b/package.json
@@ -61,7 +61,8 @@
     "prisma": "^5.8.0",
     "redis": "^4.6.8",
     "redis-mock": "^0.48.0",
-    "sinon": "^17.0.1"
+    "sinon": "^17.0.1",
+    "sqlite3": "^5.1.7"
   },
   "browser": {
     "cluster": false,
diff --git a/test/RateLimiterSQLite.test.js b/test/RateLimiterSQLite.test.js
new file mode 100644
index 0000000..7466822
--- /dev/null
+++ b/test/RateLimiterSQLite.test.js
@@ -0,0 +1,183 @@
+const { describe, it, beforeEach, afterEach } = require("mocha");
+const { expect } = require("chai");
+const sqlite3 = require("sqlite3");
+const { RateLimiterSQLite } = require("../index");
+
+describe("RateLimiterSQLite", () => {
+  let db;
+  let rateLimiter;
+
+  beforeEach((done) => {
+    // Use in-memory SQLite database for testing
+    db = new sqlite3.Database(":memory:");
+    rateLimiter = new RateLimiterSQLite({
+      storeClient: db,
+      tableName: "rate_limiter_test",
+      points: 5,
+      duration: 5,
+    });
+    // Wait for table creation
+    setTimeout(done, 100);
+  });
+
+  afterEach((done) => {
+    db.close(() => {
+      done();
+    });
+  });
+
+  describe("basic functionality", () => {
+    it("should consume points", async () => {
+      const res = await rateLimiter.consume("test");
+      expect(res.consumedPoints).to.equal(1);
+      expect(res.remainingPoints).to.equal(4);
+    });
+
+    it("should reject when too many points consumed", async () => {
+      try {
+        await rateLimiter.consume("test", 6);
+        expect.fail("should have thrown");
+      } catch (err) {
+        expect(err.remainingPoints).to.equal(0);
+        expect(err.consumedPoints).to.equal(6);
+      }
+    });
+
+    it("should respect points and duration", async () => {
+      const res1 = await rateLimiter.consume("test");
+      const res2 = await rateLimiter.consume("test");
+      expect(res1.consumedPoints).to.equal(1);
+      expect(res2.consumedPoints).to.equal(2);
+      expect(res2.remainingPoints).to.equal(3);
+    });
+  });
+
+  describe("block functionality", () => {
+    it("should block key for specified duration", async () => {
+      await rateLimiter.block("test", 1);
+
+      try {
+        await rateLimiter.consume("test");
+        expect.fail("should have thrown");
+      } catch (err) {
+        expect(err.msBeforeNext).to.be.at.least(900);
+      }
+    });
+  });
+
+  describe("get and delete operations", () => {
+    it("should get points consumed", async () => {
+      await rateLimiter.consume("test", 2);
+      const res = await rateLimiter.get("test");
+      expect(res.consumedPoints).to.equal(2);
+    });
+
+    it("should return null when getting non-existent key", async () => {
+      const res = await rateLimiter.get("nonexistent");
+      expect(res).to.be.null;
+    });
+
+    it("should delete key", async () => {
+      await rateLimiter.consume("test");
+      const deleted = await rateLimiter.delete("test");
+      expect(deleted).to.be.true;
+      const res = await rateLimiter.get("test");
+      expect(res).to.be.null;
+    });
+
+    it("should return false when deleting non-existent key", async () => {
+      const deleted = await rateLimiter.delete("nonexistent");
+      expect(deleted).to.be.false;
+    });
+  });
+
+  describe("expiration", () => {
+    it("should expire points after duration", (done) => {
+      const shortLimiter = new RateLimiterSQLite({
+        storeClient: db,
+        tableName: "rate_limiter_test_short",
+        points: 5,
+        duration: 1, // 1 second duration
+      });
+
+      setTimeout(() => {
+        shortLimiter
+          .consume("test")
+          .then(async () => {
+            // Wait for expiration
+            await new Promise((resolve) => setTimeout(resolve, 1100));
+            const res = await shortLimiter.get("test");
+            expect(res).to.be.null;
+            done();
+          })
+          .catch(done);
+      }, 100); // Wait for table creation
+    });
+  });
+
+  describe("error handling", () => {
+    it("should handle database errors gracefully", async () => {
+      // Close the database to simulate errors
+      await new Promise((resolve) => db.close(resolve));
+
+      try {
+        await rateLimiter.consume("test");
+        expect.fail("should have thrown");
+      } catch (err) {
+        expect(err).to.be.an("error");
+      }
+    });
+
+    it("should reject table when not valid", async () => {
+      try {
+        const invalidLimiter = new RateLimiterSQLite({
+          storeClient: db,
+          tableName: "invalid table name with spaces",
+          points: 5,
+          duration: 5,
+        });
+
+        expect.fail("should have thrown");
+      } catch (err) {
+        expect(err.message).to.equal(
+          "Table name must contain only letters and numbers"
+        );
+      }
+    });
+  });
+
+  describe("concurrent operations", () => {
+    it("should handle multiple concurrent requests", async () => {
+      const promises = [];
+      for (let i = 0; i < 3; i++) {
+        promises.push(rateLimiter.consume("test"));
+      }
+
+      const results = await Promise.all(promises);
+      expect(results).to.have.lengthOf(3);
+      expect(results[2].consumedPoints).to.equal(3);
+      expect(results[2].remainingPoints).to.equal(2);
+    });
+  });
+
+  describe("cleanup", () => {
+    it("should clear expired records", async () => {
+      const cleanupLimiter = new RateLimiterSQLite({
+        storeClient: db,
+        tableName: "rate_limiter_cleanup",
+        points: 5,
+        duration: 1,
+        clearExpiredByTimeout: true,
+      });
+      // wait for table creation
+      await new Promise((resolve) => setTimeout(resolve, 500));
+
+      await cleanupLimiter.consume("test");
+      await new Promise((resolve) => setTimeout(resolve, 1100));
+      await cleanupLimiter.clearExpired(Date.now());
+
+      const res = await cleanupLimiter.get("test");
+      expect(res).to.be.null;
+    });
+  });
+});


Reviews:
user: login: animir
state: COMMENTED
author_association: OWNER
submitted_at: 2025-03-12T10:23:26Z
user: login: animir
state: COMMENTED
author_association: OWNER
submitted_at: 2025-03-12T10:29:13Z
user: login: animir
state: APPROVED
author_association: OWNER
submitted_at: 2025-03-13T11:56:57Z

Review comments:
diff_hunk: @@ -0,0 +1,214 @@
+const RateLimiterStoreAbstract = require("./RateLimiterStoreAbstract");
+const RateLimiterRes = require("./RateLimiterRes");
+
+class RateLimiterSQLite extends RateLimiterStoreAbstract {
+  /**
+   * @callback callback
+   * @param {Object} err
+   *
+   * @param {Object} opts
+   * @param {callback} cb
+   * Defaults {
+   *   ... see other in RateLimiterStoreAbstract
+   *
+   *   storeClient: sqliteClient, // SQLite database instance
+   *   tableName: 'string',
+   * }
+   */
+  constructor(opts, cb = null) {
+    super(opts);
+
+    this.client = opts.storeClient;
+    this.tableName = opts.tableName;
+    this.tableCreated = opts.tableCreated || false;
+    this.clearExpiredByTimeout = opts.clearExpiredByTimeout;
+
+    // Create table on constructor
+    if (!/^[A-Za-z0-9_]*$/.test(this.tableName)) {
+      throw new Error("Table name must contain only letters and numbers");
path: lib/RateLimiterSQLite.js
user: login: animir
body: @muco-rolle Should we call `cb(new Error(...))` if `cb` is function? This will help to implement better error handling.
created_at: 2025-03-12T10:23:26Z
author_association: OWNER
diff_hunk: @@ -0,0 +1,214 @@
+const RateLimiterStoreAbstract = require("./RateLimiterStoreAbstract");
+const RateLimiterRes = require("./RateLimiterRes");
+
+class RateLimiterSQLite extends RateLimiterStoreAbstract {
+  /**
+   * @callback callback
+   * @param {Object} err
+   *
+   * @param {Object} opts
+   * @param {callback} cb
+   * Defaults {
+   *   ... see other in RateLimiterStoreAbstract
+   *
+   *   storeClient: sqliteClient, // SQLite database instance
+   *   tableName: 'string',
+   * }
+   */
+  constructor(opts, cb = null) {
+    super(opts);
+
+    this.client = opts.storeClient;
+    this.tableName = opts.tableName;
+    this.tableCreated = opts.tableCreated || false;
+    this.clearExpiredByTimeout = opts.clearExpiredByTimeout;
+
+    // Create table on constructor
+    if (!/^[A-Za-z0-9_]*$/.test(this.tableName)) {
+      throw new Error("Table name must contain only letters and numbers");
+    }
+    if (!this.tableCreated) {
+      this._createDbAndTable()
+        .then(() => {
+          this.tableCreated = true;
+
+          if (this.clearExpiredByTimeout) {
+            this._clearExpiredHourAgo();
+          }
+          if (typeof cb === "function") {
+            cb();
+          }
+        })
+        .catch((err) => {
+          if (typeof cb === "function") {
+            cb(err);
+          } else {
+            throw err;
+          }
+        });
+    } else {
+      if (this.clearExpiredByTimeout) {
+        this._clearExpiredHourAgo();
+      }
+      if (typeof cb === "function") {
+        cb();
+      }
+    }
+  }
+  async _createDbAndTable() {
+    return new Promise((resolve, reject) => {
+      const createTableSQL = this._getCreateTableSQL();
+      this.client.run(createTableSQL, (err) => {
+        if (err) {
+          reject(err);
+        } else {
+          resolve();
+        }
+      });
+    });
+  }
+
+  _getCreateTableSQL() {
+    return `CREATE TABLE IF NOT EXISTS ${this.tableName} (
+      key TEXT PRIMARY KEY,
+      points INTEGER NOT NULL DEFAULT 0,
+      expire INTEGER
+    )`;
+  }
+
+  _clearExpiredHourAgo() {
+    if (this._clearExpiredTimeoutId) {
+      clearTimeout(this._clearExpiredTimeoutId);
+    }
+    this._clearExpiredTimeoutId = setTimeout(() => {
+      this.clearExpired(Date.now() - 3600000) // Never rejected
+        .then(() => {
+          this._clearExpiredHourAgo();
+        });
+    }, 300000);
+    this._clearExpiredTimeoutId.unref();
+  }
+
+  clearExpired(nowMs) {
+    return new Promise((resolve) => {
+      this.client.run(
+        `DELETE FROM ${this.tableName} WHERE expire < ?`,
+        [nowMs],
+        () => resolve()
+      );
+    });
+  }
+
+  _getRateLimiterRes(rlKey, changedPoints, result) {
+    const res = new RateLimiterRes();
+    res.isFirstInDuration = changedPoints === result.points;
+    res.consumedPoints = res.isFirstInDuration ? changedPoints : result.points;
+    res.remainingPoints = Math.max(this.points - res.consumedPoints, 0);
+    res.msBeforeNext = result.expire
+      ? Math.max(result.expire - Date.now(), 0)
+      : -1;
+
+    return res;
+  }
+
+  _upsert(rlKey, points, msDuration, forceExpire = false) {
+    if (!this.tableCreated) {
+      return Promise.reject(Error("Table is not created yet"));
+    }
+
+    return new Promise((resolve, reject) => {
+      const dateNow = Date.now();
+      const newExpire = msDuration > 0 ? dateNow + msDuration : null;
+
+      this.client.serialize(() => {
+        if (forceExpire) {
+          this.client.run(
+            `INSERT OR REPLACE INTO ${this.tableName} (key, points, expire) VALUES (?, ?, ?)`,
+            [rlKey, points, newExpire],
+            (err) => {
+              if (err) {
+                reject(err);
+              } else {
+                resolve({ points, expire: newExpire });
+              }
+            }
+          );
+        } else {
+          // Increment points if exists and not expired, otherwise insert
+          this.client.run(
+            `
+              INSERT INTO ${this.tableName} (key, points, expire)
+              VALUES (?, ?, ?)
+              ON CONFLICT(key) DO UPDATE SET
+                points =  CASE
+                  WHEN expire is NULL OR expire > ? THEN points + excluded.points
+                  ELSE excluded.points
+                END,
+                expire = CASE
+                  WHEN expire is NULL OR expire > ? THEN expire
+                  ELSE excluded.expire
+                END
+
+            `,
+            [rlKey, points, newExpire, dateNow, dateNow],
+            (err) => {
+              if (err) {
+                reject(err);
+              }
+
+              this.client.get(
path: lib/RateLimiterSQLite.js
user: login: animir
body: @muco-rolle This query should be running in transaction with the upsert query. Otherwise it will lead to race conditions. The `_upsert` function should start a transaction, make two above queries and commit transaction. If there is an error on any step it should rollback transaction.
One good example is MySQL limiter https://github.com/animir/node-rate-limiter-flexible/blob/0a2a27df6be4027c7350d88d1f19dfde287d08eb/lib/RateLimiterMySQL.js
created_at: 2025-03-12T10:29:13Z
author_association: OWNER

PR comments:
user: login: animir
created_at: 2025-03-12T10:45:54Z
body: @muco-rolle Hi, thank you. I put a couple of comments. This PR is close to completion.
author_association: OWNER
user: login: muco-rolle
created_at: 2025-03-12T23:28:11Z
body: @animir 

I found out that SQLite supports the `RETURNING` clause, so I took the opportunity to integrate it. I also implemented `SAVEPOINT` for transactions to support nested transactions, ensuring proper handling of multiple concurrent requests.  

It should be good now.  

Feel free to let me know if you need any adjustments to the implementation.
author_association: CONTRIBUTOR
user: login: animir
created_at: 2025-03-13T11:58:02Z
body: @muco-rolle Thanks. It looks good now.
author_association: OWNER
user: login: animir
created_at: 2025-03-13T13:07:28Z
body: @muco-rolle Hi, I released it in version `6.1.0`. Please, check this page created for SQLite limiter https://github.com/animir/node-rate-limiter-flexible/wiki/SQLite. Let me know if you would like to add anything there.
author_association: OWNER