Repository: scrapy/scrapyd

Issue:
Configurable spider queue class
I have started to implement a custom job queue so i can use a shared job queue with Postgre.
I tried to use this setting SPIDER_QUEUE_CLASS but i found out that its missing from Scrapyd. Is it possible to have this setting back cause i want to avoid patch scrapyd code? I think implementation of this feature again, it is important when someone want to use Scrapyd in a multi-server environment.

Discussion:

Patch:
diff --git a/docs/config.rst b/docs/config.rst
index b8173dc..975bf62 100644
--- a/docs/config.rst
+++ b/docs/config.rst
@@ -142,6 +142,16 @@ and services.
 
 For more info see `Twisted Application Framework`_
 
+.. _spiderqueue:
+
+spiderqueue
+-----------
+
+The scheduler enqueues crawls in per-project spider queues, for the poller to pick.
+You can define a custom spider queue class that implements the ISpiderQueue interface.
+
+Defaults to ``scrapyd.spiderqueue.SqliteSpiderQueue``.
+
 .. _webroot:
 
 webroot
diff --git a/docs/news.rst b/docs/news.rst
index c70d724..e478630 100644
--- a/docs/news.rst
+++ b/docs/news.rst
@@ -3,6 +3,21 @@
 Release notes
 =============
 
+Unreleased
+----------
+
+Added
+~~~~~
+
+- Add ``spiderqueue`` configuration option for custom spider queue.
+
+Changed
+~~~~~~~
+
+- ``scrapyd.spiderqueue.SqliteSpiderQueue`` is initialized with a ``scrapyd.config.Config`` object and a project name, rather than a SQLite connection string (i.e. database file path).
+- If ``dbs_dir`` is set to ``:memory`` or to a URL, it is passed through without modification and without creating a directory to ``scrapyd.jobstorage.SqliteJobStorage`` and ``scrapyd.spiderqueue.SqliteSpiderQueue``.
+- ``scrapyd.utils.get_spider_queues`` defers the creation of the ``dbs_dir`` directory to the spider queue implementation.
+
 1.4.1 (2023-02-10)
 ------------------
 
diff --git a/scrapyd/default_scrapyd.conf b/scrapyd/default_scrapyd.conf
index 763ed72..456e363 100644
--- a/scrapyd/default_scrapyd.conf
+++ b/scrapyd/default_scrapyd.conf
@@ -17,6 +17,7 @@ runner      = scrapyd.runner
 jobstorage  = scrapyd.jobstorage.MemoryJobStorage
 application = scrapyd.app.application
 launcher    = scrapyd.launcher.Launcher
+spiderqueue = scrapyd.spiderqueue.SqliteSpiderQueue
 webroot     = scrapyd.website.Root
 eggstorage  = scrapyd.eggstorage.FilesystemEggStorage
 
diff --git a/scrapyd/jobstorage.py b/scrapyd/jobstorage.py
index ffaa950..7c94177 100644
--- a/scrapyd/jobstorage.py
+++ b/scrapyd/jobstorage.py
@@ -1,10 +1,10 @@
-import os
 from datetime import datetime
 
 from zope.interface import implementer
 
 from scrapyd.interfaces import IJobStorage
 from scrapyd.sqlite import SqliteFinishedJobs
+from scrapyd.utils import sqlite_connection_string
 
 
 def job_log_url(job):
@@ -50,11 +50,7 @@ class MemoryJobStorage(object):
 class SqliteJobStorage(object):
 
     def __init__(self, config):
-        dbsdir = config.get('dbs_dir', 'dbs')
-        if not os.path.exists(dbsdir):
-            os.makedirs(dbsdir)
-        dbpath = os.path.join(dbsdir, 'jobs.db')
-        self.jstorage = SqliteFinishedJobs(dbpath, "finished_jobs")
+        self.jstorage = SqliteFinishedJobs(sqlite_connection_string(config, 'jobs'), "finished_jobs")
         self.finished_to_keep = config.getint('finished_to_keep', 100)
 
     def add(self, job):
diff --git a/scrapyd/spiderqueue.py b/scrapyd/spiderqueue.py
index 1c88943..ef129a6 100644
--- a/scrapyd/spiderqueue.py
+++ b/scrapyd/spiderqueue.py
@@ -2,13 +2,14 @@ from zope.interface import implementer
 
 from scrapyd.interfaces import ISpiderQueue
 from scrapyd.sqlite import JsonSqlitePriorityQueue
+from scrapyd.utils import sqlite_connection_string
 
 
 @implementer(ISpiderQueue)
 class SqliteSpiderQueue(object):
 
-    def __init__(self, database=None, table='spider_queue'):
-        self.q = JsonSqlitePriorityQueue(database, table)
+    def __init__(self, config, project, table='spider_queue'):
+        self.q = JsonSqlitePriorityQueue(sqlite_connection_string(config, project), table)
 
     def add(self, name, priority=0.0, **spider_args):
         d = spider_args.copy()
diff --git a/scrapyd/utils.py b/scrapyd/utils.py
index 16e6800..baf26fc 100644
--- a/scrapyd/utils.py
+++ b/scrapyd/utils.py
@@ -2,13 +2,13 @@ import json
 import os
 import sys
 from subprocess import PIPE, Popen
+from urllib.parse import urlsplit
 
 from packaging.version import InvalidVersion, Version
 from scrapy.utils.misc import load_object
 from twisted.web import resource
 
 from scrapyd.config import Config
-from scrapyd.spiderqueue import SqliteSpiderQueue
 from scrapyd.sqlite import JsonSqliteDict
 
 
@@ -55,14 +55,17 @@ class UtilsCache:
 
 def get_spider_queues(config):
     """Return a dict of Spider Queues keyed by project name"""
-    dbsdir = config.get('dbs_dir', 'dbs')
-    if not os.path.exists(dbsdir):
-        os.makedirs(dbsdir)
-    d = {}
-    for project in get_project_list(config):
-        dbpath = os.path.join(dbsdir, '%s.db' % project)
-        d[project] = SqliteSpiderQueue(dbpath)
-    return d
+    spiderqueue = load_object(config.get('spiderqueue', 'scrapyd.spiderqueue.SqliteSpiderQueue'))
+    return {project: spiderqueue(config, project) for project in get_project_list(config)}
+
+
+def sqlite_connection_string(config, database):
+    dbs_dir = config.get('dbs_dir', 'dbs')
+    if dbs_dir == ':memory:' or urlsplit(dbs_dir).scheme:
+        return dbs_dir
+    if not os.path.exists(dbs_dir):
+        os.makedirs(dbs_dir)
+    return os.path.join(dbs_dir, f'{database}.db')
 
 
 def get_project_list(config):


Test patch:
diff --git a/scrapyd/tests/test_spiderqueue.py b/scrapyd/tests/test_spiderqueue.py
index 69e1905..5f86e5a 100644
--- a/scrapyd/tests/test_spiderqueue.py
+++ b/scrapyd/tests/test_spiderqueue.py
@@ -3,6 +3,7 @@ from twisted.trial import unittest
 from zope.interface.verify import verifyObject
 
 from scrapyd import spiderqueue
+from scrapyd.config import Config
 from scrapyd.interfaces import ISpiderQueue
 
 
@@ -11,7 +12,7 @@ class SpiderQueueTest(unittest.TestCase):
     """
 
     def setUp(self):
-        self.q = spiderqueue.SqliteSpiderQueue(':memory:')
+        self.q = spiderqueue.SqliteSpiderQueue(Config(values={'dbs_dir': ':memory:'}), 'quotesbot')
         self.name = 'spider1'
         self.priority = 5
         self.args = {