Repository: scrapy/scrapyd

Issue:
API - add items and logs url to listjobs.json
After making a call to listjobs.json, I'd like to download the items or logs related to the finished jobs. Would it make sense to add something like below?

2 new attributes:
* items_url 
* logs_url

```json
{
    "status": "ok",
    "pending": [],
    "running": [],
    "finished": [
        {
            "id": "2f16646cfcaf11e1b0090800272a6d06",
            "project": "myproject", "spider": "spider3",
            "start_time": "2012-09-12 10:14:03.594664",
            "end_time": "2012-09-12 10:24:03.594664",
            "items_url": "/items/myproject/spider3/2f16646cfcaf11e1b0090800272a6d06.jl",
            "log_url": "/logs/myproject/spider3/2f16646cfcaf11e1b0090800272a6d06.log"
        }
    ]
}
```

If so, I could submit a PR for it.

Comments:

Patch:
diff --git a/docs/api.rst b/docs/api.rst
index 085f345..b896cf7 100644
--- a/docs/api.rst
+++ b/docs/api.rst
@@ -212,7 +212,9 @@ Example response::
                 "id": "2f16646cfcaf11e1b0090800272a6d06",
                 "project": "myproject", "spider": "spider3",
                 "start_time": "2012-09-12 10:14:03.594664",
-                "end_time": "2012-09-12 10:24:03.594664"
+                "end_time": "2012-09-12 10:24:03.594664",
+                "log_url": "/logs/myproject/spider3/2f16646cfcaf11e1b0090800272a6d06.log",
+                "items_url": "/items/myproject/spider3/2f16646cfcaf11e1b0090800272a6d06.jl"
             }
         ]
     }
diff --git a/docs/news.rst b/docs/news.rst
index 1e53ed8..6256801 100644
--- a/docs/news.rst
+++ b/docs/news.rst
@@ -9,7 +9,8 @@ Unreleased
 Added
 ~~~~~
 
-- Python 3.1 support.
+- Add ``item_url`` and ``log_url`` to the response from the listjobs.json webservice. (@mxdev88)
+- Python 3.11 support.
 
 Removed
 ~~~~~~~
@@ -20,9 +21,9 @@ Removed
 Fixed
 ~~~~~
 
-- Use ``packaging.version.Version`` instead of ``distutils.LooseVersion``.
-- Print Scrapyd's version instead of Twisted's version with ``--version`` (``-v``) flag.
-- Override Scrapy's ``LOG_STDOUT`` to ``False`` to suppress logging output for listspiders.json webservice.
+- Use ``packaging.version.Version`` instead of ``distutils.LooseVersion``. (@pawelmhm)
+- Print Scrapyd's version instead of Twisted's version with ``--version`` (``-v``) flag. (@niuguy)
+- Override Scrapy's ``LOG_STDOUT`` to ``False`` to suppress logging output for listspiders.json webservice. (@Lucioric2000)
 
 1.3.0 (2022-01-12)
 ------------------
diff --git a/scrapyd/jobstorage.py b/scrapyd/jobstorage.py
index 565b12d..ffaa950 100644
--- a/scrapyd/jobstorage.py
+++ b/scrapyd/jobstorage.py
@@ -7,6 +7,14 @@ from scrapyd.interfaces import IJobStorage
 from scrapyd.sqlite import SqliteFinishedJobs
 
 
+def job_log_url(job):
+    return f"/logs/{job.project}/{job.spider}/{job.job}.log"
+
+
+def job_items_url(job):
+    return f"/items/{job.project}/{job.spider}/{job.job}.jl"
+
+
 class Job(object):
     def __init__(self, project, spider, job=None, start_time=None, end_time=None):
         self.project = project
diff --git a/scrapyd/webservice.py b/scrapyd/webservice.py
index b345bc2..f9c2320 100644
--- a/scrapyd/webservice.py
+++ b/scrapyd/webservice.py
@@ -5,6 +5,7 @@ from io import BytesIO
 
 from twisted.python import log
 
+from scrapyd.jobstorage import job_items_url, job_log_url
 from scrapyd.utils import JsonResource, UtilsCache, get_spider_list, native_stringify_dict
 
 
@@ -148,7 +149,9 @@ class ListJobs(WsResource):
                 "project": s.project,
                 "spider": s.spider, "id": s.job,
                 "start_time": str(s.start_time),
-                "end_time": str(s.end_time)
+                "end_time": str(s.end_time),
+                "log_url": job_log_url(s),
+                "items_url": job_items_url(s),
             } for s in self.root.launcher.finished
             if project is None or s.project == project
         ]
diff --git a/scrapyd/website.py b/scrapyd/website.py
index 49bf5c6..f71842e 100644
--- a/scrapyd/website.py
+++ b/scrapyd/website.py
@@ -7,6 +7,7 @@ from twisted.application.service import IServiceCollection
 from twisted.web import resource, static
 
 from scrapyd.interfaces import IEggStorage, IPoller, ISpiderScheduler
+from scrapyd.jobstorage import job_items_url, job_log_url
 
 
 class Root(resource.Resource):
@@ -17,15 +18,15 @@ class Root(resource.Resource):
         self.runner = config.get('runner')
         logsdir = config.get('logs_dir')
         itemsdir = config.get('items_dir')
-        local_items = itemsdir and (urlparse(itemsdir).scheme.lower() in ['', 'file'])
+        self.local_items = itemsdir and (urlparse(itemsdir).scheme.lower() in ['', 'file'])
         self.app = app
         self.nodename = config.get('node_name', socket.gethostname())
-        self.putChild(b'', Home(self, local_items))
+        self.putChild(b'', Home(self, self.local_items))
         if logsdir:
             self.putChild(b'logs', static.File(logsdir.encode('ascii', 'ignore'), 'text/plain'))
-        if local_items:
+        if self.local_items:
             self.putChild(b'items', static.File(itemsdir, 'text/plain'))
-        self.putChild(b'jobs', Jobs(self, local_items))
+        self.putChild(b'jobs', Jobs(self, self.local_items))
         services = config.items('services', ())
         for servName, servClsName in services:
             servCls = load_object(servClsName)
@@ -206,8 +207,8 @@ class Jobs(resource.Resource):
                 "PID": p.pid,
                 "Start": microsec_trunc(p.start_time),
                 "Runtime": microsec_trunc(datetime.now() - p.start_time),
-                "Log": '<a href="/logs/%s/%s/%s.log">Log</a>' % (p.project, p.spider, p.job),
-                "Items": '<a href="/items/%s/%s/%s.jl">Items</a>' % (p.project, p.spider, p.job),
+                "Log": f'<a href="{job_log_url(p)}">Log</a>',
+                "Items": f'<a href="{job_items_url(p)}">Items</a>',
                 "Cancel": self.cancel_button(project=p.project, jobid=p.job),
             })
             for p in self.root.launcher.processes.values()
@@ -222,8 +223,8 @@ class Jobs(resource.Resource):
                 "Start": microsec_trunc(p.start_time),
                 "Runtime": microsec_trunc(p.end_time - p.start_time),
                 "Finish": microsec_trunc(p.end_time),
-                "Log": '<a href="/logs/%s/%s/%s.log">Log</a>' % (p.project, p.spider, p.job),
-                "Items": '<a href="/items/%s/%s/%s.jl">Items</a>' % (p.project, p.spider, p.job),
+                "Log": f'<a href="{job_log_url(p)}">Log</a>',
+                "Items": f'<a href="{job_items_url(p)}">Items</a>',
             })
             for p in self.root.launcher.finished
         )


Test patch:
diff --git a/scrapyd/tests/conftest.py b/scrapyd/tests/conftest.py
index 632957a..e0ef5c9 100644
--- a/scrapyd/tests/conftest.py
+++ b/scrapyd/tests/conftest.py
@@ -47,9 +47,15 @@ def txrequest():
     return Request(http_channel)
 
 
-def common_app_fixture(request):
-    config = Config()
+@pytest.fixture(params=[None, ('scrapyd', 'items_dir', 'items')], ids=["default", "default_with_local_items"])
+def fxt_config(request):
+    conf = Config()
+    if request.param:
+        conf.cp.set(*request.param)
+    return conf
 
+
+def common_app_fixture(request, config):
     app = application(config)
     project, version = 'quotesbot', '0.1'
     storage = app.getComponent(IEggStorage)
@@ -65,14 +71,14 @@ def common_app_fixture(request):
 
 
 @pytest.fixture
-def site_no_egg(request):
-    root, storage = common_app_fixture(request)
+def site_no_egg(request, fxt_config):
+    root, storage = common_app_fixture(request, fxt_config)
     return root
 
 
 @pytest.fixture
-def site_with_egg(request):
-    root, storage = common_app_fixture(request)
+def site_with_egg(request, fxt_config):
+    root, storage = common_app_fixture(request, fxt_config)
 
     egg_path = Path(__file__).absolute().parent / "quotesbot.egg"
     project, version = 'quotesbot', '0.1'
diff --git a/scrapyd/tests/test_webservice.py b/scrapyd/tests/test_webservice.py
index e20fe61..ab4b4e8 100644
--- a/scrapyd/tests/test_webservice.py
+++ b/scrapyd/tests/test_webservice.py
@@ -2,6 +2,11 @@ from pathlib import Path
 from unittest import mock
 
 from scrapyd.interfaces import IEggStorage
+from scrapyd.jobstorage import Job
+
+
+def fake_list_jobs(*args, **kwargs):
+    yield Job('proj1', 'spider-a', 'id1234')
 
 
 def fake_list_spiders(*args, **kwargs):
@@ -49,6 +54,23 @@ class TestWebservice:
 
         assert content['projects'] == ['quotesbot']
 
+    def test_list_jobs(self, txrequest, site_with_egg):
+        txrequest.args = {}
+        endpoint = b'listjobs.json'
+        content = site_with_egg.children[endpoint].render_GET(txrequest)
+
+        assert set(content) == {'node_name', 'status', 'pending', 'running', 'finished'}
+
+    @mock.patch('scrapyd.jobstorage.MemoryJobStorage.__iter__', new=fake_list_jobs)
+    def test_list_jobs_finished(self, txrequest, site_with_egg):
+        txrequest.args = {}
+        endpoint = b'listjobs.json'
+        content = site_with_egg.children[endpoint].render_GET(txrequest)
+
+        assert set(content['finished'][0]) == {
+            'project', 'spider', 'id', 'start_time', 'end_time', 'log_url', 'items_url'
+        }
+
     def test_delete_version(self, txrequest, site_with_egg):
         endpoint = b'delversion.json'
         txrequest.args = {
diff --git a/scrapyd/tests/test_website.py b/scrapyd/tests/test_website.py
index 5fa21f2..fad881e 100644
--- a/scrapyd/tests/test_website.py
+++ b/scrapyd/tests/test_website.py
@@ -3,22 +3,37 @@ class TestWebsite:
         content = site_no_egg.children[b'jobs'].render(txrequest)
         expect_headers = {
             b'Content-Type': [b'text/html; charset=utf-8'],
-            b'Content-Length': [b'643']
+            b'Content-Length': [b'643'],
         }
-        headers = txrequest.responseHeaders.getAllRawHeaders()
-        initial = (
+        if site_no_egg.local_items:
+            expect_headers[b'Content-Length'] = [b'601']
+
+        headers = dict(txrequest.responseHeaders.getAllRawHeaders())
+
+        assert headers == expect_headers
+        assert content.decode().startswith(
             '<html><head><title>Scrapyd</title><style type="text/css">'
             '#jobs>thead td {text-align: center; font-weight'
         )
-
-        assert dict(headers) == expect_headers
-        assert content.decode().startswith(initial)
+        if site_no_egg.local_items:
+            assert b'display: none' not in content
+        else:
+            assert b'display: none' in content
 
     def test_render_home(self, txrequest, site_no_egg):
         content = site_no_egg.children[b''].render_GET(txrequest)
+        expect_headers = {
+            b'Content-Type': [b'text/html; charset=utf-8'],
+            b'Content-Length': [b'708'],
+        }
+        if site_no_egg.local_items:
+            expect_headers[b'Content-Length'] = [b'744']
+
         headers = dict(txrequest.responseHeaders.getAllRawHeaders())
 
+        assert headers == expect_headers
         assert b'Available projects' in content
-        assert headers[b'Content-Type'] == [b'text/html; charset=utf-8']
-        # content-length different between my localhost and build environment
-        assert b'Content-Length' in headers
+        if site_no_egg.local_items:
+            assert b'Items' in content
+        else:
+            assert b'Items' not in content