Repository: scrapy/scrapy

Issue:
Improve structure of Scrapy settings documentation
#515 got me thinking that some Scrapy settings are missed because they aren't documented in the main settings page (`docs/topics/settings.rst`).

One approach I came up for fixing this was to generate a list of all settings (regardless of which file/doc they're documented) but i couldn't find a way in sphinx to generate a list of all entities of a single role (in this case, "setting").

An alternative approach would to adopt the policy/convention of documenting all settings in the `settings.rst` file (perhaps splitting them in one section per component/middleware/etc) and refer to that file from the other docs (for example, donwloader middlewares in `downloader-middleware.rst` would refer to `settings.rst` for their settings documentation).

Currently, we used a mixed (probably half-transitioned) approach where we document settings separately (in the file where their component is documented) and also mix core & component settings in `settings.rst`. This proposal also aims to resolve that inconsistency.

Thoughts?



Discussion:
+1, a good suggestion

So, it seems it isn't very straightforward to create a custom index for a role with Sphinx. The built-in functionality for indices only gives control over the global index and the modules index, no way to create a custom index.

I couldn't find any Sphinx extension that helps with it either, I'll look into building an extension for this myself.


Resolving patch:
diff --git a/docs/_ext/scrapydocs.py b/docs/_ext/scrapydocs.py
--- a/docs/_ext/scrapydocs.py
+++ b/docs/_ext/scrapydocs.py
@@ -1,5 +1,82 @@
 from docutils.parsers.rst.roles import set_classes
 from docutils import nodes
+from sphinx.util.compat import Directive
+from sphinx.util.nodes import make_refnode
+from operator import itemgetter
+
+
+class settingslist_node(nodes.General, nodes.Element):
+    pass
+
+
+class SettingsListDirective(Directive):
+    def run(self):
+        return [settingslist_node('')]
+
+
+def is_setting_index(node):
+    if node.tagname == 'index':
+        # index entries for setting directives look like:
+        # [(u'pair', u'SETTING_NAME; setting', u'std:setting-SETTING_NAME', '')]
+        entry_type, info, refid, _ = node['entries'][0]
+        return entry_type == 'pair' and info.endswith('; setting')
+    return False
+
+
+def get_setting_target(node):
+    # target nodes are placed next to the node in the doc tree
+    return node.parent[node.parent.index(node) + 1]
+
+
+def get_setting_name_and_refid(node):
+    """Extract setting name from directive index node"""
+    entry_type, info, refid, _ = node['entries'][0]
+    return info.replace('; setting', ''), refid
+
+
+def collect_scrapy_settings_refs(app, doctree):
+    env = app.builder.env
+
+    if not hasattr(env, 'scrapy_all_settings'):
+        env.scrapy_all_settings = []
+
+    for node in doctree.traverse(is_setting_index):
+        targetnode = get_setting_target(node)
+        assert isinstance(targetnode, nodes.target), "Next node is not a target"
+
+        setting_name, refid = get_setting_name_and_refid(node)
+
+        env.scrapy_all_settings.append({
+            'docname': env.docname,
+            'setting_name': setting_name,
+            'refid': refid,
+        })
+
+
+def make_setting_element(setting_data, app, fromdocname):
+    refnode = make_refnode(app.builder, fromdocname,
+                           todocname=setting_data['docname'],
+                           targetid=setting_data['refid'],
+                           child=nodes.Text(setting_data['setting_name']))
+    p = nodes.paragraph()
+    p += refnode
+
+    item = nodes.list_item()
+    item += p
+    return item
+
+
+def replace_settingslist_nodes(app, doctree, fromdocname):
+    env = app.builder.env
+
+    for node in doctree.traverse(settingslist_node):
+        settings_list = nodes.bullet_list()
+        settings_list.extend([make_setting_element(d, app, fromdocname)
+                              for d in sorted(env.scrapy_all_settings,
+                                              key=itemgetter('setting_name'))
+                              if fromdocname != d['docname']])
+        node.replace_self(settings_list)
+
 
 def setup(app):
     app.add_crossref_type(
@@ -27,6 +104,12 @@ def setup(app):
     app.add_role('issue', issue_role)
     app.add_role('rev', rev_role)
 
+    app.add_node(settingslist_node)
+    app.add_directive('settingslist', SettingsListDirective)
+
+    app.connect('doctree-read', collect_scrapy_settings_refs)
+    app.connect('doctree-resolved', replace_settingslist_nodes)
+
 def source_role(name, rawtext, text, lineno, inliner, options={}, content=[]):
     ref = 'https://github.com/scrapy/scrapy/blob/master/' + text
     set_classes(options)


Test patch: