Repository: dask/dask

Issue:
[Documentation Request] Configuration  Reference
It would be really great if we had a page describing all of the possible configuration options. I am new to Dask. I am still figuring out the configuration options. I usually google around. It'd be great if that knowledge about the configuration settings, possible values, and explanations were centralized.  A great example of this is [Django's Settings Reference](https://docs.djangoproject.com/en/2.1/ref/settings/).

Thank you for listening.


Discussion:
Thanks for raising the issue @vinayan3 .

Are there options not listed in http://docs.dask.org/en/latest/configuration.html that you think should be?

Or are you asking for the config file itself, something like https://github.com/dask/distributed/blob/master/distributed/distributed.yaml for the dask.distributed library
One small thing I had to dig around for and found on a git issue was:
* `temporary_directory`

The configuration page is great at describing how configuration works, where to put the values. and the precedence. However, it'd be really awesome to have all the possible options listed out on a page with explanations.

For the distributed case that would be awesome to have it as part of the docs.dask.org.

Thanks for getting back to me @mrocklin 
Yes, I agree.  Given that config options can change and docs can get out of
date, I suggest that a first simple solution to this problem would be to
add links to common config files from the configuration doc page to the
various config files in the common projects (dask, distributed, jobqueue,
kubernetes, yarn, ml, ...) and then make sure that the config files
themselves are well documented with single-line explanations.

On Mon, Dec 10, 2018 at 6:43 PM Vinay Anantharaman <notifications@github.com>
wrote:

> One small one I had to dig around for and found on a git issue was:
>
>    - temporary_directory
>
> The configuration page is great at describing how configuration works,
> where to put the values. and the precedence. However, it'd be really
> awesome to have all the possible options listed out on a page with
> explanations.
>
> For the distributed case that would be awesome to have it as part of the
> docs.dask.org.
>
> Thanks for getting back to me @mrocklin <https://github.com/mrocklin>
>
> —
> You are receiving this because you were mentioned.
> Reply to this email directly, view it on GitHub
> <https://github.com/dask/dask/issues/4286#issuecomment-446017560>, or mute
> the thread
> <https://github.com/notifications/unsubscribe-auth/AASszEUQCxqzTs6GGWELHc5LoOvi3FXNks5u3vGxgaJpZM4ZMTSX>
> .
>

cc @jcrist 

Resolving patch:
diff --git a/dask/optimization.py b/dask/optimization.py
--- a/dask/optimization.py
+++ b/dask/optimization.py
@@ -447,7 +447,7 @@ def fuse(
     dependencies: dict, optional
         {key: [list-of-keys]}.  Must be a list to provide count of each key
         This optional input often comes from ``cull``
-    ave_width: float (default 2)
+    ave_width: float (default 1)
         Upper limit for ``width = num_nodes / height``, a good measure of
         parallelizability
     max_width: int
diff --git a/docs/source/conf.py b/docs/source/conf.py
--- a/docs/source/conf.py
+++ b/docs/source/conf.py
@@ -25,6 +25,11 @@
 
 # Add any Sphinx extension module names here, as strings. They can be extensions
 # coming with Sphinx (named 'sphinx.ext.*') or your custom ones.
+import sys
+
+source_dir = os.path.dirname(__file__)
+sys.path.insert(0, os.path.join(source_dir, "ext"))
+
 extensions = [
     "sphinx.ext.autodoc",
     "sphinx.ext.mathjax",
@@ -33,6 +38,7 @@
     "sphinx.ext.extlinks",
     "numpydoc",
     "sphinx_click.ext",
+    "dask_config_sphinx_ext",
 ]
 
 numpydoc_show_class_members = False
diff --git a/docs/source/ext/dask_config_sphinx_ext.py b/docs/source/ext/dask_config_sphinx_ext.py
new file mode 100644
--- /dev/null
+++ b/docs/source/ext/dask_config_sphinx_ext.py
@@ -0,0 +1,94 @@
+import requests
+import yaml
+
+from docutils import nodes
+from docutils.parsers.rst import Directive, directives
+
+
+def get_remote_yaml(url):
+    r = requests.get(url)
+    return yaml.safe_load(r.text)
+
+
+class DaskConfigDirective(Directive):
+
+    option_spec = {
+        "location": directives.unchanged,
+        "schema": directives.uri,
+        "config": directives.uri,
+    }
+
+    def run(self):
+        location = self.options["location"]
+        config = self.options["config"]
+        schema = self.options["schema"]
+
+        config = get_remote_yaml(config)
+        schema = get_remote_yaml(schema)
+
+        for k in location.split("."):
+            # dask config does not have a top level key
+            # we need to pass full schema and config
+            if k == "dask":
+                schema = schema
+                config = config
+            else:
+                config = config[k]
+                schema = schema["properties"].get(k, {})
+        html = generate_html(config, schema, location)
+        return [nodes.raw("", html, format="html")]
+
+
+def setup(app):
+    app.add_directive("dask-config-block", DaskConfigDirective)
+
+    return {
+        "version": "0.1",
+        "parallel_read_safe": True,
+        "parallel_write_safe": True,
+    }
+
+
+def dask_config_to_html(key, value, schema, prefix=""):
+    if isinstance(value, dict):
+        return sum(
+            [
+                dask_config_to_html(
+                    k,
+                    v,
+                    schema.get("properties", {}).get(k, {"properties": {}}),
+                    prefix=prefix + key + ".",
+                )
+                for k, v in value.items()
+            ],
+            [],
+        )
+
+    else:
+
+        try:
+            description = schema["description"]
+            description = description.strip()
+        except KeyError:
+            description = "No Comment"
+
+        key = prefix + key
+        value = str(value)
+        node = f"""<dl class="py data">
+                     <dt id="{key}">
+                       <code class="sig-prename descclassname">{key}</code>
+                       <em class="property">&nbsp;&nbsp;{value}</p></em>
+                       <a class="headerlink" href="#{key}" title="Permalink to this definition">¶</a>
+                     </dt>
+                    <dd><p>{description}</p></dd>
+                    </dl>
+
+                """
+        return [node]
+
+
+def generate_html(config, schema, location):
+    nested_html = dask_config_to_html(
+        key="", value=config, schema=schema, prefix=location
+    )
+    return "".join(nested_html)


Test patch:
diff --git a/dask/tests/test_config.py b/dask/tests/test_config.py
--- a/dask/tests/test_config.py
+++ b/dask/tests/test_config.py
@@ -420,6 +420,52 @@ def test_core_file():
     assert "shuffle-compression" in dask.config.get("dataframe")
 
 
+def test_schema():
+    jsonschema = pytest.importorskip("jsonschema")
+
+    config_fn = os.path.join(os.path.dirname(__file__), "..", "dask.yaml")
+    schema_fn = os.path.join(os.path.dirname(__file__), "..", "dask-schema.yaml")
+
+    with open(config_fn) as f:
+        config = yaml.safe_load(f)
+
+    with open(schema_fn) as f:
+        schema = yaml.safe_load(f)
+
+    jsonschema.validate(config, schema)
+
+
+def test_schema_is_complete():
+    config_fn = os.path.join(os.path.dirname(__file__), "..", "dask.yaml")
+    schema_fn = os.path.join(os.path.dirname(__file__), "..", "dask-schema.yaml")
+
+    with open(config_fn) as f:
+        config = yaml.safe_load(f)
+
+    with open(schema_fn) as f:
+        schema = yaml.safe_load(f)
+
+    def test_matches(c, s):
+        for k, v in c.items():
+            if list(c) != list(s["properties"]):
+                raise ValueError(
+                    "\nThe dask.yaml and dask-schema.yaml files are not in sync.\n"
+                    "This usually happens when we add a new configuration value,\n"
+                    "but don't add the schema of that value to the dask-schema.yaml file\n"
+                    "Please modify these files to include the missing values: \n\n"
+                    "    dask.yaml:        {}\n"
+                    "    dask-schema.yaml: {}\n\n"
+                    "Examples in these files should be a good start, \n"
+                    "even if you are not familiar with the jsonschema spec".format(
+                        sorted(c), sorted(s["properties"])
+                    )
+                )
+            if isinstance(v, dict):
+                test_matches(c[k], s["properties"][k])
+
+    test_matches(config, schema)
+
+
 def test_deprecations():
     with pytest.warns(Warning) as info:
         with dask.config.set(fuse_ave_width=123):