Repository: natcap/invest (python)

Issue:
Move args spec formatting code over from UG
The arg spec formatting code used in the `:investspec:` Sphinx extension could live in `invest` rather than `invest.users-guide`. Benefits include:
- Logical place for it to be reused in both `invest.users-guide` and `invest-workbench`
- gettext boilerplate can be consolidated with `invest`'s rather than maintaining separately

I think @davemfish suggested this before, and I'm not sure why I didn't do it then!

Pull request:
Move over arg spec formatting logic 
# Description
Fixes #777. Goes with natcap/invest.users-guide#60. Prerequisite for natcap/invest#854.

* Copied over formatting logic and tests
* Added a test
* Slightly refactored and renamed formatting functions

# Checklist
- [ ] ~Updated HISTORY.rst (if these changes are user-facing)~
- [ ] ~Updated the user's guide (if needed)~


Patch:
diff --git a/src/natcap/invest/spec_utils.py b/src/natcap/invest/spec_utils.py
index e72d420ac..416bc5997 100644
--- a/src/natcap/invest/spec_utils.py
+++ b/src/natcap/invest/spec_utils.py
@@ -1,3 +1,4 @@
+import importlib
 import json
 import os
 
@@ -217,3 +218,302 @@ def serialize_args_spec(spec):
         raise TypeError(f'fallback serializer is missing for {type(obj)}')
 
     return json.dumps(spec, default=fallback_serializer)
+
+
+# accepted geometries for a vector will be displayed in this order
+GEOMETRY_ORDER = [
+    _('POINT'),
+    _('MULTIPOINT'),
+    _('LINESTRING'),
+    _('MULTILINESTRING'),
+    _('POLYGON'),
+    _('MULTIPOLYGON')]
+
+INPUT_TYPES_HTML_FILE = 'input_types.html'
+
+
+def format_required_string(required):
+    """Represent an arg's required status as a user-friendly string.
+
+    Args:
+        required (bool | str | None): required property of an arg. May be
+            `True`, `False`, `None`, or a conditional string.
+
+    Returns:
+        string
+    """
+    if required is None or required is True:
+        return _('required')
+    elif required is False:
+        return _('optional')
+    else:
+        # assume that the about text will describe the conditional
+        return _('conditionally required')
+
+
+def format_geometries_string(geometries):
+    """Represent a set of allowed vector geometries as user-friendly text.
+
+    Args:
+        geometries (set(str)): set of geometry names
+
+    Returns:
+        string
+    """
+    # sort the geometries so they always display in a consistent order
+    sorted_geoms = sorted(
+        geometries,
+        key=lambda g: GEOMETRY_ORDER.index(g))
+    return '/'.join(geom.lower() for geom in sorted_geoms)
+
+
+def format_permissions_string(permissions):
+    """Represent a rwx-style permissions string as user-friendly text.
+
+    Args:
+        permissions (str): rwx-style permissions string
+
+    Returns:
+        string
+    """
+    permissions_strings = []
+    if 'r' in permissions:
+        permissions_strings.append(_('read'))
+    if 'w' in permissions:
+        permissions_strings.append(_('write'))
+    if 'x' in permissions:
+        permissions_strings.append(_('execute'))
+    return ', '.join(permissions_strings)
+
+
+def format_options_string_from_dict(options):
+    """Represent a dictionary of option: description pairs as a bulleted list.
+
+    Args:
+        options (dict): the dictionary of options to document, where keys are
+            options and values are descriptions of the options
+
+    Returns:
+        list of RST-formatted strings, where each is a line in a bullet list
+    """
+    lines = []
+    # casefold() is a more aggressive version of lower() that may work better
+    # for some languages to remove all case distinctions
+    sorted_options = sorted(
+        list(options.keys()),
+        key=lambda option: option.casefold()
+    )
+    for option in sorted_options:
+        lines.append(f'- {option}: {options[option]}')
+    return lines
+
+
+def format_options_string_from_list(options):
+    """Represent options as a comma-separated list.
+
+    Args:
+        options (list[str]): the set of options to document
+
+    Returns:
+        string of comma-separated options
+    """
+    return ', '.join(options)
+
+
+def capitalize(title):
+    """Capitalize a string into title case.
+
+    Args:
+        title (str): string to capitalize
+
+    Returns:
+        capitalized string (each word capitalized except linking words)
+    """
+
+    def capitalize_word(word):
+        """Capitalize a word, if appropriate."""
+        if word in {'of', 'the'}:
+            return word
+        else:
+            return word[0].upper() + word[1:]
+
+    title = ' '.join([capitalize_word(word) for word in title.split(' ')])
+    title = '/'.join([capitalize_word(word) for word in title.split('/')])
+    return title
+
+
+def format_type_string(arg_type):
+    """Represent an arg type as a user-friendly string.
+
+    Args:
+        arg_type (str|set(str)): the type to format. May be a single type or a
+            set of types.
+
+    Returns:
+        formatted string that links to a description of the input type(s)
+    """
+    # some types need a more user-friendly name
+    # all types are listed here so that they can be marked up for translation
+    type_names = {
+        'boolean': _('true/false'),
+        'csv': _('CSV'),
+        'directory': _('directory'),
+        'file': _('file'),
+        'freestyle_string': _('text'),
+        'integer': _('integer'),
+        'number': _('number'),
+        'option_string': _('option'),
+        'percent': _('percent'),
+        'raster': _('raster'),
+        'ratio': _('ratio'),
+        'vector': _('vector')
+    }
+
+    def format_single_type(arg_type):
+        """Represent a type as a link to the corresponding Input Types section.
+
+        Args:
+            arg_type (str): the type to format.
+
+        Returns:
+            formatted string that links to a description of the input type
+        """
+        # Represent the type as a string. Some need a more user-friendly name.
+        # we can only use standard docutils features here, so no :ref:
+        # this syntax works to link to a section in a different page, but it
+        # isn't universally supported and depends on knowing the built page name.
+        if arg_type == 'freestyle_string':
+            section_name = 'text'
+        elif arg_type == 'option_string':
+            section_name = 'option'
+        elif arg_type == 'boolean':
+            section_name = 'truefalse'
+        elif arg_type == 'csv':
+            section_name = 'csv'
+        else:
+            section_name = arg_type
+
+        return f'`{type_names[arg_type]} <{INPUT_TYPES_HTML_FILE}#{section_name}>`__'
+
+    if isinstance(arg_type, set):
+        return ' or '.join(format_single_type(t) for t in sorted(arg_type))
+    else:
+        return format_single_type(arg_type)
+
+
+def describe_arg_from_spec(name, spec):
+    """Generate RST documentation for an arg, given an arg spec.
+
+    This is used for documenting:
+        - a single top-level arg
+        - a row or column in a CSV
+        - a field in a vector
+        - an item in a directory
+
+    Args:
+        name (str): Name to give the section. For top-level args this is
+            arg['name']. For nested args it's typically their key in the
+            dictionary one level up.
+        spec (dict): A arg spec dictionary that conforms to the InVEST args
+            spec specification. It must at least have the key `'type'`, and
+            whatever other keys are expected for that type.
+    Returns:
+        list of strings, where each string is a line of RST-formatted text.
+        The first line has the arg name, type, required state, description,
+        and units if applicable. Depending on the type, there may be additional
+        lines that are indented, that describe details of the arg such as
+        vector fields and geometries, option_string options, etc.
+    """
+    type_string = format_type_string(spec['type'])
+    in_parentheses = [type_string]
+
+    # For numbers and rasters that have units, display the units
+    units = None
+    if spec['type'] == 'number':
+        units = spec['units']
+    elif spec['type'] == 'raster' and spec['bands'][1]['type'] == 'number':
+        units = spec['bands'][1]['units']
+    if units:
+        units_string = format_unit(units)
+        if units_string and units_string != 'none':
+            in_parentheses.append(f'units: **{units_string}**')
+
+    if spec['type'] == 'vector':
+        in_parentheses.append(format_geometries_string(spec["geometries"]))
+
+    # Represent the required state as a string, defaulting to required
+    # It doesn't make sense to include this for boolean checkboxes
+    if spec['type'] != 'boolean':
+        # get() returns None if the key doesn't exist in the dictionary
+        required_string = format_required_string(spec.get('required'))
+        in_parentheses.append(f'*{required_string}*')
+
+    # Nested args may not have an about section
+    if 'about' in spec:
+        about_string = f': {spec["about"]}'
+    else:
+        about_string = ''
+
+    first_line = f"**{name}** ({', '.join(in_parentheses)}){about_string}"
+
+    # Add details for the types that have them
+    indented_block = []
+    if spec['type'] == 'option_string':
+        # may be either a dict or set. if it's empty, the options are
+        # dynamically generated. don't try to document them.
+        if spec['options']:
+            if isinstance(spec['options'], dict):
+                indented_block.append(_('Options:'))
+                indented_block += format_options_string_from_dict(spec['options'])
+            else:
+                formatted_options = format_options_string_from_list(spec['options'])
+                indented_block.append(_('Options:') + f' {formatted_options}')
+
+    elif spec['type'] == 'csv':
+        if 'columns' not in spec and 'rows' not in spec:
+            first_line += _(
+                ' Please see the sample data table for details on the format.')
+
+    # prepend the indent to each line in the indented block
+    return [first_line] + ['\t' + line for line in indented_block]
+
+
+def describe_arg_from_name(module_name, *arg_keys):
+    """Generate RST documentation for an arg, given its model and name.
+
+    Args:
+        module_name (str): invest model module containing the arg.
+        *arg_keys: one or more strings that are nested arg keys.
+
+    Returns:
+        String describing the arg in RST format
+    """
+    # import the specified module (that should have an ARGS_SPEC attribute)
+    module = importlib.import_module(module_name)
+    # start with the spec for all args
+    # narrow down to the nested spec indicated by the sequence of arg keys
+    spec = module.ARGS_SPEC['args']
+    for i, key in enumerate(arg_keys):
+        # convert raster band numbers to ints
+        if arg_keys[i - 1] == 'bands':
+            key = int(key)
+        try:
+            spec = spec[key]
+        except KeyError:
+            keys_so_far = '.'.join(arg_keys[:i + 1])
+            raise ValueError(
+                f"Could not find the key '{keys_so_far}' in the "
+                f"{module_name} model's ARGS_SPEC")
+
+    # format spec into an RST formatted description string
+    if isinstance(spec, dict):
+        if 'name' in spec:
+            arg_name = capitalize(spec['name'])
+        else:
+            arg_name = arg_keys[-1]
+        rst = '\n\n'.join(describe_arg_from_spec(arg_name, spec))
+    elif isinstance(spec, pint.Unit):
+        rst = format_unit(spec)
+    else:
+        rst = str(spec)
+    return rst


Test patch:
diff --git a/tests/test_spec_utils.py b/tests/test_spec_utils.py
new file mode 100644
index 000000000..9677ad06a
--- /dev/null
+++ b/tests/test_spec_utils.py
@@ -0,0 +1,246 @@
+import unittest
+
+from natcap.invest import spec_utils
+from natcap.invest.spec_utils import u
+
+
+class TestSpecUtils(unittest.TestCase):
+
+    def test_number_spec(self):
+        spec = {
+            "name": "Bar",
+            "about": "Description",
+            "type": "number",
+            "units": u.meter**3/u.month,
+            "expression": "value >= 0"
+        }
+        out = spec_utils.describe_arg_from_spec(spec['name'], spec)
+        expected_rst = ([
+            '**Bar** (`number <input_types.html#number>`__, '
+            'units: **m³/month**, *required*): Description'])
+        self.assertEqual(repr(out), repr(expected_rst))
+
+    def test_ratio_spec(self):
+        spec = {
+            "name": "Bar",
+            "about": "Description",
+            "type": "ratio"
+        }
+        out = spec_utils.describe_arg_from_spec(spec['name'], spec)
+        expected_rst = (['**Bar** (`ratio <input_types.html#ratio>`__, '
+                         '*required*): Description'])
+        self.assertEqual(repr(out), repr(expected_rst))
+
+    def test_percent_spec(self):
+        spec = {
+            "name": "Bar",
+            "about": "Description",
+            "type": "percent",
+            "required": False
+        }
+        out = spec_utils.describe_arg_from_spec(spec['name'], spec)
+        expected_rst = (['**Bar** (`percent <input_types.html#percent>`__, '
+                         '*optional*): Description'])
+        self.assertEqual(repr(out), repr(expected_rst))
+
+    def test_code_spec(self):
+        spec = {
+            "name": "Bar",
+            "about": "Description",
+            "type": "integer",
+            "required": True
+        }
+        out = spec_utils.describe_arg_from_spec(spec['name'], spec)
+        expected_rst = (['**Bar** (`integer <input_types.html#integer>`__, '
+                         '*required*): Description'])
+        self.assertEqual(repr(out), repr(expected_rst))
+
+    def test_boolean_spec(self):
+        spec = {
+            "name": "Bar",
+            "about": "Description",
+            "type": "boolean"
+        }
+        out = spec_utils.describe_arg_from_spec(spec['name'], spec)
+        expected_rst = (['**Bar** (`true/false <input_types.html#truefalse>'
+                         '`__): Description'])
+        self.assertEqual(repr(out), repr(expected_rst))
+
+    def test_freestyle_string_spec(self):
+        spec = {
+            "name": "Bar",
+            "about": "Description",
+            "type": "freestyle_string"
+        }
+        out = spec_utils.describe_arg_from_spec(spec['name'], spec)
+        expected_rst = (['**Bar** (`text <input_types.html#text>`__, '
+                         '*required*): Description'])
+        self.assertEqual(repr(out), repr(expected_rst))
+
+    def test_option_string_spec_dictionary(self):
+        spec = {
+            "name": "Bar",
+            "about": "Description",
+            "type": "option_string",
+            "options": {
+                "option_a": "do something",
+                "Option_b": "do something else"
+            }
+        }
+        # expect that option case is ignored
+        # otherwise, Option_b would sort before option_a
+        out = spec_utils.describe_arg_from_spec(spec['name'], spec)
+        expected_rst = ([
+            '**Bar** (`option <input_types.html#option>`__, *required*): Description',
+            '\tOptions:',
+            '\t- option_a: do something',
+            '\t- Option_b: do something else'
+        ])
+        self.assertEqual(repr(out), repr(expected_rst))
+
+    def test_option_string_spec_list(self):
+        spec = {
+            "name": "Bar",
+            "about": "Description",
+            "type": "option_string",
+            "options": ["option_a", "Option_b"]
+        }
+        out = spec_utils.describe_arg_from_spec(spec['name'], spec)
+        expected_rst = ([
+            '**Bar** (`option <input_types.html#option>`__, *required*): Description',
+            '\tOptions: option_a, Option_b'
+        ])
+        self.assertEqual(repr(out), repr(expected_rst))
+
+    def test_raster_spec(self):
+        spec = {
+            "type": "raster",
+            "bands": {1: {"type": "integer"}},
+            "about": "Description",
+            "name": "Bar"
+        }
+        out = spec_utils.describe_arg_from_spec(spec['name'], spec)
+        expected_rst = ([
+            '**Bar** (`raster <input_types.html#raster>`__, *required*): Description'
+        ])
+        self.assertEqual(repr(out), repr(expected_rst))
+
+        spec = {
+            "type": "raster",
+            "bands": {1: {
+                "type": "number",
+                "units": u.millimeter/u.year
+            }},
+            "about": "Description",
+            "name": "Bar"
+        }
+        out = spec_utils.describe_arg_from_spec(spec['name'], spec)
+        expected_rst = ([
+            '**Bar** (`raster <input_types.html#raster>`__, units: **mm/year**, *required*): Description'
+        ])
+        self.assertEqual(repr(out), repr(expected_rst))
+
+    def test_vector_spec(self):
+        spec = {
+            "type": "vector",
+            "fields": {},
+            "geometries": {"LINESTRING"},
+            "about": "Description",
+            "name": "Bar"
+        }
+        out = spec_utils.describe_arg_from_spec(spec['name'], spec)
+        expected_rst = ([
+            '**Bar** (`vector <input_types.html#vector>`__, linestring, *required*): Description'
+        ])
+        self.assertEqual(repr(out), repr(expected_rst))
+
+        spec = {
+            "type": "vector",
+            "fields": {
+                "id": {
+                    "type": "integer",
+                    "about": "Unique identifier for each feature"
+                },
+                "precipitation": {
+                    "type": "number",
+                    "units": u.millimeter/u.year,
+                    "about": "Average annual precipitation over the area"
+                }
+            },
+            "geometries": {"POLYGON", "MULTIPOLYGON"},
+            "about": "Description",
+            "name": "Bar"
+        }
+        out = spec_utils.describe_arg_from_spec(spec['name'], spec)
+        expected_rst = ([
+            '**Bar** (`vector <input_types.html#vector>`__, polygon/multipolygon, *required*): Description',
+        ])
+        self.assertEqual(repr(out), repr(expected_rst))
+
+    def test_csv_spec(self):
+        spec = {
+            "type": "csv",
+            "about": "Description.",
+            "name": "Bar"
+        }
+        out = spec_utils.describe_arg_from_spec(spec['name'], spec)
+        expected_rst = ([
+            '**Bar** (`CSV <input_types.html#csv>`__, *required*): Description. '
+            'Please see the sample data table for details on the format.'
+        ])
+        self.assertEqual(repr(out), repr(expected_rst))
+
+        # Test every type that can be nested in a CSV column:
+        # number, ratio, percent, code,
+        spec = {
+            "type": "csv",
+            "about": "Description",
+            "name": "Bar",
+            "columns": {
+                "b": {"type": "ratio", "about": "description"}
+            }
+        }
+        out = spec_utils.describe_arg_from_spec(spec['name'], spec)
+        expected_rst = ([
+            '**Bar** (`CSV <input_types.html#csv>`__, *required*): Description'
+        ])
+        self.assertEqual(repr(out), repr(expected_rst))
+
+    def test_directory_spec(self):
+        self.maxDiff = None
+        spec = {
+            "type": "directory",
+            "about": "Description",
+            "name": "Bar",
+            "contents": {}
+        }
+        out = spec_utils.describe_arg_from_spec(spec['name'], spec)
+        expected_rst = ([
+            '**Bar** (`directory <input_types.html#directory>`__, *required*): Description'
+        ])
+        self.assertEqual(repr(out), repr(expected_rst))
+
+    def test_multi_type_spec(self):
+        spec = {
+            "type": {"raster", "vector"},
+            "about": "Description",
+            "name": "Bar",
+            "bands": {1: {"type": "integer"}},
+            "geometries": {"POLYGON"},
+            "fields": {}
+        }
+        out = spec_utils.describe_arg_from_spec(spec['name'], spec)
+        expected_rst = ([
+            '**Bar** (`raster <input_types.html#raster>`__ or `vector <input_types.html#vector>`__, *required*): Description'
+        ])
+        self.assertEqual(repr(out), repr(expected_rst))
+
+    def test_real_model_spec(self):
+        from natcap.invest import carbon
+        out = spec_utils.describe_arg_from_name(
+            'natcap.invest.carbon', 'carbon_pools_path', 'columns', 'lucode')
+        expected_rst = (
+            '**lucode** (`integer <input_types.html#integer>`__, *required*): ' +
+            carbon.ARGS_SPEC['args']['carbon_pools_path']['columns']['lucode']['about']
+        )
+        self.assertEqual(repr(out), repr(expected_rst))