Repository: natcap/invest (python)

Issue:
validation tests for option_string should test the new ARGS_SPEC options
I think `test_validation.py` needs some minor updates to better test the new spec for args with `type: option_string` where `options` should be a set or a dict. Right now these probably pass by coincidence because they test on lists.

```
    def test_valid_option(self):
        """Validation: test that a string is a valid option."""
        from natcap.invest import validation
        self.assertEqual(None, validation.check_option_string(
            'foo', options=['foo', 'bar', 'Baz']))

    def test_invalid_option(self):
        """Validation: test when a string is not a valid option."""
        from natcap.invest import validation
        error_msg = validation.check_option_string(
            'FOO', options=['foo', 'bar', 'Baz'])
        self.assertTrue('must be one of' in error_msg)
```

Pull request:
Unify our two release branches: release/workbench-alpha & release/3.10
We decided it's no longer necessary to keep a separate release branch devoted to changes needed for the Workbench. We're confident we'll want all those changes going forward, so we're merging them into `release/3.10`. A quick test revealed many conflicts in `ARGS_SPECs`, but no conflicts anywhere else. So I did the following:

* Created this `task` branch from `release/workbench-alpha`
* Reverted the commits in #486 on the `workbench` branch. I believe all that was superseded by the work in #526 on the `3.10` branch. And many of those changes would have passed through without conflict, effectively undoing the work from #526.
* `git merge release/3.10 --strategy-option theirs` -- if there were conflicts from this merge, it will automatically take the changes from `release/3.10`
* Manually updated CV & Finfish `ARGS_SPEC`s that actually did have changes in the Workbench branch that we want to keep.
* Updated HISTORY to replace the Workbench section with a 3.10 section - we haven't had 3.10 section up until now.
* Minor changes to `test_validation` to fix #610 

# Checklist
- [x] Updated HISTORY.rst (if these changes are user-facing)

- [ ] Updated the user's guide (if needed)


Patch:
diff --git a/HISTORY.rst b/HISTORY.rst
index 6c6c26d0a..0e840e379 100644
--- a/HISTORY.rst
+++ b/HISTORY.rst
@@ -33,6 +33,14 @@
 
 .. :changelog:
 
+Unreleased Changes (3.10)
+-------------------------
+* General:
+    * Added ``invest serve`` entry-point to the CLI. This launches a Flask app
+      and server on the localhost, to support the workbench.
+    * Major updates to each model's ``ARGS_SPEC`` (and some related validation)
+      to facilitate re-use & display in the Workbench and User's Guide.
+
 Unreleased Changes (3.9.1)
 --------------------------
 * General:
diff --git a/Makefile b/Makefile
index 4b3d803b3..bb8c7c66f 100644
--- a/Makefile
+++ b/Makefile
@@ -38,6 +38,7 @@ ifeq ($(OS),Windows_NT)
 	.DEFAULT_GOAL := windows_installer
 	RM_DATA_DIR := $(RMDIR) $(DATA_DIR)
 	/ := '\'
+	OSNAME = 'windows'
 else
 	NULL := /dev/null
 	PROGRAM_CHECK_SCRIPT := ./scripts/check_required_programs.sh
@@ -56,6 +57,7 @@ else
 
 	ifeq ($(shell sh -c 'uname -s 2>/dev/null || echo not'),Darwin)  # mac OSX
 		.DEFAULT_GOAL := mac_dmg
+		OSNAME = 'mac'
 	else
 		.DEFAULT_GOAL := binaries
 	endif
@@ -110,6 +112,7 @@ TEST_DATAVALIDATOR := $(PYTHON) -m pytest -vs scripts/invest-autovalidate.py
 
 # Target names.
 INVEST_BINARIES_DIR := $(DIST_DIR)/invest
+INVEST_BINARIES_DIR_ZIP := $(OSNAME)_invest_binaries.zip
 
 APIDOCS_BUILD_DIR := $(BUILD_DIR)/sphinx/apidocs
 APIDOCS_TARGET_DIR := $(DIST_DIR)/apidocs
@@ -396,6 +399,7 @@ signcode_windows:
 	@echo "Installer was signed with signtool"
 
 deploy:
+	-(cd $(INVEST_BINARIES_DIR) && $(ZIP) -r ../$(INVEST_BINARIES_DIR_ZIP) .)
 	-$(GSUTIL) -m rsync $(DIST_DIR) $(DIST_URL_BASE)
 	-$(GSUTIL) -m rsync -r $(DIST_DIR)/data $(DIST_URL_BASE)/data
 	-$(GSUTIL) -m rsync -r $(DIST_DIR)/userguide $(DIST_URL_BASE)/userguide
diff --git a/requirements-docs.yml b/requirements-docs.yml
index 4ee9246fd..073eecec3 100644
--- a/requirements-docs.yml
+++ b/requirements-docs.yml
@@ -9,6 +9,7 @@ channels:
 dependencies:
 - chardet>=3.0.4
 - Cython
+- Flask
 - GDAL>=3.1.2
 - numpy>=1.11.0,!=1.16.0
 - pandas>=1.0,<1.2.0
diff --git a/requirements-gui.txt b/requirements-gui.txt
index de041e85d..f6a1b6816 100644
--- a/requirements-gui.txt
+++ b/requirements-gui.txt
@@ -11,3 +11,4 @@ qtpy>1.3  # pip-only
 qtawesome  # pip-only
 requests
 PySide2!=5.15.0  # pip-only
+Flask
diff --git a/src/natcap/invest/cli.py b/src/natcap/invest/cli.py
index 82c3f206d..3d35127e4 100644
--- a/src/natcap/invest/cli.py
+++ b/src/natcap/invest/cli.py
@@ -349,7 +349,7 @@ class SelectModelAction(argparse.Action):
 
 
 def main(user_args=None):
-    """CLI entry point for launching InVEST runs.
+    """CLI entry point for launching InVEST runs and other useful utilities.
 
     This command-line interface supports two methods of launching InVEST models
     from the command-line:
@@ -452,6 +452,12 @@ def main(user_args=None):
         help=('The model for which the spec should be fetched.  Use "invest '
               'list" to list the available models.'))
 
+    serve_subparser = subparsers.add_parser(
+        'serve', help=('Start the flask app on the localhost.'))
+    serve_subparser.add_argument(
+        '--port', type=int, default=56789,
+        help='Port number for the Flask server')
+
     export_py_subparser = subparsers.add_parser(
         'export-py', help=('Save a python script that executes a model.'))
     export_py_subparser.add_argument(
@@ -511,8 +517,7 @@ def main(user_args=None):
             name=parsed_datastack.model_name)
 
         try:
-            validation_result = getattr(
-                model_module, 'validate')(parsed_datastack.args)
+            validation_result = model_module.validate(parsed_datastack.args)
         except KeyError as missing_keys_error:
             if args.json:
                 message = json.dumps(
@@ -588,7 +593,7 @@ def main(user_args=None):
             # We're deliberately not validating here because the user
             # can just call ``invest validate <datastack>`` to validate.
             try:
-                getattr(model_module, 'execute')(parsed_datastack.args)
+                model_module.execute(parsed_datastack.args)
             # Graceful exit from this program, otherwise the pyinstaller-made
             # executeable will exit with 'Failed to execute script cli' on an
             # uncaught exception. And that only adds unhelpful noise to stderr.
@@ -607,7 +612,7 @@ def main(user_args=None):
         # Creating this warning for future us to alert us to potential issues
         # if/when we forget to define QT_MAC_WANTS_LAYER at runtime.
         if (platform.system() == "Darwin" and
-                "QT_MAC_WANTS_LAYER"  not in os.environ):
+                "QT_MAC_WANTS_LAYER" not in os.environ):
             warnings.warn(
                 "Mac OS X Big Sur may require the 'QT_MAC_WANTS_LAYER' "
                 "environment variable to be defined in order to run.  If "
@@ -657,6 +662,11 @@ def main(user_args=None):
             parser.exit(app_exitcode,
                         'App terminated with exit code %s\n' % app_exitcode)
 
+    if args.subcommand == 'serve':
+        import natcap.invest.ui_server
+        natcap.invest.ui_server.app.run(port=args.port)
+        parser.exit(0)
+
     if args.subcommand == 'export-py':
         target_filepath = args.filepath
         if not args.filepath:
diff --git a/src/natcap/invest/coastal_vulnerability.py b/src/natcap/invest/coastal_vulnerability.py
index a48871fbe..d014b2d3a 100644
--- a/src/natcap/invest/coastal_vulnerability.py
+++ b/src/natcap/invest/coastal_vulnerability.py
@@ -269,7 +269,8 @@ ARGS_SPEC = {
             "name": "Sea Level Rise Vector"
         },
         "slr_field": {
-            "type": "freestyle_string",
+            "type": "option_string",
+            "options": {},
             "required": "slr_vector_path",
             "about": (
                 "The name of a field in the SLR vector table from which to "
diff --git a/src/natcap/invest/finfish_aquaculture/finfish_aquaculture.py b/src/natcap/invest/finfish_aquaculture/finfish_aquaculture.py
index 61d6d0573..bcc7291bb 100644
--- a/src/natcap/invest/finfish_aquaculture/finfish_aquaculture.py
+++ b/src/natcap/invest/finfish_aquaculture/finfish_aquaculture.py
@@ -46,7 +46,8 @@ ARGS_SPEC = {
                 "at farm tables). Additionally, the numbers underneath this "
                 "farm identifier name must be unique integers for all the "
                 "inputs."),
-            "type": "freestyle_string",
+            "type": "option_string",
+            "options": {},
         },
         "g_param_a": {
             "name": "Fish Growth Parameter (a)",
diff --git a/src/natcap/invest/ui_server.py b/src/natcap/invest/ui_server.py
new file mode 100644
index 000000000..1b1a9a6b1
--- /dev/null
+++ b/src/natcap/invest/ui_server.py
@@ -0,0 +1,240 @@
+"""A Flask app with HTTP endpoints used by the InVEST Workbench."""
+import codecs
+import collections
+from datetime import datetime
+import importlib
+import json
+import logging
+from osgeo import gdal
+import pprint
+import textwrap
+
+from flask import Flask
+from flask import request
+import natcap.invest.cli
+import natcap.invest.datastack
+
+logging.basicConfig(level=logging.DEBUG)
+LOGGER = logging.getLogger(__name__)
+
+app = Flask(__name__)
+
+# Lookup names to pass to `invest run` based on python module names
+_UI_META = collections.namedtuple('UIMeta', ['run_name', 'human_name'])
+MODULE_MODELRUN_MAP = {
+    v.pyname: _UI_META(
+        run_name=k,
+        human_name=v.humanname)
+    for k, v in natcap.invest.cli._MODEL_UIS.items()}
+
+
+def shutdown_server():
+    """Shutdown the flask server."""
+    func = request.environ.get('werkzeug.server.shutdown')
+    if func is None:
+        raise RuntimeError('Not running with the Werkzeug Server')
+    func()
+
+
+@app.route('/ready', methods=['GET'])
+def get_is_ready():
+    """Returns something simple to confirm the server is open."""
+    return 'Flask ready'
+
+
+@app.route('/shutdown', methods=['GET'])
+def shutdown():
+    """A request to this endpoint shuts down the server."""
+    shutdown_server()
+    return 'Flask server shutting down...'
+
+
+@app.route('/models', methods=['GET'])
+def get_invest_models():
+    """Gets a list of available InVEST models.
+
+    Returns:
+        A JSON string
+    """
+    LOGGER.debug('get model list')
+    return natcap.invest.cli.build_model_list_json()
+
+
+@app.route('/getspec', methods=['POST'])
+def get_invest_getspec():
+    """Gets the ARGS_SPEC dict from an InVEST model.
+
+    Body (JSON string): "carbon"
+
+    Returns:
+        A JSON string.
+    """
+    target_model = request.get_json()
+    target_module = natcap.invest.cli._MODEL_UIS[target_model].pyname
+    model_module = importlib.import_module(name=target_module)
+    LOGGER.debug(model_module.__file__)
+    spec = model_module.ARGS_SPEC
+    return json.dumps(spec)
+
+
+@app.route('/validate', methods=['POST'])
+def get_invest_validate():
+    """Gets the return value of an InVEST model's validate function.
+
+    Body (JSON string):
+        model_module: string (e.g. natcap.invest.carbon)
+        args: JSON string of InVEST model args keys and values
+
+    Returns:
+        A JSON string.
+    """
+    payload = request.get_json()
+    LOGGER.debug(payload)
+    target_module = payload['model_module']
+    args_dict = json.loads(payload['args'])
+    LOGGER.debug(args_dict)
+    try:
+        limit_to = payload['limit_to']
+    except KeyError:
+        limit_to = None
+    model_module = importlib.import_module(name=target_module)
+    results = model_module.validate(args_dict, limit_to=limit_to)
+    LOGGER.debug(results)
+    return json.dumps(results)
+
+
+@app.route('/colnames', methods=['POST'])
+def get_vector_colnames():
+    """Get a list of column names from a vector.
+    This is used to fill in dropdown menu options in a couple models.
+
+    Body (JSON string):
+        vector_path (string): path to a vector file
+
+    Returns:
+        a JSON string.
+    """
+    payload = request.get_json()
+    LOGGER.debug(payload)
+    vector_path = payload['vector_path']
+    # a lot of times the path will be empty so don't even try to open it
+    if vector_path:
+        try:
+            vector = gdal.OpenEx(vector_path, gdal.OF_VECTOR)
+            colnames = [defn.GetName() for defn in vector.GetLayer().schema]
+            LOGGER.debug(colnames)
+            return json.dumps(colnames)
+        except Exception as e:
+            LOGGER.exception(
+                f'Could not read column names from {vector_path}. ERROR: {e}')
+    else:
+        LOGGER.error('Empty vector path.')
+    # 422 Unprocessable Entity: the server understands the content type
+    # of the request entity, and the syntax of the request entity is
+    # correct, but it was unable to process the contained instructions.
+    return json.dumps([]), 422
+
+
+@app.route('/post_datastack_file', methods=['POST'])
+def post_datastack_file():
+    """Extracts InVEST model args from json, logfiles, or datastacks.
+
+    Body (JSON string): path to file
+
+    Returns:
+        A JSON string.
+    """
+    filepath = request.get_json()
+    stack_type, stack_info = natcap.invest.datastack.get_datastack_info(
+        filepath)
+    run_name, human_name = MODULE_MODELRUN_MAP[stack_info.model_name]
+    result_dict = {
+        'type': stack_type,
+        'args': stack_info.args,
+        'module_name': stack_info.model_name,
+        'model_run_name': run_name,
+        'model_human_name': human_name,
+        'invest_version': stack_info.invest_version
+    }
+    LOGGER.debug(result_dict)
+    return json.dumps(result_dict)
+
+
+@app.route('/write_parameter_set_file', methods=['POST'])
+def write_parameter_set_file():
+    """Writes InVEST model args keys and values to a datastack JSON file.
+
+    Body (JSON string):
+        parameterSetPath: string
+        moduleName: string(e.g. natcap.invest.carbon)
+        args: JSON string of InVEST model args keys and values
+        relativePaths: boolean
+
+    Returns:
+        A string.
+    """
+    payload = request.get_json()
+    filepath = payload['parameterSetPath']
+    modulename = payload['moduleName']
+    args = json.loads(payload['args'])
+    relative_paths = payload['relativePaths']
+
+    natcap.invest.datastack.build_parameter_set(
+        args, modulename, filepath, relative=relative_paths)
+    return 'parameter set saved'
+
+
+# Borrowed this function from natcap.invest.model because I assume
+# that module won't persist if we eventually deprecate the Qt UI.
+@app.route('/save_to_python', methods=['POST'])
+def save_to_python():
+    """Writes a python script with a call to an InVEST model execute function.
+
+    Body (JSON string):
+        filepath: string
+        modelname: string (e.g. carbon)
+        pyname: string (e.g. natcap.invest.carbon)
+        args_dict: JSON string of InVEST model args keys and values
+
+    Returns:
+        A string.
+    """
+    payload = request.get_json()
+    save_filepath = payload['filepath']
+    modelname = payload['modelname']
+    pyname = payload['pyname']
+    args_dict = json.loads(payload['args'])
+
+    script_template = textwrap.dedent("""\
+    # coding=UTF-8
+    # -----------------------------------------------
+    # Generated by InVEST {invest_version} on {today}
+    # Model: {modelname}
+
+    import {py_model}
+
+    args = {model_args}
+
+    if __name__ == '__main__':
+        {py_model}.execute(args)
+    """)
+
+    with codecs.open(save_filepath, 'w', encoding='utf-8') as py_file:
+        # cast_args = dict((unicode(key), value) for (key, value)
+        #                  in args_dict.items())
+        args = pprint.pformat(args_dict, indent=4)  # 4 spaces
+
+        # Tweak formatting from pprint:
+        # * Bump parameter inline with starting { to next line
+        # * add trailing comma to last item item pair
+        # * add extra space to spacing before first item
+        args = args.replace('{', '{\n ')
+        args = args.replace('}', ',\n}')
+        py_file.write(script_template.format(
+            invest_version=natcap.invest.cli.__version__,
+            today=datetime.now().strftime('%c'),
+            modelname=modelname,
+            py_model=pyname,
+            model_args=args))
+
+    return 'python script saved'
diff --git a/src/natcap/invest/validation.py b/src/natcap/invest/validation.py
index 5273a637d..74bbb4cf0 100644
--- a/src/natcap/invest/validation.py
+++ b/src/natcap/invest/validation.py
@@ -383,18 +383,21 @@ def check_freestyle_string(value, regexp=None, **kwargs):
 
 
 def check_option_string(value, options, **kwargs):
-    """Validate that a string is in a list of options.
+    """Validate that a string is in a set of options.
 
     Args:
         value (string): The string value to test.
-        options (list): A list of strings to test against.
+        options (set | dict): strings to test against.
+            If a dict, test against the keys.
 
     Returns:
         A string error message if ``value`` is not in ``options``.  ``None``
         otherwise.
 
     """
-    if value not in options:
+    # if options is an empty set, that means it's dynamically populated
+    # so validation should be left to the model's validate function.
+    if options and value not in options:
         return "Value must be one of: %s" % sorted(options)
 
 


Test patch:
diff --git a/tests/test_cli.py b/tests/test_cli.py
index 12648cff9..0b5024089 100644
--- a/tests/test_cli.py
+++ b/tests/test_cli.py
@@ -342,6 +342,26 @@ class CLIHeadlessTests(unittest.TestCase):
         # are warnings.
         self.assertEqual(exit_cm.exception.code, 0)
 
+    def test_serve(self):
+        """CLI: serve entry-point exists; flask app can import."""
+        from natcap.invest import cli
+
+        with unittest.mock.patch('natcap.invest.ui_server.app.run',
+                                 return_value=None) as patched_app:
+            with self.assertRaises(SystemExit) as exit_cm:
+                cli.main(['serve'])
+        self.assertEqual(exit_cm.exception.code, 0)
+
+    def test_serve_port_argument(self):
+        """CLI: serve entry-point parses port subargument."""
+        from natcap.invest import cli
+
+        with unittest.mock.patch('natcap.invest.ui_server.app.run',
+                                 return_value=None) as patched_app:
+            with self.assertRaises(SystemExit) as exit_cm:
+                cli.main(['serve', '--port', '12345'])
+        self.assertEqual(exit_cm.exception.code, 0)
+
     def test_export_python(self):
         """CLI: Export a python script for a given model."""
         from natcap.invest import cli
diff --git a/tests/test_ui_server.py b/tests/test_ui_server.py
new file mode 100644
index 000000000..c8e125874
--- /dev/null
+++ b/tests/test_ui_server.py
@@ -0,0 +1,36 @@
+import os
+import json
+import unittest
+
+TEST_DATA_PATH = os.path.join(
+    os.path.dirname(__file__), '..', 'data', 'invest-test-data')
+
+
+class EndpointFunctionTests(unittest.TestCase):
+    """Tests for some UI server endpoint functions.
+
+    Other endpoints have tests in the workbench repo. These ones need
+    vector/raster test data so it makes more sense to have them here.
+    """
+
+    def test_get_vector_colnames(self):
+        """UI server: getVectorColnames endpoint"""
+        from natcap.invest import ui_server
+        test_client = ui_server.app.test_client()
+        # an empty path
+        response = test_client.post('/colnames', json={'vector_path': ''})
+        colnames = json.loads(response.get_data(as_text=True))
+        self.assertEqual(response.status_code, 422)
+        self.assertEqual(colnames, [])
+        # a vector with one column
+        path = os.path.join(
+            TEST_DATA_PATH, 'aquaculture', 'Input', 'Finfish_Netpens.shp')
+        response = test_client.post('/colnames', json={'vector_path': path})
+        colnames = json.loads(response.get_data(as_text=True))
+        self.assertEqual(colnames, ['FarmID'])
+        # a non-vector file
+        path = os.path.join(TEST_DATA_PATH, 'ndr', 'input', 'dem.tif')
+        response = test_client.post('/colnames', json={'vector_path': path})
+        colnames = json.loads(response.get_data(as_text=True))
+        self.assertEqual(response.status_code, 422)
+        self.assertEqual(colnames, [])
diff --git a/tests/test_validation.py b/tests/test_validation.py
index dd2fc1a57..6ec84e69a 100644
--- a/tests/test_validation.py
+++ b/tests/test_validation.py
@@ -545,17 +545,30 @@ class FreestyleStringValidation(unittest.TestCase):
 class OptionStringValidation(unittest.TestCase):
     """Test Option String Validation."""
 
-    def test_valid_option(self):
-        """Validation: test that a string is a valid option."""
+    def test_valid_option_set(self):
+        """Validation: test that a string is a valid option in a set."""
         from natcap.invest import validation
         self.assertEqual(None, validation.check_option_string(
-            'foo', options=['foo', 'bar', 'Baz']))
+            'foo', options={'foo', 'bar', 'Baz'}))
 
-    def test_invalid_option(self):
-        """Validation: test when a string is not a valid option."""
+    def test_invalid_option_set(self):
+        """Validation: test when a string is not a valid option in a set."""
         from natcap.invest import validation
         error_msg = validation.check_option_string(
-            'FOO', options=['foo', 'bar', 'Baz'])
+            'FOO', options={'foo', 'bar', 'Baz'})
+        self.assertTrue('must be one of' in error_msg)
+
+    def test_valid_option_dict(self):
+        """Validation: test that a string is a valid option in a dict."""
+        from natcap.invest import validation
+        self.assertEqual(None, validation.check_option_string(
+            'foo', options={'foo': 'desc', 'bar': 'desc', 'Baz': 'desc'}))
+
+    def test_invalid_option_dict(self):
+        """Validation: test when a string is not a valid option in a dict."""
+        from natcap.invest import validation
+        error_msg = validation.check_option_string(
+            'FOO', options={'foo': 'desc', 'bar': 'desc', 'Baz': 'desc'})
         self.assertTrue('must be one of' in error_msg)