Repository: scikit-hep/pyhf

Issue:
docs: List options for optimizers in CLI API docs and add short aliases
# Description

In the [Command Line API docs for `cls`](https://scikit-hep.org/pyhf/cli.html#pyhf-cls) there are available options shown for `--teststat` and `--backend` but not for `--optimizer`.

![cls_API_docs](https://user-images.githubusercontent.com/5142394/79806132-4b668b80-832d-11ea-9779-bfc7c78a0b0f.png)

The available options should also be shown for `--optimizer`.

In addition, the options all [currently require](https://github.com/scikit-hep/pyhf/blob/eb9944c043f7f8a75b9cfe6f8bd781bd1dd7014b/src/pyhf/optimize/__init__.py) that the  word "_optimizer" be added to the end (`scipy_optimizer`, `minuit_optimizer`). It would be better to additionally alias these to be just the name as someone at the CLI is already specifying the `--optimizer` flag so having to write `_optimizer` again seems strange. It is nicer to write

```
pyhf cls --patch patch.json --optimizer minuit BkgOnly.json
```

than

```
pyhf cls --patch patch.json --optimizer minuit_optimizer BkgOnly.json
```

though we should keep the `X_optimizer` name available of course.


Discussion:

Resolving patch:
diff --git a/src/pyhf/cli/cli.py b/src/pyhf/cli/cli.py
--- a/src/pyhf/cli/cli.py
+++ b/src/pyhf/cli/cli.py
@@ -29,6 +29,7 @@ def pyhf():
 pyhf.add_command(spec.sort)
 
 # pyhf.add_command(infer.cli)
+pyhf.add_command(infer.fit)
 pyhf.add_command(infer.cls)
 
 pyhf.add_command(patchset.cli)
diff --git a/src/pyhf/cli/infer.py b/src/pyhf/cli/infer.py
--- a/src/pyhf/cli/infer.py
+++ b/src/pyhf/cli/infer.py
@@ -6,6 +6,7 @@
 
 from ..utils import EqDelimStringParamType
 from ..infer import hypotest
+from ..infer import mle
 from ..workspace import Workspace
 from .. import get_backend, set_backend, optimize
 
@@ -17,6 +18,119 @@ def cli():
     """Infererence CLI group."""
 
 
+@cli.command()
+@click.argument("workspace", default="-")
+@click.option(
+    "--output-file",
+    help="The location of the output json file. If not specified, prints to screen.",
+    default=None,
+)
+@click.option("--measurement", default=None)
+@click.option("-p", "--patch", multiple=True)
+@click.option(
+    "--value",
+    help="Flag for returning the fitted value of the objective function.",
+    default=False,
+    is_flag=True,
+)
+@click.option(
+    "--backend",
+    type=click.Choice(["numpy", "pytorch", "tensorflow", "jax", "np", "torch", "tf"]),
+    help="The tensor backend used for the calculation.",
+    default="numpy",
+)
+@click.option(
+    "--optimizer",
+    type=click.Choice(["scipy", "minuit"]),
+    help="The optimizer used for the calculation.",
+    default="scipy",
+)
+@click.option("--optconf", type=EqDelimStringParamType(), multiple=True)
+def fit(
+    workspace,
+    output_file,
+    measurement,
+    patch,
+    value,
+    backend,
+    optimizer,
+    optconf,
+):
+    """
+    Perform a maximum likelihood fit for a given pyhf workspace.
+
+    Example:
+
+    .. code-block:: shell
+
+        $ curl -sL https://git.io/JJYDE | pyhf fit --value
+
+        \b
+        {
+            "mle_parameters": {
+                "mu": [
+                    0.00017298628839781602
+                ],
+                "uncorr_bkguncrt": [
+                    1.0000015671710816,
+                    0.9999665895859197
+                ]
+            },
+            "twice_nll": 23.19636590468879
+        }
+    """
+    # set the backend if not NumPy
+    if backend in ["pytorch", "torch"]:
+        set_backend("pytorch", precision="64b")
+    elif backend in ["tensorflow", "tf"]:
+        set_backend("tensorflow", precision="64b")
+    elif backend in ["jax"]:
+        set_backend("jax")
+    tensorlib, _ = get_backend()
+
+    optconf = {k: v for item in optconf for k, v in item.items()}
+
+    # set the new optimizer
+    if optimizer:
+        new_optimizer = getattr(optimize, optimizer) or getattr(
+            optimize, f"{optimizer}_optimizer"
+        )
+        set_backend(tensorlib, new_optimizer(**optconf))
+
+    with click.open_file(workspace, "r") as specstream:
+        spec = json.load(specstream)
+    ws = Workspace(spec)
+    patches = [json.loads(click.open_file(pfile, "r").read()) for pfile in patch]
+
+    model = ws.model(
+        measurement_name=measurement,
+        patches=patches,
+        modifier_settings={
+            "normsys": {"interpcode": "code4"},
+            "histosys": {"interpcode": "code4p"},
+        },
+    )
+    data = ws.data(model)
+
+    fit_result = mle.fit(data, model, return_fitted_val=value)
+
+    _pars = fit_result if not value else fit_result[0]
+    bestfit_pars = {
+        k: tensorlib.tolist(_pars[v["slice"]]) for k, v in model.config.par_map.items()
+    }
+
+    result = {"mle_parameters": bestfit_pars}
+    if value:
+        result["twice_nll"] = tensorlib.tolist(fit_result[-1])
+
+    if output_file is None:
+        click.echo(json.dumps(result, indent=4, sort_keys=True))
+    else:
+        with open(output_file, "w+") as out_file:
+            json.dump(result, out_file, indent=4, sort_keys=True)
+        log.debug("Written to {0:s}".format(output_file))
+
+
 @cli.command()
 @click.argument('workspace', default='-')
 @click.option(
@@ -34,7 +148,12 @@ def cli():
     help='The tensor backend used for the calculation.',
     default='numpy',
 )
-@click.option('--optimizer')
+@click.option(
+    "--optimizer",
+    type=click.Choice(["scipy", "minuit"]),
+    help="The optimizer used for the calculation.",
+    default="scipy",
+)
 @click.option('--optconf', type=EqDelimStringParamType(), multiple=True)
 def cls(
     workspace,


Test patch:
diff --git a/tests/test_scripts.py b/tests/test_scripts.py
--- a/tests/test_scripts.py
+++ b/tests/test_scripts.py
@@ -55,6 +55,49 @@ def test_import_prepHistFactory_stdout(tmpdir, script_runner):
     assert d
 
 
+def test_import_prepHistFactory_and_fit(tmpdir, script_runner):
+    temp = tmpdir.join("parsed_output.json")
+    command = "pyhf xml2json validation/xmlimport_input/config/example.xml --basedir validation/xmlimport_input/ --output-file {0:s}".format(
+        temp.strpath
+    )
+    ret = script_runner.run(*shlex.split(command))
+
+    command = "pyhf fit {0:s}".format(temp.strpath)
+    ret = script_runner.run(*shlex.split(command))
+
+    assert ret.success
+    ret_json = json.loads(ret.stdout)
+    assert ret_json
+    assert "mle_parameters" in ret_json
+    assert "twice_nll" not in ret_json
+
+    for measurement in [
+        "GaussExample",
+        "GammaExample",
+        "LogNormExample",
+        "ConstExample",
+    ]:
+        command = "pyhf fit {0:s} --value --measurement {1:s}".format(
+            temp.strpath, measurement
+        )
+        ret = script_runner.run(*shlex.split(command))
+
+        assert ret.success
+        ret_json = json.loads(ret.stdout)
+        assert ret_json
+        assert "mle_parameters" in ret_json
+        assert "twice_nll" in ret_json
+
+        tmp_out = tmpdir.join("{0:s}_output.json".format(measurement))
+        # make sure output file works too
+        command += " --output-file {0:s}".format(tmp_out.strpath)
+        ret = script_runner.run(*shlex.split(command))
+        assert ret.success
+        ret_json = json.load(tmp_out)
+        assert "mle_parameters" in ret_json
+        assert "twice_nll" in ret_json
+
+
 def test_import_prepHistFactory_and_cls(tmpdir, script_runner):
     temp = tmpdir.join("parsed_output.json")
     command = 'pyhf xml2json validation/xmlimport_input/config/example.xml --basedir validation/xmlimport_input/ --output-file {0:s}'.format(
@@ -96,6 +139,23 @@ def test_import_prepHistFactory_and_cls(tmpdir, script_runner):
         assert 'CLs_exp' in d
 
 
+@pytest.mark.parametrize("backend", ["numpy", "tensorflow", "pytorch", "jax"])
+def test_fit_backend_option(tmpdir, script_runner, backend):
+    temp = tmpdir.join("parsed_output.json")
+    command = "pyhf xml2json validation/xmlimport_input/config/example.xml --basedir validation/xmlimport_input/ --output-file {0:s}".format(
+        temp.strpath
+    )
+    ret = script_runner.run(*shlex.split(command))
+
+    command = "pyhf fit --backend {0:s} {1:s}".format(backend, temp.strpath)
+    ret = script_runner.run(*shlex.split(command))
+
+    assert ret.success
+    ret_json = json.loads(ret.stdout)
+    assert ret_json
+    assert "mle_parameters" in ret_json
+
+
 @pytest.mark.parametrize("backend", ["numpy", "tensorflow", "pytorch", "jax"])
 def test_cls_backend_option(tmpdir, script_runner, backend):
     temp = tmpdir.join("parsed_output.json")
@@ -234,9 +294,25 @@ def test_testpoi(tmpdir, script_runner):
     assert len(list(set(results_obs))) == len(pois)
 
 
+@pytest.mark.parametrize("optimizer", ["scipy", "minuit"])
 @pytest.mark.parametrize(
-    'optimizer', ['scipy', 'minuit', 'scipy_optimizer', 'minuit_optimizer']
+    "opts,success", [(["maxiter=1000"], True), (["maxiter=1"], False)]
 )
+def test_fit_optimizer(tmpdir, script_runner, optimizer, opts, success):
+    temp = tmpdir.join("parsed_output.json")
+    command = "pyhf xml2json validation/xmlimport_input/config/example.xml --basedir validation/xmlimport_input/ --output-file {0:s}".format(
+        temp.strpath
+    )
+    ret = script_runner.run(*shlex.split(command))
+
+    optconf = " ".join(f"--optconf {opt}" for opt in opts)
+    command = f"pyhf fit --optimizer {optimizer} {optconf} {temp.strpath}"
+    ret = script_runner.run(*shlex.split(command))
+
+    assert ret.success == success
+
+
+@pytest.mark.parametrize('optimizer', ['scipy', 'minuit'])
 @pytest.mark.parametrize(
     'opts,success', [(['maxiter=1000'], True), (['maxiter=1'], False)]
 )