Repository: mwouts/jupytext (python)

Issue:
Mysterious failure in --pipe mode
I am trying to run jupytext --pipe black *.ipynb and it is working fine on my local machine but failing in Travis CI, where it is very difficult to debug the problem. All I have is the error message:
```
[jupytext] Reading examples/Example of parameter usage.ipynb
Traceback (most recent call last):
  File "/home/travis/virtualenv/python3.7.1/bin/jupytext", line 10, in <module>
    sys.exit(jupytext())
  File "/home/travis/virtualenv/python3.7.1/lib/python3.7/site-packages/jupytext/cli.py", line 228, in jupytext
    exit_code += jupytext_single_file(nb_file, args, log)
  File "/home/travis/virtualenv/python3.7.1/lib/python3.7/site-packages/jupytext/cli.py", line 336, in jupytext_single_file
    notebook = pipe_notebook(notebook, cmd, args.pipe_fmt)
  File "/home/travis/virtualenv/python3.7.1/lib/python3.7/site-packages/jupytext/cli.py", line 528, in pipe_notebook
    fmt = long_form_one_format(fmt, notebook.metadata)
  File "/home/travis/virtualenv/python3.7.1/lib/python3.7/site-packages/jupytext/formats.py", line 470, in long_form_one_format
    raise JupytextFormatError("No language information in this notebook. Please replace 'auto' with "
jupytext.formats.JupytextFormatError: No language information in this notebook. Please replace 'auto' with an actual script extension.
```
The problem is plainly not one of failing to provide a specific extension. My guess is that for some reason black is producing no output? Some more meaningful error message out of --pipe would make this vastly easier to track down.

The full output is visible here:

https://travis-ci.org/nanograv/PINT/jobs/600008676

Pull request description:
Improve cli messages
- Document default values (and add line breaks) in `jupytext --help`
- Compute the auto extension from the kernel specs, when language_info is not available
- Confirm with a test that `jupytext --pipe black *.md` works
- If really there's nothing we can do to compute the `.auto` extension, jupytext gives a more explicit message, and points out which option should be modified.

Closes #358

Patch:
diff --git a/jupytext/cli.py b/jupytext/cli.py
index 052fe805..6b530e29 100644
--- a/jupytext/cli.py
+++ b/jupytext/cli.py
@@ -49,55 +49,78 @@ def str2bool(value):
 
 def parse_jupytext_args(args=None):
     """Command line parser for jupytext"""
+
+    class RawTextArgumentDefaultsHelpFormatter(argparse.RawTextHelpFormatter,
+                                               argparse.ArgumentDefaultsHelpFormatter):
+        """Keep the raw formatting in command line help, plus show the default values"""
+        pass
+
     parser = argparse.ArgumentParser(
         description='Jupyter notebooks as markdown documents, Julia, Python or R scripts',
-        formatter_class=argparse.RawTextHelpFormatter)
+        formatter_class=RawTextArgumentDefaultsHelpFormatter)
 
     # Input
     parser.add_argument('notebooks',
-                        help='One or more notebook(s). Notebook is read from stdin when this argument is empty',
+                        help='One or more notebook(s).\n'
+                             'Notebook is read from stdin when this argument\n'
+                             'is empty.',
                         nargs='*')
     parser.add_argument('--from',
                         dest='input_format',
-                        help='Optional: jupytext format for the input(s). '
-                             'Inferred from the file extension and content when missing.')
+                        help='Jupytext format for the input(s). Inferred from the\n'
+                             'file extension and content when missing.')
     parser.add_argument('--pre-commit',
                         action='store_true',
-                        help='Ignore the notebook argument, and instead apply Jupytext on the notebooks found '
-                             'in the git index, which have an extension that matches the (optional) --from argument.')
+                        help='Ignore the notebook argument, and instead apply Jupytext\n'
+                             'on the notebooks found in the git index, which have an\n'
+                             'extension that matches the (optional) --from argument.\n')
     # Destination format & act on metadata
     parser.add_argument('--to',
-                        help="Destination format: either one of 'notebook', 'markdown', 'rmarkdown', 'script', any "
-                             "valid notebook extension, or a full format '[prefix_path//][suffix.]ext[:format_name]")
+                        help="Destination format: either one of 'notebook', 'markdown',\n"
+                             "'rmarkdown', 'script', any valid notebook extension, or a\n"
+                             "full format description, i.e.\n"
+                             "'[prefix_path//][suffix.]ext[:format_name]")
     parser.add_argument('--format-options', '--opt',
                         action='append',
-                        help='Set format options with e.g. --opt comment_magics=true '
-                             '--opt notebook_metadata_filter=-kernelspec.')
+                        help='Set format options with e.g.\n'
+                             '    --opt comment_magics=true\n'
+                             '    --opt notebook_metadata_filter=-kernelspec.\n')
     parser.add_argument('--set-formats',
                         type=str,
-                        help='Set jupytext.formats metadata to the given value. Use this to activate pairing on a '
-                             'notebook, with e.g. --set-formats ipynb,py:light. The --set-formats option also triggers '
-                             'the creation/update of all paired files')
+                        help='Set jupytext.formats metadata to the given value. Use this to\n'
+                             'activate pairing on a notebook, with e.g.\n'
+                             '    --set-formats ipynb,py:light.\n'
+                             'The --set-formats option also triggers the creation/update\n'
+                             'of all paired files')
     parser.add_argument('--set-kernel', '-k',
                         type=str,
-                        help="Set the kernel with the given name on the notebook. Use '--set-kernel -' to set "
-                             "a kernel matching the current environment on Python notebooks, and matching the "
-                             "notebook language otherwise "
-                             "(get the list of available kernels with 'jupyter kernelspec list')")
+                        help="Set the kernel with the given name on the notebook.\n"
+                             "Use\n"
+                             "    --set-kernel -\n"
+                             "to set a kernel matching the current\n"
+                             "environment on Python notebooks, and matching the notebook\n"
+                             "language otherwise (get the list of available kernels with\n"
+                             "'jupyter kernelspec list')")
     parser.add_argument('--update-metadata',
                         default={},
                         type=json.loads,
-                        help='Update the notebook metadata with the desired dictionary. Argument must be given in JSON '
-                             'format. For instance, if you want to activate a pairing in the generated file, '
-                             """use e.g. '{"jupytext":{"formats":"ipynb,py:light"}}'""")
+                        help='Update the notebook metadata with the desired dictionary.\n'
+                             'Argument must be given in JSON format. For instance, if you\n'
+                             'want to activate a pairing in the generated file, use e.g.\n'
+                             """    --update-metadata '{"jupytext":{"formats":"ipynb,py:light"}}'\n"""
+                             "See also the '--opt' and '--set-formats' options for other ways\n"
+                             "to operate on the Jupytext metadata."
+                        )
 
     # Destination file
     parser.add_argument('-o', '--output',
-                        help="Destination file. Defaults to the original file, with prefix/suffix/extension changed"
-                             "according to the destination format. Use '-' to print the notebook on stdout.")
+                        help="Destination file. Defaults to the original file,\n"
+                             "with prefix/suffix/extension changed according to\n"
+                             "the destination format.\n"
+                             "Use '-' to print the notebook on stdout.")
     parser.add_argument('--update', action='store_true',
-                        help='Preserve the output cells when the destination notebook is a .ipynb file '
-                             'that already exists')
+                        help='Preserve the output cells when the destination\n'
+                             'notebook is a .ipynb file that already exists')
 
     # Action: convert(default)/version/list paired paths/sync/apply/test
     action = parser.add_mutually_exclusive_group()
@@ -105,46 +128,52 @@ def parse_jupytext_args(args=None):
                         action='store_true',
                         help="Show jupytext's version number and exit")
     action.add_argument('--paired-paths', '-p',
-                        help='List the locations of the alternative representations for this notebook.',
+                        help='List the locations of the alternative representations\n'
+                             'for this notebook.',
                         action='store_true')
     action.add_argument('--sync', '-s',
-                        help='Synchronize the content of the paired representations of the given notebook. '
-                             'Input cells are taken from the file that was last modified, and outputs are read '
-                             'from the ipynb file, if present.',
+                        help='Synchronize the content of the paired representations of\n'
+                             'the given notebook. Input cells are taken from the file that\n'
+                             'was last modified, and outputs are read from the ipynb file,\n'
+                             'if present.',
                         action='store_true')
     action.add_argument('--warn-only', '-w',
                         action='store_true',
-                        help='Only issue a warning and continue processing other notebooks when the conversion of a '
-                             'given notebook fails')
+                        help='Only issue a warning and continue processing other notebooks\n'
+                             'when the conversion of a given notebook fails')
     action.add_argument('--test',
                         action='store_true',
-                        help='Test that notebook is stable under a round trip conversion, up to the expected changes')
+                        help='Test that notebook is stable under a round trip conversion,\n'
+                             'up to the expected changes')
     action.add_argument('--test-strict',
                         action='store_true',
-                        help='Test that notebook is strictly stable under a round trip conversion')
+                        help='Test that notebook is strictly stable under a round trip\n'
+                             'conversion')
     parser.add_argument('--stop', '-x',
                         dest='stop_on_first_error',
                         action='store_true',
-                        help='In --test mode, stop on first round trip conversion error, and report stack traceback')
+                        help='In --test mode, stop on first round trip conversion error, and report\n'
+                             'stack traceback')
 
     # Pipe notebook inputs into other commands
     parser.add_argument('--pipe',
                         action='append',
-                        help='Pipe the text representation of the notebook into another program, and read the'
-                             'notebook back. For instance, reformat your notebook with:'
-                             "    jupytext notebook.ipynb --pipe black"
-                             'If you want to reformat it and sync the paired representation, execute:'
-                             "    jupytext notebook.ipynb --sync --pipe black")
+                        help='Pipe the text representation of the notebook into another\n'
+                             'program, and read the notebook back. For instance, reformat\n'
+                             'your notebook with:\n'
+                             "    jupytext notebook.ipynb --pipe black\n"
+                             'If you want to reformat it and sync the paired representation, execute:\n'
+                             "    jupytext notebook.ipynb --sync --pipe black\n")
     parser.add_argument('--check',
                         action='append',
-                        help='Pipe the text representation of the notebook into another program, and test that '
-                             'the returned value is non zero. For instance, test that your notebook is pep8 compliant '
-                             'with:'
-                             "    jupytext notebook.ipynb --check flake8")
+                        help='Pipe the text representation of the notebook into another program,\n'
+                             'and test that the returned value is non zero. For instance,\n'
+                             'test that your notebook is pep8 compliant with:\n'
+                             "    jupytext notebook.ipynb --check flake8\n")
     parser.add_argument('--pipe-fmt',
                         default='auto:percent',
-                        help='The format in which the notebook should be piped to other programs, when using the '
-                             '--pipe and/or --check commands.')
+                        help='The format in which the notebook should be piped to other programs,\n'
+                             'when using the --pipe and/or --check commands.')
 
     # Execute the notebook
     parser.add_argument('--execute',
@@ -154,7 +183,8 @@ def parse_jupytext_args(args=None):
     parser.add_argument('--quiet', '-q',
                         action='store_true',
                         default=False,
-                        help='Quiet mode: do not comment about files being updated or created')
+                        help='Quiet mode: do not comment about files being updated\n'
+                             'or created')
 
     return parser.parse_args(args)
 
@@ -196,12 +226,15 @@ def jupytext(args=None):
         print_paired_paths(args.notebooks[0], args.input_format)
         return 1
 
+    if (args.test or args.test_strict) and not args.to and not args.output and not args.sync:
+        raise ValueError('Please provide one of --to, --output or --sync')
+
     if not args.to and not args.output and not args.sync \
             and not args.pipe and not args.check \
-            and not args.test and not args.test_strict \
             and not args.update_metadata and not args.set_kernel \
             and not args.execute:
-        raise ValueError('Please select an action')
+        raise ValueError('Please provide one of --to, --output, --sync, --pipe, '
+                         '--check, --update_metadata, --set-kernel or --execute')
 
     if args.output and len(args.notebooks) != 1:
         raise ValueError('Please input a single notebook when using --output')
@@ -271,11 +304,7 @@ def jupytext_single_file(nb_file, args, log):
     # Compute actual extension when using script/auto, and update nb_dest if necessary
     dest_fmt = args.to
     if dest_fmt and dest_fmt['extension'] == '.auto':
-        auto_ext = auto_ext_from_metadata(notebook.metadata)
-        if not auto_ext:
-            raise ValueError('The notebook has no language information. '
-                             'Please provide an explicit script extension.')
-        dest_fmt['extension'] = auto_ext
+        check_auto_ext(dest_fmt, notebook.metadata, '--to')
         if not args.output and nb_file != '-':
             nb_dest = full_path(base_path(nb_file, args.input_format), dest_fmt)
 
@@ -540,13 +569,29 @@ def load_paired_notebook(notebook, fmt, nb_file, log):
     return notebook, latest_inputs, latest_outputs
 
 
+def check_auto_ext(fmt, metadata, option):
+    """Raise a ValueError when the .auto extension cannot be determined"""
+    if fmt['extension'] != '.auto':
+        return
+
+    auto_ext = auto_ext_from_metadata(metadata)
+    if auto_ext:
+        fmt['extension'] = auto_ext
+        return
+
+    raise ValueError("The notebook does not have a 'language_info' metadata. "
+                     "Please replace 'auto' with the actual language extension in the {} option (currently {})."
+                     .format(option, short_form_one_format(fmt)))
+
+
 def pipe_notebook(notebook, command, fmt='py:percent', update=True):
     """Pipe the notebook, in the desired representation, to the given command. Update the notebook
     with the returned content if desired."""
     if command in ['black', 'flake8', 'autopep8']:
         command = command + ' -'
 
-    fmt = long_form_one_format(fmt, notebook.metadata)
+    fmt = long_form_one_format(fmt, notebook.metadata, auto_ext_requires_language_info=False)
+    check_auto_ext(fmt, notebook.metadata, '--pipe-fmt')
     text = writes(notebook, fmt)
     process = subprocess.Popen(command.split(' '), stdout=subprocess.PIPE, stdin=subprocess.PIPE)
     cmd_output, err = process.communicate(input=text.encode('utf-8'))
diff --git a/jupytext/formats.py b/jupytext/formats.py
index c45284be..3182950e 100644
--- a/jupytext/formats.py
+++ b/jupytext/formats.py
@@ -15,7 +15,7 @@ from .cell_to_text import MarkdownCellExporter, RMarkdownCellExporter, \
     HydrogenCellExporter, SphinxGalleryCellExporter
 from .metadata_filter import metadata_filter_as_string
 from .stringparser import StringParser
-from .languages import _SCRIPT_EXTENSIONS, _COMMENT_CHARS
+from .languages import _SCRIPT_EXTENSIONS, _COMMENT_CHARS, same_language
 from .pandoc import pandoc_version, is_pandoc_available
 from .magics import is_magic
 
@@ -563,8 +563,18 @@ def validate_one_format(jupytext_format):
 
 
 def auto_ext_from_metadata(metadata):
-    """Script extension from kernel information"""
+    """Script extension from notebook metadata"""
     auto_ext = metadata.get('language_info', {}).get('file_extension')
+
+    if auto_ext is None:
+        kernel_language = metadata.get('kernelspec', {}).get('language')
+        if kernel_language:
+            for ext in _SCRIPT_EXTENSIONS:
+                if same_language(kernel_language, _SCRIPT_EXTENSIONS[ext]['language']):
+                    auto_ext = ext
+                    break
+
     if auto_ext == '.r':
         return '.R'
+
     return auto_ext
diff --git a/jupytext/kernels.py b/jupytext/kernels.py
index ed736b69..55caf382 100644
--- a/jupytext/kernels.py
+++ b/jupytext/kernels.py
@@ -2,6 +2,7 @@
 
 import sys
 from .reraise import reraise
+from .languages import same_language
 
 try:
     # I prefer not to take a dependency on jupyter_client
@@ -39,7 +40,7 @@ def kernelspec_from_language(language):
 
         for name in find_kernel_specs():
             kernel_specs = get_kernel_spec(name)
-            if kernel_specs.language == language or (language == 'c++' and kernel_specs.language.startswith('C++')):
+            if same_language(kernel_specs.language, language):
                 return {'name': name, 'language': language, 'display_name': kernel_specs.display_name}
     except (KeyError, ValueError):
         pass
diff --git a/jupytext/languages.py b/jupytext/languages.py
index 208a75a9..eb611461 100644
--- a/jupytext/languages.py
+++ b/jupytext/languages.py
@@ -49,6 +49,19 @@ def default_language_from_metadata_and_ext(metadata, ext):
     return language.lower()
 
 
+def same_language(kernel_language, language):
+    """Are those the same language?"""
+    if kernel_language == language:
+        return True
+    if kernel_language.lower() == language:
+        return True
+    if kernel_language.startswith('C++') and language == 'c++':
+        return True
+    if kernel_language == 'octave' and language == 'matlab':
+        return True
+    return False
+
+
 def set_main_and_cell_language(metadata, cells, ext):
     """Set main language for the given collection of cells, and
     use magics for cells that use other languages"""


Test patch:
diff --git a/tests/test_auto_ext.py b/tests/test_auto_ext.py
index 6ecc5140..0813e8bd 100644
--- a/tests/test_auto_ext.py
+++ b/tests/test_auto_ext.py
@@ -18,10 +18,24 @@ def test_auto_in_fmt(nb_file):
     assert nb2.metadata['jupytext']['text_representation']['format_name'] == 'percent'
 
     del nb.metadata['language_info']
+    del nb.metadata['kernelspec']
     with pytest.raises(JupytextFormatError):
         writes(nb, 'auto:percent')
 
 
+@pytest.mark.parametrize('nb_file', list_notebooks('ipynb_all'))
+def test_auto_from_kernelspecs_works(nb_file):
+    nb = read(nb_file)
+    language_info = nb.metadata.pop('language_info')
+    expected_ext = language_info.get('file_extension')
+    if not expected_ext:
+        pytest.skip('No file_extension in language_info')
+    if expected_ext == '.r':
+        expected_ext = '.R'
+    auto_ext = auto_ext_from_metadata(nb.metadata)
+    assert auto_ext == expected_ext
+
+
 @pytest.mark.parametrize('nb_file', list_notebooks('ipynb_R') + list_notebooks('ipynb_py', skip='(World|plotly)'))
 def test_auto_in_formats(nb_file):
     nb = read(nb_file)
@@ -40,6 +54,7 @@ def test_auto_in_formats(nb_file):
     assert nb2.metadata['jupytext']['formats'] == expected_formats
 
     del nb.metadata['language_info']
+    del nb.metadata['kernelspec']
     with pytest.raises(JupytextFormatError):
         writes(nb, 'ipynb')
     with pytest.raises(JupytextFormatError):
diff --git a/tests/test_black.py b/tests/test_black.py
index e365b72c..edacd863 100644
--- a/tests/test_black.py
+++ b/tests/test_black.py
@@ -151,3 +151,39 @@ def test_apply_black_and_sync_on_paired_notebook(tmpdir, nb_file):
     nb_black.metadata = {key: nb_black.metadata[key] for key in nb_black.metadata
                          if key in _DEFAULT_NOTEBOOK_METADATA}
     compare(nb_black, nb_now)
+
+
+@requires_black
+def test_apply_black_on_markdown_notebook(tmpdir):
+    text = """---
+jupyter:
+  kernelspec:
+    display_name: Python 3
+    language: python
+    name: python3
+  language_info:
+    codemirror_mode:
+      name: ipython
+      version: 3
+    file_extension: .py
+    mimetype: text/x-python
+    name: python
+    nbconvert_exporter: python
+    pygments_lexer: ipython3
+    version: 3.7.4
+---
+
+```python
+1    +     \
+2+3\
++4
+```
+"""
+    tmp_md = str(tmpdir.join('test.md'))
+    with open(tmp_md, 'w') as fp:
+        fp.write(text)
+
+    jupytext([tmp_md, '--pipe', 'black'])
+
+    nb = read(tmp_md)
+    compare(nb.cells, [new_code_cell('1 + 2 + 3 + 4')])
diff --git a/tests/test_cli.py b/tests/test_cli.py
index 6ec83d5d..d5692eaa 100644
--- a/tests/test_cli.py
+++ b/tests/test_cli.py
@@ -180,7 +180,7 @@ def test_error_not_same_ext(tmp_ipynb, tmpdir):
 
 
 def test_error_no_action(tmp_ipynb):
-    with pytest.raises(ValueError, match="Please select an action"):
+    with pytest.raises(ValueError, match="Please provide one of"):
         jupytext([tmp_ipynb])
 
 
@@ -982,3 +982,25 @@ cat = 42
 
     with mock.patch('jupytext.magics.is_magic', erroneous_is_magic):
         assert jupytext([tmp_py, '--to', 'ipynb', '--test-strict']) != 0
+
+
+def test_339_require_to(tmpdir):
+    """Test that the `--to` argument is asked for when a `--test` command is provided"""
+    tmp_py = str(tmpdir.join('test.py'))
+
+    with pytest.raises(ValueError, match='--to'):
+        jupytext([tmp_py, '--test-strict'])
+
+
+@requires_black
+def test_detailed_message_missing_language_info(tmpdir):
+    text = """```python
+1 + 1
+```
+"""
+    tmp_md = str(tmpdir.join('test.md'))
+    with open(tmp_md, 'w') as fp:
+        fp.write(text)
+
+    with pytest.raises(ValueError, match='--pipe-fmt'):
+        jupytext([tmp_md, '--pipe', 'black'])