Repository: mwouts/jupytext
Pull request #476 (2020-03-30 17:48:44)

Additional test/fix for the string parser
`""""` (a quadruple double quote) at the end of a multiline string should not cause any issue (#460)

codecov[bot]: # [Codecov](https://codecov.io/gh/mwouts/jupytext/pull/476?src=pr&el=h1) Report
> Merging [#476](https://codecov.io/gh/mwouts/jupytext/pull/476?src=pr&el=desc) into [master](https://codecov.io/gh/mwouts/jupytext/commit/c2b3bfb5ff7829495b3c65381abbced5048075ec&el=desc) will **decrease** coverage by `0.01%`.
> The diff coverage is `98.26%`.

[![Impacted file tree graph](https://codecov.io/gh/mwouts/jupytext/pull/476/graphs/tree.svg?width=650&height=150&src=pr&token=a1pNjsiILi)](https://codecov.io/gh/mwouts/jupytext/pull/476?src=pr&el=tree)

```diff
@@            Coverage Diff             @@
##           master     #476      +/-   ##
==========================================
- Coverage   98.86%   98.84%   -0.02%     
==========================================
  Files          80       80              
  Lines        7899     7993      +94     
==========================================
+ Hits         7809     7901      +92     
- Misses         90       92       +2     
```


| [Impacted Files](https://codecov.io/gh/mwouts/jupytext/pull/476?src=pr&el=tree) | Coverage Δ | |
|---|---|---|
| [tests/test\_combine.py](https://codecov.io/gh/mwouts/jupytext/pull/476/diff?src=pr&el=tree#diff-dGVzdHMvdGVzdF9jb21iaW5lLnB5) | `97.26% <95.34%> (-2.74%)` | :arrow_down: |
| [jupytext/combine.py](https://codecov.io/gh/mwouts/jupytext/pull/476/diff?src=pr&el=tree#diff-anVweXRleHQvY29tYmluZS5weQ==) | `100.00% <100.00%> (ø)` | |
| [jupytext/stringparser.py](https://codecov.io/gh/mwouts/jupytext/pull/476/diff?src=pr&el=tree#diff-anVweXRleHQvc3RyaW5ncGFyc2VyLnB5) | `95.65% <100.00%> (+0.19%)` | :arrow_up: |
| [tests/test\_stringparser.py](https://codecov.io/gh/mwouts/jupytext/pull/476/diff?src=pr&el=tree#diff-dGVzdHMvdGVzdF9zdHJpbmdwYXJzZXIucHk=) | `100.00% <100.00%> (ø)` | |

------

[Continue to review full report at Codecov](https://codecov.io/gh/mwouts/jupytext/pull/476?src=pr&el=continue).
> **Legend** - [Click here to learn more](https://docs.codecov.io/docs/codecov-delta)
> `Δ = absolute <relative> (impact)`, `ø = not affected`, `? = missing data`
> Powered by [Codecov](https://codecov.io/gh/mwouts/jupytext/pull/476?src=pr&el=footer). Last update [c2b3bfb...e5c58c6](https://codecov.io/gh/mwouts/jupytext/pull/476?src=pr&el=lastupdated). Read the [comment docs](https://docs.codecov.io/docs/pull-request-comments).


Patch:
diff --git a/CHANGELOG.md b/CHANGELOG.md
index f22230eb..a06b0ad4 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -3,6 +3,7 @@
 
 **Changed**
 - Use `os.path.samefile` when searching for the kernel that corresponds to the current environment (`--set-kernel -`)
+- The outputs from the `.ipynb` file are matched with the input cells from the text file with less strict rules. In this version, a search and replace on the text file will not remove the outputs any more (#464).
 
 **Fixed**
 - Fixed the CLI example for not commenting out magic commands: `--opt comment_magics=false`. In addition, most of the `jupytext` commands in `using-cli.md` are now tested! (#465)
diff --git a/jupytext/combine.py b/jupytext/combine.py
index c30751ae..c77d0c9b 100644
--- a/jupytext/combine.py
+++ b/jupytext/combine.py
@@ -19,18 +19,15 @@ def black_invariant(text, chars=None):
     return text
 
 
-def same_content(ref, test):
+def same_content(ref, test, endswith=False):
     """Is the content of two cells the same, up to reformating by black"""
+    if endswith:
+        return black_invariant(ref).endswith(black_invariant(test))
     return black_invariant(ref) == black_invariant(test)
 
 
 def combine_inputs_with_outputs(nb_source, nb_outputs, fmt=None):
-    """Copy outputs and metadata of the second notebook into
-    the first one, for cells that have matching inputs"""
-
-    output_code_cells = [cell for cell in nb_outputs.cells if cell.cell_type == 'code']
-    output_other_cells = [cell for cell in nb_outputs.cells if cell.cell_type != 'code']
-
+    """Copy outputs and metadata of the second notebook into the first one."""
     # nbformat version number taken from the notebook with outputs
     assert nb_outputs.nbformat == nb_source.nbformat, \
         "The notebook with outputs is in format {}.{}, please upgrade it to {}.x".format(
@@ -60,33 +57,89 @@ def combine_inputs_with_outputs(nb_source, nb_outputs, fmt=None):
     else:
         cell_metadata_filter = nb_source.metadata.get('jupytext', {}).get('cell_metadata_filter')
 
-    for cell in nb_source.cells:
+    outputs_map = map_outputs_to_inputs(nb_source.cells, nb_outputs.cells)
+
+    for cell, j in zip(nb_source.cells, outputs_map):
         # Remove outputs to warranty that trust of returned notebook is that of second notebook
         if cell.cell_type == 'code':
             cell.execution_count = None
             cell.outputs = []
 
-            for i, ocell in enumerate(output_code_cells):
-                if same_content(cell.source, ocell.source):
-                    # Fill outputs with that of second notebook
-                    cell.execution_count = ocell.execution_count
-                    cell.outputs = ocell.outputs
-
-                    # Restore the filtered output cell metadata
-                    cell.metadata = restore_filtered_metadata(cell.metadata, ocell.metadata,
-                                                              cell_metadata_filter, _IGNORE_CELL_METADATA)
+            if j is not None:
+                ocell = nb_outputs.cells[j]
+                cell.execution_count = ocell.execution_count
+                cell.outputs = ocell.outputs
 
-                    output_code_cells = output_code_cells[(i + 1):]
-                    break
+                # Restore the filtered output cell metadata
+                cell.metadata = restore_filtered_metadata(cell.metadata, ocell.metadata,
+                                                          cell_metadata_filter, _IGNORE_CELL_METADATA)
         else:
-            for i, ocell in enumerate(output_other_cells):
-                if cell.cell_type == ocell.cell_type and same_content(cell.source, ocell.source):
-                    # The 'spin' format does not allow metadata on non-code cells
-                    cell.metadata = restore_filtered_metadata(cell.metadata, ocell.metadata,
-                                                              '-all' if format_name == 'spin' else cell_metadata_filter,
-                                                              _IGNORE_CELL_METADATA)
-
-                    output_other_cells = output_other_cells[(i + 1):]
-                    break
+            if j is not None:
+                ocell = nb_outputs.cells[j]
+
+                # The 'spin' format does not allow metadata on non-code cells
+                cell.metadata = restore_filtered_metadata(cell.metadata, ocell.metadata,
+                                                          '-all' if format_name == 'spin' else cell_metadata_filter,
+                                                          _IGNORE_CELL_METADATA)
 
     return nb_source
+
+
+def map_outputs_to_inputs(cells_inputs, cells_outputs):
+    """Returns a map i->(j or None) that maps the cells with outputs to the input cells"""
+    n_in = len(cells_inputs)
+    n_out = len(cells_outputs)
+    outputs_map = [None] * n_in
+
+    # First rule: match based on cell type, content, in increasing order
+    current_j = 0
+    for i in range(n_in):
+        cell_input = cells_inputs[i]
+        for j in range(current_j, n_out):
+            cell_output = cells_outputs[j]
+            if cell_input.cell_type == cell_output.cell_type and same_content(cell_input.source, cell_output.source):
+                outputs_map[i] = j
+                current_j = j + 1
+                break
+
+    # Second rule: match unused outputs based on cell type and content
+    # Third rule: is the new cell the final part of a previous cell with outputs?
+    unused_ouputs = set(range(n_out)).difference(outputs_map)
+    for endswith in [False, True]:
+        if not unused_ouputs:
+            return outputs_map
+
+        for i in range(n_in):
+            if outputs_map[i] is not None:
+                continue
+            cell_input = cells_inputs[i]
+            for j in unused_ouputs:
+                cell_output = cells_outputs[j]
+                if cell_input.cell_type == cell_output.cell_type and \
+                        same_content(cell_output.source, cell_input.source, endswith):
+                    outputs_map[i] = j
+                    unused_ouputs.remove(j)
+                    break
+
+    # Fourth rule: match based on increasing index, for non-empty cells
+    if not unused_ouputs:
+        return outputs_map
+
+    prev_j = -1
+    for i in range(n_in):
+        if outputs_map[i] is not None:
+            prev_j = outputs_map[i]
+            continue
+
+        j = prev_j + 1
+        if j not in unused_ouputs:
+            continue
+
+        cell_input = cells_inputs[i]
+        cell_output = cells_outputs[j]
+        if cell_input.cell_type == cell_output.cell_type and cell_input.source:
+            outputs_map[i] = j
+            unused_ouputs.remove(j)
+            prev_j = j
+
+    return outputs_map
diff --git a/jupytext/stringparser.py b/jupytext/stringparser.py
index 1b3c618d..bf38f7ee 100644
--- a/jupytext/stringparser.py
+++ b/jupytext/stringparser.py
@@ -48,16 +48,26 @@ class StringParser:
             if not self.python:
                 continue
 
-            if self.triple == char:
-                if line[i - 2:i + 1] == 3 * char and i >= self.triple_start + 3:
+            if line[i - 2:i + 1] == 3 * char and i >= self.triple_start + 3:
+                # End of a triple quote
+                if self.triple == char:
                     self.triple = None
+                    self.triple_start = i
                     continue
-            if self.triple is not None:
-                continue
-            if line[i - 2:i + 1] == 3 * char:
+
+                # Are we looking for a different triple quote?
+                if self.triple is not None:
+                    continue
+
+                # Triple quote starting
                 self.triple = char
                 self.triple_start = i
                 continue
+
+            # Inside a multiline quote
+            if self.triple is not None:
+                continue
+
             self.single = char
 
         # Line ended


Test patch:
diff --git a/tests/test_combine.py b/tests/test_combine.py
index d6b27a5e..0c801993 100644
--- a/tests/test_combine.py
+++ b/tests/test_combine.py
@@ -1,6 +1,10 @@
+import pytest
+from copy import deepcopy
 from nbformat.v4.nbbase import new_code_cell, new_markdown_cell, new_notebook
 from jupytext.combine import combine_inputs_with_outputs
+from jupytext.compare import compare_notebooks
 import jupytext
+from .utils import list_notebooks
 
 
 def test_combine():
@@ -114,3 +118,86 @@ def test_read_text_and_combine_with_outputs(tmpdir):
     assert nb.cells[2]['outputs']
 
     assert len(nb.cells) == 3
+
+
+@pytest.mark.parametrize('nb_file', list_notebooks('ipynb_all'))
+def test_combine_stable(nb_file):
+    nb_org = jupytext.read(nb_file)
+    nb_source = deepcopy(nb_org)
+    nb_outputs = deepcopy(nb_org)
+
+    for cell in nb_source.cells:
+        try:
+            cell.outputs = []
+        except KeyError:
+            continue
+
+    combine_inputs_with_outputs(nb_source, nb_outputs)
+    compare_notebooks(nb_source, nb_org)
+
+
+def test_combine_reorder():
+    nb_source = new_notebook(
+        cells=[new_markdown_cell('Markdown text'),
+               new_code_cell('1+1'),
+               new_code_cell('2+2'),
+               new_code_cell('3+3'),
+               new_markdown_cell('Markdown text'),
+               new_code_cell('4+4')])
+
+    nb_outputs = new_notebook(
+        cells=[new_markdown_cell('Markdown text'),
+               new_code_cell('2+2'),
+               new_code_cell('4+4'),
+               new_code_cell('1+1'),
+               new_code_cell('3+3'),
+               new_markdown_cell('Markdown text')])
+
+    nb_outputs.cells[1].outputs = ['4']
+    nb_outputs.cells[2].outputs = ['8']
+    nb_outputs.cells[3].outputs = ['2']
+    nb_outputs.cells[4].outputs = ['6']
+
+    combine_inputs_with_outputs(nb_source, nb_outputs)
+
+    assert nb_source.cells[1].outputs == ['2']
+    assert nb_source.cells[2].outputs == ['4']
+    assert nb_source.cells[3].outputs == ['6']
+    assert nb_source.cells[5].outputs == ['8']
+
+
+def test_combine_split():
+    nb_source = new_notebook(
+        cells=[new_code_cell('1+1'),
+               new_code_cell('2+2')])
+
+    nb_outputs = new_notebook(
+        cells=[new_code_cell('1+1\n2+2')])
+
+    nb_outputs.cells[0].outputs = ['4']
+
+    combine_inputs_with_outputs(nb_source, nb_outputs)
+
+    assert nb_source.cells[0].outputs == []
+    assert nb_source.cells[1].outputs == ['4']
+
+
+def test_combine_refactor():
+    nb_source = new_notebook(
+        cells=[new_code_cell('a=1'),
+               new_code_cell('a+1'),
+               new_code_cell('a+2')])
+
+    nb_outputs = new_notebook(
+        cells=[new_code_cell('b=1'),
+               new_code_cell('b+1'),
+               new_code_cell('b+2')])
+
+    nb_outputs.cells[1].outputs = ['2']
+    nb_outputs.cells[2].outputs = ['3']
+
+    combine_inputs_with_outputs(nb_source, nb_outputs)
+
+    assert nb_source.cells[0].outputs == []
+    assert nb_source.cells[1].outputs == ['2']
+    assert nb_source.cells[2].outputs == ['3']
diff --git a/tests/test_stringparser.py b/tests/test_stringparser.py
index 36307dc1..9ed0f13e 100644
--- a/tests/test_stringparser.py
+++ b/tests/test_stringparser.py
@@ -53,3 +53,19 @@ comment that starts with four quotes
         sp.read_line(line)
 
     assert quoted == [1, 2]
+
+
+def test_long_string_ends_with_four_quotes(text="""'''This is a multiline
+comment that ends with four quotes
+''''
+
+1 + 1
+"""):
+    quoted = []
+    sp = StringParser('python')
+    for i, line in enumerate(text.splitlines()):
+        if sp.is_quoted():
+            quoted.append(i)
+        sp.read_line(line)
+
+    assert quoted == [1, 2]