Repository: pyout/pyout

Issue:
breeding emtpy lines while listing output which leaves the screen
just do smth like
```
datalad install ///openfmri
datalad install openfmri/ds000{011,11*}
datalad ls -rLa openfmri 
```

- there is a bunch of empty lines at the end of the listing before "EXITED" reported by datalad exiting the loop.  Feels like a counter of some kind is incorrect while accounting for stuff which left the screen already
- if you scroll terminal up - you will see some full terminal height empty spaces with one line of the listing at the bottom.  Ideally I expect that output either be completely redrawn (thus repeating previously output lines) or just continuing listing with possibly adjusted formatting (e.g. the widths of the fields).
- when I have used it on my local `openfmri` superdataset installation then there were also smaller periods of emptiness if you scroll terminal up

Discussion:

Patch:
diff --git a/CHANGELOG.md b/CHANGELOG.md
index f7307f7..bd4a517 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -8,6 +8,9 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
 
 ### Fixed
 
+- The output was corrupted when a callback function tried to update a
+  line that was no longer visible on the screen.
+
 - When a table did not include a summary, "incremental" and "final"
   mode added "None" as the last row.
 
diff --git a/pyout/interface.py b/pyout/interface.py
index d40097c..71d8687 100644
--- a/pyout/interface.py
+++ b/pyout/interface.py
@@ -30,6 +30,9 @@ class Stream(object):
     def width(self):
         """Maximum line width.
         """
+    @abc.abstractproperty
+    def height(self):
+        """Maximum number of rows that are visible."""
 
     @abc.abstractmethod
     def write(self, text):
@@ -190,12 +193,34 @@ class Writer(object):
             # possible for the summary lines to shrink.
             lgr.debug("Clearing summary")
             self._stream.clear_last_lines(last_summary_len)
+        else:
+            last_summary_len = 0
+
         content, status, summary = self._content.update(row, style)
+
+        single_row_updated = False
         if isinstance(status, int):
-            lgr.debug("Overwriting line %d with %r", status, row)
-            self._stream.overwrite_line(self._last_content_len - status,
-                                        content)
-        else:
+            height = self._stream.height
+            if height is None:  # non-tty
+                n_visible = self._last_content_len
+            else:
+                n_visible = min(
+                    height - last_summary_len - 1,  # -1 for current line.
+                    self._last_content_len)
+
+            n_back = self._last_content_len - status
+            if n_back > n_visible:
+                lgr.debug("Cannot move back %d rows for update; "
+                          "only %d visible rows",
+                          n_back, n_visible)
+                status = "repaint"
+                content = six.text_type(self._content)
+            else:
+                lgr.debug("Overwriting line %d with %r", status, row)
+                self._stream.overwrite_line(n_back, content)
+                single_row_updated = True
+
+        if not single_row_updated:
             if status == "repaint":
                 lgr.debug("Repainting the whole thing.  Blame row %r", row)
                 self._stream.move_to(self._last_content_len)
diff --git a/pyout/tabular.py b/pyout/tabular.py
index f5068f2..52cf315 100644
--- a/pyout/tabular.py
+++ b/pyout/tabular.py
@@ -24,6 +24,7 @@ class TerminalStream(interface.Stream):
 
     def __init__(self):
         self.term = Terminal()
+        self._height = self.term.height
 
     @property
     def width(self):
@@ -31,6 +32,12 @@ class TerminalStream(interface.Stream):
         """
         return self.term.width
 
+    @property
+    def height(self):
+        """Terminal height.
+        """
+        return self._height
+
     def write(self, text):
         """Write `text` to terminal.
         """


Test patch:
diff --git a/pyout/tests/test_tabular.py b/pyout/tests/test_tabular.py
index 8a4d172..8664f0f 100644
--- a/pyout/tests/test_tabular.py
+++ b/pyout/tests/test_tabular.py
@@ -37,14 +37,36 @@ class Terminal(blessings.Terminal):
     def width(self):
         return 100
 
+    @property
+    def height(self):
+        return 20
+
+
+class TerminalNonInteractive(Terminal):
+
+    @property
+    def width(self):
+        return None
+
+    @property
+    def height(self):
+        return None
+
 
 class Tabular(TheRealTabular):
     """Test-specific subclass of pyout.Tabular.
     """
 
     def __init__(self, *args, **kwargs):
-        with patch("pyout.interface.sys.stdout.isatty", return_value=True):
-            with patch("pyout.tabular.Terminal", Terminal):
+        interactive = kwargs.pop("interactive", True)
+        if interactive:
+            term = Terminal
+        else:
+            term = TerminalNonInteractive
+
+        with patch("pyout.interface.sys.stdout.isatty",
+                   return_value=interactive):
+            with patch("pyout.tabular.Terminal", term):
                 super(Tabular, self).__init__(*args, **kwargs)
 
     @property
@@ -894,6 +916,53 @@ def test_tabular_write_callable_values_multi_return():
     assert_contains_nc(lines, "foo done /tmp/a")
 
 
+@pytest.mark.timeout(10)
+@pytest.mark.parametrize("nrows", [20, 21])
+@pytest.mark.parametrize("interactive", [True, False])
+def test_tabular_callback_to_hidden_row(nrows, interactive):
+    if sys.version_info < (3,):
+        try:
+            import jsonschema
+        except ImportError:
+            pass
+        else:
+            # Ideally we'd also check if the tests are running under
+            # coverage, but I don't know a way to do that.
+            pytest.xfail(
+                "Hangs for unknown reason in py2/coverage/jsonschema run")
+
+    delay = Delayed("OK")
+    out = Tabular(interactive=interactive,
+                  style={"status": {"aggregate": len}})
+    # Make sure we're in update mode even for interactive=False.
+    out.mode = "update"
+    with out:
+        for i in range(1, nrows + 1):
+            status = delay.run if i == 3 else "s{:02d}".format(i)
+            out(OrderedDict([("name", "foo{:02d}".format(i)),
+                             ("status", status)]))
+        delay.now = True
+
+    lines = out.stdout.splitlines()
+    # The test terminal height is 20.  The summary line takes up one
+    # line and the current line takes up another, so we have 18
+    # available rows. Row 3 goes off the screen when we have 21 rows.
+
+    if nrows > 20 and interactive:
+        # No leading escape codes because it was part of a whole repaint.
+        nexpected_plain = 1
+        nexpected_updated = 0
+    else:
+        nexpected_plain = 0
+        nexpected_updated = 1
+
+    assert len([l for l in lines if l == "foo03 OK "]) == nexpected_plain
+
+    cuu1 = unicode_cap("cuu1")
+    updated = [l for l in lines if l.startswith(cuu1) and "foo03 OK " in l]
+    assert len(updated) == nexpected_updated
+
+
 @pytest.mark.timeout(10)
 @pytest.mark.parametrize("result",
                          [{"status": "done", "path": "/tmp/a"},