Repository: isi-vista/vistautils

Issue:
Add methods for windowed iteration over sequences
I use these all the time, but for some reason they are not in the standard library (not even itertools)

Comments:

Patch:
diff --git a/docs/index.rst b/docs/index.rst
index 5dfe475..ac42df9 100644
--- a/docs/index.rst
+++ b/docs/index.rst
@@ -16,7 +16,7 @@ Welcome to vistautils's documentation!
    vistautils.checkpoints
    vistautils.collection_utils
    vistautils.io_utils
-   vistautils.iterutils
+   vistautils.iter_utils
    vistautils.key_value
    vistautils.logging_utils
    vistautils.misc_utils
diff --git a/vistautils/iter_utils.py b/vistautils/iter_utils.py
new file mode 100644
index 0000000..24987d5
--- /dev/null
+++ b/vistautils/iter_utils.py
@@ -0,0 +1,76 @@
+from collections import deque
+from itertools import islice
+from typing import Iterable, TypeVar, Tuple
+
+from vistautils.preconditions import check_arg
+
+_T = TypeVar("_T")
+
+
+def windowed(iterable: Iterable[_T], window_size: int, *,
+             partial_windows: bool = False) -> Iterable[Tuple[_T, ...]]:
+    check_arg(window_size >= 1)
+    if partial_windows:
+        return _possibly_incomplete_windows(iterable, window_size)
+    else:
+        return _complete_windows(iterable, window_size)
+
+
+def _complete_windows(iterable: Iterable[_T], window_size: int) -> Iterable[Tuple[_T, ...]]:
+    """
+    Complete sliding windows of the given size over an iterable.
+
+    For a sequence ``(x_0, x_1, ...., x_n)`` and window size ``w``, this will return
+    the sequence ``(x_0, ..., x_{0+w}), (x_1, ..., x_{1+w}), ...``. This will *not* return
+    incomplete windows at the end of an iterable (that is, if ``x_{i+w}`` would be outside
+    the input sequence).
+
+    Implementation borrowed from Jerry Kindall and ShadowRange(?) at
+    https://stackoverflow.com/a/40937024/413345
+    """
+    it = iter(iterable)
+    win = deque(islice(it, window_size), window_size)
+    if len(win) < window_size:
+        return
+    # cache method access for slight speed boost
+    append = win.append
+    yield tuple(win)
+    for e in it:
+        append(e)
+        yield tuple(win)
+
+
+def _possibly_incomplete_windows(iterable: Iterable[_T], window_size: int) \
+        -> Iterable[Tuple[_T, ...]]:
+    """
+    All sliding windows of the given size over an iterable.
+
+    For a sequence ``(x_0, x_1, ...., x_n)`` and window size ``w``, this will return
+    the sequence ``(x_0, ..., x_{0+w}), (x_1, ..., x_{1+w}), ...``. This *will* return
+    incomplete windows at the end of an iterable (that is, if ``x_{i+w}`` would be outside
+    the input sequence); the positions in a window beyond an iterable will be filled with
+    ``fill_value``.
+
+    Adapted from complete windowing implementation borrowed from Jerry Kindall and ShadowRange(?) at
+    https://stackoverflow.com/a/40937024/413345
+    """
+    it = iter(iterable)
+    win = deque(islice(it, window_size), window_size)
+    if not win:
+        return
+    # cache method access for slight speed boost
+    append = win.append
+    yield tuple(win)
+    for e in it:
+        append(e)
+        yield tuple(win)
+    # add incomplete windows at the end
+    popleft = win.popleft
+    for _ in range(window_size - 1):
+        popleft()
+        if win:
+            yield tuple(win)
+        else:
+            # if the window size exceeds the sequence size, we need to stop popping early
+            # or we will have a bunch of empty tuples at the end
+            break
diff --git a/vistautils/iterutils.py b/vistautils/iterutils.py
index 4bab8fd..08cacc2 100644
--- a/vistautils/iterutils.py
+++ b/vistautils/iterutils.py
@@ -1,11 +1,12 @@
-# why does itertools think these are useful enough to put in the documentation but not in
-# the library?
-from itertools import tee
+from typing import TypeVar, Iterable, Tuple
 
+import vistautils.iter_utils
 
-# from itertools docs
-def tile_with_pairs(iterable):
-    "s -> (s0,s1), (s1,s2), (s2, s3), ..."
-    a, b = tee(iterable)
-    next(b, None)
-    return zip(a, b)
+# deprecated, present for backwards compatibility
+_T = TypeVar('_T')
+
+
+# noinspection PyTypeHints
+def tile_with_pairs(iterable: Iterable[_T]) -> Iterable[Tuple[_T, _T]]:
+    # noinspection PyTypeChecker
+    return vistautils.iter_utils.windowed(iterable, 2, partial_windows=False)  # type: ignore
diff --git a/vistautils/newsfragments/17.feature b/vistautils/newsfragments/17.feature
new file mode 100644
index 0000000..4386d0f
--- /dev/null
+++ b/vistautils/newsfragments/17.feature
@@ -0,0 +1,1 @@
+Adds `windowed` for iterating over sliding windows over a sequence
\ No newline at end of file


Test patch:
diff --git a/tests/test_iter_utils.py b/tests/test_iter_utils.py
new file mode 100644
index 0000000..ac970d4
--- /dev/null
+++ b/tests/test_iter_utils.py
@@ -0,0 +1,32 @@
+from unittest import TestCase
+
+from vistautils.iter_utils import windowed
+
+
+class TestIterUtils(TestCase):
+    def test_windowed_bad_window_size(self):
+        # need positive window size
+        with self.assertRaises(ValueError):
+            windowed(range(5), 0)
+
+    def test_single_element_window(self):
+        self.assertEqual([], list(windowed([], 1)))
+        self.assertEqual([(0,), (1,), (2,)], list(windowed(range(3), 1)))
+        self.assertEqual([(0,), (1,), (2,)], list(windowed(range(3), 1,
+                                                           partial_windows=True)))
+
+    def test_two_element_window(self):
+        self.assertEqual([(0, 1), (1, 2)], list(windowed(range(3), 2)))
+        self.assertEqual([(0, 1), (1, 2), (2,)],
+                         list(windowed(range(3), 2, partial_windows=True)))
+
+    def test_window_size_equals_sequence_size(self):
+        self.assertEqual([(0, 1, 2)], list(windowed(range(3), 3)))
+        self.assertEqual([(0, 1, 2), (1, 2), (2, )],
+                         list(windowed(range(3), 3, partial_windows=True)))
+
+    def test_window_size_exceeds_sequence_size(self):
+        # window size exceeds sequence length
+        self.assertEqual([], list(windowed(range(3), 4)))
+        self.assertEqual([(0, 1, 2), (1, 2), (2, )],
+                         list(windowed(range(3), 4, partial_windows=True)))