Repository: dask/dask

Issue:
DataFrame tabular repr
Like normal `pd.DataFrame`, it's nice to check `DataFrame` overview via tabular format. 

My idea is adding a table with `divisions`, `columns` and `dtype` to existing string repr as below (without calling `.compute()`). It should also work on console accordingly.

![2016-10-02 10 50 43](https://cloud.githubusercontent.com/assets/1696302/19018044/275b5f60-888e-11e6-861a-97e23f0f6e04.png)

Any ideas?



Discussion:

Resolving patch:
diff --git a/dask/dataframe/core.py b/dask/dataframe/core.py
--- a/dask/dataframe/core.py
+++ b/dask/dataframe/core.py
@@ -23,9 +23,9 @@
 from ..array.core import partial_by_order
 from .. import threaded
 from ..compatibility import apply, operator_div, bind_method, PY3
-from ..utils import (repr_long_list, random_state_data,
+from ..utils import (random_state_data,
                      pseudorandom, derived_from, funcname, memory_repr,
-                     put_lines, M)
+                     put_lines, M, key_split)
 from ..base import Base, compute, tokenize, normalize_token
 from ..async import get_sync
 from . import methods
@@ -277,16 +277,6 @@ def copy(self):
     def _keys(self):
         return [(self._name, i) for i in range(self.npartitions)]
 
-    def __repr__(self):
-        name = self._name if len(self._name) < 10 else self._name[:7] + '...'
-        if self.known_divisions:
-            div_text = ', divisions=%s' % repr_long_list(self.divisions)
-        else:
-            div_text = ''
-
-        return ("dd.%s<%s, npartitions=%s%s>" %
-                (self.__class__.__name__, name, self.npartitions, div_text))
-
     def __array__(self, dtype=None, **kwargs):
         self._computed = self.compute()
         x = np.array(self._computed)
@@ -299,6 +289,29 @@ def __array_wrap__(self, array, context=None):
     def _elemwise(self):
         return elemwise
 
+    @property
+    def _repr_data(self):
+        raise NotImplementedError
+
+    @property
+    def _repr_divisions(self):
+        name = "npartitions={0}".format(self.npartitions)
+        if self.known_divisions:
+            divisions = pd.Index(self.divisions, name=name)
+        else:
+            # avoid to be converted to NaN
+            divisions = pd.Index(['None'] * (self.npartitions + 1),
+                                 name=name)
+        return divisions
+
+    def __repr__(self):
+        data = self._repr_data.to_string(max_rows=5, show_dimensions=False)
+        return """Dask {klass} Structure:
+{data}
+Dask Name: {name}, {task} tasks""".format(klass=self.__class__.__name__,
+                                          data=data, name=key_split(self._name),
+                                          task=len(self.dask))
+
     @property
     def index(self):
         """Return dask Index instance"""
@@ -1596,6 +1609,28 @@ def nbytes(self):
         return self.reduction(methods.nbytes, np.sum, token='nbytes',
                               meta=int, split_every=False)
 
+    @cache_readonly
+    def _repr_data(self):
+        values = [str(self.dtype)] + ['...'] * self.npartitions
+        return pd.Series(values, index=self._repr_divisions, name=self.name)
+
+    def __repr__(self):
+        """ have to overwrite footer """
+        if self.name is not None:
+            footer = "Name: {name}, dtype: {dtype}".format(name=self.name,
+                                                           dtype=self.dtype)
+        else:
+            footer = "dtype: {dtype}".format(dtype=self.dtype)
+
+        return """Dask {klass} Structure:
+{data}
+{footer}
+Dask Name: {name}, {task} tasks""".format(klass=self.__class__.__name__,
+                                          data=self.to_string(),
+                                          footer=footer,
+                                          name=key_split(self._name),
+                                          task=len(self.dask))
+
     @derived_from(pd.Series)
     def round(self, decimals=0):
         return elemwise(M.round, self, decimals)
@@ -1761,6 +1796,11 @@ def to_frame(self, name=None):
         return self.map_partitions(M.to_frame, name,
                                    meta=self._meta.to_frame(name))
 
+    @derived_from(pd.Series)
+    def to_string(self, max_rows=5):
+        # option_context doesn't affect
+        return self._repr_data.to_string(max_rows=max_rows)
+
     @classmethod
     def _bind_operator_method(cls, name, op):
         """ bind operator method like DataFrame.add to this class """
@@ -2338,6 +2378,12 @@ def to_bag(self, index=False):
         from .io import to_bag
         return to_bag(self, index)
 
+    @derived_from(pd.DataFrame)
+    def to_string(self, max_rows=5):
+        # option_context doesn't affect
+        return self._repr_data.to_string(max_rows=max_rows,
+                                         show_dimensions=False)
+
     def _get_numeric_data(self, how='any', subset=None):
         # calculate columns to avoid unnecessary calculation
         numerics = self._meta._get_numeric_data()
@@ -2653,6 +2699,33 @@ def to_records(self, index=False):
         from .io import to_records
         return to_records(self)
 
+    @derived_from(pd.DataFrame)
+    def to_html(self, max_rows=5):
+        # pd.Series doesn't have html repr
+        data = self._repr_data.to_html(max_rows=max_rows,
+                                       show_dimensions=False)
+        return self._HTML_FMT.format(data=data, name=key_split(self._name),
+                                     task=len(self.dask))
+
+    @cache_readonly
+    def _repr_data(self):
+        dtypes = self.dtypes
+        values = {key: [value] + ['...'] * self.npartitions for key, value
+                  in zip(dtypes.index, dtypes.values)}
+        return pd.DataFrame(values,
+                            index=self._repr_divisions,
+                            columns=self.columns)
+
+    _HTML_FMT = """<div><strong>Dask DataFrame Structure:</strong></div>
+{data}
+<div>Dask Name: {name}, {task} tasks</div>"""
+
+    def _repr_html_(self):
+        data = self._repr_data.to_html(max_rows=5,
+                                       show_dimensions=False, notebook=True)
+        return self._HTML_FMT.format(data=data, name=key_split(self._name),
+                                     task=len(self.dask))
+
 
 # bind operators
 for op in [operator.abs, operator.add, operator.and_, operator_div,
@@ -3681,5 +3754,9 @@ def to_delayed(df):
     return [Delayed(k, [df.dask]) for k in df._keys()]
 
 
+def _escape_html_tag(s):
+    return s.replace('<', r'&lt;', 1).replace('>', r'&gt;', 1)
+
+
 if PY3:
     _Frame.to_delayed.__doc__ = to_delayed.__doc__


Test patch:
diff --git a/dask/dataframe/tests/test_dataframe.py b/dask/dataframe/tests/test_dataframe.py
--- a/dask/dataframe/tests/test_dataframe.py
+++ b/dask/dataframe/tests/test_dataframe.py
@@ -108,18 +108,6 @@ def test_Series():
     assert isinstance(d.a, dd.Series)
     assert isinstance(d.a + 1, dd.Series)
     assert_eq((d + 1), full + 1)
-    assert repr(d.a).startswith('dd.Series')
-
-
-def test_repr():
-    df = pd.DataFrame({'x': list(range(100))})
-    ddf = dd.from_pandas(df, 3)
-
-    for x in [ddf, ddf.index, ddf.x]:
-        assert type(x).__name__ in repr(x)
-        assert x._name[:5] in repr(x)
-        assert str(x.npartitions) in repr(x)
-        assert len(repr(x)) < 80
 
 
 def test_Index():
@@ -129,7 +117,6 @@ def test_Index():
                                                   periods=10))]:
         ddf = dd.from_pandas(case, 3)
         assert_eq(ddf.index, case.index)
-        assert repr(ddf.index).startswith('dd.Index')
         pytest.raises(AttributeError, lambda: ddf.index.index)
 
 
diff --git a/dask/dataframe/tests/test_format.py b/dask/dataframe/tests/test_format.py
new file mode 100644
--- /dev/null
+++ b/dask/dataframe/tests/test_format.py
@@ -0,0 +1,422 @@
+# coding: utf-8
+import pandas as pd
+
+import dask.dataframe as dd
+
+
+def test_repr():
+    df = pd.DataFrame({'x': list(range(100))})
+    ddf = dd.from_pandas(df, 3)
+
+    for x in [ddf, ddf.index, ddf.x]:
+        assert type(x).__name__ in repr(x)
+        assert str(x.npartitions) in repr(x)
+
+
+def test_dataframe_format():
+    df = pd.DataFrame({'A': [1, 2, 3, 4, 5, 6, 7, 8],
+                       'B': list('ABCDEFGH'),
+                       'C': pd.Categorical(list('AAABBBCC'))})
+    ddf = dd.from_pandas(df, 3)
+    exp = ("Dask DataFrame Structure:\n"
+           "                   A       B         C\n"
+           "npartitions=3                         \n"
+           "0              int64  object  category\n"
+           "3                ...     ...       ...\n"
+           "6                ...     ...       ...\n"
+           "7                ...     ...       ...\n"
+           "Dask Name: from_pandas, 3 tasks")
+    assert repr(ddf) == exp
+    assert str(ddf) == exp
+
+    exp = ("                   A       B         C\n"
+           "npartitions=3                         \n"
+           "0              int64  object  category\n"
+           "3                ...     ...       ...\n"
+           "6                ...     ...       ...\n"
+           "7                ...     ...       ...")
+    assert ddf.to_string() == exp
+
+    exp_table = """<table border="1" class="dataframe">
+  <thead>
+    <tr style="text-align: right;">
+      <th></th>
+      <th>A</th>
+      <th>B</th>
+      <th>C</th>
+    </tr>
+    <tr>
+      <th>npartitions=3</th>
+      <th></th>
+      <th></th>
+      <th></th>
+    </tr>
+  </thead>
+  <tbody>
+    <tr>
+      <th>0</th>
+      <td>int64</td>
+      <td>object</td>
+      <td>category</td>
+    </tr>
+    <tr>
+      <th>3</th>
+      <td>...</td>
+      <td>...</td>
+      <td>...</td>
+    </tr>
+    <tr>
+      <th>6</th>
+      <td>...</td>
+      <td>...</td>
+      <td>...</td>
+    </tr>
+    <tr>
+      <th>7</th>
+      <td>...</td>
+      <td>...</td>
+      <td>...</td>
+    </tr>
+  </tbody>
+</table>"""
+
+    exp = """<div><strong>Dask DataFrame Structure:</strong></div>
+{exp_table}
+<div>Dask Name: from_pandas, 3 tasks</div>""".format(exp_table=exp_table)
+    assert ddf.to_html() == exp
+
+    # table is boxed with div
+    exp = """<div><strong>Dask DataFrame Structure:</strong></div>
+<div>
+{exp_table}
+</div>
+<div>Dask Name: from_pandas, 3 tasks</div>""".format(exp_table=exp_table)
+    assert ddf._repr_html_() == exp
+
+
+def test_dataframe_format_with_index():
+    df = pd.DataFrame({'A': [1, 2, 3, 4, 5, 6, 7, 8],
+                       'B': list('ABCDEFGH'),
+                       'C': pd.Categorical(list('AAABBBCC'))},
+                      index=list('ABCDEFGH'))
+    ddf = dd.from_pandas(df, 3)
+    exp = ("Dask DataFrame Structure:\n"
+           "                   A       B         C\n"
+           "npartitions=3                         \n"
+           "A              int64  object  category\n"
+           "D                ...     ...       ...\n"
+           "G                ...     ...       ...\n"
+           "H                ...     ...       ...\n"
+           "Dask Name: from_pandas, 3 tasks")
+    assert repr(ddf) == exp
+    assert str(ddf) == exp
+
+    exp_table = """<table border="1" class="dataframe">
+  <thead>
+    <tr style="text-align: right;">
+      <th></th>
+      <th>A</th>
+      <th>B</th>
+      <th>C</th>
+    </tr>
+    <tr>
+      <th>npartitions=3</th>
+      <th></th>
+      <th></th>
+      <th></th>
+    </tr>
+  </thead>
+  <tbody>
+    <tr>
+      <th>A</th>
+      <td>int64</td>
+      <td>object</td>
+      <td>category</td>
+    </tr>
+    <tr>
+      <th>D</th>
+      <td>...</td>
+      <td>...</td>
+      <td>...</td>
+    </tr>
+    <tr>
+      <th>G</th>
+      <td>...</td>
+      <td>...</td>
+      <td>...</td>
+    </tr>
+    <tr>
+      <th>H</th>
+      <td>...</td>
+      <td>...</td>
+      <td>...</td>
+    </tr>
+  </tbody>
+</table>"""
+
+    exp = """<div><strong>Dask DataFrame Structure:</strong></div>
+{exp_table}
+<div>Dask Name: from_pandas, 3 tasks</div>""".format(exp_table=exp_table)
+    assert ddf.to_html() == exp
+
+    # table is boxed with div
+    exp = """<div><strong>Dask DataFrame Structure:</strong></div>
+<div>
+{exp_table}
+</div>
+<div>Dask Name: from_pandas, 3 tasks</div>""".format(exp_table=exp_table)
+    assert ddf._repr_html_() == exp
+
+
+def test_dataframe_format_unknown_divisions():
+    df = pd.DataFrame({'A': [1, 2, 3, 4, 5, 6, 7, 8],
+                       'B': list('ABCDEFGH'),
+                       'C': pd.Categorical(list('AAABBBCC'))})
+    ddf = dd.from_pandas(df, 3)
+    ddf = ddf.clear_divisions()
+    assert not ddf.known_divisions
+
+    exp = ("Dask DataFrame Structure:\n"
+           "                   A       B         C\n"
+           "npartitions=3                         \n"
+           "None           int64  object  category\n"
+           "None             ...     ...       ...\n"
+           "None             ...     ...       ...\n"
+           "None             ...     ...       ...\n"
+           "Dask Name: from_pandas, 3 tasks")
+    assert repr(ddf) == exp
+    assert str(ddf) == exp
+
+    exp = ("                   A       B         C\n"
+           "npartitions=3                         \n"
+           "None           int64  object  category\n"
+           "None             ...     ...       ...\n"
+           "None             ...     ...       ...\n"
+           "None             ...     ...       ...")
+    assert ddf.to_string() == exp
+
+    exp_table = """<table border="1" class="dataframe">
+  <thead>
+    <tr style="text-align: right;">
+      <th></th>
+      <th>A</th>
+      <th>B</th>
+      <th>C</th>
+    </tr>
+    <tr>
+      <th>npartitions=3</th>
+      <th></th>
+      <th></th>
+      <th></th>
+    </tr>
+  </thead>
+  <tbody>
+    <tr>
+      <th>None</th>
+      <td>int64</td>
+      <td>object</td>
+      <td>category</td>
+    </tr>
+    <tr>
+      <th>None</th>
+      <td>...</td>
+      <td>...</td>
+      <td>...</td>
+    </tr>
+    <tr>
+      <th>None</th>
+      <td>...</td>
+      <td>...</td>
+      <td>...</td>
+    </tr>
+    <tr>
+      <th>None</th>
+      <td>...</td>
+      <td>...</td>
+      <td>...</td>
+    </tr>
+  </tbody>
+</table>"""
+
+    exp = """<div><strong>Dask DataFrame Structure:</strong></div>
+{exp_table}
+<div>Dask Name: from_pandas, 3 tasks</div>""".format(exp_table=exp_table)
+    assert ddf.to_html() == exp
+
+    # table is boxed with div
+    exp = """<div><strong>Dask DataFrame Structure:</strong></div>
+<div>
+{exp_table}
+</div>
+<div>Dask Name: from_pandas, 3 tasks</div>""".format(exp_table=exp_table)
+    assert ddf._repr_html_() == exp
+
+
+def test_dataframe_format_long():
+    df = pd.DataFrame({'A': [1, 2, 3, 4, 5, 6, 7, 8] * 10,
+                       'B': list('ABCDEFGH') * 10,
+                       'C': pd.Categorical(list('AAABBBCC') * 10)})
+    ddf = dd.from_pandas(df, 10)
+    exp = ('Dask DataFrame Structure:\n'
+           '                    A       B         C\n'
+           'npartitions=10                         \n'
+           '0               int64  object  category\n'
+           '8                 ...     ...       ...\n'
+           '...               ...     ...       ...\n'
+           '72                ...     ...       ...\n'
+           '79                ...     ...       ...\n'
+           'Dask Name: from_pandas, 10 tasks')
+    assert repr(ddf) == exp
+    assert str(ddf) == exp
+
+    exp = ("                    A       B         C\n"
+           "npartitions=10                         \n"
+           "0               int64  object  category\n"
+           "8                 ...     ...       ...\n"
+           "...               ...     ...       ...\n"
+           "72                ...     ...       ...\n"
+           "79                ...     ...       ...")
+    assert ddf.to_string() == exp
+
+    exp_table = """<table border="1" class="dataframe">
+  <thead>
+    <tr style="text-align: right;">
+      <th></th>
+      <th>A</th>
+      <th>B</th>
+      <th>C</th>
+    </tr>
+    <tr>
+      <th>npartitions=10</th>
+      <th></th>
+      <th></th>
+      <th></th>
+    </tr>
+  </thead>
+  <tbody>
+    <tr>
+      <th>0</th>
+      <td>int64</td>
+      <td>object</td>
+      <td>category</td>
+    </tr>
+    <tr>
+      <th>8</th>
+      <td>...</td>
+      <td>...</td>
+      <td>...</td>
+    </tr>
+    <tr>
+      <th>...</th>
+      <td>...</td>
+      <td>...</td>
+      <td>...</td>
+    </tr>
+    <tr>
+      <th>72</th>
+      <td>...</td>
+      <td>...</td>
+      <td>...</td>
+    </tr>
+    <tr>
+      <th>79</th>
+      <td>...</td>
+      <td>...</td>
+      <td>...</td>
+    </tr>
+  </tbody>
+</table>"""
+
+    exp = """<div><strong>Dask DataFrame Structure:</strong></div>
+{exp_table}
+<div>Dask Name: from_pandas, 10 tasks</div>""".format(exp_table=exp_table)
+    assert ddf.to_html() == exp
+
+    # table is boxed with div
+    exp = u"""<div><strong>Dask DataFrame Structure:</strong></div>
+<div>
+{exp_table}
+</div>
+<div>Dask Name: from_pandas, 10 tasks</div>""".format(exp_table=exp_table)
+    assert ddf._repr_html_() == exp
+
+
+def test_series_format():
+    s = pd.Series([1, 2, 3, 4, 5, 6, 7, 8],
+                  index=list('ABCDEFGH'))
+    ds = dd.from_pandas(s, 3)
+    exp = """Dask Series Structure:
+npartitions=3
+A    int64
+D      ...
+G      ...
+H      ...
+dtype: int64
+Dask Name: from_pandas, 3 tasks"""
+    assert repr(ds) == exp
+    assert str(ds) == exp
+
+    exp = """npartitions=3
+A    int64
+D      ...
+G      ...
+H      ..."""
+    assert ds.to_string() == exp
+
+    s = pd.Series([1, 2, 3, 4, 5, 6, 7, 8],
+                  index=list('ABCDEFGH'), name='XXX')
+    ds = dd.from_pandas(s, 3)
+    exp = """Dask Series Structure:
+npartitions=3
+A    int64
+D      ...
+G      ...
+H      ...
+Name: XXX, dtype: int64
+Dask Name: from_pandas, 3 tasks"""
+    assert repr(ds) == exp
+    assert str(ds) == exp
+
+
+def test_series_format_long():
+    s = pd.Series([1, 2, 3, 4, 5, 6, 7, 8, 9, 10] * 10,
+                  index=list('ABCDEFGHIJ') * 10)
+    ds = dd.from_pandas(s, 10)
+    exp = ("Dask Series Structure:\nnpartitions=10\nA    int64\nB      ...\n"
+           "     ...  \nJ      ...\nJ      ...\ndtype: int64\n"
+           "Dask Name: from_pandas, 10 tasks")
+    assert repr(ds) == exp
+    assert str(ds) == exp
+
+    exp = "npartitions=10\nA    int64\nB      ...\n     ...  \nJ      ...\nJ      ..."
+    assert ds.to_string() == exp
+
+
+def test_index_format():
+    s = pd.Series([1, 2, 3, 4, 5, 6, 7, 8],
+                  index=list('ABCDEFGH'))
+    ds = dd.from_pandas(s, 3)
+    exp = """Dask Index Structure:
+npartitions=3
+A    object
+D       ...
+G       ...
+H       ...
+dtype: object
+Dask Name: from_pandas, 6 tasks"""
+    assert repr(ds.index) == exp
+    assert str(ds.index) == exp
+
+    s = pd.Series([1, 2, 3, 4, 5, 6, 7, 8],
+                  index=pd.CategoricalIndex([1, 2, 3, 4, 5, 6, 7, 8], name='YYY'))
+    ds = dd.from_pandas(s, 3)
+    exp = """Dask Index Structure:
+npartitions=3
+1    category
+4         ...
+7         ...
+8         ...
+Name: YYY, dtype: category
+Dask Name: from_pandas, 6 tasks"""
+    assert repr(ds.index) == exp
+    assert str(ds.index) == exp