Repository: astanin/python-tabulate

Issue:
Ignore empty strings when deducing column types
An empty cell (not a cell containing None, but containing `""` or `b""`) forces the column type to be `str`, even if the rest of the cells contain eg. numeric data.  A cell containing None is ignored for type deduction; and, is later replaced by the `missingval` value (such as `"?"`).

This makes it difficult to format sparse tables sensibly (columns always forced to `str` type).  One can remove all blank data and replace with None, but:

It makes it impossible to distinguish between "missing" (`None`) and "blank" (`""`) cells, without messing up the typing of the rest of the column.  When fixed, missing cells would be denoted by missingval eg. "?", while empty cells would simply ... remain empty.  

This makes it easy to display sparse tables where a few cells still require input, while many cells are ignored/unimportant (for example), while maintaining proper column formatting for the remaining data.

The idea of a blank value being of a certain type (`str`), instead of an empty instance of the deduced type of the rest of the column's values seems draconian.

This is an easy 3-line fix, and doesn't interfere with any existing unit tests.

Pull forthcoming...

Discussion:

Solution patch:
diff --git a/tabulate/__init__.py b/tabulate/__init__.py
index c349a79..d6a977a 100644
--- a/tabulate/__init__.py
+++ b/tabulate/__init__.py
@@ -933,8 +933,13 @@ def _isbool(string):
 def _type(string, has_invisible=True, numparse=True):
     """The least generic type (type(None), int, float, str, unicode).
 
+    Treats empty string as missing for the purposes of type deduction, so as to not influence
+    the type of an otherwise complete column; does *not* result in missingval replacement!
+
     >>> _type(None) is type(None)
     True
+    >>> _type("") is type(None)
+    True
     >>> _type("foo") is type("")
     True
     >>> _type("1") is type(1)
@@ -949,15 +954,26 @@ def _type(string, has_invisible=True, numparse=True):
     if has_invisible and isinstance(string, (str, bytes)):
         string = _strip_ansi(string)
 
-    if string is None:
+    if string is None or (isinstance(string, (bytes, str)) and not string):
         return type(None)
     elif hasattr(string, "isoformat"):  # datetime.datetime, date, and time
         return str
     elif _isbool(string):
         return bool
-    elif _isint(string) and numparse:
+    elif numparse and (
+        _isint(string) or (
+            isinstance(string, str)
+            and _isnumber_with_thousands_separator(string)
+            and '.' not in string 
+        )
+    ):
         return int
-    elif _isnumber(string) and numparse:
+    elif numparse and (
+        _isnumber(string) or (
+            isinstance(string, str)
+            and _isnumber_with_thousands_separator(string)
+        )
+    ):
         return float
     elif isinstance(string, bytes):
         return bytes
@@ -1251,7 +1267,7 @@ def _column_type(strings, has_invisible=True, numparse=True):
 
 
 def _format(val, valtype, floatfmt, intfmt, missingval="", has_invisible=True):
-    """Format a value according to its type.
+    """Format a value according to its deduced type.  Empty values are deemed valid for any type.
 
     Unicode is supported:
 
@@ -1264,6 +1280,8 @@ def _format(val, valtype, floatfmt, intfmt, missingval="", has_invisible=True):
     """  # noqa
     if val is None:
         return missingval
+    if isinstance(val, (bytes, str)) and not val:
+        return ""
 
     if valtype is str:
         return f"{val}"
@@ -1298,6 +1316,8 @@ def _format(val, valtype, floatfmt, intfmt, missingval="", has_invisible=True):
             formatted_val = format(float(raw_val), floatfmt)
             return val.replace(raw_val, formatted_val)
         else:
+            if isinstance(val,str) and ',' in val:
+                val = val.replace(',', '')  # handle thousands-separators
             return format(float(val), floatfmt)
     else:
         return f"{val}"
@@ -1592,9 +1612,10 @@ def _wrap_text_to_colwidths(list_of_lists, colwidths, numparses=True):
 
             if width is not None:
                 wrapper = _CustomTextWrap(width=width)
-                # Cast based on our internal type handling
-                # Any future custom formatting of types (such as datetimes)
-                # may need to be more explicit than just `str` of the object
+                # Cast based on our internal type handling. Any future custom
+                # formatting of types (such as datetimes) may need to be more
+                # explicit than just `str` of the object. Also doesn't work for
+                # custom floatfmt/intfmt, nor with any missing/blank cells.
                 casted_cell = (
                     str(cell) if _isnumber(cell) else _type(cell, numparse)(cell)
                 )
@@ -2549,7 +2570,6 @@ def _format_table(
         append_row = _append_basic_row
 
     padded_headers = pad_row(headers, pad)
-    padded_rows = [pad_row(row, pad) for row in rows]
 
     if fmt.lineabove and "lineabove" not in hidden:
         _append_line(lines, padded_widths, colaligns, fmt.lineabove)
@@ -2559,18 +2579,18 @@ def _format_table(
         if fmt.linebelowheader and "linebelowheader" not in hidden:
             _append_line(lines, padded_widths, colaligns, fmt.linebelowheader)
 
-    if padded_rows and fmt.linebetweenrows and "linebetweenrows" not in hidden:
+    if rows and fmt.linebetweenrows and "linebetweenrows" not in hidden:
         # initial rows with a line below
-        for row, ralign in zip(padded_rows[:-1], rowaligns):
+        for row, ralign in zip(rows[:-1], rowaligns):
             if row != SEPARATING_LINE:
                 append_row(
-                    lines, row, padded_widths, colaligns, fmt.datarow, rowalign=ralign
+                    lines, pad_row(row, pad), padded_widths, colaligns, fmt.datarow, rowalign=ralign
                 )
             _append_line(lines, padded_widths, colaligns, fmt.linebetweenrows)
         # the last row without a line below
         append_row(
             lines,
-            padded_rows[-1],
+            pad_row(rows[-1], pad),
             padded_widths,
             colaligns,
             fmt.datarow,
@@ -2584,13 +2604,13 @@ def _format_table(
             or fmt.lineabove
             or Line("", "", "", "")
         )
-        for row in padded_rows:
+        for row in rows:
             # test to see if either the 1st column or the 2nd column (account for showindex) has
             # the SEPARATING_LINE flag
             if _is_separating_line(row):
                 _append_line(lines, padded_widths, colaligns, separating_line)
             else:
-                append_row(lines, row, padded_widths, colaligns, fmt.datarow)
+                append_row(lines, pad_row(row, pad), padded_widths, colaligns, fmt.datarow)
 
     if fmt.linebelow and "linebelow" not in hidden:
         _append_line(lines, padded_widths, colaligns, fmt.linebelow)


Test patch:
diff --git a/test/test_output.py b/test/test_output.py
index 68b5e55..bba7622 100644
--- a/test/test_output.py
+++ b/test/test_output.py
@@ -261,6 +261,21 @@ def test_simple_with_sep_line():
     assert_equal(expected, result)
 
 
+def test_orgtbl_with_sep_line():
+    "Output: orgtbl with headers and separating line"
+    expected = "\n".join(
+        [
+            "| strings   |   numbers |",
+            "|-----------+-----------|",
+            "| spam      |   41.9999 |",
+            "|-----------+-----------|",
+            "| eggs      |  451      |",
+        ]
+    )
+    result = tabulate(_test_table_with_sep_line, _test_table_headers, tablefmt="orgtbl")
+    assert_equal(expected, result)
+
+
 def test_readme_example_with_sep():
     table = [["Earth", 6371], ["Mars", 3390], SEPARATING_LINE, ["Moon", 1737]]
     expected = "\n".join(
@@ -315,6 +330,28 @@ def test_simple_multiline_2_with_sep_line():
     assert_equal(expected, result)
 
 
+def test_orgtbl_multiline_2_with_sep_line():
+    "Output: simple with multiline cells"
+    expected = "\n".join(
+        [
+            "|  key  |   value   |",
+            "|-------+-----------|",
+            "|  foo  |    bar    |",
+            "|-------+-----------|",
+            "| spam  | multiline |",
+            "|       |   world   |",
+        ]
+    )
+    table = [
+        ["key", "value"],
+        ["foo", "bar"],
+        SEPARATING_LINE,
+        ["spam", "multiline\nworld"],
+    ]
+    result = tabulate(table, headers="firstrow", stralign="center", tablefmt="orgtbl")
+    assert_equal(expected, result)
+
+
 def test_simple_headerless():
     "Output: simple without headers"
     expected = "\n".join(
@@ -2824,6 +2861,13 @@ def test_floatfmt():
     assert_equal(expected, result)
 
 
+def test_floatfmt_thousands():
+    "Output: floating point format"
+    result = tabulate([["1.23456789"], [1.0], ["1,234.56"]], floatfmt=".3f", tablefmt="plain")
+    expected = "   1.235\n   1.000\n1234.560"
+    assert_equal(expected, result)
+
+
 def test_floatfmt_multi():
     "Output: floating point format different for each column"
     result = tabulate(
@@ -2964,6 +3008,32 @@ def test_missingval_multi():
     assert_equal(expected, result)
 
 
+def test_column_emptymissing_deduction():
+    "Missing or empty/blank values shouldn't change type deduction of rest of column"
+    from fractions import Fraction
+
+    test_table = [
+        [None, "1.23423515351", Fraction(1, 3)],
+        [Fraction(56789, 1000000), 12345.1, b"abc"],
+        ["", b"", None],
+        [Fraction(10000, 3), None, ""],
+    ]
+    result = tabulate(
+        test_table,
+        floatfmt=",.5g",
+        missingval="?",
+    )
+    print(f"\n{result}")
+    expected = """\
+------------  -----------  ---
+    ?              1.2342  1/3
+    0.056789  12,345       abc
+                           ?
+3,333.3            ?
+------------  -----------  ---"""
+    assert_equal(expected, result)
+
+
 def test_column_alignment():
     "Output: custom alignment for text and numbers"
     expected = "\n".join(["-----  ---", "Alice   1", "  Bob  333", "-----  ---"])