Repository: jazzband/tablib (2023-09-06 ~ 2024-07-10)

problem_statement: Databook unable to load xls
I'm trying to load an xls file however it fails with the following error
```
with open("xls_file", "rb") as fh: 
    imported_data = Databook().load(fh, "xls")
```
```
Traceback (most recent call last):
  File ".\test.py", line 5, in <module>
    imported_data = Databook().load(fh, "xls")
  File "lib\site-packages\tablib\core.py", line 872, in load
    fmt.import_book(self, stream, **kwargs)
  File "lib\site-packages\tablib\formats\_xls.py", line 103, in import_book
    xls_book = xlrd.open_workbook(file_contents=in_stream)
  File "lib\site-packages\xlrd\__init__.py", line 166, in open_workbook
    file_format = inspect_format(filename, file_contents)
  File "lib\site-packages\xlrd\__init__.py", line 57, in inspect_format
    peek = content[:PEEK_SIZE]
TypeError: '_io.BufferedReader' object is not subscriptable
```
date: 2023-09-06
patch: diff --git a/src/tablib/formats/_xls.py b/src/tablib/formats/_xls.py
index 6c8dce7..2c6c527 100644
--- a/src/tablib/formats/_xls.py
+++ b/src/tablib/formats/_xls.py
@@ -100,7 +100,7 @@ class XLSFormat:
 
         dbook.wipe()
 
-        xls_book = xlrd.open_workbook(file_contents=in_stream)
+        xls_book = xlrd.open_workbook(file_contents=in_stream.read())
 
         for sheet in xls_book.sheets():
             data = tablib.Dataset()

test_patch: diff --git a/tests/test_tablib.py b/tests/test_tablib.py
index 9830d78..d557d25 100755
--- a/tests/test_tablib.py
+++ b/tests/test_tablib.py
@@ -1,19 +1,18 @@
 #!/usr/bin/env python
 """Tests for Tablib."""
 
-import datetime
+import datetime as dt
 import doctest
 import json
 import pickle
 import tempfile
 import unittest
-from collections import OrderedDict
 from decimal import Decimal
 from io import BytesIO, StringIO
 from pathlib import Path
 from uuid import uuid4
 
-import pytest
+import xlrd
 from openpyxl.reader.excel import load_workbook
 
 import tablib
@@ -56,7 +55,7 @@ class TablibTestCase(BaseTestCase):
             'latex', 'df', 'rst',
         ]
         for format_ in all_formats:
-            if format_ in exclude:
+            if format_ in exclude or (format_ == 'df' and pandas is None):
                 continue
             dataset.export(format_)
 
@@ -64,11 +63,15 @@ class TablibTestCase(BaseTestCase):
         with self.assertRaises(UnsupportedFormat):
             data.export('??')
         # A known format but uninstalled
-        del registry._formats['ods']
-        msg = (r"The 'ods' format is not available. You may want to install the "
-               "odfpy package \\(or `pip install \"tablib\\[ods\\]\"`\\).")
-        with self.assertRaisesRegex(UnsupportedFormat, msg):
-            data.export('ods')
+        saved_registry = registry._formats.copy()
+        try:
+            del registry._formats['ods']
+            msg = (r"The 'ods' format is not available. You may want to install the "
+                   "odfpy package \\(or `pip install \"tablib\\[ods\\]\"`\\).")
+            with self.assertRaisesRegex(UnsupportedFormat, msg):
+                data.export('ods')
+        finally:
+            registry._formats = saved_registry
 
     def test_empty_append(self):
         """Verify append() correctly adds tuple with no headers."""
@@ -115,10 +118,19 @@ class TablibTestCase(BaseTestCase):
 
         # With Headers
         data.headers = ('fname', 'lname')
-        new_col = [21, 22]
-        data.append_col(new_col, header='age')
+        age_col = [21, 22]
+        data.append_col(age_col, header='age')
+        size_col = [1.65, 1.86]
+        data.insert_col(1, size_col, header='size')
 
-        self.assertEqual(data['age'], new_col)
+        self.assertEqual(data['age'], age_col)
+        self.assertEqual(data['size'], size_col)
+
+    def test_add_column_no_data_with_headers(self):
+        """Verify adding empty column when dataset has only headers."""
+        data.headers = ('fname', 'lname')
+        data.insert_col(1, [], header='size')
+        self.assertEqual(data.headers, ['fname', 'size', 'lname'])
 
     def test_add_column_no_data_no_headers(self):
         """Verify adding new column with no headers."""
@@ -178,10 +190,42 @@ class TablibTestCase(BaseTestCase):
     def test_add_callable_column(self):
         """Verify adding column with values specified as callable."""
 
-        def new_col(x):
-            return x[0]
+        def new_col(row):
+            return row[0]
+
+        def initials(row):
+            return f"{row[0][0]}{row[1][0]}"
 
         self.founders.append_col(new_col, header='first_again')
+        self.founders.append_col(initials, header='initials')
+
+        # A new row can still be appended, and the dynamic column value generated.
+        self.founders.append(('Some', 'One', 71))
+        # Also acceptable when all dynamic column values are provided.
+        self.founders.append(('Other', 'Second', 84, 'Other', 'OS'))
+
+        self.assertEqual(self.founders[3], ('Some', 'One', 71, 'Some', 'SO'))
+        self.assertEqual(self.founders[4], ('Other', 'Second', 84, 'Other', 'OS'))
+        self.assertEqual(
+            self.founders['first_again'],
+            ['John', 'George', 'Thomas', 'Some', 'Other']
+        )
+        self.assertEqual(
+            self.founders['initials'],
+            ['JA', 'GW', 'TJ', 'SO', 'OS']
+        )
+
+        # However only partial dynamic values provided is not accepted.
+        with self.assertRaises(tablib.InvalidDimensions):
+            self.founders.append(('Should', 'Crash', 60, 'Partial'))
+
+        # Add a new row after dynamic column deletion
+        del self.founders['first_again']
+        self.founders.append(('After', 'Deletion', 75))
+        self.assertEqual(
+            self.founders['initials'],
+            ['JA', 'GW', 'TJ', 'SO', 'OS', 'AD']
+        )
 
     def test_header_slicing(self):
         """Verify slicing by headers."""
@@ -286,7 +330,6 @@ class TablibTestCase(BaseTestCase):
             'c|3'
         ])
 
-    @pytest.mark.skipif(pandas is None, reason="pandas is not installed")
     def test_unicode_append(self):
         """Passes in a single unicode character and exports."""
 
@@ -295,19 +338,17 @@ class TablibTestCase(BaseTestCase):
         data.append(new_row)
         self._test_export_data_in_all_formats(data)
 
-    @pytest.mark.skipif(pandas is None, reason="pandas is not installed")
     def test_datetime_append(self):
         """Passes in a single datetime and a single date and exports."""
 
         new_row = (
-            datetime.datetime.now(),
-            datetime.datetime.today(),
+            dt.datetime.now(),
+            dt.datetime.today(),
         )
 
         data.append(new_row)
         self._test_export_data_in_all_formats(data)
 
-    @pytest.mark.skipif(pandas is None, reason="pandas is not installed")
     def test_separator_append(self):
         for _ in range(3):
             data.append_separator('foobar')
@@ -353,7 +394,6 @@ class TablibTestCase(BaseTestCase):
         dset = tablib.Dataset().load(tmp_file, 'yaml')
         self.assertEqual(dset.json, '[]')
 
-    @pytest.mark.skipif(pandas is None, reason="pandas is not installed")
     def test_auto_format_detect(self):
         """Test auto format detection."""
         # html, jira, latex, rst are export only.
@@ -367,8 +407,9 @@ class TablibTestCase(BaseTestCase):
         _ods = self.founders.export('ods')
         self.assertEqual(tablib.detect_format(_ods), 'ods')
 
-        _df = self.founders.export('df')
-        self.assertEqual(tablib.detect_format(_df), 'df')
+        if pandas is not None:
+            _df = self.founders.export('df')
+            self.assertEqual(tablib.detect_format(_df), 'df')
 
         _yaml = '- {age: 90, first_name: John, last_name: Adams}'
         self.assertEqual(tablib.detect_format(_yaml), 'yaml')
@@ -533,12 +574,34 @@ class TablibTestCase(BaseTestCase):
         """Confirm formatters are being triggered."""
 
         def _formatter(cell_value):
-            return str(cell_value).upper()
+            return str(cell_value)[1:]
 
         self.founders.add_formatter('last_name', _formatter)
 
-        for name in [r['last_name'] for r in self.founders.dict]:
-            self.assertTrue(name.isupper())
+        expected = [
+            {'first_name': 'John', 'last_name': 'dams', 'gpa': 90},
+            {'first_name': 'George', 'last_name': 'ashington', 'gpa': 67},
+            {'first_name': 'Thomas', 'last_name': 'efferson', 'gpa': 50},
+        ]
+        self.assertEqual(self.founders.dict, expected)
+        # Test once more as the result should be the same
+        self.assertEqual(self.founders.dict, expected)
+
+    def test_formatters_all_cols(self):
+        """
+        Passing None as first add_formatter param apply formatter to all columns.
+        """
+
+        def _formatter(cell_value):
+            return str(cell_value).upper()
+
+        self.founders.add_formatter(None, _formatter)
+
+        self.assertEqual(self.founders.dict, [
+            {'first_name': 'JOHN', 'last_name': 'ADAMS', 'gpa': '90'},
+            {'first_name': 'GEORGE', 'last_name': 'WASHINGTON', 'gpa': '67'},
+            {'first_name': 'THOMAS', 'last_name': 'JEFFERSON', 'gpa': '50'},
+        ])
 
     def test_unicode_renders_markdown_table(self):
         # add another entry to test right field width for
@@ -1103,13 +1166,53 @@ class TSVTests(BaseTestCase):
 
 
 class ODSTests(BaseTestCase):
-    def test_ods_export_datatypes(self):
-        date_time = datetime.datetime(2019, 10, 4, 12, 30, 8)
-        data.append(('string', '004', 42, 21.55, Decimal('34.5'), date_time))
-        data.headers = ('string', 'start0', 'integer', 'float', 'decimal', 'date/time')
-        # ODS is currently write-only, just test that output doesn't crash.
-        assert data.ods is not None
-        assert len(data.ods)
+    def test_ods_export_import_set(self):
+        date = dt.date(2019, 10, 4)
+        date_time = dt.datetime(2019, 10, 4, 12, 30, 8)
+        time = dt.time(14, 30)
+        data.append(('string', '004', 42, 21.55, Decimal('34.5'), date, time, date_time, None))
+        data.headers = (
+            'string', 'start0', 'integer', 'float', 'decimal', 'date', 'time', 'date/time', 'None'
+        )
+        _ods = data.ods
+        data.ods = _ods
+        self.assertEqual(data.dict[0]['string'], 'string')
+        self.assertEqual(data.dict[0]['start0'], '004')
+        self.assertEqual(data.dict[0]['integer'], 42)
+        self.assertEqual(data.dict[0]['float'], 21.55)
+        self.assertEqual(data.dict[0]['decimal'], 34.5)
+        self.assertEqual(data.dict[0]['date'], date)
+        self.assertEqual(data.dict[0]['time'], time)
+        self.assertEqual(data.dict[0]['date/time'], date_time)
+        self.assertEqual(data.dict[0]['None'], '')
+
+    def test_ods_import_book(self):
+        ods_source = Path(__file__).parent / 'files' / 'book.ods'
+        with ods_source.open('rb') as fh:
+            dbook = tablib.Databook().load(fh, 'ods')
+        self.assertEqual(len(dbook.sheets()), 2)
+
+    def test_ods_import_set_skip_lines(self):
+        data.append(('garbage', 'line', ''))
+        data.append(('', '', ''))
+        data.append(('id', 'name', 'description'))
+        _ods = data.ods
+        new_data = tablib.Dataset().load(_ods, skip_lines=2)
+        self.assertEqual(new_data.headers, ['id', 'name', 'description'])
+
+    def test_ods_import_set_ragged(self):
+        ods_source = Path(__file__).parent / 'files' / 'ragged.ods'
+        with ods_source.open('rb') as fh:
+            dataset = tablib.Dataset().load(fh, 'ods')
+        self.assertEqual(dataset.pop(), (1, '', True, ''))
+
+    def test_ods_unknown_value_type(self):
+        # The ods file was trafficked to contain:
+        # <table:table-cell office:value-type="unknown" calcext:value-type="string">
+        ods_source = Path(__file__).parent / 'files' / 'unknown_value_type.ods'
+        with ods_source.open('rb') as fh:
+            dataset = tablib.Dataset().load(fh, 'ods')
+        self.assertEqual(dataset.pop(), ('abcd',))
 
 
 class XLSTests(BaseTestCase):
@@ -1122,7 +1225,7 @@ class XLSTests(BaseTestCase):
         xls_source = Path(__file__).parent / 'files' / 'dates.xls'
         with xls_source.open('rb') as fh:
             dset = tablib.Dataset().load(fh, 'xls')
-        self.assertEqual(dset.dict[0]['birth_date'], datetime.datetime(2015, 4, 12, 0, 0))
+        self.assertEqual(dset.dict[0]['birth_date'], dt.datetime(2015, 4, 12, 0, 0))
 
     def test_xlsx_import_set_skip_lines(self):
         data.append(('garbage', 'line', ''))
@@ -1139,7 +1242,7 @@ class XLSTests(BaseTestCase):
             data = tablib.Dataset().load(fh.read())
         self.assertEqual(
             data.dict[0],
-            OrderedDict([
+            dict([
                 ('div by 0', '#DIV/0!'),
                 ('name unknown', '#NAME?'),
                 ('not available (formula)', '#N/A'),
@@ -1147,6 +1250,28 @@ class XLSTests(BaseTestCase):
             ])
         )
 
+    def test_book_import_from_stream(self):
+        in_stream = self.founders.xls
+        book = tablib.Databook().load(in_stream, 'xls')
+        self.assertEqual(book.sheets()[0].title, 'Founders')
+
+    def test_xls_export_with_dates(self):
+        date = dt.date(2019, 10, 4)
+        time = dt.time(14, 30)
+        date_time = dt.datetime(2019, 10, 4, 12, 30, 8)
+        data.append((date, time, date_time))
+        data.headers = ('date', 'time', 'date/time')
+        _xls = data.xls
+        xls_book = xlrd.open_workbook(file_contents=_xls, formatting_info=True)
+        row = xls_book.sheet_by_index(0).row(1)
+
+        def get_format_str(cell):
+            return xls_book.format_map[xls_book.xf_list[cell.xf_index].format_key].format_str
+
+        self.assertEqual('m/d/yy', get_format_str(row[0]))
+        self.assertEqual('h:mm:ss', get_format_str(row[1]))
+        self.assertEqual('m/d/yy h:mm', get_format_str(row[2]))
+
 
 class XLSXTests(BaseTestCase):
     def test_xlsx_format_detect(self):
@@ -1155,7 +1280,7 @@ class XLSXTests(BaseTestCase):
         self.assertEqual(detect_format(in_stream), 'xlsx')
 
     def test_xlsx_import_set(self):
-        date_time = datetime.datetime(2019, 10, 4, 12, 30, 8)
+        date_time = dt.datetime(2019, 10, 4, 12, 30, 8)
         data.append(('string', '004', 42, 21.55, date_time))
         data.headers = ('string', 'start0', 'integer', 'float', 'date/time')
         _xlsx = data.xlsx


----

problem_statement: Output formatter corrupts original data
version: 3.5.0
python 3.11

I'm trying to export the same dataset several times in different file formats. I've found that specifying a formatter for a column data alters the underlying data. The issue occurs if exporting the same format too.

```python
import tablib

dataset = tablib.Dataset()
dataset.headers = ["text"]
dataset.append(["aAbBcC"])
dataset.add_formatter("text", lambda s: s[2:])

print(dataset.export("csv"))
print(dataset.export("csv"))
```
Output:
```
text
bBcC

text
cC
```

date: 2024-02-25
patch: diff --git a/src/tablib/core.py b/src/tablib/core.py
index a7ebff2..70e26f0 100644
--- a/src/tablib/core.py
+++ b/src/tablib/core.py
@@ -8,7 +8,6 @@
     :license: MIT, see LICENSE for more details.
 """
 
-from collections import OrderedDict
 from copy import copy
 from operator import itemgetter
 
@@ -74,6 +73,9 @@ class Row:
     def insert(self, index, value):
         self._row.insert(index, value)
 
+    def copy(self):
+        return Row(self._row.copy(), self.tags.copy())
+
     def __contains__(self, item):
         return item in self._row
 
@@ -155,6 +157,9 @@ class Dataset:
         # (column, callback) tuples
         self._formatters = []
 
+        # {col_index: col_func}
+        self._dynamic_columns = {}
+
         self.headers = kwargs.get('headers')
 
         self.title = kwargs.get('title')
@@ -187,6 +192,8 @@ class Dataset:
 
                 pos = self.headers.index(key)
                 del self.headers[pos]
+                if pos in self._dynamic_columns:
+                    del self._dynamic_columns[pos]
 
                 for i, row in enumerate(self._data):
 
@@ -238,7 +245,13 @@ class Dataset:
     def _validate(self, row=None, col=None, safety=False):
         """Assures size of every row in dataset is of proper proportions."""
         if row:
-            is_valid = (len(row) == self.width) if self.width else True
+            if self.width:
+                is_valid = (
+                    len(row) == self.width or
+                    len(row) == (self.width - len(self._dynamic_columns))
+                )
+            else:
+                is_valid = True
         elif col:
             if len(col) < 1:
                 is_valid = True
@@ -254,38 +267,31 @@ class Dataset:
                 raise InvalidDimensions
             return False
 
-    def _package(self, dicts=True, ordered=True):
+    def _package(self, dicts=True):
         """Packages Dataset into lists of dictionaries for transmission."""
         # TODO: Dicts default to false?
 
         _data = list(self._data)
 
-        if ordered:
-            dict_pack = OrderedDict
-        else:
-            dict_pack = dict
-
-        # Execute formatters
-        if self._formatters:
-            for row_i, row in enumerate(_data):
+        def format_row(row):
+            # Execute formatters
+            if self._formatters:
+                row = row.copy()  # To not mutate internal data structure
                 for col, callback in self._formatters:
-                    try:
-                        if col is None:
-                            for j, c in enumerate(row):
-                                _data[row_i][j] = callback(c)
-                        else:
-                            _data[row_i][col] = callback(row[col])
-                    except IndexError:
-                        raise InvalidDatasetIndex
+                    if col is None:
+                        # Apply formatter to all cells
+                        row = [callback(cell) for cell in row]
+                    else:
+                        row[col] = callback(row[col])
+            return list(row)
 
         if self.headers:
             if dicts:
-                data = [dict_pack(list(zip(self.headers, data_row))) for data_row in _data]
+                data = [dict(list(zip(self.headers, format_row(row)))) for row in _data]
             else:
-                data = [list(self.headers)] + list(_data)
+                data = [list(self.headers)] + [format_row(row) for row in _data]
         else:
-            data = [list(row) for row in _data]
-
+            data = [format_row(row) for row in _data]
         return data
 
     def _get_headers(self):
@@ -300,10 +306,7 @@ class Dataset:
         """Validating headers setter."""
         self._validate(collection)
         if collection:
-            try:
-                self.__headers = list(collection)
-            except TypeError:
-                raise TypeError
+            self.__headers = list(collection)
         else:
             self.__headers = None
 
@@ -446,9 +449,17 @@ class Dataset:
 
         The default behaviour is to insert the given row to the :class:`Dataset`
         object at the given index.
-       """
+
+        You can add :ref:`tags <tags>` to the row you are inserting.
+        This gives you the ability to :method:`filter <Dataset.filter>` your
+        :class:`Dataset` later.
+        """
 
         self._validate(row)
+        if len(row) < self.width:
+            for pos, func in self._dynamic_columns.items():
+                row = list(row)
+                row.insert(pos, func(row))
         self._data.insert(index, Row(row, tags=tags))
 
     def rpush(self, row, tags=()):
@@ -529,24 +540,14 @@ class Dataset:
         that row.
 
         See :ref:`dyncols` for an in-depth example.
-
-        .. versionchanged:: 0.9.0
-           If inserting a column, and :attr:`Dataset.headers` is set, the
-           header attribute must be set, and will be considered the header for
-           that row.
-
-        .. versionadded:: 0.9.0
-           If inserting a row, you can add :ref:`tags <tags>` to the row you are inserting.
-           This gives you the ability to :method:`filter <Dataset.filter>` your
-           :class:`Dataset` later.
-
         """
 
         if col is None:
             col = []
 
         # Callable Columns...
-        if hasattr(col, '__call__'):
+        if callable(col):
+            self._dynamic_columns[self.width] = col
             col = list(map(col, self._data))
 
         col = self._clean_col(col)
@@ -622,9 +623,8 @@ class Dataset:
     def add_formatter(self, col, handler):
         """Adds a formatter to the :class:`Dataset`.
 
-        .. versionadded:: 0.9.5
-
-        :param col: column to. Accepts index int or header str.
+        :param col: column to. Accepts index int, header str, or None to apply
+                    the formatter to all columns.
         :param handler: reference to callback function to execute against
                         each cell value.
         """
@@ -635,7 +635,7 @@ class Dataset:
             else:
                 raise KeyError
 
-        if not col > self.width:
+        if col is None or col <= self.width:
             self._formatters.append((col, handler))
         else:
             raise InvalidDatasetIndex
@@ -876,20 +876,15 @@ class Databook:
         else:
             raise InvalidDatasetType
 
-    def _package(self, ordered=True):
+    def _package(self):
         """Packages :class:`Databook` for delivery."""
         collector = []
 
-        if ordered:
-            dict_pack = OrderedDict
-        else:
-            dict_pack = dict
-
         for dset in self._datasets:
-            collector.append(dict_pack(
-                title=dset.title,
-                data=dset._package(ordered=ordered)
-            ))
+            collector.append({
+                'title': dset.title,
+                'data': dset._package()
+            })
         return collector
 
     @property


----

problem_statement: Question / feature request: Excel - format as date 
I can feed a tablib cell a datetime, and formatting that cell as date in Excel works - the number shown initially gets converted to a valid date.

Can i tell tablib to format the cell as datetime right away, so the Excel file shows date formatting right after being opened?

(Looks like XlsxWriter and xlwt can do that, but i'm reluctant to switch.)


date: 2024-07-10
patch: diff --git a/src/tablib/formats/_xls.py b/src/tablib/formats/_xls.py
index 2c6c527..947625d 100644
--- a/src/tablib/formats/_xls.py
+++ b/src/tablib/formats/_xls.py
@@ -1,6 +1,6 @@
 """ Tablib - XLS Support.
 """
-
+import datetime
 from io import BytesIO
 
 import xlrd
@@ -12,6 +12,9 @@ import tablib
 # special styles
 wrap = xlwt.easyxf("alignment: wrap on")
 bold = xlwt.easyxf("font: bold on")
+datetime_style = xlwt.easyxf(num_format_str='M/D/YY h:mm')
+date_style = xlwt.easyxf(num_format_str='M/D/YY')
+time_style = xlwt.easyxf(num_format_str='h:mm:ss')
 
 
 class XLSFormat:
@@ -138,6 +141,13 @@ class XLSFormat:
                 elif len(row) < dataset.width:
                     ws.write(i, j, col, bold)
 
+                # format date types
+                elif isinstance(col, datetime.datetime):
+                    ws.write(i, j, col, datetime_style)
+                elif isinstance(col, datetime.date):
+                    ws.write(i, j, col, date_style)
+                elif isinstance(col, datetime.time):
+                    ws.write(i, j, col, time_style)
                 # wrap the rest
                 else:
                     try: