Repository: pandas-dev/pandas

Issue:
ER: In HDFStore create warnings for mode, quiet
- If `mode == 'w'` and `warn==True` then weird `raw_input` is requested. Change this to raise/warn as appropriate
- change the `_quiet` field (default is `False`) to a warning (about duplicates in readback)



Discussion:
When I was looking at this before, was thinking that store could change mode from `w` after opening so that it doesn't overwrite on a second write. (i.e., at end of the handle open part, just change mode to whatever seems appropriate - user can change mode explicitly if they want...)

I always open as append mode (personally)

I simulate w by removing the file

I think that might be the way to go here

then u don't need to track status of the mode


Resolving patch:
diff --git a/pandas/io/pytables.py b/pandas/io/pytables.py
--- a/pandas/io/pytables.py
+++ b/pandas/io/pytables.py
@@ -61,9 +61,14 @@ def _ensure_encoding(encoding):
     return encoding
 
 
-class IncompatibilityWarning(Warning):
+class PossibleDataLossError(Exception):
+    pass
+
+class ClosedFileError(Exception):
     pass
 
+class IncompatibilityWarning(Warning):
+    pass
 
 incompatibility_doc = """
 where criteria is being ignored as this version [%s] is too old (or
@@ -71,16 +76,20 @@ class IncompatibilityWarning(Warning):
 the copy_to method)
 """
 
-
 class AttributeConflictWarning(Warning):
     pass
 
-
 attribute_conflict_doc = """
 the [%s] attribute of the existing index is [%s] which conflicts with the new
 [%s], resetting the attribute to None
 """
 
+class DuplicateWarning(Warning):
+    pass
+
+duplicate_doc = """
+duplicate entries in table, taking most recently appended
+"""
 
 performance_doc = """
 your performance may suffer as PyTables will pickle object types that it cannot
@@ -263,7 +272,6 @@ class HDFStore(StringMixin):
     >>> bar = store['foo']   # retrieve
     >>> store.close()
     """
-    _quiet = False
 
     def __init__(self, path, mode=None, complevel=None, complib=None,
                  fletcher32=False):
@@ -281,11 +289,12 @@ def __init__(self, path, mode=None, complevel=None, complib=None,
         self._complib = complib
         self._fletcher32 = fletcher32
         self._filters = None
-        self.open(mode=mode, warn=False)
+        self.open(mode=mode)
 
     @property
     def root(self):
         """ return the root node """
+        self._check_if_open()
         return self._handle.root
 
     def __getitem__(self, key):
@@ -299,6 +308,7 @@ def __delitem__(self, key):
 
     def __getattr__(self, name):
         """ allow attribute access to get stores """
+        self._check_if_open()
         try:
             return self.get(name)
         except:
@@ -321,24 +331,26 @@ def __len__(self):
 
     def __unicode__(self):
         output = '%s\nFile path: %s\n' % (type(self), pprint_thing(self._path))
-
-        if len(list(self.keys())):
-            keys   = []
-            values = []
-
-            for k in self.keys():
-                try:
-                    s = self.get_storer(k)
-                    if s is not None:
-                        keys.append(pprint_thing(s.pathname or k))
-                        values.append(pprint_thing(s or 'invalid_HDFStore node'))
-                except Exception as detail:
-                    keys.append(k)
-                    values.append("[invalid_HDFStore node: %s]" % pprint_thing(detail))
-
-            output += adjoin(12, keys, values)
+        if self.is_open:
+            if len(list(self.keys())):
+                keys   = []
+                values = []
+
+                for k in self.keys():
+                    try:
+                        s = self.get_storer(k)
+                        if s is not None:
+                            keys.append(pprint_thing(s.pathname or k))
+                            values.append(pprint_thing(s or 'invalid_HDFStore node'))
+                    except Exception as detail:
+                        keys.append(k)
+                        values.append("[invalid_HDFStore node: %s]" % pprint_thing(detail))
+
+                output += adjoin(12, keys, values)
+            else:
+                output += 'Empty'
         else:
-            output += 'Empty'
+            output += "File is CLOSED"
 
         return output
 
@@ -358,7 +370,7 @@ def items(self):
 
     iteritems = items
 
-    def open(self, mode='a', warn=True):
+    def open(self, mode='a'):
         """
         Open the file in the specified mode
 
@@ -367,19 +379,23 @@ def open(self, mode='a', warn=True):
         mode : {'a', 'w', 'r', 'r+'}, default 'a'
             See HDFStore docstring or tables.openFile for info about modes
         """
-        self._mode = mode
-        if warn and mode == 'w':  # pragma: no cover
-            while True:
-                if compat.PY3:
-                    raw_input = input
-                response = raw_input("Re-opening as mode='w' will delete the "
-                                     "current file. Continue (y/n)?")
-                if response == 'y':
-                    break
-                elif response == 'n':
-                    return
-        if self._handle is not None and self._handle.isopen:
-            self._handle.close()
+        if self._mode != mode:
+
+            # if we are chaning a write mode to read, ok
+            if self._mode in ['a','w'] and mode in ['r','r+']:
+                pass
+            elif mode in ['w']:
+
+                # this would truncate, raise here
+                if self.is_open:
+                    raise PossibleDataLossError("Re-opening the file [{0}] with mode [{1}] "
+                                                "will delete the current file!".format(self._path,self._mode))
+
+            self._mode = mode
+
+        # close and reopen the handle
+        if self.is_open:
+            self.close()
 
         if self._complib is not None:
             if self._complevel is None:
@@ -401,13 +417,24 @@ def close(self):
         """
         Close the PyTables file handle
         """
-        self._handle.close()
+        if self._handle is not None:
+            self._handle.close()
+        self._handle = None
+
+    @property
+    def is_open(self):
+        """
+        return a boolean indicating whether the file is open
+        """
+        if self._handle is None: return False
+        return bool(self._handle.isopen)
 
     def flush(self):
         """
         Force all buffered modifications to be written to disk
         """
-        self._handle.flush()
+        if self._handle is not None:
+            self._handle.flush()
 
     def get(self, key):
         """
@@ -748,11 +775,13 @@ def create_table_index(self, key, **kwargs):
     def groups(self):
         """ return a list of all the top-level nodes (that are not themselves a pandas storage object) """
         _tables()
+        self._check_if_open()
         return [ g for g in self._handle.walkNodes() if getattr(g._v_attrs,'pandas_type',None) or getattr(
             g,'table',None) or (isinstance(g,_table_mod.table.Table) and g._v_name != u('table')) ]
 
     def get_node(self, key):
         """ return the node with the key or None if it does not exist """
+        self._check_if_open()
         try:
             if not key.startswith('/'):
                 key = '/' + key
@@ -811,6 +840,9 @@ def copy(self, file, mode = 'w', propindexes = True, keys = None, complib = None
         return new_store
 
     ###### private methods ######
+    def _check_if_open(self):
+        if not self.is_open:
+            raise ClosedFileError("{0} file is not open!".format(self._path))
 
     def _create_storer(self, group, value = None, table = False, append = False, **kwargs):
         """ return a suitable Storer class to operate """
@@ -1647,10 +1679,6 @@ def pathname(self):
     def _handle(self):
         return self.parent._handle
 
-    @property
-    def _quiet(self):
-        return self.parent._quiet
-
     @property
     def _filters(self):
         return self.parent._filters
@@ -2918,9 +2946,7 @@ def read(self, where=None, columns=None, **kwargs):
                 objs.append(obj)
 
         else:
-            if not self._quiet:  # pragma: no cover
-                print ('Duplicate entries in table, taking most recently '
-                       'appended')
+            warnings.warn(duplicate_doc, DuplicateWarning)
 
             # reconstruct
             long_index = MultiIndex.from_arrays(


Test patch:
diff --git a/pandas/io/tests/test_pytables.py b/pandas/io/tests/test_pytables.py
--- a/pandas/io/tests/test_pytables.py
+++ b/pandas/io/tests/test_pytables.py
@@ -14,7 +14,8 @@
                     date_range, Index)
 from pandas.io.pytables import (HDFStore, get_store, Term, read_hdf,
                                 IncompatibilityWarning, PerformanceWarning,
-                                AttributeConflictWarning)
+                                AttributeConflictWarning, DuplicateWarning,
+                                PossibleDataLossError, ClosedFileError)
 import pandas.util.testing as tm
 from pandas.tests.test_series import assert_series_equal
 from pandas.tests.test_frame import assert_frame_equal
@@ -78,6 +79,13 @@ def _maybe_remove(store, key):
     except:
         pass
 
+def compat_assert_produces_warning(w,f):
+    """ don't produce a warning under PY3 """
+    if compat.PY3:
+        f()
+    else:
+        with tm.assert_produces_warning(expected_warning=w):
+            f()
 
 class TestHDFStore(unittest.TestCase):
 
@@ -174,7 +182,10 @@ def test_repr(self):
             df['datetime2']  = datetime.datetime(2001,1,3,0,0)
             df.ix[3:6,['obj1']] = np.nan
             df = df.consolidate().convert_objects()
+
+            warnings.filterwarnings('ignore', category=PerformanceWarning)
             store['df'] = df
+            warnings.filterwarnings('always', category=PerformanceWarning)
 
             # make a random group in hdf space
             store._handle.createGroup(store._handle.root,'bah')
@@ -197,10 +208,9 @@ def test_contains(self):
             self.assert_('bar' not in store)
 
             # GH 2694
-            warnings.filterwarnings('ignore', category=tables.NaturalNameWarning)
-            store['node())'] = tm.makeDataFrame()
+            with tm.assert_produces_warning(expected_warning=tables.NaturalNameWarning):
+                store['node())'] = tm.makeDataFrame()
             self.assert_('node())' in store)
-            warnings.filterwarnings('always', category=tables.NaturalNameWarning)
 
     def test_versioning(self):
 
@@ -226,11 +236,49 @@ def test_versioning(self):
 
     def test_reopen_handle(self):
 
-        with ensure_clean(self.path) as store:
+        with tm.ensure_clean(self.path) as path:
+
+            store = HDFStore(path,mode='a')
             store['a'] = tm.makeTimeSeries()
-            store.open('w', warn=False)
-            self.assert_(store._handle.isopen)
+
+            # invalid mode change
+            self.assertRaises(PossibleDataLossError, store.open, 'w')
+            store.close()
+            self.assert_(not store.is_open)
+
+            # truncation ok here
+            store.open('w')
+            self.assert_(store.is_open)
             self.assertEquals(len(store), 0)
+            store.close()
+            self.assert_(not store.is_open)
+
+            store = HDFStore(path,mode='a')
+            store['a'] = tm.makeTimeSeries()
+
+            # reopen as read
+            store.open('r')
+            self.assert_(store.is_open)
+            self.assertEquals(len(store), 1)
+            self.assert_(store._mode == 'r')
+            store.close()
+            self.assert_(not store.is_open)
+
+            # reopen as append
+            store.open('a')
+            self.assert_(store.is_open)
+            self.assertEquals(len(store), 1)
+            self.assert_(store._mode == 'a')
+            store.close()
+            self.assert_(not store.is_open)
+
+            # reopen as append (again)
+            store.open('a')
+            self.assert_(store.is_open)
+            self.assertEquals(len(store), 1)
+            self.assert_(store._mode == 'a')
+            store.close()
+            self.assert_(not store.is_open)
 
     def test_flush(self):
 
@@ -382,11 +430,15 @@ def test_put_mixed_type(self):
 
         with ensure_clean(self.path) as store:
             _maybe_remove(store, 'df')
+
+            # cannot use assert_produces_warning here for some reason
+            # a PendingDeprecationWarning is also raised?
             warnings.filterwarnings('ignore', category=PerformanceWarning)
             store.put('df',df)
+            warnings.filterwarnings('always', category=PerformanceWarning)
+
             expected = store.get('df')
             tm.assert_frame_equal(expected,df)
-            warnings.filterwarnings('always', category=PerformanceWarning)
 
     def test_append(self):
 
@@ -408,12 +460,11 @@ def test_append(self):
             tm.assert_frame_equal(store['df3'], df)
 
             # this is allowed by almost always don't want to do it
-            warnings.filterwarnings('ignore', category=tables.NaturalNameWarning)
-            _maybe_remove(store, '/df3 foo')
-            store.append('/df3 foo', df[:10])
-            store.append('/df3 foo', df[10:])
-            tm.assert_frame_equal(store['df3 foo'], df)
-            warnings.filterwarnings('always', category=tables.NaturalNameWarning)
+            with tm.assert_produces_warning(expected_warning=tables.NaturalNameWarning):
+                _maybe_remove(store, '/df3 foo')
+                store.append('/df3 foo', df[:10])
+                store.append('/df3 foo', df[10:])
+                tm.assert_frame_equal(store['df3 foo'], df)
 
             # panel
             wp = tm.makePanel()
@@ -1705,9 +1756,8 @@ def test_tuple_index(self):
         idx = [(0., 1.), (2., 3.), (4., 5.)]
         data = np.random.randn(30).reshape((3, 10))
         DF = DataFrame(data, index=idx, columns=col)
-        warnings.filterwarnings('ignore', category=PerformanceWarning)
-        self._check_roundtrip(DF, tm.assert_frame_equal)
-        warnings.filterwarnings('always', category=PerformanceWarning)
+        with tm.assert_produces_warning(expected_warning=PerformanceWarning):
+            self._check_roundtrip(DF, tm.assert_frame_equal)
 
     def test_index_types(self):
 
@@ -1715,26 +1765,25 @@ def test_index_types(self):
 
         func = lambda l, r: tm.assert_series_equal(l, r, True, True, True)
 
-        warnings.filterwarnings('ignore', category=PerformanceWarning)
-        ser = Series(values, [0, 'y'])
-        self._check_roundtrip(ser, func)
-        warnings.filterwarnings('always', category=PerformanceWarning)
+        with tm.assert_produces_warning(expected_warning=PerformanceWarning):
+            ser = Series(values, [0, 'y'])
+            self._check_roundtrip(ser, func)
 
-        ser = Series(values, [datetime.datetime.today(), 0])
-        self._check_roundtrip(ser, func)
+        with tm.assert_produces_warning(expected_warning=PerformanceWarning):
+            ser = Series(values, [datetime.datetime.today(), 0])
+            self._check_roundtrip(ser, func)
 
-        ser = Series(values, ['y', 0])
-        self._check_roundtrip(ser, func)
+        with tm.assert_produces_warning(expected_warning=PerformanceWarning):
+            ser = Series(values, ['y', 0])
+            self._check_roundtrip(ser, func)
 
-        warnings.filterwarnings('ignore', category=PerformanceWarning)
-        ser = Series(values, [datetime.date.today(), 'a'])
-        self._check_roundtrip(ser, func)
-        warnings.filterwarnings('always', category=PerformanceWarning)
+        with tm.assert_produces_warning(expected_warning=PerformanceWarning):
+            ser = Series(values, [datetime.date.today(), 'a'])
+            self._check_roundtrip(ser, func)
 
-        warnings.filterwarnings('ignore', category=PerformanceWarning)
-        ser = Series(values, [1.23, 'b'])
-        self._check_roundtrip(ser, func)
-        warnings.filterwarnings('always', category=PerformanceWarning)
+        with tm.assert_produces_warning(expected_warning=PerformanceWarning):
+            ser = Series(values, [1.23, 'b'])
+            self._check_roundtrip(ser, func)
 
         ser = Series(values, [1, 1.53])
         self._check_roundtrip(ser, func)
@@ -1918,10 +1967,12 @@ def test_wide_table(self):
     def test_wide_table_dups(self):
         wp = tm.makePanel()
         with ensure_clean(self.path) as store:
-            store._quiet = True
             store.put('panel', wp, table=True)
             store.put('panel', wp, table=True, append=True)
-            recons = store['panel']
+
+            with tm.assert_produces_warning(expected_warning=DuplicateWarning):
+                recons = store['panel']
+
             tm.assert_panel_equal(recons, wp)
 
     def test_long(self):
@@ -2231,11 +2282,10 @@ def test_retain_index_attributes(self):
 
 
             # try to append a table with a different frequency
-            warnings.filterwarnings('ignore', category=AttributeConflictWarning)
-            df2 = DataFrame(dict(A = Series(lrange(3),
-                                            index=date_range('2002-1-1',periods=3,freq='D'))))
-            store.append('data',df2)
-            warnings.filterwarnings('always', category=AttributeConflictWarning)
+            with tm.assert_produces_warning(expected_warning=AttributeConflictWarning):
+                df2 = DataFrame(dict(A = Series(lrange(3),
+                                                index=date_range('2002-1-1',periods=3,freq='D'))))
+                store.append('data',df2)
 
             self.assert_(store.get_storer('data').info['index']['freq'] is None)
 
@@ -2251,26 +2301,28 @@ def test_retain_index_attributes2(self):
 
         with tm.ensure_clean(self.path) as path:
 
-            warnings.filterwarnings('ignore', category=AttributeConflictWarning)
+            with tm.assert_produces_warning(expected_warning=AttributeConflictWarning):
 
-            df  = DataFrame(dict(A = Series(lrange(3), index=date_range('2000-1-1',periods=3,freq='H'))))
-            df.to_hdf(path,'data',mode='w',append=True)
-            df2 = DataFrame(dict(A = Series(lrange(3), index=date_range('2002-1-1',periods=3,freq='D'))))
-            df2.to_hdf(path,'data',append=True)
+                df  = DataFrame(dict(A = Series(lrange(3), index=date_range('2000-1-1',periods=3,freq='H'))))
+                df.to_hdf(path,'data',mode='w',append=True)
+                df2 = DataFrame(dict(A = Series(lrange(3), index=date_range('2002-1-1',periods=3,freq='D'))))
+                df2.to_hdf(path,'data',append=True)
+
+                idx = date_range('2000-1-1',periods=3,freq='H')
+                idx.name = 'foo'
+                df  = DataFrame(dict(A = Series(lrange(3), index=idx)))
+                df.to_hdf(path,'data',mode='w',append=True)
 
-            idx = date_range('2000-1-1',periods=3,freq='H')
-            idx.name = 'foo'
-            df  = DataFrame(dict(A = Series(lrange(3), index=idx)))
-            df.to_hdf(path,'data',mode='w',append=True)
             self.assert_(read_hdf(path,'data').index.name == 'foo')
 
-            idx2 = date_range('2001-1-1',periods=3,freq='H')
-            idx2.name = 'bar'
-            df2 = DataFrame(dict(A = Series(lrange(3), index=idx2)))
-            df2.to_hdf(path,'data',append=True)
-            self.assert_(read_hdf(path,'data').index.name is None)
+            with tm.assert_produces_warning(expected_warning=AttributeConflictWarning):
 
-            warnings.filterwarnings('always', category=AttributeConflictWarning)
+                idx2 = date_range('2001-1-1',periods=3,freq='H')
+                idx2.name = 'bar'
+                df2 = DataFrame(dict(A = Series(lrange(3), index=idx2)))
+                df2.to_hdf(path,'data',append=True)
+
+            self.assert_(read_hdf(path,'data').index.name is None)
 
     def test_panel_select(self):
 
@@ -2611,6 +2663,95 @@ def _check_roundtrip_table(self, obj, comparator, compression=False):
             # sorted_obj = _test_sort(obj)
             comparator(retrieved, obj)
 
+    def test_multiple_open_close(self):
+        # GH 4409, open & close multiple times
+
+        with tm.ensure_clean(self.path) as path:
+
+            df = tm.makeDataFrame()
+            df.to_hdf(path,'df',mode='w',table=True)
+
+            # single
+            store = HDFStore(path)
+            self.assert_('CLOSED' not in str(store))
+            self.assert_(store.is_open)
+            store.close()
+            self.assert_('CLOSED' in str(store))
+            self.assert_(not store.is_open)
+
+            # multiples
+            store1 = HDFStore(path)
+            store2 = HDFStore(path)
+
+            self.assert_('CLOSED' not in str(store1))
+            self.assert_('CLOSED' not in str(store2))
+            self.assert_(store1.is_open)
+            self.assert_(store2.is_open)
+
+            store1.close()
+            self.assert_('CLOSED' in str(store1))
+            self.assert_(not store1.is_open)
+            self.assert_('CLOSED' not in str(store2))
+            self.assert_(store2.is_open)
+
+            store2.close()
+            self.assert_('CLOSED' in str(store1))
+            self.assert_('CLOSED' in str(store2))
+            self.assert_(not store1.is_open)
+            self.assert_(not store2.is_open)
+
+            # nested close
+            store = HDFStore(path,mode='w')
+            store.append('df',df)
+
+            store2 = HDFStore(path)
+            store2.append('df2',df)
+            store2.close()
+            self.assert_('CLOSED' in str(store2))
+            self.assert_(not store2.is_open)
+
+            store.close()
+            self.assert_('CLOSED' in str(store))
+            self.assert_(not store.is_open)
+
+            # double closing
+            store = HDFStore(path,mode='w')
+            store.append('df', df)
+
+            store2 = HDFStore(path)
+            store.close()
+            self.assert_('CLOSED' in str(store))
+            self.assert_(not store.is_open)
+
+            store2.close()
+            self.assert_('CLOSED' in str(store2))
+            self.assert_(not store2.is_open)
+
+        # ops on a closed store
+        with tm.ensure_clean(self.path) as path:
+
+            df = tm.makeDataFrame()
+            df.to_hdf(path,'df',mode='w',table=True)
+
+            store = HDFStore(path)
+            store.close()
+
+            self.assertRaises(ClosedFileError, store.keys)
+            self.assertRaises(ClosedFileError, lambda : 'df' in store)
+            self.assertRaises(ClosedFileError, lambda : len(store))
+            self.assertRaises(ClosedFileError, lambda : store['df'])
+            self.assertRaises(ClosedFileError, lambda : store.df)
+            self.assertRaises(ClosedFileError, store.select, 'df')
+            self.assertRaises(ClosedFileError, store.get, 'df')
+            self.assertRaises(ClosedFileError, store.append, 'df2', df)
+            self.assertRaises(ClosedFileError, store.put, 'df3', df)
+            self.assertRaises(ClosedFileError, store.get_storer, 'df2')
+            self.assertRaises(ClosedFileError, store.remove, 'df2')
+
+            def f():
+                store.select('df')
+            tm.assertRaisesRegexp(ClosedFileError, 'file is not open', f)
+
     def test_pytables_native_read(self):
 
         try:
@@ -2648,13 +2789,13 @@ def test_legacy_table_read(self):
             store.select('df2', typ='legacy_frame')
 
             # old version warning
-            warnings.filterwarnings('ignore', category=IncompatibilityWarning)
-            self.assertRaises(
-                Exception, store.select, 'wp1', Term('minor_axis', '=', 'B'))
+            with tm.assert_produces_warning(expected_warning=IncompatibilityWarning):
+                self.assertRaises(
+                    Exception, store.select, 'wp1', Term('minor_axis', '=', 'B'))
 
-            df2 = store.select('df2')
-            store.select('df2', Term('index', '>', df2.index[2]))
-            warnings.filterwarnings('always', category=IncompatibilityWarning)
+            with tm.assert_produces_warning(expected_warning=IncompatibilityWarning):
+                df2 = store.select('df2')
+                store.select('df2', Term('index', '>', df2.index[2]))
 
         finally:
             safe_close(store)
@@ -2813,10 +2954,11 @@ def test_tseries_indices_frame(self):
     def test_unicode_index(self):
 
         unicode_values = [u('\u03c3'), u('\u03c3\u03c3')]
-        warnings.filterwarnings('ignore', category=PerformanceWarning)
-        s = Series(np.random.randn(len(unicode_values)), unicode_values)
-        self._check_roundtrip(s, tm.assert_series_equal)
-        warnings.filterwarnings('always', category=PerformanceWarning)
+        def f():
+            s = Series(np.random.randn(len(unicode_values)), unicode_values)
+            self._check_roundtrip(s, tm.assert_series_equal)
+
+        compat_assert_produces_warning(PerformanceWarning,f)
 
     def test_store_datetime_mixed(self):