Merge pull request #6104 from jreback/pytables_3.1

COMPAT: provide compat tests and better error messages for multiple file opening on PyTables >= 3.1 (GH6047)

Author: jreback

pandas/io/pytables.py

@@ -211,11 +211,12 @@ class DuplicateWarning(Warning):
 # oh the troubles to reduce import time
 _table_mod = None
 _table_supports_index = False
-
+_table_file_open_policy_is_strict = False
 
 def _tables():
     global _table_mod
     global _table_supports_index
+    global _table_file_open_policy_is_strict
     if _table_mod is None:
         import tables
         from distutils.version import LooseVersion
@@ -225,8 +226,15 @@ def _tables():
         ver = tables.__version__
         _table_supports_index = LooseVersion(ver) >= '2.3'
 
-    return _table_mod
+        # set the file open policy
+        # return the file open policy; this changes as of pytables 3.1
+        # depending on the HDF5 version
+        try:
+            _table_file_open_policy_is_strict = tables.file._FILE_OPEN_POLICY == 'strict'
+        except:
+            pass
 
+    return _table_mod
 
 @contextmanager
 def get_store(path, **kwargs):
@@ -524,6 +532,22 @@ def open(self, mode='a', **kwargs):
                 self._handle = tables.openFile(self._path, 'r', **kwargs)
             else:
                 raise
+
+        except (ValueError) as e:
+
+            # trap PyTables >= 3.1 FILE_OPEN_POLICY exception
+            # to provide an updated message
+            if 'FILE_OPEN_POLICY' in str(e):
+
+                e = ValueError("PyTables [{version}] no longer supports opening multiple files\n"
+                               "even in read-only mode on this HDF5 version [{hdf_version}]. You can accept this\n"
+                               "and not open the same file multiple times at once,\n"
+                               "upgrade the HDF5 version, or downgrade to PyTables 3.0.0 which allows\n"
+                               "files to be opened multiple times at once\n".format(version=tables.__version__,
+                                                                                    hdf_version=tables.getHDF5Version()))
+
+            raise e
+
         except (Exception) as e:
 
             # trying to read from a non-existant file causes an error which

pandas/io/tests/test_pytables.py

@@ -15,6 +15,7 @@
                                 IncompatibilityWarning, PerformanceWarning,
                                 AttributeConflictWarning, DuplicateWarning,
                                 PossibleDataLossError, ClosedFileError)
+from pandas.io import pytables as pytables
 import pandas.util.testing as tm
 from pandas.util.testing import (assert_panel4d_equal,
                                  assert_panel_equal,
@@ -3683,53 +3684,66 @@ def test_multiple_open_close(self):
             self.assert_('CLOSED' in str(store))
             self.assert_(not store.is_open)
 
-            # multiples
-            store1 = HDFStore(path)
-            store2 = HDFStore(path)
-
-            self.assert_('CLOSED' not in str(store1))
-            self.assert_('CLOSED' not in str(store2))
-            self.assert_(store1.is_open)
-            self.assert_(store2.is_open)
-
-            store1.close()
-            self.assert_('CLOSED' in str(store1))
-            self.assert_(not store1.is_open)
-            self.assert_('CLOSED' not in str(store2))
-            self.assert_(store2.is_open)
-
-            store2.close()
-            self.assert_('CLOSED' in str(store1))
-            self.assert_('CLOSED' in str(store2))
-            self.assert_(not store1.is_open)
-            self.assert_(not store2.is_open)
-
-            # nested close
-            store = HDFStore(path,mode='w')
-            store.append('df',df)
+        with ensure_clean_path(self.path) as path:
 
-            store2 = HDFStore(path)
-            store2.append('df2',df)
-            store2.close()
-            self.assert_('CLOSED' in str(store2))
-            self.assert_(not store2.is_open)
+            if pytables._table_file_open_policy_is_strict:
 
-            store.close()
-            self.assert_('CLOSED' in str(store))
-            self.assert_(not store.is_open)
+                # multiples
+                store1 = HDFStore(path)
+                def f():
+                    HDFStore(path)
+                self.assertRaises(ValueError, f)
+                store1.close()
 
-            # double closing
-            store = HDFStore(path,mode='w')
-            store.append('df', df)
+            else:
 
-            store2 = HDFStore(path)
-            store.close()
-            self.assert_('CLOSED' in str(store))
-            self.assert_(not store.is_open)
+                # multiples
+                store1 = HDFStore(path)
+                store2 = HDFStore(path)
+
+                self.assert_('CLOSED' not in str(store1))
+                self.assert_('CLOSED' not in str(store2))
+                self.assert_(store1.is_open)
+                self.assert_(store2.is_open)
+
+                store1.close()
+                self.assert_('CLOSED' in str(store1))
+                self.assert_(not store1.is_open)
+                self.assert_('CLOSED' not in str(store2))
+                self.assert_(store2.is_open)
+
+                store2.close()
+                self.assert_('CLOSED' in str(store1))
+                self.assert_('CLOSED' in str(store2))
+                self.assert_(not store1.is_open)
+                self.assert_(not store2.is_open)
+
+                # nested close
+                store = HDFStore(path,mode='w')
+                store.append('df',df)
+
+                store2 = HDFStore(path)
+                store2.append('df2',df)
+                store2.close()
+                self.assert_('CLOSED' in str(store2))
+                self.assert_(not store2.is_open)
+
+                store.close()
+                self.assert_('CLOSED' in str(store))
+                self.assert_(not store.is_open)
+
+                # double closing
+                store = HDFStore(path,mode='w')
+                store.append('df', df)
+
+                store2 = HDFStore(path)
+                store.close()
+                self.assert_('CLOSED' in str(store))
+                self.assert_(not store.is_open)
 
-            store2.close()
-            self.assert_('CLOSED' in str(store2))
-            self.assert_(not store2.is_open)
+                store2.close()
+                self.assert_('CLOSED' in str(store2))
+                self.assert_(not store2.is_open)
 
         # ops on a closed store
         with ensure_clean_path(self.path) as path: