BUG: hashtable memory error causes test_factorize_nan crash

Author: Mike Kelly

doc/source/release.rst

@@ -527,6 +527,7 @@ Bug Fixes
   (:issue:`7105`)
 - Bug in ``DatetimeIndex`` specifying ``freq`` raises ``ValueError`` when passed value is too short (:issue:`7098`)
 - Fixed a bug with the `info` repr not honoring the `display.max_info_columns` setting (:issue:`6939`)
+- Fixed a memory error in the hashtable implementation that caused crashes while running tests (:issue:`7157`)
 
 pandas 0.13.1
 -------------

pandas/hashtable.pyx

@@ -66,11 +66,12 @@ cdef class ObjectVector:
 
     def to_array(self):
         self.ao.resize(self.n)
+        self.m = self.n
         return self.ao
 
     cdef inline append(self, object o):
         if self.n == self.m:
-            self.m = self.m * 2
+            self.m = max(self.m * 2, _INIT_VEC_CAP)
             self.ao.resize(self.m)
             self.data = <PyObject**> self.ao.data
 
@@ -97,11 +98,12 @@ cdef class Int64Vector:
 
     def to_array(self):
         self.ao.resize(self.n)
+        self.m = self.n
         return self.ao
 
     cdef inline append(self, int64_t x):
         if self.n == self.m:
-            self.m = self.m * 2
+            self.m = max(self.m * 2, _INIT_VEC_CAP)
             self.ao.resize(self.m)
             self.data = <int64_t*> self.ao.data
 
@@ -126,11 +128,12 @@ cdef class Float64Vector:
 
     def to_array(self):
         self.ao.resize(self.n)
+        self.m = self.n
         return self.ao
 
     cdef inline append(self, float64_t x):
         if self.n == self.m:
-            self.m = self.m * 2
+            self.m = max(self.m * 2, _INIT_VEC_CAP)
             self.ao.resize(self.m)
             self.data = <float64_t*> self.ao.data
 

pandas/tests/test_hashtable.py

@@ -5,6 +5,7 @@
 import pandas as pd
 
 class TestFactorizer(unittest.TestCase):
+
     def test_factorize_nan(self):
         # nan should map to na_sentinel, not reverse_indexer[na_sentinel]
         # rizer.factorize should not raise an exception if na_sentinel indexes
@@ -25,6 +26,29 @@ def test_factorize_nan(self):
         self.assertEqual(len(set(key)), len(set(expected)))
         self.assert_(np.array_equal(pd.isnull(key), expected == na_sentinel))        
 
+    def test_vector_resize(self):
+        # Test for memory errors after internal vector
+        # reallocations (pull request #7157)
+
+        def _test_vector_resize(htable, uniques, dtype, nvals):
+            vals = np.array(np.random.randn(1000), dtype=dtype)
+            # get_labels appends to the vector
+            htable.get_labels(vals[:nvals], uniques, 0, -1)
+            # to_array resizes the vector
+            uniques.to_array()
+            htable.get_labels(vals, uniques, 0, -1)
+
+        test_cases = [
+            (_hash.PyObjectHashTable, _hash.ObjectVector, 'object'),
+            (_hash.Float64HashTable, _hash.Float64Vector, 'float64'),
+            (_hash.Int64HashTable, _hash.Int64Vector, 'int64')]
+
+        for (tbl, vect, dtype) in test_cases:
+            # resizing to empty is a special case
+            _test_vector_resize(tbl(), vect(), dtype, 0)
+            _test_vector_resize(tbl(), vect(), dtype, 10)
+
+
 if __name__ == '__main__':
     nose.runmodule(argv=[__file__, '-vvs', '-x', '--pdb', '--pdb-failure'],
                    exit=False)