Merge pull request #6109 from cpcloud/ne-version

TST: add tests for old version of numexpr

Author: cpcloud

ci/requirements-2.6.txt

@@ -5,3 +5,4 @@ pytz==2013b
 http://www.crummy.com/software/BeautifulSoup/bs4/download/4.2/beautifulsoup4-4.2.0.tar.gz
 html5lib==1.0b2
 bigquery==2.0.17
+numexpr==1.4.2

ci/script.sh

@@ -5,9 +5,12 @@ echo "inside $0"
 if [ -n "$LOCALE_OVERRIDE" ]; then
     export LC_ALL="$LOCALE_OVERRIDE";
     echo "Setting LC_ALL to $LOCALE_OVERRIDE"
+    curdir="$(pwd)"
+    cd /tmp
     pycmd='import pandas; print("pandas detected console encoding: %s" % pandas.get_option("display.encoding"))'
     python -c "$pycmd"
+    cd "$curdir"
 fi
 
-echo nosetests -v --exe -w /tmp -A "$NOSE_ARGS" pandas --with-xunit --xunit-file=/tmp/nosetests.xml
-nosetests -v --exe -w /tmp -A "$NOSE_ARGS" pandas --with-xunit --xunit-file=/tmp/nosetests.xml
+echo nosetests --exe -w /tmp -A "$NOSE_ARGS" pandas --with-xunit --xunit-file=/tmp/nosetests.xml
+nosetests --exe -w /tmp -A "$NOSE_ARGS" pandas --with-xunit --xunit-file=/tmp/nosetests.xml

pandas/computation/eval.py

@@ -3,13 +3,11 @@
 """Top level ``eval`` module.
 """
 
-import numbers
-import numpy as np
 
 from pandas.core import common as com
-from pandas.compat import string_types
 from pandas.computation.expr import Expr, _parsers, _ensure_scope
 from pandas.computation.engines import _engines
+from distutils.version import LooseVersion
 
 
 def _check_engine(engine):
@@ -38,7 +36,13 @@ def _check_engine(engine):
             import numexpr
         except ImportError:
             raise ImportError("'numexpr' not found. Cannot use "
-                              "engine='numexpr' if 'numexpr' is not installed")
+                              "engine='numexpr' for query/eval "
+                              "if 'numexpr' is not installed")
+        else:
+            ne_version = numexpr.__version__
+            if ne_version < LooseVersion('2.0'):
+                raise ImportError("'numexpr' version is %s, "
+                                  "must be >= 2.0" % ne_version)
 
 
 def _check_parser(parser):

pandas/computation/expressions.py

@@ -8,10 +8,11 @@
 
 import numpy as np
 from pandas.core.common import _values_from_object
+from distutils.version import LooseVersion
 
 try:
     import numexpr as ne
-    _NUMEXPR_INSTALLED = True
+    _NUMEXPR_INSTALLED = ne.__version__ >= LooseVersion('2.0')
 except ImportError:  # pragma: no cover
     _NUMEXPR_INSTALLED = False
 

pandas/computation/tests/test_eval.py

@@ -2,6 +2,7 @@
 
 import functools
 from itertools import product
+from distutils.version import LooseVersion
 
 import nose
 from nose.tools import assert_raises, assert_true, assert_false, assert_equal
@@ -17,13 +18,13 @@
 from pandas.util.testing import makeCustomDataframe as mkdf
 
 from pandas.computation import pytables
-from pandas.computation.expressions import _USE_NUMEXPR
 from pandas.computation.engines import _engines
 from pandas.computation.expr import PythonExprVisitor, PandasExprVisitor
-from pandas.computation.ops import (_binary_ops_dict, _unary_ops_dict,
+from pandas.computation.ops import (_binary_ops_dict,
                                     _special_case_arith_ops_syms,
                                     _arith_ops_syms, _bool_ops_syms)
 from pandas.computation.common import NameResolutionError
+
 import pandas.computation.expr as expr
 import pandas.util.testing as tm
 from pandas.util.testing import (assert_frame_equal, randbool,
@@ -35,11 +36,6 @@
 _scalar_skip = 'in', 'not in'
 
 
-def skip_if_no_ne(engine='numexpr'):
-    if not _USE_NUMEXPR and engine == 'numexpr':
-        raise nose.SkipTest("numexpr engine not installed or disabled")
-
-
 def engine_has_neg_frac(engine):
     return _engines[engine].has_neg_frac
 
@@ -108,7 +104,7 @@ class TestEvalNumexprPandas(tm.TestCase):
     @classmethod
     def setUpClass(cls):
         super(TestEvalNumexprPandas, cls).setUpClass()
-        skip_if_no_ne()
+        tm.skip_if_no_ne()
         import numexpr as ne
         cls.ne = ne
         cls.engine = 'numexpr'
@@ -426,7 +422,7 @@ def check_single_invert_op(self, lhs, cmp1, rhs):
             assert_array_equal(expected, result)
 
             for engine in self.current_engines:
-                skip_if_no_ne(engine)
+                tm.skip_if_no_ne(engine)
                 assert_array_equal(result, pd.eval('~elb', engine=engine,
                                                    parser=self.parser))
 
@@ -457,7 +453,7 @@ def check_compound_invert_op(self, lhs, cmp1, rhs):
 
             # make sure the other engines work the same as this one
             for engine in self.current_engines:
-                skip_if_no_ne(engine)
+                tm.skip_if_no_ne(engine)
                 ev = pd.eval(ex, engine=self.engine, parser=self.parser)
                 assert_array_equal(ev, result)
 
@@ -709,7 +705,7 @@ class TestEvalNumexprPython(TestEvalNumexprPandas):
     @classmethod
     def setUpClass(cls):
         super(TestEvalNumexprPython, cls).setUpClass()
-        skip_if_no_ne()
+        tm.skip_if_no_ne()
         import numexpr as ne
         cls.ne = ne
         cls.engine = 'numexpr'
@@ -788,7 +784,7 @@ class TestAlignment(object):
     lhs_index_types = index_types + ('s',)  # 'p'
 
     def check_align_nested_unary_op(self, engine, parser):
-        skip_if_no_ne(engine)
+        tm.skip_if_no_ne(engine)
         s = 'df * ~2'
         df = mkdf(5, 3, data_gen_f=f)
         res = pd.eval(s, engine=engine, parser=parser)
@@ -799,7 +795,7 @@ def test_align_nested_unary_op(self):
             yield self.check_align_nested_unary_op, engine, parser
 
     def check_basic_frame_alignment(self, engine, parser):
-        skip_if_no_ne(engine)
+        tm.skip_if_no_ne(engine)
         args = product(self.lhs_index_types, self.index_types,
                        self.index_types)
         for lr_idx_type, rr_idx_type, c_idx_type in args:
@@ -815,7 +811,7 @@ def test_basic_frame_alignment(self):
             yield self.check_basic_frame_alignment, engine, parser
 
     def check_frame_comparison(self, engine, parser):
-        skip_if_no_ne(engine)
+        tm.skip_if_no_ne(engine)
         args = product(self.lhs_index_types, repeat=2)
         for r_idx_type, c_idx_type in args:
             df = mkdf(10, 10, data_gen_f=f, r_idx_type=r_idx_type,
@@ -833,7 +829,7 @@ def test_frame_comparison(self):
             yield self.check_frame_comparison, engine, parser
 
     def check_medium_complex_frame_alignment(self, engine, parser):
-        skip_if_no_ne(engine)
+        tm.skip_if_no_ne(engine)
         args = product(self.lhs_index_types, self.index_types,
                        self.index_types, self.index_types)
 
@@ -850,7 +846,7 @@ def test_medium_complex_frame_alignment(self):
             yield self.check_medium_complex_frame_alignment, engine, parser
 
     def check_basic_frame_series_alignment(self, engine, parser):
-        skip_if_no_ne(engine)
+        tm.skip_if_no_ne(engine)
 
         def testit(r_idx_type, c_idx_type, index_name):
             df = mkdf(10, 10, data_gen_f=f, r_idx_type=r_idx_type,
@@ -878,7 +874,7 @@ def test_basic_frame_series_alignment(self):
             yield self.check_basic_frame_series_alignment, engine, parser
 
     def check_basic_series_frame_alignment(self, engine, parser):
-        skip_if_no_ne(engine)
+        tm.skip_if_no_ne(engine)
 
         def testit(r_idx_type, c_idx_type, index_name):
             df = mkdf(10, 7, data_gen_f=f, r_idx_type=r_idx_type,
@@ -911,7 +907,7 @@ def test_basic_series_frame_alignment(self):
             yield self.check_basic_series_frame_alignment, engine, parser
 
     def check_series_frame_commutativity(self, engine, parser):
-        skip_if_no_ne(engine)
+        tm.skip_if_no_ne(engine)
         args = product(self.lhs_index_types, self.index_types, ('+', '*'),
                        ('index', 'columns'))
         for r_idx_type, c_idx_type, op, index_name in args:
@@ -934,7 +930,7 @@ def test_series_frame_commutativity(self):
             yield self.check_series_frame_commutativity, engine, parser
 
     def check_complex_series_frame_alignment(self, engine, parser):
-        skip_if_no_ne(engine)
+        tm.skip_if_no_ne(engine)
 
         import random
         args = product(self.lhs_index_types, self.index_types,
@@ -978,7 +974,7 @@ def test_complex_series_frame_alignment(self):
             yield self.check_complex_series_frame_alignment, engine, parser
 
     def check_performance_warning_for_poor_alignment(self, engine, parser):
-        skip_if_no_ne(engine)
+        tm.skip_if_no_ne(engine)
         df = DataFrame(randn(1000, 10))
         s = Series(randn(10000))
         if engine == 'numexpr':
@@ -1034,7 +1030,7 @@ class TestOperationsNumExprPandas(tm.TestCase):
     @classmethod
     def setUpClass(cls):
         super(TestOperationsNumExprPandas, cls).setUpClass()
-        skip_if_no_ne()
+        tm.skip_if_no_ne()
         cls.engine = 'numexpr'
         cls.parser = 'pandas'
         cls.arith_ops = expr._arith_ops_syms + expr._cmp_ops_syms
@@ -1194,7 +1190,7 @@ def test_assignment_fails(self):
                           local_dict={'df': df, 'df2': df2})
 
     def test_assignment_column(self):
-        skip_if_no_ne('numexpr')
+        tm.skip_if_no_ne('numexpr')
         df = DataFrame(np.random.randn(5, 2), columns=list('ab'))
         orig_df = df.copy()
 
@@ -1345,10 +1341,9 @@ class TestOperationsNumExprPython(TestOperationsNumExprPandas):
     @classmethod
     def setUpClass(cls):
         super(TestOperationsNumExprPython, cls).setUpClass()
-        if not _USE_NUMEXPR:
-            raise nose.SkipTest("numexpr engine not installed")
         cls.engine = 'numexpr'
         cls.parser = 'python'
+        tm.skip_if_no_ne(cls.engine)
         cls.arith_ops = expr._arith_ops_syms + expr._cmp_ops_syms
         cls.arith_ops = filter(lambda x: x not in ('in', 'not in'),
                                cls.arith_ops)
@@ -1435,7 +1430,7 @@ def setUpClass(cls):
 class TestScope(object):
 
     def check_global_scope(self, e, engine, parser):
-        skip_if_no_ne(engine)
+        tm.skip_if_no_ne(engine)
         assert_array_equal(_var_s * 2, pd.eval(e, engine=engine,
                                                parser=parser))
 
@@ -1445,7 +1440,7 @@ def test_global_scope(self):
             yield self.check_global_scope, e, engine, parser
 
     def check_no_new_locals(self, engine, parser):
-        skip_if_no_ne(engine)
+        tm.skip_if_no_ne(engine)
         x = 1
         lcls = locals().copy()
         pd.eval('x + 1', local_dict=lcls, engine=engine, parser=parser)
@@ -1458,7 +1453,7 @@ def test_no_new_locals(self):
             yield self.check_no_new_locals, engine, parser
 
     def check_no_new_globals(self, engine, parser):
-        skip_if_no_ne(engine)
+        tm.skip_if_no_ne(engine)
         x = 1
         gbls = globals().copy()
         pd.eval('x + 1', engine=engine, parser=parser)
@@ -1471,21 +1466,21 @@ def test_no_new_globals(self):
 
 
 def test_invalid_engine():
-    skip_if_no_ne()
+    tm.skip_if_no_ne()
     assertRaisesRegexp(KeyError, 'Invalid engine \'asdf\' passed',
                        pd.eval, 'x + y', local_dict={'x': 1, 'y': 2},
                        engine='asdf')
 
 
 def test_invalid_parser():
-    skip_if_no_ne()
+    tm.skip_if_no_ne()
     assertRaisesRegexp(KeyError, 'Invalid parser \'asdf\' passed',
                        pd.eval, 'x + y', local_dict={'x': 1, 'y': 2},
                        parser='asdf')
 
 
 def check_is_expr_syntax(engine):
-    skip_if_no_ne(engine)
+    tm.skip_if_no_ne(engine)
     s = 1
     valid1 = 's + 1'
     valid2 = '__y + _xx'
@@ -1494,7 +1489,7 @@ def check_is_expr_syntax(engine):
 
 
 def check_is_expr_names(engine):
-    skip_if_no_ne(engine)
+    tm.skip_if_no_ne(engine)
     r, s = 1, 2
     valid = 's + r'
     invalid = '__y + __x'
@@ -1517,7 +1512,7 @@ def test_is_expr_names():
 
 
 def check_disallowed_nodes(engine, parser):
-    skip_if_no_ne(engine)
+    tm.skip_if_no_ne(engine)
     VisitorClass = _parsers[parser]
     uns_ops = VisitorClass.unsupported_nodes
     inst = VisitorClass('x + 1', engine, parser)
@@ -1532,7 +1527,7 @@ def test_disallowed_nodes():
 
 
 def check_syntax_error_exprs(engine, parser):
-    skip_if_no_ne(engine)
+    tm.skip_if_no_ne(engine)
     e = 's +'
     assert_raises(SyntaxError, pd.eval, e, engine=engine, parser=parser)
 
@@ -1543,7 +1538,7 @@ def test_syntax_error_exprs():
 
 
 def check_name_error_exprs(engine, parser):
-    skip_if_no_ne(engine)
+    tm.skip_if_no_ne(engine)
     e = 's + t'
     assert_raises(NameError, pd.eval, e, engine=engine, parser=parser)
 
@@ -1553,6 +1548,33 @@ def test_name_error_exprs():
         yield check_name_error_exprs, engine, parser
 
 
+def check_invalid_numexpr_version(engine, parser):
+    def testit():
+        a, b = 1, 2
+        res = pd.eval('a + b', engine=engine, parser=parser)
+        tm.assert_equal(res, 3)
+
+    if engine == 'numexpr':
+        try:
+            import numexpr as ne
+        except ImportError:
+            raise nose.SkipTest("no numexpr")
+        else:
+            if ne.__version__ < LooseVersion('2.0'):
+                with tm.assertRaisesRegexp(ImportError, "'numexpr' version is "
+                                           ".+, must be >= 2.0"):
+                    testit()
+            else:
+                testit()
+    else:
+        testit()
+
+
+def test_invalid_numexpr_version():
+    for engine, parser in ENGINES_PARSERS:
+        yield check_invalid_numexpr_version, engine, parser
+
+
 if __name__ == '__main__':
     nose.runmodule(argv=[__file__, '-vvs', '-x', '--pdb', '--pdb-failure'],
                    exit=False)