Create MypyStatusItem

Author: dmtucker

src/pytest_mypy.py

@@ -90,6 +90,24 @@ def pytest_collect_file(path, parent):
     return None
 
 
+@pytest.hookimpl(hookwrapper=True, trylast=True)
+def pytest_collection_modifyitems(session, config, items):
+    """
+    Add a MypyStatusItem if any MypyFileItems were collected.
+
+    Since mypy might check files that were not collected,
+    pytest could pass even though mypy failed!
+    To prevent that, add an explicit check for the mypy exit status.
+
+    This should execute as late as possible to avoid missing any
+    MypyFileItems injected by other pytest_collection_modifyitems
+    implementations.
+    """
+    yield
+    if any(isinstance(item, MypyFileItem) for item in items):
+        items.append(MypyStatusItem(nodeid_name, session, config, session))
+
+
 class MypyItem(pytest.Item):
 
     """A Mypy-related test Item."""
@@ -131,6 +149,21 @@ def reportinfo(self):
         )
 
 
+class MypyStatusItem(MypyItem):
+
+    """A check for a non-zero mypy exit status."""
+
+    def runtest(self):
+        """Raise a MypyError if mypy exited with a non-zero status."""
+        results = _mypy_results(self.session)
+        if results['status']:
+            raise MypyError(
+                'mypy exited with status {status}.'.format(
+                    status=results['status'],
+                ),
+            )
+
+
 def _mypy_results(session):
     """Get the cached mypy results for the session, or generate them."""
     return _cached_json_results(

tests/test_pytest_mypy.py

@@ -29,7 +29,10 @@ def pyfunc(x: int) -> int:
     result = testdir.runpytest_subprocess(*xdist_args)
     result.assert_outcomes()
     result = testdir.runpytest_subprocess('--mypy', *xdist_args)
-    result.assert_outcomes(passed=pyfile_count)
+    mypy_file_checks = pyfile_count
+    mypy_status_check = 1
+    mypy_checks = mypy_file_checks + mypy_status_check
+    result.assert_outcomes(passed=mypy_checks)
     assert result.ret == 0
 
 
@@ -42,7 +45,10 @@ def pyfunc(x: int) -> str:
     result = testdir.runpytest_subprocess(*xdist_args)
     result.assert_outcomes()
     result = testdir.runpytest_subprocess('--mypy', *xdist_args)
-    result.assert_outcomes(failed=1)
+    mypy_file_checks = 1
+    mypy_status_check = 1
+    mypy_checks = mypy_file_checks + mypy_status_check
+    result.assert_outcomes(failed=mypy_checks)
     result.stdout.fnmatch_lines([
         '2: error: Incompatible return value*',
     ])
@@ -62,7 +68,10 @@ def test_mypy_ignore_missings_imports(testdir, xdist_args):
             pass
     '''.format(module_name=module_name))
     result = testdir.runpytest_subprocess('--mypy', *xdist_args)
-    result.assert_outcomes(failed=1)
+    mypy_file_checks = 1
+    mypy_status_check = 1
+    mypy_checks = mypy_file_checks + mypy_status_check
+    result.assert_outcomes(failed=mypy_checks)
     result.stdout.fnmatch_lines([
         "2: error: Cannot find *module named '{module_name}'".format(
             module_name=module_name,
@@ -73,7 +82,7 @@ def test_mypy_ignore_missings_imports(testdir, xdist_args):
         '--mypy-ignore-missing-imports',
         *xdist_args
     )
-    result.assert_outcomes(passed=1)
+    result.assert_outcomes(passed=mypy_checks)
     assert result.ret == 0
 
 
@@ -84,10 +93,14 @@ def test_fails():
             assert False
     ''')
     result = testdir.runpytest_subprocess('--mypy', *xdist_args)
-    result.assert_outcomes(failed=1, passed=1)
+    test_count = 1
+    mypy_file_checks = 1
+    mypy_status_check = 1
+    mypy_checks = mypy_file_checks + mypy_status_check
+    result.assert_outcomes(failed=test_count, passed=mypy_checks)
     assert result.ret != 0
     result = testdir.runpytest_subprocess('--mypy', '-m', 'mypy', *xdist_args)
-    result.assert_outcomes(passed=1)
+    result.assert_outcomes(passed=mypy_checks)
     assert result.ret == 0
 
 
@@ -107,7 +120,12 @@ def runtest(self):
     result = testdir.runpytest_subprocess(*xdist_args)
     result.assert_outcomes()
     result = testdir.runpytest_subprocess('--mypy', *xdist_args)
-    result.assert_outcomes(failed=1)
+    mypy_file_checks = 1  # conftest.py
+    mypy_status_check = 1
+    result.assert_outcomes(
+        failed=mypy_file_checks,  # patched to raise an Exception
+        passed=mypy_status_check,  # conftest.py has no type errors.
+    )
     result.stdout.fnmatch_lines(['*' + message])
     assert result.ret != 0
 
@@ -171,6 +189,9 @@ def test_pytest_collection_modifyitems(testdir, xdist_args):
     Verify that collected files which are removed in a
     pytest_collection_modifyitems implementation are not
     checked by mypy.
+
+    This would also fail if a MypyStatusItem were injected
+    despite there being no MypyFileItems.
     """
     testdir.makepyfile(conftest='''
         def pytest_collection_modifyitems(session, config, items):
@@ -190,5 +211,49 @@ def test_pass():
             pass
     ''')
     result = testdir.runpytest_subprocess('--mypy', *xdist_args)
-    result.assert_outcomes(passed=1)
+    test_count = 1
+    result.assert_outcomes(passed=test_count)
     assert result.ret == 0
+
+
+def test_mypy_indirect(testdir, xdist_args):
+    """Verify that uncollected files checked by mypy cause a failure."""
+    testdir.makepyfile(bad='''
+        def pyfunc(x: int) -> str:
+            return x * 2
+    ''')
+    testdir.makepyfile(good='''
+        import bad
+    ''')
+    xdist_args.append('good.py')  # Nothing may come after xdist_args in py34.
+    result = testdir.runpytest_subprocess('--mypy', *xdist_args)
+    assert result.ret != 0
+
+
+def test_mypy_indirect_inject(testdir, xdist_args):
+    """
+    Verify that uncollected files checked by mypy because of a MypyFileItem
+    injected in pytest_collection_modifyitems cause a failure.
+    """
+    testdir.makepyfile(bad='''
+        def pyfunc(x: int) -> str:
+            return x * 2
+    ''')
+    testdir.makepyfile(good='''
+        import bad
+    ''')
+    testdir.makepyfile(conftest='''
+        import py
+        import pytest
+
+        @pytest.hookimpl(trylast=True)  # Inject as late as possible.
+        def pytest_collection_modifyitems(session, config, items):
+            plugin = config.pluginmanager.getplugin('mypy')
+            items.append(
+                plugin.MypyFileItem(py.path.local('good.py'), session),
+            )
+    ''')
+    testdir.mkdir('empty')
+    xdist_args.append('empty')  # Nothing may come after xdist_args in py34.
+    result = testdir.runpytest_subprocess('--mypy', *xdist_args)
+    assert result.ret != 0