Add basic python logging to pyt with -v -vv -vvv

Very basic python logging added to pyt.

Very useful when you want to see:

-  which files are being processed
-  if your imports are not being inspected
-  which file crashed pyt

Author: bcaller

pyt/__main__.py

@@ -1,5 +1,6 @@
 """The comand line module of PyT."""
 
+import logging
 import os
 import sys
 from collections import defaultdict
@@ -26,6 +27,8 @@
     is_function_without_leading_
 )
 
+log = logging.getLogger(__name__)
+
 
 def discover_files(targets, excluded_files, recursive=False):
     included_files = list()
@@ -37,11 +40,13 @@ def discover_files(targets, excluded_files, recursive=False):
                     if file.endswith('.py') and file not in excluded_list:
                         fullpath = os.path.join(root, file)
                         included_files.append(fullpath)
+                        log.debug('Discovered file: %s', fullpath)
                 if not recursive:
                     break
         else:
             if target not in excluded_list:
                 included_files.append(target)
+                log.debug('Discovered file: %s', target)
     return included_files
 
 
@@ -60,6 +65,14 @@ def retrieve_nosec_lines(
 def main(command_line_args=sys.argv[1:]):  # noqa: C901
     args = parse_args(command_line_args)
 
+    logging_level = (
+        logging.ERROR if not args.verbose else
+        logging.WARN if args.verbose == 1 else
+        logging.INFO if args.verbose == 2 else
+        logging.DEBUG
+    )
+    logging.basicConfig(level=logging_level, format='[%(levelname)s] %(name)s: %(message)s')
+
     files = discover_files(
         args.targets,
         args.excluded_paths,
@@ -74,6 +87,7 @@ def main(command_line_args=sys.argv[1:]):  # noqa: C901
 
     cfg_list = list()
     for path in sorted(files):
+        log.info("Processing %s", path)
         if not args.ignore_nosec:
             nosec_lines[path] = retrieve_nosec_lines(path)
 

pyt/cfg/stmt_visitor.py

@@ -1,6 +1,8 @@
 import ast
 import itertools
+import logging
 import os.path
+from pkgutil import iter_modules
 
 from .alias_helper import (
     as_alias_handler,
@@ -52,6 +54,9 @@
     remove_breaks
 )
 
+log = logging.getLogger(__name__)
+uninspectable_modules = {module.name for module in iter_modules()}  # Don't warn about failing to import these
+
 
 class StmtVisitor(ast.NodeVisitor):
     def __init__(self, allow_local_directory_imports=True):
@@ -429,9 +434,12 @@ def visit_Assign(self, node):
             else:
                 label = LabelVisitor()
                 label.visit(node)
-                print('Assignment not properly handled.',
-                      'Could result in not finding a vulnerability.',
-                      'Assignment:', label.result)
+                log.warn(
+                    'Assignment not properly handled in %s. Could result in not finding a vulnerability.'
+                    'Assignment: %s',
+                    getattr(self, 'filenames', ['?'])[0],
+                    self.label.result,
+                )
                 return self.append_node(AssignmentNode(
                     label.result,
                     label.result,
@@ -1022,6 +1030,10 @@ def visit_Import(self, node):
                         name.asname,
                         retrieve_import_alias_mapping(node.names)
                     )
+        for alias in node.names:
+            if alias.name not in uninspectable_modules:
+                log.warn("Cannot inspect module %s", alias.name)
+                uninspectable_modules.add(alias.name)  # Don't repeatedly warn about this
         return IgnoredNode()
 
     def visit_ImportFrom(self, node):
@@ -1061,4 +1073,7 @@ def visit_ImportFrom(self, node):
                         retrieve_import_alias_mapping(node.names),
                         from_from=True
                     )
+        if node.module not in uninspectable_modules:
+            log.warn("Cannot inspect module %s", node.module)
+            uninspectable_modules.add(node.module)
         return IgnoredNode()

pyt/core/ast_helper.py

@@ -2,25 +2,25 @@
 Useful when working with the ast module."""
 
 import ast
+import logging
 import os
 import subprocess
 from functools import lru_cache
 
 from .transformer import PytTransformer
 
-
+log = logging.getLogger(__name__)
 BLACK_LISTED_CALL_NAMES = ['self']
 recursive = False
 
 
 def _convert_to_3(path):  # pragma: no cover
     """Convert python 2 file to python 3."""
     try:
-        print('##### Trying to convert file to Python 3. #####')
+        log.warn('##### Trying to convert %s to Python 3. #####', path)
         subprocess.call(['2to3', '-w', path])
     except subprocess.SubprocessError:
-        print('Check if 2to3 is installed. '
-              'https://docs.python.org/2/library/2to3.html')
+        log.exception('Check if 2to3 is installed. https://docs.python.org/2/library/2to3.html')
         exit(1)
 
 

pyt/usage.py

@@ -30,7 +30,11 @@ def _add_required_group(parser):
 
 def _add_optional_group(parser):
     optional_group = parser.add_argument_group('optional arguments')
-
+    optional_group.add_argument(
+        '-v', '--verbose',
+        action='count',
+        help='Increase logging verbosity. Can repeated e.g. -vvv',
+    )
     optional_group.add_argument(
         '-a', '--adaptor',
         help='Choose a web framework adaptor: '

tests/usage_test.py

@@ -25,7 +25,7 @@ def test_no_args(self):
 
         self.maxDiff = None
 
-        EXPECTED = """usage: python -m pyt [-h] [-a ADAPTOR] [-pr PROJECT_ROOT]
+        EXPECTED = """usage: python -m pyt [-h] [-v] [-a ADAPTOR] [-pr PROJECT_ROOT]
                      [-b BASELINE_JSON_FILE] [-t TRIGGER_WORD_FILE]
                      [-m BLACKBOX_MAPPING_FILE] [-i] [-o OUTPUT_FILE]
                      [--ignore-nosec] [-r] [-x EXCLUDED_PATHS]
@@ -36,6 +36,7 @@ def test_no_args(self):
   targets               source file(s) or directory(s) to be scanned
 
 optional arguments:
+  -v, --verbose         Increase logging verbosity. Can repeated e.g. -vvv
   -a ADAPTOR, --adaptor ADAPTOR
                         Choose a web framework adaptor: Flask(Default),
                         Django, Every or Pylons
@@ -74,7 +75,7 @@ def test_valid_args_but_no_targets(self):
             with capture_sys_output() as (_, stderr):
                 parse_args(['-j'])
 
-        EXPECTED = """usage: python -m pyt [-h] [-a ADAPTOR] [-pr PROJECT_ROOT]
+        EXPECTED = """usage: python -m pyt [-h] [-v] [-a ADAPTOR] [-pr PROJECT_ROOT]
                      [-b BASELINE_JSON_FILE] [-t TRIGGER_WORD_FILE]
                      [-m BLACKBOX_MAPPING_FILE] [-i] [-o OUTPUT_FILE]
                      [--ignore-nosec] [-r] [-x EXCLUDED_PATHS]