Filter the directories/files hashed when using commands with patterns

Author: flora-five

package.py

@@ -243,32 +243,50 @@ def generate_content_hash(source_paths, hash_func=hashlib.sha256, log=None):
 
     if log:
         log = log.getChild("hash")
+    _log = log if log.isEnabledFor(DEBUG3) else None
 
     hash_obj = hash_func()
 
-    for source_path in source_paths:
-        if os.path.isdir(source_path):
-            source_dir = source_path
-            _log = log if log.isEnabledFor(DEBUG3) else None
-            for source_file in list_files(source_dir, log=_log):
+    for source_path, pf, prefix in source_paths:
+        if pf is not None:
+            for path_from_pattern in pf.filter(source_path, prefix):
+                if os.path.isdir(path_from_pattern):
+                    # Hash only the path of the directory
+                    source_dir = path_from_pattern
+                    source_file = None
+                else:
+                    source_dir = os.path.dirname(path_from_pattern)
+                    source_file = os.path.relpath(path_from_pattern, source_dir)
                 update_hash(hash_obj, source_dir, source_file)
                 if log:
-                    log.debug(os.path.join(source_dir, source_file))
+                    log.debug(path_from_pattern)
         else:
-            source_dir = os.path.dirname(source_path)
-            source_file = os.path.relpath(source_path, source_dir)
-            update_hash(hash_obj, source_dir, source_file)
-            if log:
-                log.debug(source_path)
+            if os.path.isdir(source_path):
+                source_dir = source_path
+                for source_file in list_files(source_dir, log=_log):
+                    update_hash(hash_obj, source_dir, source_file)
+                    if log:
+                        log.debug(os.path.join(source_dir, source_file))
+            else:
+                source_dir = os.path.dirname(source_path)
+                source_file = os.path.relpath(source_path, source_dir)
+                update_hash(hash_obj, source_dir, source_file)
+                if log:
+                    log.debug(source_path)
 
     return hash_obj
 
 
-def update_hash(hash_obj, file_root, file_path):
+def update_hash(hash_obj, file_root, file_path=None):
     """
-    Update a hashlib object with the relative path and contents of a file.
+    Update a hashlib object with the relative path and, if the given
+    file_path is not None, its content.
     """
 
+    if file_path is None:
+        hash_obj.update(file_root.encode())
+        return
+
     relative_path = os.path.join(file_root, file_path)
     hash_obj.update(relative_path.encode())
 
@@ -562,7 +580,6 @@ class ZipContentFilter:
     def __init__(self, args):
         self._args = args
         self._rules = None
-        self._excludes = set()
         self._log = logging.getLogger("zip")
 
     def compile(self, patterns):
@@ -668,7 +685,7 @@ def hash(self, extra_paths):
         if not self._source_paths:
             raise ValueError("BuildPlanManager.plan() should be called first")
 
-        content_hash_paths = self._source_paths + extra_paths
+        content_hash_paths = self._source_paths + [(p, None, None) for p in extra_paths]
 
         # Generate a hash based on file names and content. Also use the
         # runtime value, build command, and content of the build paths
@@ -677,7 +694,7 @@ def hash(self, extra_paths):
         content_hash = generate_content_hash(content_hash_paths, log=self._log)
         return content_hash
 
-    def plan(self, source_path, query):
+    def plan(self, source_path, query, log=None):
         claims = source_path
         if not isinstance(source_path, list):
             claims = [source_path]
@@ -686,11 +703,14 @@ def plan(self, source_path, query):
         build_plan = []
         build_step = []
 
+        if log:
+            log = log.getChild("plan")
+
         def step(*x):
             build_step.append(x)
 
-        def hash(path):
-            source_paths.append(path)
+        def hash(path, patterns=None, prefix=None):
+            source_paths.append((path, patterns, prefix))
 
         def pip_requirements_step(path, prefix=None, required=False, tmp_dir=None):
             command = runtime
@@ -759,7 +779,7 @@ def npm_requirements_step(path, prefix=None, required=False, tmp_dir=None):
                 step("npm", runtime, requirements, prefix, tmp_dir)
                 hash(requirements)
 
-        def commands_step(path, commands):
+        def commands_step(path, commands, patterns):
             if not commands:
                 return
 
@@ -773,8 +793,6 @@ def commands_step(path, commands):
             for c in commands:
                 if isinstance(c, str):
                     if c.startswith(":zip"):
-                        if path:
-                            hash(path)
                         if batch:
                             step("sh", "\n".join(batch))
                             batch.clear()
@@ -785,12 +803,18 @@ def commands_step(path, commands):
                             prefix = prefix.strip()
                             _path = os.path.normpath(_path)
                             step("zip:embedded", _path, prefix)
+                            if path:
+                                hash(path, patterns, prefix)
                         elif n == 2:
                             _, _path = c
                             _path = os.path.normpath(_path)
                             step("zip:embedded", _path)
+                            if path:
+                                hash(path, patterns=patterns)
                         elif n == 1:
                             step("zip:embedded")
+                            if path:
+                                hash(path, patterns=patterns)
                         else:
                             raise ValueError(
                                 ":zip invalid call signature, use: "
@@ -829,7 +853,7 @@ def commands_step(path, commands):
                 if patterns:
                     step("set:filter", patterns_list(self._args, patterns))
                 if commands:
-                    commands_step(path, commands)
+                    commands_step(path, commands, patterns)
                 else:
                     prefix = claim.get("prefix_in_zip")
                     pip_requirements = claim.get("pip_requirements")
@@ -883,23 +907,26 @@ def commands_step(path, commands):
                             )
                     if path:
                         step("zip", path, prefix)
-                        if patterns:
-                            # Take patterns into account when computing hash
-                            pf = ZipContentFilter(args=self._args)
-                            pf.compile(patterns)
-
-                            for path_from_pattern in pf.filter(path, prefix):
-                                hash(path_from_pattern)
-                        else:
-                            hash(path)
+                        hash(path, patterns, prefix)
             else:
                 raise ValueError("Unsupported source_path item: {}".format(claim))
 
             if build_step:
                 build_plan.append(build_step)
                 build_step = []
 
-        self._source_paths = source_paths
+        if log.isEnabledFor(DEBUG3):
+            log.debug("source_paths: %s", json.dumps(source_paths, indent=2))
+
+        for p, patterns, prefix in source_paths:
+            if self._source_paths is None:
+                self._source_paths = []
+            pf = None
+            if patterns is not None:
+                pf = ZipContentFilter(args=self._args)
+                pf.compile(patterns)
+            self._source_paths.append((p, pf, prefix))
+
         return build_plan
 
     def execute(self, build_plan, zip_stream, query):
@@ -1713,7 +1740,7 @@ def prepare_command(args):
     docker = query.docker
 
     bpm = BuildPlanManager(args, log=log)
-    build_plan = bpm.plan(source_path, query)
+    build_plan = bpm.plan(source_path, query, log)
 
     if log.isEnabledFor(DEBUG2):
         log.debug("BUILD_PLAN: %s", json.dumps(build_plan, indent=2))