Merge pull request #85 from gethari/crawling-progress

#feat: Report progress of the crawling #64

Author: zachary62

nodes.py

@@ -61,12 +61,23 @@ def exec(self, prep_res):
             )
         else:
             print(f"Crawling directory: {prep_res['local_dir']}...")
+
+            def progress_callback(processed, total):
+                percentage = (processed / total) * 100 if total > 0 else 0
+                rounded_percentage = int(percentage)
+                if rounded_percentage > progress_callback.last_reported:
+                    progress_callback.last_reported = rounded_percentage
+                    print(f"\033[92mProgress: {processed}/{total} files ({rounded_percentage}%)\033[0m")
+
+            progress_callback.last_reported = -1
+
             result = crawl_local_files(
                 directory=prep_res["local_dir"],
                 include_patterns=prep_res["include_patterns"],
                 exclude_patterns=prep_res["exclude_patterns"],
                 max_file_size=prep_res["max_file_size"],
                 use_relative_paths=prep_res["use_relative_paths"],
+                progress_callback=progress_callback
             )
 
         # Convert dict to list of tuples: [(path, content), ...]

utils/crawl_local_files.py

@@ -2,13 +2,13 @@
 import fnmatch
 import pathspec
 
-
 def crawl_local_files(
     directory,
     include_patterns=None,
     exclude_patterns=None,
     max_file_size=None,
     use_relative_paths=True,
+    progress_callback=None,
 ):
     """
     Crawl files in a local directory with similar interface as crawl_github_files.
@@ -18,6 +18,7 @@ def crawl_local_files(
         exclude_patterns (set): File patterns to exclude (e.g. {"tests/*"})
         max_file_size (int): Maximum file size in bytes
         use_relative_paths (bool): Whether to use paths relative to directory
+        progress_callback (callable): Function to report progress, takes (processed, total) as arguments
 
     Returns:
         dict: {"files": {filepath: content}}
@@ -34,91 +35,84 @@ def crawl_local_files(
         try:
             with open(gitignore_path, "r", encoding="utf-8") as f:
                 gitignore_patterns = f.readlines()
-            gitignore_spec = pathspec.PathSpec.from_lines(
-                "gitwildmatch", gitignore_patterns
-            )
+            gitignore_spec = pathspec.PathSpec.from_lines("gitwildmatch", gitignore_patterns)
             print(f"Loaded .gitignore patterns from {gitignore_path}")
         except Exception as e:
-            print(
-                f"Warning: Could not read or parse .gitignore file {gitignore_path}: {e}"
-            )
-    # --- End Load .gitignore ---
+            print(f"Warning: Could not read or parse .gitignore file {gitignore_path}: {e}")
 
+    all_files = []
     for root, dirs, files in os.walk(directory):
-        # Filter directories using .gitignore and exclude_patterns early to avoid descending
-        # Need to process dirs list *in place* for os.walk to respect it
+        # Filter directories using .gitignore and exclude_patterns early
         excluded_dirs = set()
         for d in dirs:
             dirpath_rel = os.path.relpath(os.path.join(root, d), directory)
 
-            # Check against .gitignore (important for directories)
             if gitignore_spec and gitignore_spec.match_file(dirpath_rel):
                 excluded_dirs.add(d)
-                continue  # Skip further checks if gitignored
+                continue
 
-            # Check against standard exclude_patterns
             if exclude_patterns:
                 for pattern in exclude_patterns:
-                    # Match pattern against full relative path or directory name itself
-                    if fnmatch.fnmatch(dirpath_rel, pattern) or fnmatch.fnmatch(
-                        d, pattern
-                    ):
+                    if fnmatch.fnmatch(dirpath_rel, pattern) or fnmatch.fnmatch(d, pattern):
                         excluded_dirs.add(d)
                         break
 
-        # Modify dirs in-place: remove excluded ones
-        # Iterate over a copy (.copy()) because we are modifying the list during iteration
         for d in dirs.copy():
             if d in excluded_dirs:
                 dirs.remove(d)
 
-        # Now process files in the non-excluded directories
         for filename in files:
             filepath = os.path.join(root, filename)
+            all_files.append(filepath)
+
+    total_files = len(all_files)
+    processed_files = 0
+
+    for filepath in all_files:
+        relpath = os.path.relpath(filepath, directory) if use_relative_paths else filepath
+
+        # --- Exclusion check ---
+        excluded = False
+        if gitignore_spec and gitignore_spec.match_file(relpath):
+            excluded = True
+
+        if not excluded and exclude_patterns:
+            for pattern in exclude_patterns:
+                if fnmatch.fnmatch(relpath, pattern):
+                    excluded = True
+                    break
+
+        included = False
+        if include_patterns:
+            for pattern in include_patterns:
+                if fnmatch.fnmatch(relpath, pattern):
+                    included = True
+                    break
+        else:
+            included = True
+
+        if not included or excluded:
+            processed_files += 1
+            if progress_callback:
+                progress_callback(processed_files, total_files)
+            continue
+
+        if max_file_size and os.path.getsize(filepath) > max_file_size:
+            processed_files += 1
+            if progress_callback:
+                progress_callback(processed_files, total_files)
+            continue
 
-            # Get path relative to directory if requested
-            if use_relative_paths:
-                relpath = os.path.relpath(filepath, directory)
-            else:
-                relpath = filepath
-
-            # --- Exclusion check ---
-            excluded = False
-            # 1. Check .gitignore first
-            if gitignore_spec and gitignore_spec.match_file(relpath):
-                excluded = True
-
-            # 2. Check standard exclude_patterns if not already excluded by .gitignore
-            if not excluded and exclude_patterns:
-                for pattern in exclude_patterns:
-                    if fnmatch.fnmatch(relpath, pattern):
-                        excluded = True
-                        break
-
-            included = False
-            if include_patterns:
-                for pattern in include_patterns:
-                    if fnmatch.fnmatch(relpath, pattern):
-                        included = True
-                        break
-            else:
-                # If no include patterns, include everything *not excluded*
-                included = True
-
-            # Skip if not included or if excluded (by either method)
-            if not included or excluded:
-                continue
-
-            # Check file size
-            if max_file_size and os.path.getsize(filepath) > max_file_size:
-                continue
+        try:
+            with open(filepath, "r", encoding="utf-8") as f:
+                content = f.read()
+            files_dict[relpath] = content
+        except Exception as e:
+            print(f"Warning: Could not read file {filepath}: {e}")
 
-            try:
-                with open(filepath, "r", encoding="utf-8") as f:
-                    content = f.read()
-                files_dict[relpath] = content
-            except Exception as e:
-                print(f"Warning: Could not read file {filepath}: {e}")
+        processed_files += 1
+        if progress_callback:
+            progress_callback(processed_files, total_files)
 
     return {"files": files_dict}
 
@@ -138,4 +132,4 @@ def crawl_local_files(
     )
     print(f"Found {len(files_data['files'])} files:")
     for path in files_data["files"]:
-        print(f"  {path}")
+        print(f"  {path}")