Merge remote-tracking branch 'upstream/main' into crawling-progress

Author: gethari

.env.sample

@@ -1,2 +1,5 @@
 GEMINI_PROJECT_ID=<GEMINI_PROJECT_ID>
-GITHUB_TOKEN=<GITHUB_TOKEN>
+GEMINI_API_KEY=<GEMINI_API_KEY>
+GITHUB_TOKEN=<GITHUB_TOKEN>
+OPENROUTER_API_KEY = <OPENROUTER_API_KEY>
+OPENROUTER_MODEL = <OPENROUTER_MODEL>

.gitignore

@@ -99,4 +99,11 @@ coverage/
 llm_cache.json
 
 # Output files
-output/
+output/
+
+# uv manage
+pyproject.toml
+uv.lock
+
+docs/*.pdf
+docs/design-cn.md

README.md

@@ -73,13 +73,16 @@ This is a tutorial project of [Pocket Flow](https://github.com/The-Pocket/Pocket
 ## 🚀 Getting Started
 
 1. Clone this repository
+   ```bash
+   git clone https://github.com/The-Pocket/PocketFlow-Tutorial-Codebase-Knowledge
+   ```
 
-2. Install dependencies:
+3. Install dependencies:
    ```bash
    pip install -r requirements.txt
    ```
 
-3. Set up LLM in [`utils/call_llm.py`](./utils/call_llm.py) by providing credentials. By default, you can use the AI Studio key with this client for Gemini Pro 2.5:
+4. Set up LLM in [`utils/call_llm.py`](./utils/call_llm.py) by providing credentials. By default, you can use the AI Studio key with this client for Gemini Pro 2.5:
 
    ```python
    client = genai.Client(
@@ -92,7 +95,7 @@ This is a tutorial project of [Pocket Flow](https://github.com/The-Pocket/Pocket
    python utils/call_llm.py
    ```
 
-4. Generate a complete codebase tutorial by running the main script:
+5. Generate a complete codebase tutorial by running the main script:
     ```bash
     # Analyze a GitHub repository
     python main.py --repo https://github.com/username/repo --include "*.py" "*.js" --exclude "tests/*" --max-size 50000
@@ -112,6 +115,8 @@ This is a tutorial project of [Pocket Flow](https://github.com/The-Pocket/Pocket
     - `-e, --exclude` - Files to exclude (e.g., "tests/*" "docs/*")
     - `-s, --max-size` - Maximum file size in bytes (default: 100KB)
     - `--language` - Language for the generated tutorial (default: "english")
+    - `--max-abstractions` - Maximum number of abstractions to identify (default: 10)
+    - `--no-cache` - Disable LLM response caching (default: caching enabled)
 
 The application will crawl the repository, analyze the codebase structure, generate tutorial content in the specified language, and save the output in the specified directory (default: ./output).
 

docs/_config.yml

@@ -1,5 +1,5 @@
 # Basic site settings
-title: Codebase2Tutorial
+title: Pocket Flow
 
 # Theme settings
 remote_theme: just-the-docs/just-the-docs

main.py

@@ -14,8 +14,10 @@
 }
 
 DEFAULT_EXCLUDE_PATTERNS = {
+    "assets/*", "data/*", "examples/*", "images/*", "public/*", "static/*", "temp/*",
+    "docs/*", 
     "venv/*", ".venv/*", "*test*", "tests/*", "docs/*", "examples/*", "v1/*",
-    "dist/*", "build/*", "experimental/*", "deprecated/*",
+    "dist/*", "build/*", "experimental/*", "deprecated/*", "misc/*", 
     "legacy/*", ".git/*", ".github/*", ".next/*", ".vscode/*", "obj/*", "bin/*", "node_modules/*", "*.log"
 }
 
@@ -36,6 +38,10 @@ def main():
     parser.add_argument("-s", "--max-size", type=int, default=100000, help="Maximum file size in bytes (default: 100000, about 100KB).")
     # Add language parameter for multi-language support
     parser.add_argument("--language", default="english", help="Language for the generated tutorial (default: english)")
+    # Add use_cache parameter to control LLM caching
+    parser.add_argument("--no-cache", action="store_true", help="Disable LLM response caching (default: caching enabled)")
+    # Add max_abstraction_num parameter to control the number of abstractions
+    parser.add_argument("--max-abstractions", type=int, default=10, help="Maximum number of abstractions to identify (default: 10)")
 
     args = parser.parse_args()
 
@@ -61,6 +67,12 @@ def main():
 
         # Add language for multi-language support
         "language": args.language,
+        
+        # Add use_cache flag (inverse of no-cache flag)
+        "use_cache": not args.no_cache,
+        
+        # Add max_abstraction_num parameter
+        "max_abstraction_num": args.max_abstractions,
 
         # Outputs will be populated by the nodes
         "files": [],
@@ -73,6 +85,7 @@ def main():
 
     # Display starting message with repository/directory and language
     print(f"Starting tutorial generation for: {args.repo or args.dir} in {args.language.capitalize()} language")
+    print(f"LLM caching: {'Disabled' if args.no_cache else 'Enabled'}")
 
     # Create the flow instance
     tutorial_flow = create_tutorial_flow()