diff --git a/api/api.py b/api/api.py index d40e73f9..28b54ea7 100644 --- a/api/api.py +++ b/api/api.py @@ -1,3 +1,4 @@ +import asyncio import os import logging from fastapi import FastAPI, HTTPException, Query, Request, WebSocket @@ -8,7 +9,7 @@ from datetime import datetime from pydantic import BaseModel, Field import google.generativeai as genai -import asyncio +from aiofiles import os as aioos, open # Configure logging from api.logging_config import setup_logging @@ -36,6 +37,10 @@ def get_adalflow_default_root_path(): return os.path.expanduser(os.path.join("~", ".adalflow")) +async def awalk_collect(path: str): + # Вернёт list[(root, dirs, files)] + return await asyncio.to_thread(lambda: list(os.walk(path))) + # --- Pydantic Models --- class WikiPage(BaseModel): """ @@ -281,7 +286,8 @@ async def get_local_repo_structure(path: str = Query(None, description="Path to content={"error": "No path provided. Please provide a 'path' query parameter."} ) - if not os.path.isdir(path): + is_dir = await aioos.path.isdir(path) + if not is_dir: return JSONResponse( status_code=404, content={"error": f"Directory not found: {path}"} @@ -292,7 +298,7 @@ async def get_local_repo_structure(path: str = Query(None, description="Path to file_tree_lines = [] readme_content = "" - for root, dirs, files in os.walk(path): + for root, dirs, files in await awalk_collect(path): # Exclude hidden dirs/files and virtual envs dirs[:] = [d for d in dirs if not d.startswith('.') and d != '__pycache__' and d != 'node_modules' and d != '.venv'] for file in files: @@ -304,8 +310,8 @@ async def get_local_repo_structure(path: str = Query(None, description="Path to # Find README.md (case-insensitive) if file.lower() == 'readme.md' and not readme_content: try: - with open(os.path.join(root, file), 'r', encoding='utf-8') as f: - readme_content = f.read() + async with open(os.path.join(root, file), 'r', encoding='utf-8') as f: + readme_content = await f.read() except Exception as e: logger.warning(f"Could not read README.md: {str(e)}") readme_content = "" @@ -413,11 +419,13 @@ def get_wiki_cache_path(owner: str, repo: str, repo_type: str, language: str) -> async def read_wiki_cache(owner: str, repo: str, repo_type: str, language: str) -> Optional[WikiCacheData]: """Reads wiki cache data from the file system.""" cache_path = get_wiki_cache_path(owner, repo, repo_type, language) - if os.path.exists(cache_path): + path_exists = await aioos.path.exists(cache_path) + if path_exists: try: - with open(cache_path, 'r', encoding='utf-8') as f: - data = json.load(f) - return WikiCacheData(**data) + async with open(cache_path, 'r', encoding='utf-8') as f: + file_content = await f.read() + data = json.loads(file_content) + return WikiCacheData(**data) except Exception as e: logger.error(f"Error reading wiki cache from {cache_path}: {e}") return None @@ -445,8 +453,8 @@ async def save_wiki_cache(data: WikiCacheRequest) -> bool: logger.info(f"Writing cache file to: {cache_path}") - with open(cache_path, 'w', encoding='utf-8') as f: - json.dump(payload.model_dump(), f, indent=2) + async with open(cache_path, 'w', encoding='utf-8') as f: + await f.write(json.dumps(payload.model_dump(), indent=2)) logger.info(f"Wiki cache successfully saved to {cache_path}") return True except IOError as e: @@ -525,9 +533,10 @@ async def delete_wiki_cache( logger.info(f"Attempting to delete wiki cache for {owner}/{repo} ({repo_type}), lang: {language}") cache_path = get_wiki_cache_path(owner, repo, repo_type, language) - if os.path.exists(cache_path): + path_exists = await aioos.path.exists(cache_path) + if path_exists: try: - os.remove(cache_path) + await aioos.remove(cache_path) logger.info(f"Successfully deleted wiki cache: {cache_path}") return {"message": f"Wiki cache for {owner}/{repo} ({language}) deleted successfully"} except Exception as e: @@ -584,18 +593,19 @@ async def get_processed_projects(): # WIKI_CACHE_DIR is already defined globally in the file try: - if not os.path.exists(WIKI_CACHE_DIR): + path_exists = await aioos.path.exists(WIKI_CACHE_DIR) + if not path_exists: logger.info(f"Cache directory {WIKI_CACHE_DIR} not found. Returning empty list.") return [] logger.info(f"Scanning for project cache files in: {WIKI_CACHE_DIR}") - filenames = await asyncio.to_thread(os.listdir, WIKI_CACHE_DIR) # Use asyncio.to_thread for os.listdir + filenames = await aioos.listdir(WIKI_CACHE_DIR) for filename in filenames: if filename.startswith("deepwiki_cache_") and filename.endswith(".json"): file_path = os.path.join(WIKI_CACHE_DIR, filename) try: - stats = await asyncio.to_thread(os.stat, file_path) # Use asyncio.to_thread for os.stat + stats = await aioos.stat(file_path) parts = filename.replace("deepwiki_cache_", "").replace(".json", "").split('_') # Expecting repo_type_owner_repo_language diff --git a/api/main.py b/api/main.py index 791e31b7..c79417b7 100644 --- a/api/main.py +++ b/api/main.py @@ -75,5 +75,6 @@ def patched_watch(*args, **kwargs): host="0.0.0.0", port=port, reload=is_development, + workers=os.cpu_count() or 1, # In this place os.cpu_count() selected because we can have case when vCPU used reload_excludes=["**/logs/*", "**/__pycache__/*", "**/*.pyc"] if is_development else None, ) diff --git a/api/poetry.lock b/api/poetry.lock index a2446bba..52edfe99 100644 --- a/api/poetry.lock +++ b/api/poetry.lock @@ -45,6 +45,18 @@ sqlalchemy = ["sqlalchemy (>=2.0.30)"] together = ["together (>=1.3.14)"] torch = ["torch (>=2.3.1)"] +[[package]] +name = "aiofiles" +version = "25.1.0" +description = "File support for asyncio." +optional = false +python-versions = ">=3.9" +groups = ["main"] +files = [ + {file = "aiofiles-25.1.0-py3-none-any.whl", hash = "sha256:abe311e527c862958650f9438e859c1fa7568a141b22abcd015e120e86a85695"}, + {file = "aiofiles-25.1.0.tar.gz", hash = "sha256:a8d728f0a29de45dc521f18f07297428d56992a742f0cd2701ba86e44d23d5b2"}, +] + [[package]] name = "aiohappyeyeballs" version = "2.6.1" @@ -215,6 +227,18 @@ files = [ frozenlist = ">=1.1.0" typing-extensions = {version = ">=4.2", markers = "python_version < \"3.13\""} +[[package]] +name = "annotated-doc" +version = "0.0.3" +description = "Document parameters, class attributes, return types, and variables inline, with Annotated." +optional = false +python-versions = ">=3.8" +groups = ["main"] +files = [ + {file = "annotated_doc-0.0.3-py3-none-any.whl", hash = "sha256:348ec6664a76f1fd3be81f43dffbee4c7e8ce931ba71ec67cc7f4ade7fbbb580"}, + {file = "annotated_doc-0.0.3.tar.gz", hash = "sha256:e18370014c70187422c33e945053ff4c286f453a984eba84d0dbfa0c935adeda"}, +] + [[package]] name = "annotated-types" version = "0.7.0" @@ -329,18 +353,18 @@ files = [ [[package]] name = "boto3" -version = "1.40.56" +version = "1.40.58" description = "The AWS SDK for Python" optional = false python-versions = ">=3.9" groups = ["main"] files = [ - {file = "boto3-1.40.56-py3-none-any.whl", hash = "sha256:8985a840d57671aa3c6124b0c178e79be97e3447de4b5819156071793f82ee5c"}, - {file = "boto3-1.40.56.tar.gz", hash = "sha256:c1afdb04dd27418fc58400434ab8e05998bb452b69c428168d9ada344fe6b93e"}, + {file = "boto3-1.40.58-py3-none-any.whl", hash = "sha256:951515c1ea0ae9e99e56c3b6f408a2f59e1b57fab4d96dab737e73956f729177"}, + {file = "boto3-1.40.58.tar.gz", hash = "sha256:5a99c0bd2e282af4afde1af10d8838b397120722b6b685f0c0fa6b8cac351304"}, ] [package.dependencies] -botocore = ">=1.40.56,<1.41.0" +botocore = ">=1.40.58,<1.41.0" jmespath = ">=0.7.1,<2.0.0" s3transfer = ">=0.14.0,<0.15.0" @@ -349,14 +373,14 @@ crt = ["botocore[crt] (>=1.21.0,<2.0a0)"] [[package]] name = "botocore" -version = "1.40.56" +version = "1.40.58" description = "Low-level, data-driven core of boto 3." optional = false python-versions = ">=3.9" groups = ["main"] files = [ - {file = "botocore-1.40.56-py3-none-any.whl", hash = "sha256:0962dfc9bfb0afa1855042a88a72cc722cc7f9c08f51d2c5c88181d525a59a27"}, - {file = "botocore-1.40.56.tar.gz", hash = "sha256:b29df3418a299609632cab240ee79275463b176ebeb3adc841ba367a3fa0c4db"}, + {file = "botocore-1.40.58-py3-none-any.whl", hash = "sha256:2571ca3aec8150e1b5a597794da6fd06284de72f29d3ea806804b798755f2e5a"}, + {file = "botocore-1.40.58.tar.gz", hash = "sha256:cf2de7f5538f23c8067408a984ed32221e8b196ce98e66945a479d06b2663c33"}, ] [package.dependencies] @@ -795,17 +819,18 @@ packaging = "*" [[package]] name = "fastapi" -version = "0.119.1" +version = "0.120.0" description = "FastAPI framework, high performance, easy to learn, fast to code, ready for production" optional = false python-versions = ">=3.8" groups = ["main"] files = [ - {file = "fastapi-0.119.1-py3-none-any.whl", hash = "sha256:0b8c2a2cce853216e150e9bd4faaed88227f8eb37de21cb200771f491586a27f"}, - {file = "fastapi-0.119.1.tar.gz", hash = "sha256:a5e3426edce3fe221af4e1992c6d79011b247e3b03cc57999d697fe76cbf8ae0"}, + {file = "fastapi-0.120.0-py3-none-any.whl", hash = "sha256:84009182e530c47648da2f07eb380b44b69889a4acfd9e9035ee4605c5cfc469"}, + {file = "fastapi-0.120.0.tar.gz", hash = "sha256:6ce2c1cfb7000ac14ffd8ddb2bc12e62d023a36c20ec3710d09d8e36fab177a0"}, ] [package.dependencies] +annotated-doc = ">=0.0.2" pydantic = ">=1.7.4,<1.8 || >1.8,<1.8.1 || >1.8.1,<2.0.0 || >2.0.0,<2.0.1 || >2.0.1,<2.1.0 || >2.1.0,<3.0.0" starlette = ">=0.40.0,<0.49.0" typing-extensions = ">=4.8.0" @@ -1006,15 +1031,15 @@ grpcio-gcp = ["grpcio-gcp (>=0.2.2,<1.0.0)"] [[package]] name = "google-api-core" -version = "2.26.0" +version = "2.27.0" description = "Google API client core library" optional = false python-versions = ">=3.7" groups = ["main"] markers = "python_version < \"3.14\"" files = [ - {file = "google_api_core-2.26.0-py3-none-any.whl", hash = "sha256:2b204bd0da2c81f918e3582c48458e24c11771f987f6258e6e227212af78f3ed"}, - {file = "google_api_core-2.26.0.tar.gz", hash = "sha256:e6e6d78bd6cf757f4aee41dcc85b07f485fbb069d5daa3afb126defba1e91a62"}, + {file = "google_api_core-2.27.0-py3-none-any.whl", hash = "sha256:779a380db4e21a4ee3d717cf8efbf324e53900bf37e1ffb273e5348a9916dd42"}, + {file = "google_api_core-2.27.0.tar.gz", hash = "sha256:d32e2f5dd0517e91037169e75bf0a9783b255aff1d11730517c0b2b29e9db06a"}, ] [package.dependencies] @@ -1965,14 +1990,14 @@ pydantic = ">=2.9" [[package]] name = "openai" -version = "2.6.0" +version = "2.6.1" description = "The official Python library for the openai API" optional = false python-versions = ">=3.8" groups = ["main"] files = [ - {file = "openai-2.6.0-py3-none-any.whl", hash = "sha256:f33fa12070fe347b5787a7861c8dd397786a4a17e1c3186e239338dac7e2e743"}, - {file = "openai-2.6.0.tar.gz", hash = "sha256:f119faf7fc07d7e558c1e7c32c873e241439b01bd7480418234291ee8c8f4b9d"}, + {file = "openai-2.6.1-py3-none-any.whl", hash = "sha256:904e4b5254a8416746a2f05649594fa41b19d799843cd134dac86167e094edef"}, + {file = "openai-2.6.1.tar.gz", hash = "sha256:27ae704d190615fca0c0fc2b796a38f8b5879645a3a52c9c453b23f97141bb49"}, ] [package.dependencies] @@ -3404,4 +3429,4 @@ propcache = ">=0.2.1" [metadata] lock-version = "2.1" python-versions = "^3.11" -content-hash = "b558e94d5d8bdcc4273f47c52c8bfa6f4e003df0cf754f56340b8b98283d4a8d" +content-hash = "ceb57e5bf4e658b25939c59858f1fc9aa4fdcc0f1c51c4a8d654fec7a0b682b1" diff --git a/api/pyproject.toml b/api/pyproject.toml index 09760f8b..17808449 100644 --- a/api/pyproject.toml +++ b/api/pyproject.toml @@ -24,6 +24,7 @@ python-dotenv = ">=1.0.0" openai = ">=1.76.2" ollama = ">=0.4.8" aiohttp = ">=3.8.4" +aiofiles = ">=25.1.0" boto3 = ">=1.34.0" websockets = ">=11.0.3" azure-identity = ">=1.12.0" diff --git a/src/app/[owner]/[repo]/page.tsx b/src/app/[owner]/[repo]/page.tsx index 81a33760..dddfb83d 100644 --- a/src/app/[owner]/[repo]/page.tsx +++ b/src/app/[owner]/[repo]/page.tsx @@ -1094,7 +1094,7 @@ IMPORTANT: console.log(`Starting generation for ${pages.length} pages with controlled concurrency`); // Maximum concurrent requests - const MAX_CONCURRENT = 1; + const MAX_CONCURRENT = 3; // Create a queue of pages const queue = [...pages];