diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f48484df..b2d726a2 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -47,6 +47,35 @@ jobs: - name: Run tests run: pytest -p no:cacheprovider -o addopts="" tests/ -q + test-windows: + name: Test (Windows) + runs-on: windows-latest + timeout-minutes: 20 + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + cache: pip + + - name: Install dependencies + shell: pwsh + run: | + python -m pip install -r requirements.txt + python -m pip install pytest pytest-asyncio pytest-cov ruff + python -m pip install -e . --no-deps + + - name: Run Windows-compatible tests + shell: pwsh + run: | + python -m pytest -p no:cacheprovider -o addopts="" ` + tests/test_caw_windows_compat.py ` + tests/test_gitignore_filtering.py ` + tests/test_parse_timeout.py -q + lint: name: Lint runs-on: ubuntu-latest diff --git a/README.md b/README.md index d38a6a89..3cef36f5 100644 --- a/README.md +++ b/README.md @@ -46,6 +46,22 @@ overview, with Mermaid diagrams. It works on codebases from a few thousand to over a million lines, in 11 languages. It needs an LLM API key, or a Claude or Codex subscription, or an AI IDE that speaks MCP. +## Windows support + +CodeWiki supports Windows 11 with Python 3.12 or newer. In PowerShell, create +and activate a virtual environment with: + +```powershell +py -3.12 -m venv .venv +.\.venv\Scripts\Activate.ps1 +python -m pip install -e . +``` + +Git must be installed and available on `PATH` for repository-aware filtering. +The `claude-code` provider also requires the Claude CLI and its normal +authentication. CodeWiki adapts the subscription backend's POSIX file locking +on Windows; no `fcntl` package or Unix shell is required. + ## Quick start **1. Install** diff --git a/codewiki/mcp/tools/doc_writer.py b/codewiki/mcp/tools/doc_writer.py index d7450904..27b00035 100644 --- a/codewiki/mcp/tools/doc_writer.py +++ b/codewiki/mcp/tools/doc_writer.py @@ -99,7 +99,7 @@ async def handle_write_doc_file( {"error": f"File already exists: {filename}. Use edit_doc_file to modify it."} ) - await asyncio.to_thread(doc_path.write_text, content, "utf-8") + await asyncio.to_thread(doc_path.write_text, content, encoding="utf-8", newline="\n") session.docs_written += 1 # Mermaid validation @@ -143,7 +143,7 @@ async def handle_edit_doc_file( old_content = path_history.pop() history[str(doc_path)] = path_history session.registry["file_history"] = history - await asyncio.to_thread(doc_path.write_text, old_content, "utf-8") + await asyncio.to_thread(doc_path.write_text, old_content, encoding="utf-8", newline="\n") # Validate Mermaid after undo mermaid_result = await _validate_mermaid(str(doc_path), filename, repair=False) @@ -181,7 +181,7 @@ async def handle_edit_doc_file( # Save history only for edits that actually happen, so undo never # pops a no-op entry left behind by a failed/rejected command. _save_history(session, doc_path, current_content) - await asyncio.to_thread(doc_path.write_text, new_content, "utf-8") + await asyncio.to_thread(doc_path.write_text, new_content, encoding="utf-8", newline="\n") # Snippet around the edit replacement_line = current_content.split(old_str)[0].count("\n") @@ -202,7 +202,7 @@ async def handle_edit_doc_file( lines = lines[:insert_line] + new_str_lines + lines[insert_line:] new_content = "\n".join(lines) _save_history(session, doc_path, current_content) - await asyncio.to_thread(doc_path.write_text, new_content, "utf-8") + await asyncio.to_thread(doc_path.write_text, new_content, encoding="utf-8", newline="\n") start = max(0, insert_line - 4) end = min(len(lines), start + len(new_str_lines) + 8) diff --git a/codewiki/src/be/agent_tools/doc_guard.py b/codewiki/src/be/agent_tools/doc_guard.py index 5f0a11ea..0aac77c1 100644 --- a/codewiki/src/be/agent_tools/doc_guard.py +++ b/codewiki/src/be/agent_tools/doc_guard.py @@ -105,6 +105,13 @@ def summarize_tool_activity(messages: list[Any]) -> str: if isinstance(part, ToolReturnPart) and part.tool_call_id in writes: lines = str(part.content).strip().splitlines() results[part.tool_call_id] = _excerpt(lines[0] if lines else "") + if isinstance(part.content, list) and any( + isinstance(item, dict) and item.get("type") == "missing" + for item in part.content + ): + rejected.append( + f"{getattr(part, 'tool_name', 'unknown')}: {_excerpt(part.content)}" + ) elif isinstance(part, RetryPromptPart) and part.tool_name: rejected.append(f"{part.tool_name}: {_excerpt(part.model_response())}") diff --git a/codewiki/src/be/agent_tools/str_replace_editor.py b/codewiki/src/be/agent_tools/str_replace_editor.py index 2a8a4f8e..3a954354 100644 --- a/codewiki/src/be/agent_tools/str_replace_editor.py +++ b/codewiki/src/be/agent_tools/str_replace_editor.py @@ -7,7 +7,6 @@ import logging import os import re -import shlex import subprocess import sys from collections import defaultdict @@ -229,9 +228,15 @@ def flake8(file_path: str) -> str: """Run flake8 on a given file and return the output as a string""" if Path(file_path).suffix != ".py": return "" - cmd = "flake8 --isolated --select=F821,F822,F831,E111,E112,E113,E999,E902 {file_path}" out = subprocess.run( - cmd.format(file_path=file_path), shell=True, check=False, capture_output=True + [ + "flake8", + "--isolated", + "--select=F821,F822,F831,E111,E112,E113,E999,E902", + str(file_path), + ], + check=False, + capture_output=True, ) # Use errors="replace" so non-UTF-8 bytes (e.g. GBK-encoded paths on Windows) don't crash decoding. return out.stdout.decode("utf-8", errors="replace") @@ -560,21 +565,29 @@ def view(self, path: Path, view_range: list[int] | None = None): # Hidden entries are skipped except `.github` (CI workflows are # documentation-relevant artifacts the agent must be able to find). - out = subprocess.run( - rf"find {shlex.quote(str(path))} -maxdepth 2 " - r"\( -not -path '*/.*' -o -name .github -o -path '*/.github/*' \)", - shell=True, - check=False, - capture_output=True, + display_root = self._get_display_path(path) + entries = [display_root] + try: + for child in sorted(path.iterdir(), key=lambda p: p.name): + if child.name.startswith(".") and child.name != ".github": + continue + entries.append(f"{display_root}/{child.name}") + if child.is_dir(): + try: + for grandchild in sorted(child.iterdir(), key=lambda p: p.name): + if grandchild.name.startswith(".") and child.name != ".github": + continue + entries.append(f"{display_root}/{child.name}/{grandchild.name}") + except OSError: + pass + except OSError as exc: + self.logs.append(f"Error listing directory {display_root}: {exc}") + return + + stdout = "\n".join(entries) + self.logs.append( + f"Here's the files and directories up to 2 levels deep in {display_root}, excluding hidden items (except .github):\n{stdout}\n" ) - # Use errors="replace" so non-UTF-8 bytes (e.g. GBK-encoded filenames on Windows) don't crash decoding. - stdout = out.stdout.decode("utf-8", errors="replace") - stderr = out.stderr.decode("utf-8", errors="replace") - - if not stderr: - stdout = stdout.replace(str(path), self._get_display_path(path)) - stdout = f"Here's the files and directories up to 2 levels deep in {self._get_display_path(path)}, excluding hidden items (except .github):\n{stdout}\n" - self.logs.append(stdout) return file_content = self.read_file(path) diff --git a/codewiki/src/be/caw_backend.py b/codewiki/src/be/caw_backend.py index b8e5de4a..4383a481 100644 --- a/codewiki/src/be/caw_backend.py +++ b/codewiki/src/be/caw_backend.py @@ -22,15 +22,20 @@ import asyncio import logging import os +import sys import time import shutil -from typing import Any +from typing import Any, Callable -from caw import Agent as CawAgent -from caw import ToolGroup +from codewiki.src.be.caw_compat import CawAgent, ToolGroup from codewiki.src.be.agent_tools.deps import CodeWikiDeps -from codewiki.src.be.agent_tools.doc_guard import DocumentationWriteError, doc_written +from codewiki.src.be.agent_tools.doc_guard import ( + DOC_WRITE_REMINDERS, + DocumentationWriteError, + doc_written, + reminder_prompt, +) from codewiki.src.be.backend import AgentReply, LLMBackend from codewiki.src.be.module_scope import ( can_split_module, @@ -62,13 +67,16 @@ "codex": "codex", } -# Disable WRITER (Write/Edit/NotebookEdit) so the agent must use CodeWiki's -# str_replace_editor and Mermaid validation runs. INTERACTION (AskUserQuestion) -# and WEB (WebFetch/WebSearch) are also off. PARALLEL (Task) stays enabled: -# Claude Code can fan out Read-heavy exploration without affecting -# documentation correctness. +# INTERACTION (AskUserQuestion) and WEB (WebFetch/WebSearch) are off. +# PARALLEL (Task) stays enabled so Claude Code can fan out Read-heavy +# exploration without affecting documentation correctness. _AGENT_TOOL_GROUP = ToolGroup.READER | ToolGroup.PARALLEL +# caw's default usage-limit detector relies on provider response text and can +# misclassify normal documentation as a limit, then sleep for 60 minutes. +# CodeWiki must surface the turn and let its caller decide how to handle it. +_CAW_AUTO_WAIT = False + def _agent_tool_group_for_provider(provider: str) -> ToolGroup: """Return the caw tool group needed for a module-agent session.""" @@ -140,8 +148,10 @@ def _patched(self) -> list[str]: # https://code.claude.com/docs/en/settings#permission-rule-syntax # `--allowedTools` flag: https://code.claude.com/docs/en/cli-reference # caw's ClaudeCodeSession only ever emits `--disallowedTools`, so rewrite its -# `claude` command to add `--allowedTools mcp__` for every server in -# the --mcp-config. The patch swaps the `subprocess` module reference INSIDE +# `claude` command to add explicit `mcp____` rules for every +# CodeWiki toolkit tool. A server-only rule does not expose the tools to +# Claude Code's non-interactive permission resolver. The patch swaps the +# `subprocess` module reference INSIDE # caw.providers.claude_code for a thin proxy — the global subprocess.Popen # class stays untouched (isinstance / subclass safe) and no other claude # invocation in this process is affected. Belongs upstream in caw; remove @@ -150,7 +160,7 @@ def _patched(self) -> list[str]: def _with_allowed_tools(cmd): - """Append ``--allowedTools mcp__,...`` to a ``claude`` command. + """Append explicit CodeWiki MCP tool rules to a ``claude`` command. Pure ``list -> list`` transform. Applies only when *cmd* is a claude invocation carrying ``--mcp-config`` and no explicit allow-list already; @@ -175,9 +185,17 @@ def _with_allowed_tools(cmd): servers = list(json.load(f).get("mcpServers", {}).keys()) if not servers: return cmd - allowed = ",".join(f"mcp__{s}" for s in servers) + tool_names = ( + "read_code_components", + "str_replace_editor", + "generate_sub_module_documentation", + ) + allowed = ",".join(f"mcp__{server}__{tool}" for server in servers for tool in tool_names) logger.info("Injected --allowedTools for MCP servers: %s", servers) - return list(cmd) + ["--allowedTools", allowed] + # caw's temporary config is the complete CodeWiki tool surface for + # this session. Ignore user/project MCP registrations so Claude's + # deferred-tool index cannot be polluted by unrelated connectors. + return list(cmd) + ["--strict-mcp-config", "--allowedTools", allowed] except Exception as e: # noqa: BLE001 — never break the spawn on a patch hiccup logger.warning("claude allowedTools patch skipped: %s", e) return cmd @@ -199,7 +217,32 @@ class _SubprocessProxy: @staticmethod def Popen(cmd, *args, **kwargs): - return subprocess.Popen(_with_allowed_tools(cmd), *args, **kwargs) + command = _with_allowed_tools(cmd) + if sys.platform == "win32" and command and command[0] == "claude": + claude_exe = shutil.which("claude.exe") + if not claude_exe: + claude_cmd = shutil.which("claude.cmd") + if claude_cmd: + from pathlib import Path + + candidate = ( + Path(claude_cmd).parent + / "node_modules" + / "@anthropic-ai" + / "claude-code" + / "bin" + / "claude.exe" + ) + if candidate.is_file(): + claude_exe = str(candidate) + else: + command = [claude_cmd, *command[1:]] + kwargs["shell"] = True + if claude_exe: + command = [claude_exe, *command[1:]] + if kwargs.get("text") and "encoding" not in kwargs: + kwargs["encoding"] = "utf-8" + return subprocess.Popen(command, *args, **kwargs) def __getattr__(self, name): return getattr(subprocess, name) @@ -265,6 +308,7 @@ def complete( model=effective_model, tools=ToolGroup.READER, system_prompt=system_prompt, + auto_wait=_CAW_AUTO_WAIT, ) started = time.time() traj = agent.completion(prompt) @@ -305,6 +349,7 @@ def _run_update_agent_sync( system_prompt=system_prompt, tools=_agent_tool_group_for_provider(self._caw_provider), tool_servers=[toolkit], + auto_wait=_CAW_AUTO_WAIT, ) original_cwd = os.getcwd() run_cwd = deps.absolute_docs_path if self._caw_provider == "codex" else self._repo_root @@ -435,15 +480,16 @@ def _run_module_agent_sync( scope_component_ids=core_component_ids, ) - toolkit = CawToolKit(deps=deps, backend=self, allow_subagent=can_delegate) - - agent = CawAgent( - provider=self._caw_provider, - model=self._model, - system_prompt=system_prompt, - tools=_agent_tool_group_for_provider(self._caw_provider), - tool_servers=[toolkit], - ) + def agent_factory() -> Any: + toolkit = CawToolKit(deps=deps, backend=self, allow_subagent=can_delegate) + return CawAgent( + provider=self._caw_provider, + model=self._model, + system_prompt=system_prompt, + tools=_agent_tool_group_for_provider(self._caw_provider), + tool_servers=[toolkit], + auto_wait=_CAW_AUTO_WAIT, + ) user_prompt = format_user_prompt( module_name=module_name, @@ -455,6 +501,13 @@ def _run_module_agent_sync( ) if not module_path: user_prompt = root_recovery_prompt(user_prompt, working_dir, module_name) + target_doc = module_doc_file(module_name, list(module_path), config_layout(config)) + user_prompt += ( + f"\n\nIMPORTANT: Use the CodeWiki MCP `str_replace_editor` as the preferred " + f"writer. For that tool, write the documentation at relative path " + f"`{target_doc}` with `working_dir=\"docs\"`; never pass an absolute path " + f"as `working_dir`. Do not use native Claude Code file-writing tools." + ) # caw forks claude / codex via subprocess.Popen without a cwd, so the # child CLI inherits Python's cwd — typically the repo root where the @@ -497,7 +550,14 @@ def _run_module_agent_sync( try: os.chdir(run_cwd) try: - traj = agent.completion(user_prompt) + traj = self._complete_module_until_written( + agent_factory, + user_prompt, + deps=deps, + module_name=module_name, + module_path=list(module_path), + layout=config_layout(config), + ) finally: os.chdir(original_cwd) file_manager.save_json(deps.module_tree, module_tree_path) @@ -535,3 +595,46 @@ def _run_module_agent_sync( except Exception as e: logger.error("Error processing module %s via caw: %s", module_name, e) raise + + @staticmethod + def _complete_module_until_written( + agent_factory: Callable[[], Any], + user_prompt: str, + *, + deps: CodeWikiDeps, + module_name: str, + module_path: list[str], + layout: str | None, + ) -> Any: + """Continue a caw conversation when the agent returns without writing.""" + doc_file = module_doc_file(module_name, module_path, layout) + for attempt in range(DOC_WRITE_REMINDERS + 1): + agent = agent_factory() + prompt = user_prompt + if attempt: + prompt += "\n\n" + reminder_prompt(module_name, doc_file) + trajectory = agent.completion(prompt) + if doc_written(deps.absolute_docs_path, module_name, module_path, deps.module_tree): + return trajectory + result = " ".join((getattr(trajectory, "result", None) or "").split()).lower() + if ( + "no matching deferred tools found" in result + or "mcp" in result + and "not available" in result + ): + logger.error( + "Agent for %s cannot access the CodeWiki MCP writer; " + "stopping retries to avoid repeated provider charges. Final answer: %s", + module_name, + result[:300] or "", + ) + return trajectory + if attempt == DOC_WRITE_REMINDERS: + return trajectory + logger.warning( + "Agent for %s finished without writing %s; requesting write reminder %d/%d", + module_name, + doc_file, + attempt + 1, + DOC_WRITE_REMINDERS, + ) diff --git a/codewiki/src/be/caw_compat.py b/codewiki/src/be/caw_compat.py new file mode 100644 index 00000000..04d0924d --- /dev/null +++ b/codewiki/src/be/caw_compat.py @@ -0,0 +1,79 @@ +"""Compatibility imports for the optional coding-agent-wrapper dependency.""" + +from __future__ import annotations + +import importlib +import sys +import types +from typing import Any + + +def _windows_fcntl() -> types.ModuleType: + import msvcrt + + module = types.ModuleType("fcntl") + module.LOCK_SH = 1 + module.LOCK_EX = 2 + module.LOCK_NB = 4 + module.LOCK_UN = 8 + + def flock(file_obj: Any, operation: int) -> None: + position = file_obj.tell() + try: + file_obj.seek(0) + if operation & module.LOCK_UN: + msvcrt.locking(file_obj.fileno(), msvcrt.LK_UNLCK, 1) + else: + mode = msvcrt.LK_NBLCK if operation & module.LOCK_NB else msvcrt.LK_LOCK + msvcrt.locking(file_obj.fileno(), mode, 1) + finally: + file_obj.seek(position) + + module.flock = flock + return module + + +def _windows_pty() -> types.ModuleType: + module = types.ModuleType("pty") + + def unsupported(*_args: Any, **_kwargs: Any) -> None: + raise OSError("Pseudo-terminals are not available on Windows") + + module.openpty = unsupported + module.spawn = unsupported + return module + + +def _install_windows_posix_compatibility() -> None: + sys.modules["fcntl"] = _windows_fcntl() + # caw imports its optional interactive TUI provider eagerly. The regular + # Claude Code provider used by CodeWiki does not need a pseudo-terminal. + sys.modules["pty"] = _windows_pty() + sys.modules["termios"] = types.ModuleType("termios") + + +def _import_with_windows_compatibility() -> Any: + sentinel = object() + previous = {name: sys.modules.get(name, sentinel) for name in ("fcntl", "pty", "termios")} + _install_windows_posix_compatibility() + try: + return importlib.import_module("caw") + finally: + for name, module in previous.items(): + if module is sentinel: + sys.modules.pop(name, None) + else: + sys.modules[name] = module + + +def load_caw() -> tuple[Any, Any]: + """Load caw, supplying only its missing Windows file-lock API when needed.""" + if sys.platform == "win32": + module = _import_with_windows_compatibility() + return module.Agent, module.ToolGroup + + module = importlib.import_module("caw") + return module.Agent, module.ToolGroup + + +CawAgent, ToolGroup = load_caw() diff --git a/codewiki/src/be/dependency_analyzer/analysis/repo_analyzer.py b/codewiki/src/be/dependency_analyzer/analysis/repo_analyzer.py index 67b6cfe3..589094f0 100644 --- a/codewiki/src/be/dependency_analyzer/analysis/repo_analyzer.py +++ b/codewiki/src/be/dependency_analyzer/analysis/repo_analyzer.py @@ -209,7 +209,9 @@ def analyze_repository_structure(self, repo_dir: str) -> dict: def _build_file_tree(self, repo_dir: str) -> dict: def build_tree(path: Path, base_path: Path) -> dict | None: relative_path = path.relative_to(base_path) - relative_path_str = str(relative_path) + # Repository artifacts use POSIX separators in JSON and prompts on + # every host; filesystem access continues to use Path objects. + relative_path_str = relative_path.as_posix() # 🚫 Reject symlinks if path.is_symlink(): diff --git a/codewiki/src/be/dependency_analyzer/analyzers/vue.py b/codewiki/src/be/dependency_analyzer/analyzers/vue.py index 4934f72c..129d64f1 100644 --- a/codewiki/src/be/dependency_analyzer/analyzers/vue.py +++ b/codewiki/src/be/dependency_analyzer/analyzers/vue.py @@ -253,12 +253,23 @@ def _run_script_analyzer(self, masked: bytes, use_ts: bool): for node in nodes: if node.id == self.component_id: continue + node.id = self._normalize_component_id(node.id) + node.component_id = self._normalize_component_id(node.component_id) + node.relative_path = Path(node.relative_path).as_posix() self.nodes.append(node) self.script_names.setdefault(node.name.split(".")[-1], node.id) for rel in relationships: + rel.caller = self._normalize_component_id(rel.caller) + rel.callee = self._normalize_component_id(rel.callee) if rel.caller != self.component_id: self.call_relationships.append(rel) + @staticmethod + def _normalize_component_id(component_id: str | None) -> str | None: + if component_id is None: + return None + return component_id.replace("\\", "/") + def _collect_imports(self, root): for stmt in root.children: if stmt.type != "import_statement": diff --git a/codewiki/src/be/dependency_analyzer/ast_parser.py b/codewiki/src/be/dependency_analyzer/ast_parser.py index 8e626112..1807d35c 100644 --- a/codewiki/src/be/dependency_analyzer/ast_parser.py +++ b/codewiki/src/be/dependency_analyzer/ast_parser.py @@ -117,7 +117,7 @@ def _build_components_from_analysis(self, call_graph_result: dict): component_id_mapping = {} for func_dict in functions: - component_id = func_dict.get("id", "") + component_id = self._normalize_component_id(func_dict.get("id", "")) if not component_id: continue @@ -128,7 +128,7 @@ def _build_components_from_analysis(self, call_graph_result: dict): "component_type", func_dict.get("node_type", "function") ), file_path=func_dict.get("file_path", ""), - relative_path=func_dict.get("relative_path", ""), + relative_path=self._normalize_relative_path(func_dict.get("relative_path", "")), source_code=func_dict.get("source_code", func_dict.get("code_snippet", "")), start_line=func_dict.get("start_line", 0), end_line=func_dict.get("end_line", 0), @@ -147,7 +147,10 @@ def _build_components_from_analysis(self, call_graph_result: dict): self.components[component_id] = node component_id_mapping[component_id] = component_id - legacy_id = f"{func_dict.get('file_path', '')}:{func_dict.get('name', '')}" + legacy_id = ( + f"{self._normalize_relative_path(func_dict.get('file_path', ''))}:" + f"{func_dict.get('name', '')}" + ) if legacy_id and legacy_id != component_id: component_id_mapping[legacy_id] = component_id @@ -163,8 +166,8 @@ def _build_components_from_analysis(self, call_graph_result: dict): processed_relationships = 0 for rel_dict in relationships: - caller_id = rel_dict.get("caller", "") - callee_id = rel_dict.get("callee", "") + caller_id = self._normalize_component_id(rel_dict.get("caller", "")) + callee_id = self._normalize_component_id(rel_dict.get("callee", "")) caller_component_id = component_id_mapping.get(caller_id) @@ -183,6 +186,16 @@ def _build_components_from_analysis(self, call_graph_result: dict): self.components[caller_component_id].depends_on.add(callee_component_id) processed_relationships += 1 + @staticmethod + def _normalize_relative_path(path: str) -> str: + """Keep repository-relative paths stable across operating systems.""" + return str(path or "").replace("\\", "/") + + @classmethod + def _normalize_component_id(cls, component_id: str) -> str: + """Normalize the path portion of a component ID without changing symbols.""" + return cls._normalize_relative_path(component_id) + def _determine_component_type(self, func_dict: dict) -> str: if func_dict.get("is_method", False): return "method" diff --git a/codewiki/src/be/doc_layout.py b/codewiki/src/be/doc_layout.py index a6b6d56f..c34779ad 100644 --- a/codewiki/src/be/doc_layout.py +++ b/codewiki/src/be/doc_layout.py @@ -73,6 +73,11 @@ def module_doc_relpath(module_path: list[str], layout: str | None) -> str: return posixpath.join(*module_path[:-1], f"{module_path[-1]}.md") +def _join_doc_relpath(docs_dir: str, relative_path: str) -> str: + """Join a repository-style POSIX path using the host OS separator.""" + return os.path.join(docs_dir, *relative_path.split("/")) + + def module_doc_file(module_name: str, module_path: list[str] | None, layout: str | None) -> str: """Docs-relative file the agent documenting ``module_name`` must write. @@ -168,7 +173,7 @@ def find_doc( if module_tree is None: module_tree = load_module_tree(docs_dir) for rel in _candidate_relpaths(name, module_tree): - path = os.path.join(docs_dir, rel) + path = _join_doc_relpath(docs_dir, rel) if os.path.isfile(path): return path if search and name != OVERVIEW_STEM: @@ -199,7 +204,7 @@ def list_doc_files(docs_dir: str, module_tree: dict[str, Any] | None = None) -> module_tree = load_module_tree(docs_dir) for path in iter_tree_paths(module_tree): rel = module_doc_relpath(path, LAYOUT_HIERARCHICAL) - if "/" in rel and os.path.isfile(os.path.join(docs_dir, rel)): + if "/" in rel and os.path.isfile(_join_doc_relpath(docs_dir, rel)): pages[path[-1]] = rel return pages @@ -223,7 +228,7 @@ def target_doc_path( module_tree = load_module_tree(docs_dir) for path in iter_tree_paths(module_tree): if path[-1] == name: - return os.path.join(docs_dir, module_doc_relpath(path, layout)) + return _join_doc_relpath(docs_dir, module_doc_relpath(path, layout)) return os.path.join(docs_dir, f"{name}.md") @@ -271,14 +276,14 @@ def relocate_docs(docs_dir: str, module_tree: dict[str, Any], layout: str | None moved = [] for path in iter_tree_paths(module_tree): target_rel = module_doc_relpath(path, layout) - target = os.path.join(docs_dir, target_rel) + target = _join_doc_relpath(docs_dir, target_rel) if os.path.isfile(target): continue for rel in ( module_doc_relpath(path, LAYOUT_HIERARCHICAL), module_doc_relpath(path, LAYOUT_FLAT), ): - source = os.path.join(docs_dir, rel) + source = _join_doc_relpath(docs_dir, rel) if rel != target_rel and os.path.isfile(source): os.makedirs(os.path.dirname(target), exist_ok=True) os.replace(source, target) @@ -304,7 +309,7 @@ def organize_docs( for stem, rel in list_doc_files(docs_dir, module_tree).items(): relpaths.setdefault(stem, rel) for rel in sorted(set(list_doc_files(docs_dir, module_tree).values())): - path = os.path.join(docs_dir, rel) + path = _join_doc_relpath(docs_dir, rel) try: with open(path, encoding="utf-8") as f: text = f.read() diff --git a/codewiki/src/be/prompt_template.py b/codewiki/src/be/prompt_template.py index f649cead..ce0b4ac6 100644 --- a/codewiki/src/be/prompt_template.py +++ b/codewiki/src/be/prompt_template.py @@ -48,7 +48,7 @@ -- `str_replace_editor`: File system operations for creating and editing documentation files, and (with `working_dir="repo"`, `view` only) for reading build, CI, container, packaging, manifest and config files such as Dockerfile, Makefile, .github/workflows/*.yml, pyproject.toml or package.json +- `str_replace_editor`: File system operations for creating and editing documentation files; use it so Mermaid validation runs - `read_code_components`: Explore additional code dependencies not included in the provided components - `generate_sub_module_documentation`: Generate detailed documentation for individual sub-modules via sub-agents @@ -81,7 +81,7 @@ -- `str_replace_editor`: File system operations for creating and editing documentation files, and (with `working_dir="repo"`, `view` only) for reading build, CI, container, packaging, manifest and config files such as Dockerfile, Makefile, .github/workflows/*.yml, pyproject.toml or package.json +- `str_replace_editor`: File system operations for creating and editing documentation files; use it so Mermaid validation runs - `read_code_components`: Explore additional code dependencies not included in the provided components {custom_instructions} diff --git a/codewiki/src/fe/background_worker.py b/codewiki/src/fe/background_worker.py index 6a2d321a..daadccd6 100644 --- a/codewiki/src/fe/background_worker.py +++ b/codewiki/src/fe/background_worker.py @@ -6,13 +6,13 @@ import os import time import threading -import subprocess import asyncio from datetime import datetime from pathlib import Path from queue import Queue from typing import Dict +from codewiki.src.be.dependency_analyzer.analysis.cloning import cleanup_repo_directory from codewiki.src.be.documentation_generator import DocumentationGenerator from codewiki.src.config import Config, MAIN_MODEL from .models import JobStatus @@ -260,6 +260,6 @@ def _process_job(self, job_id: str): # Cleanup temporary repository if "temp_repo_dir" in locals() and os.path.exists(temp_repo_dir): try: - subprocess.run(["rm", "-rf", temp_repo_dir], check=True) + cleanup_repo_directory(temp_repo_dir) except Exception as e: print(f"Failed to cleanup temp directory: {e}") diff --git a/pyproject.toml b/pyproject.toml index a96d9216..b15a5ac8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -19,6 +19,8 @@ classifiers = [ "License :: OSI Approved :: MIT License", "Programming Language :: Python :: 3", "Programming Language :: Python :: 3.12", + "Operating System :: Microsoft :: Windows", + "Operating System :: POSIX", "Topic :: Software Development :: Documentation", "Topic :: Software Development :: Libraries :: Python Modules", ] diff --git a/tests/test_caw_windows_compat.py b/tests/test_caw_windows_compat.py new file mode 100644 index 00000000..990d78fb --- /dev/null +++ b/tests/test_caw_windows_compat.py @@ -0,0 +1,17 @@ +"""Windows compatibility for the subscription backend dependency.""" + +import sys + +import pytest + + +@pytest.mark.skipif(sys.platform != "win32", reason="Windows-specific compatibility test") +def test_caw_imports_without_posix_fcntl(): + import importlib.util + + if importlib.util.find_spec("caw") is None: + pytest.skip("caw is not installed") + from codewiki.src.be.caw_compat import CawAgent, ToolGroup + + assert CawAgent is not None + assert ToolGroup is not None diff --git a/tests/test_doc_write_guard.py b/tests/test_doc_write_guard.py index fb070596..575c3110 100644 --- a/tests/test_doc_write_guard.py +++ b/tests/test_doc_write_guard.py @@ -210,7 +210,7 @@ def completion(self, prompt): return backend -@pytest.mark.parametrize("kind,expected_calls", [("pydantic", 3), ("caw", 1)]) +@pytest.mark.parametrize("kind,expected_calls", [("pydantic", 3), ("caw", 3)]) def test_backend_failure_is_explicit_and_preserves_reserved_children( tmp_path, monkeypatch, kind, expected_calls ): diff --git a/tests/test_overview_generation.py b/tests/test_overview_generation.py index 9090d5e7..dcd98e4d 100644 --- a/tests/test_overview_generation.py +++ b/tests/test_overview_generation.py @@ -44,7 +44,7 @@ def generator(tmp_path, replies, layout="flat"): }, } } - (tmp_path / "module_tree.json").write_text(json.dumps(tree)) + (tmp_path / "module_tree.json").write_text(json.dumps(tree), encoding="utf-8") gen = DocumentationGenerator.__new__(DocumentationGenerator) gen.config = SimpleNamespace( layout=layout, repo_path="/tmp/example", get_prompt_addition=lambda: "Write in Portuguese." @@ -58,8 +58,12 @@ def test_parent_receives_child_contents_and_relative_links(tmp_path, layout): gen = generator(tmp_path, [VALID], layout) child_dir = tmp_path if layout == "flat" else tmp_path / "parent" child_dir.mkdir(exist_ok=True) - (child_dir / "build_and_deployment.md").write_text("# Build\n\nCHILD_CONTENT_SENTINEL") - (child_dir / "bootstrap.md").write_text("# Bootstrap\n\nInitialize the plugin.") + (child_dir / "build_and_deployment.md").write_text( + "# Build\n\nCHILD_CONTENT_SENTINEL", encoding="utf-8" + ) + (child_dir / "bootstrap.md").write_text( + "# Bootstrap\n\nInitialize the plugin.", encoding="utf-8" + ) asyncio.run(gen.generate_parent_module_docs(["parent"], str(tmp_path))) prompt, system = gen.backend.calls[0] assert "CHILD_CONTENT_SENTINEL" in prompt @@ -67,7 +71,7 @@ def test_parent_receives_child_contents_and_relative_links(tmp_path, layout): assert '"link": "' + ("parent/" if layout == "hierarchical" else "") + "bootstrap.md" in prompt assert "read those files with your" not in prompt assert "Portuguese" in system - assert (tmp_path / "parent.md").read_text() == VALID.strip() + assert (tmp_path / "parent.md").read_text(encoding="utf-8") == VALID.strip() @pytest.mark.parametrize( @@ -86,7 +90,7 @@ def test_parent_receives_child_contents_and_relative_links(tmp_path, layout): def test_invalid_output_is_corrected_once(tmp_path, response): gen = generator(tmp_path, [response, "" + VALID + ""]) asyncio.run(gen.generate_parent_module_docs(["parent"], str(tmp_path))) - assert (tmp_path / "parent.md").read_text() == VALID.strip() + assert (tmp_path / "parent.md").read_text(encoding="utf-8") == VALID.strip() assert len(gen.backend.calls) == 2 assert "Validation failure" in gen.backend.calls[1][0] assert gen.backend.usage_ledger.summary()["n_calls"] == 2 @@ -97,11 +101,11 @@ def test_exhausted_correction_never_overwrites_old_page(tmp_path, original): gen = generator(tmp_path, [PREAMBLE, PREAMBLE]) path = tmp_path / "parent.md" if original is not None: - path.write_text(original) + path.write_text(original, encoding="utf-8") with pytest.raises(RuntimeError, match="after one correction attempt"): asyncio.run(gen.generate_parent_module_docs(["parent"], str(tmp_path))) assert len(gen.backend.calls) == 2 - assert path.read_text() == original if original is not None else not path.exists() + assert path.read_text(encoding="utf-8") == original if original is not None else not path.exists() @pytest.mark.parametrize("original", ["", PREAMBLE, PREAMBLE.replace("I'll", "I’ll")]) @@ -109,9 +113,9 @@ def test_exhausted_correction_never_overwrites_old_page(tmp_path, original): def test_recognized_old_pages_are_repaired(tmp_path, original, module_path, filename): gen = generator(tmp_path, [VALID]) path = tmp_path / filename - path.write_text(original) + path.write_text(original, encoding="utf-8") asyncio.run(gen.generate_parent_module_docs(module_path, str(tmp_path))) - assert path.read_text() == VALID.strip() + assert path.read_text(encoding="utf-8") == VALID.strip() @pytest.mark.parametrize( @@ -124,16 +128,16 @@ def test_recognized_old_pages_are_repaired(tmp_path, original, module_path, file ) def test_other_existing_prose_is_preserved(tmp_path, original): gen = generator(tmp_path, []) - (tmp_path / "parent.md").write_text(original) + (tmp_path / "parent.md").write_text(original, encoding="utf-8") asyncio.run(gen.generate_parent_module_docs(["parent"], str(tmp_path))) assert not gen.backend.calls - assert (tmp_path / "parent.md").read_text() == original + assert (tmp_path / "parent.md").read_text(encoding="utf-8") == original def test_missing_and_unreadable_children_warn_without_fabrication(tmp_path, monkeypatch, caplog): gen = generator(tmp_path, [VALID]) path = tmp_path / "bootstrap.md" - path.write_text("unreadable secret") + path.write_text("unreadable secret", encoding="utf-8") original = Path.read_text def read(path, *args, **kwargs): @@ -152,12 +156,14 @@ def read(path, *args, **kwargs): def test_sanitized_child_filename_is_read(tmp_path): gen = generator(tmp_path, [VALID]) path = tmp_path / "module_tree.json" - tree = json.loads(path.read_text()) + tree = json.loads(path.read_text(encoding="utf-8")) tree["parent"]["children"]["Build Deployment"] = tree["parent"]["children"].pop( "build_and_deployment" ) - path.write_text(json.dumps(tree)) - (tmp_path / "build_deployment.md").write_text("# Build\n\nSANITIZED_CONTENT") + path.write_text(json.dumps(tree), encoding="utf-8") + (tmp_path / "build_deployment.md").write_text( + "# Build\n\nSANITIZED_CONTENT", encoding="utf-8" + ) asyncio.run(gen.generate_parent_module_docs(["parent"], str(tmp_path))) assert "SANITIZED_CONTENT" in gen.backend.calls[0][0] @@ -205,7 +211,7 @@ def test_valid_prose_in_non_latin_script(): def test_failed_atomic_replacement_preserves_existing_file(tmp_path, monkeypatch): path = tmp_path / "overview.md" - path.write_text(PREAMBLE) + path.write_text(PREAMBLE, encoding="utf-8") def fail(*args): raise OSError("disk unavailable") @@ -213,40 +219,40 @@ def fail(*args): monkeypatch.setattr(support.os, "replace", fail) with pytest.raises(OSError, match="disk unavailable"): support.save_overview(VALID, str(path)) - assert path.read_text() == PREAMBLE + assert path.read_text(encoding="utf-8") == PREAMBLE assert list(tmp_path.iterdir()) == [path] def test_whole_repo_recovery_restores_old_page_on_failure(tmp_path): path = tmp_path / "overview.md" - path.write_text(PREAMBLE) + path.write_text(PREAMBLE, encoding="utf-8") with pytest.raises(RuntimeError), support.preserve_incomplete_overviews([str(path)]): - path.write_text("partial edit") + path.write_text("partial edit", encoding="utf-8") raise RuntimeError("agent failed") - assert path.read_text() == PREAMBLE + assert path.read_text(encoding="utf-8") == PREAMBLE def test_parent_failure_does_not_skip_independent_modules_or_pass_silently(tmp_path): gen = generator(tmp_path, []) gen.config.docs_dir = str(tmp_path) tree_path = tmp_path / "module_tree.json" - tree = json.loads(tree_path.read_text()) + tree = json.loads(tree_path.read_text(encoding="utf-8")) tree["independent"] = {"components": [], "children": {}} - tree_path.write_text(json.dumps(tree)) - (tmp_path / "parent.md").write_text(PREAMBLE) + tree_path.write_text(json.dumps(tree), encoding="utf-8") + (tmp_path / "parent.md").write_text(PREAMBLE, encoding="utf-8") async def module(module_name, **kwargs): - (tmp_path / f"{module_name}.md").write_text(VALID) + (tmp_path / f"{module_name}.md").write_text(VALID, encoding="utf-8") return tree async def overview(module_path, working_dir, components=None): if module_path == ["parent"]: raise RuntimeError("invalid parent response") - (tmp_path / "overview.md").write_text(VALID) + (tmp_path / "overview.md").write_text(VALID, encoding="utf-8") return tree gen.backend.run_module_agent = module gen.generate_parent_module_docs = overview with pytest.raises(RuntimeError, match="parent: invalid parent response"): asyncio.run(gen.generate_module_documentation({}, [])) - assert (tmp_path / "independent.md").read_text() == VALID + assert (tmp_path / "independent.md").read_text(encoding="utf-8") == VALID diff --git a/tests/test_techdocs.py b/tests/test_techdocs.py index f51ce7bc..c717cb04 100644 --- a/tests/test_techdocs.py +++ b/tests/test_techdocs.py @@ -116,8 +116,8 @@ def test_hierarchical_export(tmp_path): assert (out_docs / MARKER_FILENAME).is_file() # Links to the overview point at index.md, relative to each page - assert "[Overview](../index.md)" in (out_docs / "auth/login.md").read_text() - auth = (out_docs / "auth.md").read_text() + assert "[Overview](../index.md)" in (out_docs / "auth/login.md").read_text(encoding="utf-8") + auth = (out_docs / "auth.md").read_text(encoding="utf-8") assert "[Shop](index.md)" in auth # Links labelled with the module name show the page title assert "[Login](auth/login.md)" in auth @@ -130,13 +130,13 @@ def test_hierarchical_export(tmp_path): assert "on 2026-11-01 with `test-model`" in auth assert '??? abstract "Documented components (2)"' in auth assert f"[`login`]({REPO}/blob/{COMMIT}/src/auth.py) — `src/auth.py`" in auth - billing = (out_docs / "billing.md").read_text() + billing = (out_docs / "billing.md").read_text(encoding="utf-8") assert f"[`billing.py`]({REPO}/blob/{COMMIT}/src/billing.py)" in billing # The overview documents no components - assert "Documented components" not in (out_docs / "index.md").read_text() + assert "Documented components" not in (out_docs / "index.md").read_text(encoding="utf-8") # The source docs are untouched - assert (docs / "overview.md").read_text().startswith("# Shop\n") + assert (docs / "overview.md").read_text(encoding="utf-8").startswith("# Shop\n") def test_flat_export(tmp_path): @@ -149,9 +149,11 @@ def test_flat_export(tmp_path): assert config["nav"][1] == { "Authentication": [{"Authentication": "auth.md"}, {"Login": "login.md"}] } - assert "[Overview](index.md)" in (out / "docs/login.md").read_text() + assert "[Overview](index.md)" in (out / "docs/login.md").read_text(encoding="utf-8") # No repository URL: components are listed without links - assert "- `LoginForm` — `src/login.py`" in (out / "docs/login.md").read_text() + assert "- `LoginForm` — `src/login.py`" in ( + out / "docs/login.md" + ).read_text(encoding="utf-8") assert "repo_url" not in config diff --git a/tests/test_usage_ledger.py b/tests/test_usage_ledger.py index 2fe678c4..d8d0a83d 100644 --- a/tests/test_usage_ledger.py +++ b/tests/test_usage_ledger.py @@ -89,9 +89,12 @@ def _caw_backend(monkeypatch, usage): from codewiki.src.be import caw_backend from codewiki.src.be.caw_backend import CawBackend + captured = {} + class FakeCawAgent: def __init__(self, **kwargs): self.model = kwargs.get("model") + captured.update(kwargs) def completion(self, prompt): return SimpleNamespace( @@ -106,16 +109,17 @@ def completion(self, prompt): backend = object.__new__(CawBackend) # skip the CLI-on-PATH check backend._caw_provider = "claude_code" backend._model = None - return backend + return backend, captured def test_caw_complete_records_usage(monkeypatch): usage = _usage(input_tokens=1000, output_tokens=200, cache_read_tokens=50, cost_usd=0.12) - backend = _caw_backend(monkeypatch, usage) + backend, captured = _caw_backend(monkeypatch, usage) with backend.usage_ledger.in_phase("overview", target="repo"): assert backend.complete("prompt") == "ok" + assert captured["auto_wait"] is False assert backend.last_usage["cost_usd"] == 0.12 (call,) = backend.usage_ledger.summary()["calls"] assert call["phase"] == "overview" @@ -127,6 +131,111 @@ def test_caw_complete_records_usage(monkeypatch): assert call["cost_usd"] == 0.12 +def test_caw_allowed_tools_names_each_mcp_tool(tmp_path): + import json + import pytest + + pytest.importorskip("caw") + from codewiki.src.be.caw_backend import _with_allowed_tools + + config_path = tmp_path / "mcp.json" + config_path.write_text( + json.dumps({"mcpServers": {"codewiki_tools_abc123": {"url": "http://127.0.0.1"}}}), + encoding="utf-8", + ) + command = _with_allowed_tools(["claude", "--mcp-config", str(config_path), "--print", "prompt"]) + + allowed = command[-1] + assert "--strict-mcp-config" in command + assert allowed == ( + "mcp__codewiki_tools_abc123__read_code_components," + "mcp__codewiki_tools_abc123__str_replace_editor," + "mcp__codewiki_tools_abc123__generate_sub_module_documentation" + ) + + +def test_caw_module_agents_do_not_enable_native_writers(): + import pytest + + pytest.importorskip("caw") + from codewiki.src.be.caw_backend import _agent_tool_group_for_provider + from codewiki.src.be.caw_compat import ToolGroup + + tools = _agent_tool_group_for_provider("claude_code") + + assert not tools & ToolGroup.WRITER + + +def test_caw_module_agent_retries_when_page_is_missing(tmp_path): + import pytest + + pytest.importorskip("caw") + from codewiki.src.be.caw_backend import CawBackend + + class FakeAgent: + def __init__(self, writes): + self.prompts = [] + self.writes = writes + + def completion(self, prompt): + self.prompts.append(prompt) + if self.writes: + (tmp_path / "repo.md").write_text("# repo\n", encoding="utf-8") + return SimpleNamespace(total_usage=None, model="test") + + agents = [] + + def agent_factory(): + agent = FakeAgent(bool(agents)) + agents.append(agent) + return agent + + trajectory = CawBackend._complete_module_until_written( + agent_factory, + "generate documentation", + deps=SimpleNamespace(absolute_docs_path=str(tmp_path), module_tree={}), + module_name="repo", + module_path=[], + layout=None, + ) + + assert trajectory.model == "test" + assert len(agents) == 2 + assert len(agents[0].prompts) == 1 + assert len(agents[1].prompts) == 1 + assert 'command="create"' in agents[1].prompts[0] + + +def test_caw_module_agent_stops_when_mcp_writer_is_unavailable(tmp_path): + import pytest + + pytest.importorskip("caw") + from codewiki.src.be.caw_backend import CawBackend + + agents = [] + + class UnavailableAgent: + def completion(self, prompt): + agents.append(prompt) + return SimpleNamespace( + result="The CodeWiki MCP str_replace_editor is not available.", + total_usage=None, + model="test", + ) + + trajectory = CawBackend._complete_module_until_written( + UnavailableAgent, + "generate documentation", + deps=SimpleNamespace(absolute_docs_path=str(tmp_path), module_tree={}), + module_name="repo", + module_path=[], + layout=None, + ) + + assert trajectory.model == "test" + assert len(agents) == 1 + + class _StubBackend(LLMBackend): def complete(self, prompt, *, model=None, system_prompt=None): self._record_usage("complete", "", _usage(input_tokens=10, cost_usd=0.25))