Navegador

navegador / navegador / monorepo.py
Source Blame History 602 lines
a1b231b… lmata 1 """
a1b231b… lmata 2 Monorepo support — workspace-aware ingestion for Turborepo, Nx, Yarn/npm/pnpm
a1b231b… lmata 3 workspaces, Cargo workspaces, and Go workspaces.
a1b231b… lmata 4
a1b231b… lmata 5 Usage:
a1b231b… lmata 6 from navegador.monorepo import WorkspaceDetector, MonorepoIngester
a1b231b… lmata 7
a1b231b… lmata 8 config = WorkspaceDetector().detect("/path/to/monorepo")
a1b231b… lmata 9 if config:
a1b231b… lmata 10 ingester = MonorepoIngester(store)
a1b231b… lmata 11 stats = ingester.ingest("/path/to/monorepo")
a1b231b… lmata 12 """
a1b231b… lmata 13
a1b231b… lmata 14 from __future__ import annotations
a1b231b… lmata 15
a1b231b… lmata 16 import fnmatch
a1b231b… lmata 17 import json
a1b231b… lmata 18 import logging
e5e1a3b… lmata 19 from dataclasses import dataclass
a1b231b… lmata 20 from pathlib import Path
a1b231b… lmata 21 from typing import Any
a1b231b… lmata 22
a1b231b… lmata 23 from navegador.graph.schema import EdgeType, NodeLabel
a1b231b… lmata 24 from navegador.graph.store import GraphStore
a1b231b… lmata 25 from navegador.ingestion.parser import RepoIngester
a1b231b… lmata 26
a1b231b… lmata 27 logger = logging.getLogger(__name__)
a1b231b… lmata 28
a1b231b… lmata 29
a1b231b… lmata 30 # ── Data model ────────────────────────────────────────────────────────────────
a1b231b… lmata 31
a1b231b… lmata 32
a1b231b… lmata 33 @dataclass
a1b231b… lmata 34 class WorkspaceConfig:
a1b231b… lmata 35 """Configuration for a detected workspace."""
a1b231b… lmata 36
a1b231b… lmata 37 type: str # turborepo | nx | yarn | pnpm | cargo | go
a1b231b… lmata 38 root: Path
a1b231b… lmata 39 packages: list[Path]
a1b231b… lmata 40 name: str = ""
a1b231b… lmata 41
a1b231b… lmata 42 def __post_init__(self) -> None:
a1b231b… lmata 43 if not self.name:
a1b231b… lmata 44 self.name = self.root.name
a1b231b… lmata 45
a1b231b… lmata 46
a1b231b… lmata 47 # ── Detector ──────────────────────────────────────────────────────────────────
a1b231b… lmata 48
a1b231b… lmata 49
a1b231b… lmata 50 class WorkspaceDetector:
a1b231b… lmata 51 """Detects monorepo workspace configuration from a repository root."""
a1b231b… lmata 52
a1b231b… lmata 53 def detect(self, repo_path: str | Path) -> WorkspaceConfig | None:
a1b231b… lmata 54 """
a1b231b… lmata 55 Auto-detect workspace type and package locations.
a1b231b… lmata 56
a1b231b… lmata 57 Checks (in priority order):
a1b231b… lmata 58 turbo.json → Turborepo
a1b231b… lmata 59 nx.json → Nx
a1b231b… lmata 60 pnpm-workspace.yaml → pnpm workspaces
a1b231b… lmata 61 package.json → Yarn/npm workspaces (if "workspaces" key present)
a1b231b… lmata 62 Cargo.toml → Rust workspace (if [workspace] section present)
a1b231b… lmata 63 go.work → Go workspace
a1b231b… lmata 64
a1b231b… lmata 65 Returns None when no known workspace configuration is found.
a1b231b… lmata 66 """
a1b231b… lmata 67 root = Path(repo_path).resolve()
a1b231b… lmata 68
a1b231b… lmata 69 # Turborepo
a1b231b… lmata 70 if (root / "turbo.json").exists():
a1b231b… lmata 71 packages = self._js_workspace_packages(root)
a1b231b… lmata 72 return WorkspaceConfig(type="turborepo", root=root, packages=packages)
a1b231b… lmata 73
a1b231b… lmata 74 # Nx
a1b231b… lmata 75 if (root / "nx.json").exists():
a1b231b… lmata 76 packages = self._nx_packages(root)
a1b231b… lmata 77 return WorkspaceConfig(type="nx", root=root, packages=packages)
a1b231b… lmata 78
a1b231b… lmata 79 # pnpm workspaces
a1b231b… lmata 80 if (root / "pnpm-workspace.yaml").exists():
a1b231b… lmata 81 packages = self._pnpm_packages(root)
a1b231b… lmata 82 return WorkspaceConfig(type="pnpm", root=root, packages=packages)
a1b231b… lmata 83
a1b231b… lmata 84 # Yarn / npm workspaces
a1b231b… lmata 85 pkg_json = root / "package.json"
a1b231b… lmata 86 if pkg_json.exists():
a1b231b… lmata 87 try:
a1b231b… lmata 88 data = json.loads(pkg_json.read_text(encoding="utf-8"))
a1b231b… lmata 89 except (OSError, json.JSONDecodeError):
a1b231b… lmata 90 data = {}
a1b231b… lmata 91 if "workspaces" in data:
a1b231b… lmata 92 patterns = data["workspaces"]
a1b231b… lmata 93 # Yarn Berry stores them under workspaces.packages
a1b231b… lmata 94 if isinstance(patterns, dict):
a1b231b… lmata 95 patterns = patterns.get("packages", [])
a1b231b… lmata 96 packages = self._glob_packages(root, patterns)
a1b231b… lmata 97 return WorkspaceConfig(type="yarn", root=root, packages=packages)
a1b231b… lmata 98
a1b231b… lmata 99 # Cargo workspace
a1b231b… lmata 100 cargo_toml = root / "Cargo.toml"
a1b231b… lmata 101 if cargo_toml.exists():
a1b231b… lmata 102 packages = self._cargo_packages(root, cargo_toml)
a1b231b… lmata 103 if packages is not None:
a1b231b… lmata 104 return WorkspaceConfig(type="cargo", root=root, packages=packages)
a1b231b… lmata 105
a1b231b… lmata 106 # Go workspace
a1b231b… lmata 107 if (root / "go.work").exists():
a1b231b… lmata 108 packages = self._go_packages(root)
a1b231b… lmata 109 return WorkspaceConfig(type="go", root=root, packages=packages)
a1b231b… lmata 110
0f42b61… lmata 111 # Bare monorepo — no tooling, just a directory of apps/services
0f42b61… lmata 112 packages = self._bare_packages(root)
0f42b61… lmata 113 if len(packages) >= 2:
0f42b61… lmata 114 return WorkspaceConfig(type="bare", root=root, packages=packages)
0f42b61… lmata 115
a1b231b… lmata 116 return None
a1b231b… lmata 117
a1b231b… lmata 118 # ── JS-family helpers ─────────────────────────────────────────────────────
a1b231b… lmata 119
a1b231b… lmata 120 def _js_workspace_packages(self, root: Path) -> list[Path]:
a1b231b… lmata 121 """
a1b231b… lmata 122 Resolve workspace package paths from package.json workspaces field.
a1b231b… lmata 123 Falls back to scanning for package.json files one level down.
a1b231b… lmata 124 """
a1b231b… lmata 125 pkg_json = root / "package.json"
a1b231b… lmata 126 if pkg_json.exists():
a1b231b… lmata 127 try:
a1b231b… lmata 128 data = json.loads(pkg_json.read_text(encoding="utf-8"))
a1b231b… lmata 129 except (OSError, json.JSONDecodeError):
a1b231b… lmata 130 data = {}
a1b231b… lmata 131 patterns = data.get("workspaces", [])
a1b231b… lmata 132 if isinstance(patterns, dict):
a1b231b… lmata 133 patterns = patterns.get("packages", [])
a1b231b… lmata 134 if patterns:
a1b231b… lmata 135 return self._glob_packages(root, patterns)
a1b231b… lmata 136 return self._fallback_packages(root)
a1b231b… lmata 137
a1b231b… lmata 138 def _nx_packages(self, root: Path) -> list[Path]:
a1b231b… lmata 139 """
a1b231b… lmata 140 Nx workspaces store packages in apps/ and libs/ by convention,
a1b231b… lmata 141 or declare them in nx.json under "projects".
a1b231b… lmata 142 """
a1b231b… lmata 143 # Try reading nx.json for explicit projects
a1b231b… lmata 144 nx_json = root / "nx.json"
a1b231b… lmata 145 try:
e5e1a3b… lmata 146 json.loads(nx_json.read_text(encoding="utf-8"))
a1b231b… lmata 147 except (OSError, json.JSONDecodeError):
e5e1a3b… lmata 148 pass
a1b231b… lmata 149
a1b231b… lmata 150 # Nx 16+ uses workspaceLayout or projects in project.json files
a1b231b… lmata 151 packages: list[Path] = []
a1b231b… lmata 152 for subdir in ("apps", "libs", "packages"):
a1b231b… lmata 153 base = root / subdir
a1b231b… lmata 154 if base.is_dir():
a1b231b… lmata 155 for child in sorted(base.iterdir()):
a1b231b… lmata 156 if child.is_dir() and not child.name.startswith("."):
a1b231b… lmata 157 packages.append(child)
a1b231b… lmata 158
a1b231b… lmata 159 if packages:
a1b231b… lmata 160 return packages
a1b231b… lmata 161
a1b231b… lmata 162 # Fall through to package.json-based resolution
a1b231b… lmata 163 return self._js_workspace_packages(root)
a1b231b… lmata 164
a1b231b… lmata 165 def _pnpm_packages(self, root: Path) -> list[Path]:
a1b231b… lmata 166 """Parse pnpm-workspace.yaml for package glob patterns."""
a1b231b… lmata 167 yaml_path = root / "pnpm-workspace.yaml"
a1b231b… lmata 168 try:
a1b231b… lmata 169 text = yaml_path.read_text(encoding="utf-8")
a1b231b… lmata 170 except OSError:
a1b231b… lmata 171 return self._fallback_packages(root)
a1b231b… lmata 172
a1b231b… lmata 173 # Minimal YAML list parser — avoids a PyYAML dependency
a1b231b… lmata 174 patterns: list[str] = []
a1b231b… lmata 175 in_packages = False
a1b231b… lmata 176 for line in text.splitlines():
a1b231b… lmata 177 stripped = line.strip()
a1b231b… lmata 178 if stripped.startswith("packages:"):
a1b231b… lmata 179 in_packages = True
a1b231b… lmata 180 continue
a1b231b… lmata 181 if in_packages:
a1b231b… lmata 182 if stripped.startswith("-"):
a1b231b… lmata 183 value = stripped.lstrip("- ").strip().strip("'\"")
a1b231b… lmata 184 patterns.append(value)
a1b231b… lmata 185 elif stripped and not stripped.startswith("#"):
a1b231b… lmata 186 in_packages = False
a1b231b… lmata 187
a1b231b… lmata 188 if patterns:
a1b231b… lmata 189 return self._glob_packages(root, patterns)
a1b231b… lmata 190 return self._fallback_packages(root)
a1b231b… lmata 191
a1b231b… lmata 192 def _glob_packages(self, root: Path, patterns: list[str]) -> list[Path]:
a1b231b… lmata 193 """Expand workspace glob patterns (e.g. 'packages/*') to absolute Paths."""
a1b231b… lmata 194 packages: list[Path] = []
a1b231b… lmata 195 for pattern in patterns:
a1b231b… lmata 196 # Skip negation patterns
a1b231b… lmata 197 if pattern.startswith("!"):
a1b231b… lmata 198 continue
a1b231b… lmata 199 # Simple glob expansion using fnmatch against existing dirs
a1b231b… lmata 200 if "*" in pattern or "?" in pattern:
a1b231b… lmata 201 # Split at the first wildcard component
a1b231b… lmata 202 parts = Path(pattern).parts
a1b231b… lmata 203 base_parts: list[str] = []
a1b231b… lmata 204 for p in parts:
a1b231b… lmata 205 if "*" in p or "?" in p:
a1b231b… lmata 206 break
a1b231b… lmata 207 base_parts.append(p)
a1b231b… lmata 208 base = root / Path(*base_parts) if base_parts else root
a1b231b… lmata 209 if base.is_dir():
a1b231b… lmata 210 # The wildcard component
a1b231b… lmata 211 wildcard_idx = len(base_parts)
a1b231b… lmata 212 wild = parts[wildcard_idx] if wildcard_idx < len(parts) else "*"
a1b231b… lmata 213 for child in sorted(base.iterdir()):
a1b231b… lmata 214 if child.is_dir() and fnmatch.fnmatch(child.name, wild):
a1b231b… lmata 215 packages.append(child)
a1b231b… lmata 216 else:
a1b231b… lmata 217 resolved = (root / pattern).resolve()
a1b231b… lmata 218 if resolved.is_dir():
a1b231b… lmata 219 packages.append(resolved)
a1b231b… lmata 220 return packages
a1b231b… lmata 221
a1b231b… lmata 222 def _fallback_packages(self, root: Path) -> list[Path]:
a1b231b… lmata 223 """Scan one level down for directories containing a package.json."""
a1b231b… lmata 224 packages: list[Path] = []
a1b231b… lmata 225 for child in sorted(root.iterdir()):
a1b231b… lmata 226 if child.is_dir() and not child.name.startswith("."):
a1b231b… lmata 227 if (child / "package.json").exists():
a1b231b… lmata 228 packages.append(child)
a1b231b… lmata 229 return packages
0f42b61… lmata 230
0f42b61… lmata 231 # ── Bare monorepo helpers ─────────────────────────────────────────────────
0f42b61… lmata 232
0f42b61… lmata 233 # Manifests expected directly inside the package root
0f42b61… lmata 234 _PROJECT_MANIFESTS = (
0f42b61… lmata 235 "package.json",
0f42b61… lmata 236 "pyproject.toml",
0f42b61… lmata 237 "setup.py",
0f42b61… lmata 238 "setup.cfg",
0f42b61… lmata 239 "Cargo.toml",
0f42b61… lmata 240 "go.mod",
0f42b61… lmata 241 "pom.xml",
0f42b61… lmata 242 "build.gradle",
0f42b61… lmata 243 "build.gradle.kts",
0f42b61… lmata 244 "Gemfile",
0f42b61… lmata 245 "composer.json",
0f42b61… lmata 246 "mix.exs",
0f42b61… lmata 247 )
0f42b61… lmata 248
0f42b61… lmata 249 # Manifests that may live one subdirectory deeper (e.g. Django's manage.py
0f42b61… lmata 250 # inside an inner package dir: myapp/myapp/manage.py)
0f42b61… lmata 251 _NESTED_MANIFESTS = (
0f42b61… lmata 252 "manage.py",
0f42b61… lmata 253 "wsgi.py",
0f42b61… lmata 254 "asgi.py",
0f42b61… lmata 255 )
0f42b61… lmata 256
0f42b61… lmata 257 def _bare_packages(self, root: Path) -> list[Path]:
0f42b61… lmata 258 """
0f42b61… lmata 259 Detect a bare monorepo: a directory whose immediate children are
0f42b61… lmata 260 independent apps/services with no shared workspace tooling.
0f42b61… lmata 261
0f42b61… lmata 262 A child directory qualifies if it contains at least one recognised
0f42b61… lmata 263 project manifest directly, or a Django/WSGI manifest one level deeper
0f42b61… lmata 264 (e.g. myapp/myapp/manage.py).
0f42b61… lmata 265 Non-project dirs (docs, scripts, config-only folders) are skipped.
0f42b61… lmata 266 """
0f42b61… lmata 267 packages: list[Path] = []
0f42b61… lmata 268 for child in sorted(root.iterdir()):
0f42b61… lmata 269 if not child.is_dir() or child.name.startswith("."):
0f42b61… lmata 270 continue
0f42b61… lmata 271 # Check top-level manifests first
0f42b61… lmata 272 for manifest in self._PROJECT_MANIFESTS:
0f42b61… lmata 273 if (child / manifest).exists():
0f42b61… lmata 274 packages.append(child)
0f42b61… lmata 275 break
0f42b61… lmata 276 else:
0f42b61… lmata 277 # Fall back: look one level deeper for Django/WSGI markers
0f42b61… lmata 278 if self._has_nested_manifest(child):
0f42b61… lmata 279 packages.append(child)
0f42b61… lmata 280 return packages
0f42b61… lmata 281
0f42b61… lmata 282 def _has_nested_manifest(self, pkg_root: Path) -> bool:
0f42b61… lmata 283 """Return True if any immediate subdirectory contains a nested manifest."""
0f42b61… lmata 284 for subdir in pkg_root.iterdir():
0f42b61… lmata 285 if not subdir.is_dir() or subdir.name.startswith("."):
0f42b61… lmata 286 continue
0f42b61… lmata 287 for manifest in self._NESTED_MANIFESTS:
0f42b61… lmata 288 if (subdir / manifest).exists():
0f42b61… lmata 289 return True
0f42b61… lmata 290 return False
a1b231b… lmata 291
a1b231b… lmata 292 # ── Cargo helpers ─────────────────────────────────────────────────────────
a1b231b… lmata 293
a1b231b… lmata 294 def _cargo_packages(self, root: Path, cargo_toml: Path) -> list[Path] | None:
a1b231b… lmata 295 """
a1b231b… lmata 296 Parse Cargo.toml for a [workspace] section and return member paths.
a1b231b… lmata 297 Returns None if this is not a workspace Cargo.toml.
a1b231b… lmata 298 """
a1b231b… lmata 299 try:
a1b231b… lmata 300 text = cargo_toml.read_text(encoding="utf-8")
a1b231b… lmata 301 except OSError:
a1b231b… lmata 302 return None
a1b231b… lmata 303
a1b231b… lmata 304 if "[workspace]" not in text:
a1b231b… lmata 305 return None
a1b231b… lmata 306
a1b231b… lmata 307 # Minimal TOML parser for the members list
a1b231b… lmata 308 members: list[str] = []
a1b231b… lmata 309 in_workspace = False
a1b231b… lmata 310 in_members = False
a1b231b… lmata 311 for line in text.splitlines():
a1b231b… lmata 312 stripped = line.strip()
a1b231b… lmata 313 if stripped == "[workspace]":
a1b231b… lmata 314 in_workspace = True
a1b231b… lmata 315 in_members = False
a1b231b… lmata 316 continue
a1b231b… lmata 317 if in_workspace:
a1b231b… lmata 318 if stripped.startswith("[") and stripped != "[workspace]":
a1b231b… lmata 319 in_workspace = False
a1b231b… lmata 320 in_members = False
a1b231b… lmata 321 continue
a1b231b… lmata 322 if stripped.startswith("members"):
a1b231b… lmata 323 in_members = True
a1b231b… lmata 324 if in_members:
a1b231b… lmata 325 # Collect quoted strings from this line and continuation lines
a1b231b… lmata 326 for token in stripped.split('"'):
a1b231b… lmata 327 candidate = token.strip().strip(",").strip()
a1b231b… lmata 328 if candidate and candidate not in ("=", "[", "]", "members"):
a1b231b… lmata 329 members.append(candidate)
a1b231b… lmata 330 if stripped.endswith("]"):
a1b231b… lmata 331 in_members = False
a1b231b… lmata 332
a1b231b… lmata 333 packages: list[Path] = []
a1b231b… lmata 334 for member in members:
a1b231b… lmata 335 if "*" in member:
a1b231b… lmata 336 base = root / member.split("*")[0].rstrip("/")
a1b231b… lmata 337 if base.is_dir():
a1b231b… lmata 338 for child in sorted(base.iterdir()):
a1b231b… lmata 339 if child.is_dir():
a1b231b… lmata 340 packages.append(child)
a1b231b… lmata 341 else:
a1b231b… lmata 342 resolved = (root / member).resolve()
a1b231b… lmata 343 if resolved.is_dir():
a1b231b… lmata 344 packages.append(resolved)
a1b231b… lmata 345 return packages
a1b231b… lmata 346
a1b231b… lmata 347 # ── Go helpers ────────────────────────────────────────────────────────────
a1b231b… lmata 348
a1b231b… lmata 349 def _go_packages(self, root: Path) -> list[Path]:
a1b231b… lmata 350 """Parse go.work for module paths (use directives)."""
a1b231b… lmata 351 go_work = root / "go.work"
a1b231b… lmata 352 try:
a1b231b… lmata 353 text = go_work.read_text(encoding="utf-8")
a1b231b… lmata 354 except OSError:
a1b231b… lmata 355 return self._fallback_packages(root)
a1b231b… lmata 356
a1b231b… lmata 357 packages: list[Path] = []
a1b231b… lmata 358 for line in text.splitlines():
a1b231b… lmata 359 stripped = line.strip()
a1b231b… lmata 360 if stripped.startswith("use "):
a1b231b… lmata 361 path_str = stripped[4:].strip().strip("()")
a1b231b… lmata 362 if path_str:
a1b231b… lmata 363 resolved = (root / path_str).resolve()
a1b231b… lmata 364 if resolved.is_dir():
a1b231b… lmata 365 packages.append(resolved)
a1b231b… lmata 366 return packages
a1b231b… lmata 367
a1b231b… lmata 368
a1b231b… lmata 369 # ── Ingester ──────────────────────────────────────────────────────────────────
a1b231b… lmata 370
a1b231b… lmata 371
a1b231b… lmata 372 class MonorepoIngester:
a1b231b… lmata 373 """Ingest a monorepo respecting workspace boundaries."""
a1b231b… lmata 374
a1b231b… lmata 375 def __init__(self, store: GraphStore) -> None:
a1b231b… lmata 376 self.store = store
a1b231b… lmata 377
a1b231b… lmata 378 def ingest(
a1b231b… lmata 379 self,
a1b231b… lmata 380 repo_path: str | Path,
a1b231b… lmata 381 clear: bool = False,
a1b231b… lmata 382 ) -> dict[str, Any]:
a1b231b… lmata 383 """
a1b231b… lmata 384 Detect workspace, ingest each package, and create DEPENDS_ON edges
a1b231b… lmata 385 between packages that reference each other.
a1b231b… lmata 386
a1b231b… lmata 387 Steps:
a1b231b… lmata 388 1. Detect workspace configuration.
a1b231b… lmata 389 2. Optionally clear the graph.
a1b231b… lmata 390 3. Create a root Repository node for the monorepo.
a1b231b… lmata 391 4. For each package: create a Repository node, then ingest files.
a1b231b… lmata 392 5. Parse inter-package dependency declarations and create DEPENDS_ON edges.
a1b231b… lmata 393
a1b231b… lmata 394 Returns aggregated stats plus a "packages" count.
a1b231b… lmata 395 """
a1b231b… lmata 396 repo_path = Path(repo_path).resolve()
a1b231b… lmata 397 if not repo_path.exists():
a1b231b… lmata 398 raise FileNotFoundError(f"Repository not found: {repo_path}")
a1b231b… lmata 399
a1b231b… lmata 400 detector = WorkspaceDetector()
a1b231b… lmata 401 config = detector.detect(repo_path)
a1b231b… lmata 402
a1b231b… lmata 403 if config is None:
a1b231b… lmata 404 logger.warning(
a1b231b… lmata 405 "No workspace configuration found at %s; falling back to single-repo ingest",
a1b231b… lmata 406 repo_path,
a1b231b… lmata 407 )
a1b231b… lmata 408 ingester = RepoIngester(self.store)
a1b231b… lmata 409 stats = ingester.ingest(repo_path, clear=clear)
a1b231b… lmata 410 stats["packages"] = 0
a1b231b… lmata 411 stats["workspace_type"] = "none"
a1b231b… lmata 412 return stats
a1b231b… lmata 413
a1b231b… lmata 414 if clear:
a1b231b… lmata 415 self.store.clear()
a1b231b… lmata 416
a1b231b… lmata 417 # Root node for the whole monorepo
a1b231b… lmata 418 self.store.create_node(
a1b231b… lmata 419 NodeLabel.Repository,
a1b231b… lmata 420 {
a1b231b… lmata 421 "name": config.name,
a1b231b… lmata 422 "path": str(repo_path),
a1b231b… lmata 423 "file_path": "",
a1b231b… lmata 424 },
a1b231b… lmata 425 )
a1b231b… lmata 426
a1b231b… lmata 427 aggregate: dict[str, int] = {
a1b231b… lmata 428 "files": 0,
a1b231b… lmata 429 "functions": 0,
a1b231b… lmata 430 "classes": 0,
a1b231b… lmata 431 "edges": 0,
a1b231b… lmata 432 "skipped": 0,
a1b231b… lmata 433 }
a1b231b… lmata 434
a1b231b… lmata 435 ingested_packages: list[tuple[str, Path]] = []
a1b231b… lmata 436
a1b231b… lmata 437 for pkg_path in config.packages:
a1b231b… lmata 438 if not pkg_path.is_dir():
a1b231b… lmata 439 continue
a1b231b… lmata 440
a1b231b… lmata 441 pkg_name = pkg_path.name
a1b231b… lmata 442 logger.info("Ingesting package: %s (%s)", pkg_name, pkg_path)
a1b231b… lmata 443
a1b231b… lmata 444 # Package-level Repository node
a1b231b… lmata 445 self.store.create_node(
a1b231b… lmata 446 NodeLabel.Repository,
a1b231b… lmata 447 {
a1b231b… lmata 448 "name": pkg_name,
a1b231b… lmata 449 "path": str(pkg_path),
a1b231b… lmata 450 "file_path": "",
a1b231b… lmata 451 },
a1b231b… lmata 452 )
a1b231b… lmata 453
a1b231b… lmata 454 # Link package to monorepo root
a1b231b… lmata 455 self.store.create_edge(
a1b231b… lmata 456 from_label=NodeLabel.Repository,
a1b231b… lmata 457 from_key={"name": config.name, "path": str(repo_path)},
a1b231b… lmata 458 edge_type=EdgeType.CONTAINS,
a1b231b… lmata 459 to_label=NodeLabel.Repository,
a1b231b… lmata 460 to_key={"name": pkg_name, "path": str(pkg_path)},
a1b231b… lmata 461 )
a1b231b… lmata 462
a1b231b… lmata 463 # Ingest files in this package
a1b231b… lmata 464 pkg_ingester = RepoIngester(self.store)
a1b231b… lmata 465 try:
a1b231b… lmata 466 pkg_stats = pkg_ingester.ingest(pkg_path, clear=False)
a1b231b… lmata 467 for key in ("files", "functions", "classes", "edges", "skipped"):
a1b231b… lmata 468 aggregate[key] = aggregate.get(key, 0) + pkg_stats.get(key, 0)
a1b231b… lmata 469 ingested_packages.append((pkg_name, pkg_path))
a1b231b… lmata 470 except Exception:
a1b231b… lmata 471 logger.exception("Failed to ingest package %s", pkg_path)
a1b231b… lmata 472
a1b231b… lmata 473 # Create inter-package DEPENDS_ON edges
a1b231b… lmata 474 dep_edges = self._create_dependency_edges(config, ingested_packages)
a1b231b… lmata 475 aggregate["edges"] += dep_edges
a1b231b… lmata 476 aggregate["packages"] = len(ingested_packages)
a1b231b… lmata 477 aggregate["workspace_type"] = config.type
a1b231b… lmata 478
a1b231b… lmata 479 logger.info(
a1b231b… lmata 480 "Monorepo ingest complete (%s): %d packages, %d files",
a1b231b… lmata 481 config.type,
a1b231b… lmata 482 len(ingested_packages),
a1b231b… lmata 483 aggregate["files"],
a1b231b… lmata 484 )
a1b231b… lmata 485 return aggregate
a1b231b… lmata 486
a1b231b… lmata 487 def _create_dependency_edges(
a1b231b… lmata 488 self,
a1b231b… lmata 489 config: WorkspaceConfig,
a1b231b… lmata 490 packages: list[tuple[str, Path]],
a1b231b… lmata 491 ) -> int:
a1b231b… lmata 492 """
a1b231b… lmata 493 Parse each package's manifest (package.json / Cargo.toml / go.mod)
a1b231b… lmata 494 and create DEPENDS_ON edges for references to sibling packages.
a1b231b… lmata 495
a1b231b… lmata 496 Returns the number of edges created.
a1b231b… lmata 497 """
a1b231b… lmata 498 pkg_names = {name for name, _ in packages}
a1b231b… lmata 499 edges_created = 0
a1b231b… lmata 500
a1b231b… lmata 501 for pkg_name, pkg_path in packages:
a1b231b… lmata 502 deps = self._read_package_deps(config.type, pkg_path)
a1b231b… lmata 503 for dep_name in deps:
a1b231b… lmata 504 # Normalise: strip org scope (@scope/name → name)
a1b231b… lmata 505 bare = dep_name.lstrip("@").split("/")[-1] if "/" in dep_name else dep_name
a1b231b… lmata 506 if dep_name in pkg_names or bare in pkg_names:
a1b231b… lmata 507 target = dep_name if dep_name in pkg_names else bare
a1b231b… lmata 508 try:
a1b231b… lmata 509 self.store.create_edge(
a1b231b… lmata 510 from_label=NodeLabel.Repository,
a1b231b… lmata 511 from_key={"name": pkg_name},
a1b231b… lmata 512 edge_type=EdgeType.DEPENDS_ON,
a1b231b… lmata 513 to_label=NodeLabel.Repository,
a1b231b… lmata 514 to_key={"name": target},
a1b231b… lmata 515 )
a1b231b… lmata 516 edges_created += 1
a1b231b… lmata 517 except Exception:
e5e1a3b… lmata 518 logger.debug("Could not create DEPENDS_ON edge %s → %s", pkg_name, target)
a1b231b… lmata 519
a1b231b… lmata 520 return edges_created
a1b231b… lmata 521
a1b231b… lmata 522 def _read_package_deps(self, workspace_type: str, pkg_path: Path) -> list[str]:
a1b231b… lmata 523 """Return a flat list of declared dependency names for a package."""
a1b231b… lmata 524 if workspace_type in ("turborepo", "nx", "yarn", "pnpm"):
a1b231b… lmata 525 return self._js_deps(pkg_path)
a1b231b… lmata 526 if workspace_type == "cargo":
a1b231b… lmata 527 return self._cargo_deps(pkg_path)
a1b231b… lmata 528 if workspace_type == "go":
a1b231b… lmata 529 return self._go_deps(pkg_path)
0f42b61… lmata 530 if workspace_type == "bare":
0f42b61… lmata 531 # Try all known manifest parsers and merge results
0f42b61… lmata 532 deps: list[str] = []
0f42b61… lmata 533 deps.extend(self._js_deps(pkg_path))
0f42b61… lmata 534 deps.extend(self._cargo_deps(pkg_path))
0f42b61… lmata 535 deps.extend(self._go_deps(pkg_path))
0f42b61… lmata 536 return deps
a1b231b… lmata 537 return []
a1b231b… lmata 538
a1b231b… lmata 539 def _js_deps(self, pkg_path: Path) -> list[str]:
a1b231b… lmata 540 pkg_json = pkg_path / "package.json"
a1b231b… lmata 541 if not pkg_json.exists():
a1b231b… lmata 542 return []
a1b231b… lmata 543 try:
a1b231b… lmata 544 data = json.loads(pkg_json.read_text(encoding="utf-8"))
a1b231b… lmata 545 except (OSError, json.JSONDecodeError):
a1b231b… lmata 546 return []
a1b231b… lmata 547 all_deps: dict[str, str] = {}
a1b231b… lmata 548 for key in ("dependencies", "devDependencies", "peerDependencies"):
a1b231b… lmata 549 all_deps.update(data.get(key, {}))
a1b231b… lmata 550 return list(all_deps.keys())
a1b231b… lmata 551
a1b231b… lmata 552 def _cargo_deps(self, pkg_path: Path) -> list[str]:
a1b231b… lmata 553 cargo_toml = pkg_path / "Cargo.toml"
a1b231b… lmata 554 if not cargo_toml.exists():
a1b231b… lmata 555 return []
a1b231b… lmata 556 try:
a1b231b… lmata 557 text = cargo_toml.read_text(encoding="utf-8")
a1b231b… lmata 558 except OSError:
a1b231b… lmata 559 return []
a1b231b… lmata 560 deps: list[str] = []
a1b231b… lmata 561 in_deps = False
a1b231b… lmata 562 for line in text.splitlines():
a1b231b… lmata 563 stripped = line.strip()
a1b231b… lmata 564 if stripped in ("[dependencies]", "[dev-dependencies]", "[build-dependencies]"):
a1b231b… lmata 565 in_deps = True
a1b231b… lmata 566 continue
a1b231b… lmata 567 if stripped.startswith("[") and in_deps:
a1b231b… lmata 568 in_deps = False
a1b231b… lmata 569 continue
a1b231b… lmata 570 if in_deps and "=" in stripped and not stripped.startswith("#"):
a1b231b… lmata 571 name = stripped.split("=")[0].strip()
a1b231b… lmata 572 if name:
a1b231b… lmata 573 deps.append(name)
a1b231b… lmata 574 return deps
a1b231b… lmata 575
a1b231b… lmata 576 def _go_deps(self, pkg_path: Path) -> list[str]:
a1b231b… lmata 577 go_mod = pkg_path / "go.mod"
a1b231b… lmata 578 if not go_mod.exists():
a1b231b… lmata 579 return []
a1b231b… lmata 580 try:
a1b231b… lmata 581 text = go_mod.read_text(encoding="utf-8")
a1b231b… lmata 582 except OSError:
a1b231b… lmata 583 return []
a1b231b… lmata 584 deps: list[str] = []
a1b231b… lmata 585 in_require = False
a1b231b… lmata 586 for line in text.splitlines():
a1b231b… lmata 587 stripped = line.strip()
a1b231b… lmata 588 if stripped.startswith("require ("):
a1b231b… lmata 589 in_require = True
a1b231b… lmata 590 continue
a1b231b… lmata 591 if stripped == ")":
a1b231b… lmata 592 in_require = False
a1b231b… lmata 593 continue
a1b231b… lmata 594 if in_require and stripped and not stripped.startswith("//"):
a1b231b… lmata 595 parts = stripped.split()
a1b231b… lmata 596 if parts:
a1b231b… lmata 597 deps.append(parts[0])
a1b231b… lmata 598 elif stripped.startswith("require ") and not stripped.startswith("require ("):
a1b231b… lmata 599 parts = stripped.split()
a1b231b… lmata 600 if len(parts) >= 2:
a1b231b… lmata 601 deps.append(parts[1])
a1b231b… lmata 602 return deps

Keyboard Shortcuts

Open search /
Next entry (timeline) j
Previous entry (timeline) k
Open focused entry Enter
Show this help ?
Toggle theme Top nav button