Navegador

navegador / navegador / monorepo.py
Blame History Raw 603 lines
1
"""
2
Monorepo support — workspace-aware ingestion for Turborepo, Nx, Yarn/npm/pnpm
3
workspaces, Cargo workspaces, and Go workspaces.
4
5
Usage:
6
from navegador.monorepo import WorkspaceDetector, MonorepoIngester
7
8
config = WorkspaceDetector().detect("/path/to/monorepo")
9
if config:
10
ingester = MonorepoIngester(store)
11
stats = ingester.ingest("/path/to/monorepo")
12
"""
13
14
from __future__ import annotations
15
16
import fnmatch
17
import json
18
import logging
19
from dataclasses import dataclass
20
from pathlib import Path
21
from typing import Any
22
23
from navegador.graph.schema import EdgeType, NodeLabel
24
from navegador.graph.store import GraphStore
25
from navegador.ingestion.parser import RepoIngester
26
27
logger = logging.getLogger(__name__)
28
29
30
# ── Data model ────────────────────────────────────────────────────────────────
31
32
33
@dataclass
34
class WorkspaceConfig:
35
"""Configuration for a detected workspace."""
36
37
type: str # turborepo | nx | yarn | pnpm | cargo | go
38
root: Path
39
packages: list[Path]
40
name: str = ""
41
42
def __post_init__(self) -> None:
43
if not self.name:
44
self.name = self.root.name
45
46
47
# ── Detector ──────────────────────────────────────────────────────────────────
48
49
50
class WorkspaceDetector:
51
"""Detects monorepo workspace configuration from a repository root."""
52
53
def detect(self, repo_path: str | Path) -> WorkspaceConfig | None:
54
"""
55
Auto-detect workspace type and package locations.
56
57
Checks (in priority order):
58
turbo.json → Turborepo
59
nx.json → Nx
60
pnpm-workspace.yaml → pnpm workspaces
61
package.json → Yarn/npm workspaces (if "workspaces" key present)
62
Cargo.toml → Rust workspace (if [workspace] section present)
63
go.work → Go workspace
64
65
Returns None when no known workspace configuration is found.
66
"""
67
root = Path(repo_path).resolve()
68
69
# Turborepo
70
if (root / "turbo.json").exists():
71
packages = self._js_workspace_packages(root)
72
return WorkspaceConfig(type="turborepo", root=root, packages=packages)
73
74
# Nx
75
if (root / "nx.json").exists():
76
packages = self._nx_packages(root)
77
return WorkspaceConfig(type="nx", root=root, packages=packages)
78
79
# pnpm workspaces
80
if (root / "pnpm-workspace.yaml").exists():
81
packages = self._pnpm_packages(root)
82
return WorkspaceConfig(type="pnpm", root=root, packages=packages)
83
84
# Yarn / npm workspaces
85
pkg_json = root / "package.json"
86
if pkg_json.exists():
87
try:
88
data = json.loads(pkg_json.read_text(encoding="utf-8"))
89
except (OSError, json.JSONDecodeError):
90
data = {}
91
if "workspaces" in data:
92
patterns = data["workspaces"]
93
# Yarn Berry stores them under workspaces.packages
94
if isinstance(patterns, dict):
95
patterns = patterns.get("packages", [])
96
packages = self._glob_packages(root, patterns)
97
return WorkspaceConfig(type="yarn", root=root, packages=packages)
98
99
# Cargo workspace
100
cargo_toml = root / "Cargo.toml"
101
if cargo_toml.exists():
102
packages = self._cargo_packages(root, cargo_toml)
103
if packages is not None:
104
return WorkspaceConfig(type="cargo", root=root, packages=packages)
105
106
# Go workspace
107
if (root / "go.work").exists():
108
packages = self._go_packages(root)
109
return WorkspaceConfig(type="go", root=root, packages=packages)
110
111
# Bare monorepo — no tooling, just a directory of apps/services
112
packages = self._bare_packages(root)
113
if len(packages) >= 2:
114
return WorkspaceConfig(type="bare", root=root, packages=packages)
115
116
return None
117
118
# ── JS-family helpers ─────────────────────────────────────────────────────
119
120
def _js_workspace_packages(self, root: Path) -> list[Path]:
121
"""
122
Resolve workspace package paths from package.json workspaces field.
123
Falls back to scanning for package.json files one level down.
124
"""
125
pkg_json = root / "package.json"
126
if pkg_json.exists():
127
try:
128
data = json.loads(pkg_json.read_text(encoding="utf-8"))
129
except (OSError, json.JSONDecodeError):
130
data = {}
131
patterns = data.get("workspaces", [])
132
if isinstance(patterns, dict):
133
patterns = patterns.get("packages", [])
134
if patterns:
135
return self._glob_packages(root, patterns)
136
return self._fallback_packages(root)
137
138
def _nx_packages(self, root: Path) -> list[Path]:
139
"""
140
Nx workspaces store packages in apps/ and libs/ by convention,
141
or declare them in nx.json under "projects".
142
"""
143
# Try reading nx.json for explicit projects
144
nx_json = root / "nx.json"
145
try:
146
json.loads(nx_json.read_text(encoding="utf-8"))
147
except (OSError, json.JSONDecodeError):
148
pass
149
150
# Nx 16+ uses workspaceLayout or projects in project.json files
151
packages: list[Path] = []
152
for subdir in ("apps", "libs", "packages"):
153
base = root / subdir
154
if base.is_dir():
155
for child in sorted(base.iterdir()):
156
if child.is_dir() and not child.name.startswith("."):
157
packages.append(child)
158
159
if packages:
160
return packages
161
162
# Fall through to package.json-based resolution
163
return self._js_workspace_packages(root)
164
165
def _pnpm_packages(self, root: Path) -> list[Path]:
166
"""Parse pnpm-workspace.yaml for package glob patterns."""
167
yaml_path = root / "pnpm-workspace.yaml"
168
try:
169
text = yaml_path.read_text(encoding="utf-8")
170
except OSError:
171
return self._fallback_packages(root)
172
173
# Minimal YAML list parser — avoids a PyYAML dependency
174
patterns: list[str] = []
175
in_packages = False
176
for line in text.splitlines():
177
stripped = line.strip()
178
if stripped.startswith("packages:"):
179
in_packages = True
180
continue
181
if in_packages:
182
if stripped.startswith("-"):
183
value = stripped.lstrip("- ").strip().strip("'\"")
184
patterns.append(value)
185
elif stripped and not stripped.startswith("#"):
186
in_packages = False
187
188
if patterns:
189
return self._glob_packages(root, patterns)
190
return self._fallback_packages(root)
191
192
def _glob_packages(self, root: Path, patterns: list[str]) -> list[Path]:
193
"""Expand workspace glob patterns (e.g. 'packages/*') to absolute Paths."""
194
packages: list[Path] = []
195
for pattern in patterns:
196
# Skip negation patterns
197
if pattern.startswith("!"):
198
continue
199
# Simple glob expansion using fnmatch against existing dirs
200
if "*" in pattern or "?" in pattern:
201
# Split at the first wildcard component
202
parts = Path(pattern).parts
203
base_parts: list[str] = []
204
for p in parts:
205
if "*" in p or "?" in p:
206
break
207
base_parts.append(p)
208
base = root / Path(*base_parts) if base_parts else root
209
if base.is_dir():
210
# The wildcard component
211
wildcard_idx = len(base_parts)
212
wild = parts[wildcard_idx] if wildcard_idx < len(parts) else "*"
213
for child in sorted(base.iterdir()):
214
if child.is_dir() and fnmatch.fnmatch(child.name, wild):
215
packages.append(child)
216
else:
217
resolved = (root / pattern).resolve()
218
if resolved.is_dir():
219
packages.append(resolved)
220
return packages
221
222
def _fallback_packages(self, root: Path) -> list[Path]:
223
"""Scan one level down for directories containing a package.json."""
224
packages: list[Path] = []
225
for child in sorted(root.iterdir()):
226
if child.is_dir() and not child.name.startswith("."):
227
if (child / "package.json").exists():
228
packages.append(child)
229
return packages
230
231
# ── Bare monorepo helpers ─────────────────────────────────────────────────
232
233
# Manifests expected directly inside the package root
234
_PROJECT_MANIFESTS = (
235
"package.json",
236
"pyproject.toml",
237
"setup.py",
238
"setup.cfg",
239
"Cargo.toml",
240
"go.mod",
241
"pom.xml",
242
"build.gradle",
243
"build.gradle.kts",
244
"Gemfile",
245
"composer.json",
246
"mix.exs",
247
)
248
249
# Manifests that may live one subdirectory deeper (e.g. Django's manage.py
250
# inside an inner package dir: myapp/myapp/manage.py)
251
_NESTED_MANIFESTS = (
252
"manage.py",
253
"wsgi.py",
254
"asgi.py",
255
)
256
257
def _bare_packages(self, root: Path) -> list[Path]:
258
"""
259
Detect a bare monorepo: a directory whose immediate children are
260
independent apps/services with no shared workspace tooling.
261
262
A child directory qualifies if it contains at least one recognised
263
project manifest directly, or a Django/WSGI manifest one level deeper
264
(e.g. myapp/myapp/manage.py).
265
Non-project dirs (docs, scripts, config-only folders) are skipped.
266
"""
267
packages: list[Path] = []
268
for child in sorted(root.iterdir()):
269
if not child.is_dir() or child.name.startswith("."):
270
continue
271
# Check top-level manifests first
272
for manifest in self._PROJECT_MANIFESTS:
273
if (child / manifest).exists():
274
packages.append(child)
275
break
276
else:
277
# Fall back: look one level deeper for Django/WSGI markers
278
if self._has_nested_manifest(child):
279
packages.append(child)
280
return packages
281
282
def _has_nested_manifest(self, pkg_root: Path) -> bool:
283
"""Return True if any immediate subdirectory contains a nested manifest."""
284
for subdir in pkg_root.iterdir():
285
if not subdir.is_dir() or subdir.name.startswith("."):
286
continue
287
for manifest in self._NESTED_MANIFESTS:
288
if (subdir / manifest).exists():
289
return True
290
return False
291
292
# ── Cargo helpers ─────────────────────────────────────────────────────────
293
294
def _cargo_packages(self, root: Path, cargo_toml: Path) -> list[Path] | None:
295
"""
296
Parse Cargo.toml for a [workspace] section and return member paths.
297
Returns None if this is not a workspace Cargo.toml.
298
"""
299
try:
300
text = cargo_toml.read_text(encoding="utf-8")
301
except OSError:
302
return None
303
304
if "[workspace]" not in text:
305
return None
306
307
# Minimal TOML parser for the members list
308
members: list[str] = []
309
in_workspace = False
310
in_members = False
311
for line in text.splitlines():
312
stripped = line.strip()
313
if stripped == "[workspace]":
314
in_workspace = True
315
in_members = False
316
continue
317
if in_workspace:
318
if stripped.startswith("[") and stripped != "[workspace]":
319
in_workspace = False
320
in_members = False
321
continue
322
if stripped.startswith("members"):
323
in_members = True
324
if in_members:
325
# Collect quoted strings from this line and continuation lines
326
for token in stripped.split('"'):
327
candidate = token.strip().strip(",").strip()
328
if candidate and candidate not in ("=", "[", "]", "members"):
329
members.append(candidate)
330
if stripped.endswith("]"):
331
in_members = False
332
333
packages: list[Path] = []
334
for member in members:
335
if "*" in member:
336
base = root / member.split("*")[0].rstrip("/")
337
if base.is_dir():
338
for child in sorted(base.iterdir()):
339
if child.is_dir():
340
packages.append(child)
341
else:
342
resolved = (root / member).resolve()
343
if resolved.is_dir():
344
packages.append(resolved)
345
return packages
346
347
# ── Go helpers ────────────────────────────────────────────────────────────
348
349
def _go_packages(self, root: Path) -> list[Path]:
350
"""Parse go.work for module paths (use directives)."""
351
go_work = root / "go.work"
352
try:
353
text = go_work.read_text(encoding="utf-8")
354
except OSError:
355
return self._fallback_packages(root)
356
357
packages: list[Path] = []
358
for line in text.splitlines():
359
stripped = line.strip()
360
if stripped.startswith("use "):
361
path_str = stripped[4:].strip().strip("()")
362
if path_str:
363
resolved = (root / path_str).resolve()
364
if resolved.is_dir():
365
packages.append(resolved)
366
return packages
367
368
369
# ── Ingester ──────────────────────────────────────────────────────────────────
370
371
372
class MonorepoIngester:
373
"""Ingest a monorepo respecting workspace boundaries."""
374
375
def __init__(self, store: GraphStore) -> None:
376
self.store = store
377
378
def ingest(
379
self,
380
repo_path: str | Path,
381
clear: bool = False,
382
) -> dict[str, Any]:
383
"""
384
Detect workspace, ingest each package, and create DEPENDS_ON edges
385
between packages that reference each other.
386
387
Steps:
388
1. Detect workspace configuration.
389
2. Optionally clear the graph.
390
3. Create a root Repository node for the monorepo.
391
4. For each package: create a Repository node, then ingest files.
392
5. Parse inter-package dependency declarations and create DEPENDS_ON edges.
393
394
Returns aggregated stats plus a "packages" count.
395
"""
396
repo_path = Path(repo_path).resolve()
397
if not repo_path.exists():
398
raise FileNotFoundError(f"Repository not found: {repo_path}")
399
400
detector = WorkspaceDetector()
401
config = detector.detect(repo_path)
402
403
if config is None:
404
logger.warning(
405
"No workspace configuration found at %s; falling back to single-repo ingest",
406
repo_path,
407
)
408
ingester = RepoIngester(self.store)
409
stats = ingester.ingest(repo_path, clear=clear)
410
stats["packages"] = 0
411
stats["workspace_type"] = "none"
412
return stats
413
414
if clear:
415
self.store.clear()
416
417
# Root node for the whole monorepo
418
self.store.create_node(
419
NodeLabel.Repository,
420
{
421
"name": config.name,
422
"path": str(repo_path),
423
"file_path": "",
424
},
425
)
426
427
aggregate: dict[str, int] = {
428
"files": 0,
429
"functions": 0,
430
"classes": 0,
431
"edges": 0,
432
"skipped": 0,
433
}
434
435
ingested_packages: list[tuple[str, Path]] = []
436
437
for pkg_path in config.packages:
438
if not pkg_path.is_dir():
439
continue
440
441
pkg_name = pkg_path.name
442
logger.info("Ingesting package: %s (%s)", pkg_name, pkg_path)
443
444
# Package-level Repository node
445
self.store.create_node(
446
NodeLabel.Repository,
447
{
448
"name": pkg_name,
449
"path": str(pkg_path),
450
"file_path": "",
451
},
452
)
453
454
# Link package to monorepo root
455
self.store.create_edge(
456
from_label=NodeLabel.Repository,
457
from_key={"name": config.name, "path": str(repo_path)},
458
edge_type=EdgeType.CONTAINS,
459
to_label=NodeLabel.Repository,
460
to_key={"name": pkg_name, "path": str(pkg_path)},
461
)
462
463
# Ingest files in this package
464
pkg_ingester = RepoIngester(self.store)
465
try:
466
pkg_stats = pkg_ingester.ingest(pkg_path, clear=False)
467
for key in ("files", "functions", "classes", "edges", "skipped"):
468
aggregate[key] = aggregate.get(key, 0) + pkg_stats.get(key, 0)
469
ingested_packages.append((pkg_name, pkg_path))
470
except Exception:
471
logger.exception("Failed to ingest package %s", pkg_path)
472
473
# Create inter-package DEPENDS_ON edges
474
dep_edges = self._create_dependency_edges(config, ingested_packages)
475
aggregate["edges"] += dep_edges
476
aggregate["packages"] = len(ingested_packages)
477
aggregate["workspace_type"] = config.type
478
479
logger.info(
480
"Monorepo ingest complete (%s): %d packages, %d files",
481
config.type,
482
len(ingested_packages),
483
aggregate["files"],
484
)
485
return aggregate
486
487
def _create_dependency_edges(
488
self,
489
config: WorkspaceConfig,
490
packages: list[tuple[str, Path]],
491
) -> int:
492
"""
493
Parse each package's manifest (package.json / Cargo.toml / go.mod)
494
and create DEPENDS_ON edges for references to sibling packages.
495
496
Returns the number of edges created.
497
"""
498
pkg_names = {name for name, _ in packages}
499
edges_created = 0
500
501
for pkg_name, pkg_path in packages:
502
deps = self._read_package_deps(config.type, pkg_path)
503
for dep_name in deps:
504
# Normalise: strip org scope (@scope/name → name)
505
bare = dep_name.lstrip("@").split("/")[-1] if "/" in dep_name else dep_name
506
if dep_name in pkg_names or bare in pkg_names:
507
target = dep_name if dep_name in pkg_names else bare
508
try:
509
self.store.create_edge(
510
from_label=NodeLabel.Repository,
511
from_key={"name": pkg_name},
512
edge_type=EdgeType.DEPENDS_ON,
513
to_label=NodeLabel.Repository,
514
to_key={"name": target},
515
)
516
edges_created += 1
517
except Exception:
518
logger.debug("Could not create DEPENDS_ON edge %s → %s", pkg_name, target)
519
520
return edges_created
521
522
def _read_package_deps(self, workspace_type: str, pkg_path: Path) -> list[str]:
523
"""Return a flat list of declared dependency names for a package."""
524
if workspace_type in ("turborepo", "nx", "yarn", "pnpm"):
525
return self._js_deps(pkg_path)
526
if workspace_type == "cargo":
527
return self._cargo_deps(pkg_path)
528
if workspace_type == "go":
529
return self._go_deps(pkg_path)
530
if workspace_type == "bare":
531
# Try all known manifest parsers and merge results
532
deps: list[str] = []
533
deps.extend(self._js_deps(pkg_path))
534
deps.extend(self._cargo_deps(pkg_path))
535
deps.extend(self._go_deps(pkg_path))
536
return deps
537
return []
538
539
def _js_deps(self, pkg_path: Path) -> list[str]:
540
pkg_json = pkg_path / "package.json"
541
if not pkg_json.exists():
542
return []
543
try:
544
data = json.loads(pkg_json.read_text(encoding="utf-8"))
545
except (OSError, json.JSONDecodeError):
546
return []
547
all_deps: dict[str, str] = {}
548
for key in ("dependencies", "devDependencies", "peerDependencies"):
549
all_deps.update(data.get(key, {}))
550
return list(all_deps.keys())
551
552
def _cargo_deps(self, pkg_path: Path) -> list[str]:
553
cargo_toml = pkg_path / "Cargo.toml"
554
if not cargo_toml.exists():
555
return []
556
try:
557
text = cargo_toml.read_text(encoding="utf-8")
558
except OSError:
559
return []
560
deps: list[str] = []
561
in_deps = False
562
for line in text.splitlines():
563
stripped = line.strip()
564
if stripped in ("[dependencies]", "[dev-dependencies]", "[build-dependencies]"):
565
in_deps = True
566
continue
567
if stripped.startswith("[") and in_deps:
568
in_deps = False
569
continue
570
if in_deps and "=" in stripped and not stripped.startswith("#"):
571
name = stripped.split("=")[0].strip()
572
if name:
573
deps.append(name)
574
return deps
575
576
def _go_deps(self, pkg_path: Path) -> list[str]:
577
go_mod = pkg_path / "go.mod"
578
if not go_mod.exists():
579
return []
580
try:
581
text = go_mod.read_text(encoding="utf-8")
582
except OSError:
583
return []
584
deps: list[str] = []
585
in_require = False
586
for line in text.splitlines():
587
stripped = line.strip()
588
if stripped.startswith("require ("):
589
in_require = True
590
continue
591
if stripped == ")":
592
in_require = False
593
continue
594
if in_require and stripped and not stripped.startswith("//"):
595
parts = stripped.split()
596
if parts:
597
deps.append(parts[0])
598
elif stripped.startswith("require ") and not stripped.startswith("require ("):
599
parts = stripped.split()
600
if len(parts) >= 2:
601
deps.append(parts[1])
602
return deps
603

Keyboard Shortcuts

Open search /
Next entry (timeline) j
Previous entry (timeline) k
Open focused entry Enter
Show this help ?
Toggle theme Top nav button