|
1
|
""" |
|
2
|
VCS submodule traversal — ingest a parent repo and all its git submodules as |
|
3
|
linked Repository nodes. |
|
4
|
|
|
5
|
Issue: #61 |
|
6
|
|
|
7
|
Usage:: |
|
8
|
|
|
9
|
from navegador.submodules import SubmoduleIngester |
|
10
|
|
|
11
|
ing = SubmoduleIngester(store) |
|
12
|
submodules = ing.detect_submodules("/path/to/repo") |
|
13
|
stats = ing.ingest_with_submodules("/path/to/repo", clear=False) |
|
14
|
""" |
|
15
|
|
|
16
|
from __future__ import annotations |
|
17
|
|
|
18
|
import logging |
|
19
|
import re |
|
20
|
from pathlib import Path |
|
21
|
from typing import Any |
|
22
|
|
|
23
|
from navegador.graph.schema import EdgeType, NodeLabel |
|
24
|
from navegador.graph.store import GraphStore |
|
25
|
|
|
26
|
logger = logging.getLogger(__name__) |
|
27
|
|
|
28
|
|
|
29
|
class SubmoduleIngester: |
|
30
|
""" |
|
31
|
Detects and ingests git submodules as linked Repository nodes. |
|
32
|
|
|
33
|
After ingesting the parent repository (using RepoIngester) each submodule |
|
34
|
is also ingested and a DEPENDS_ON edge is created from the parent |
|
35
|
Repository node to each submodule Repository node. |
|
36
|
""" |
|
37
|
|
|
38
|
def __init__(self, store: GraphStore) -> None: |
|
39
|
self.store = store |
|
40
|
|
|
41
|
# ── Submodule detection ─────────────────────────────────────────────────── |
|
42
|
|
|
43
|
def detect_submodules(self, repo_path: str | Path) -> list[dict[str, Any]]: |
|
44
|
""" |
|
45
|
Parse ``.gitmodules`` and return a list of submodule descriptors. |
|
46
|
|
|
47
|
Parameters |
|
48
|
---------- |
|
49
|
repo_path: |
|
50
|
Root of the parent repository. |
|
51
|
|
|
52
|
Returns |
|
53
|
------- |
|
54
|
list of dicts with keys: |
|
55
|
``name`` — submodule logical name |
|
56
|
``path`` — relative path within the parent repo |
|
57
|
``url`` — remote URL |
|
58
|
``abs_path`` — absolute filesystem path (``repo_path / path``) |
|
59
|
""" |
|
60
|
repo_root = Path(repo_path).resolve() |
|
61
|
gitmodules = repo_root / ".gitmodules" |
|
62
|
|
|
63
|
if not gitmodules.exists(): |
|
64
|
return [] |
|
65
|
|
|
66
|
text = gitmodules.read_text(encoding="utf-8") |
|
67
|
return _parse_gitmodules(text, repo_root) |
|
68
|
|
|
69
|
# ── Ingestion ───────────────────────────────────────────────────────────── |
|
70
|
|
|
71
|
def ingest_with_submodules( |
|
72
|
self, |
|
73
|
repo_path: str | Path, |
|
74
|
clear: bool = False, |
|
75
|
) -> dict[str, Any]: |
|
76
|
""" |
|
77
|
Ingest the parent repository and all discovered submodules. |
|
78
|
|
|
79
|
For each submodule whose ``abs_path`` exists on disk the full code |
|
80
|
ingestion pipeline (``RepoIngester``) is run. Repository nodes are |
|
81
|
linked with DEPENDS_ON edges. |
|
82
|
|
|
83
|
Parameters |
|
84
|
---------- |
|
85
|
repo_path: |
|
86
|
Root of the parent (super-project) repository. |
|
87
|
clear: |
|
88
|
If ``True``, wipe the graph before ingesting the parent. |
|
89
|
|
|
90
|
Returns |
|
91
|
------- |
|
92
|
dict with keys: |
|
93
|
``parent`` — ingestion stats for the parent repo |
|
94
|
``submodules`` — dict keyed by submodule name → stats or error |
|
95
|
``total_files`` — aggregate file count |
|
96
|
""" |
|
97
|
from navegador.ingestion.parser import RepoIngester |
|
98
|
|
|
99
|
repo_root = Path(repo_path).resolve() |
|
100
|
parent_name = repo_root.name |
|
101
|
|
|
102
|
ingester = RepoIngester(self.store) |
|
103
|
|
|
104
|
# Ingest parent |
|
105
|
logger.info("SubmoduleIngester: ingesting parent %s", repo_root) |
|
106
|
parent_stats = ingester.ingest(str(repo_root), clear=clear) |
|
107
|
|
|
108
|
# Ensure parent Repository node exists |
|
109
|
self.store.create_node( |
|
110
|
NodeLabel.Repository, |
|
111
|
{ |
|
112
|
"name": parent_name, |
|
113
|
"path": str(repo_root), |
|
114
|
"language": "", |
|
115
|
"description": "parent repository", |
|
116
|
}, |
|
117
|
) |
|
118
|
|
|
119
|
submodules = self.detect_submodules(repo_root) |
|
120
|
sub_results: dict[str, Any] = {} |
|
121
|
total_files = parent_stats.get("files", 0) |
|
122
|
|
|
123
|
for sub in submodules: |
|
124
|
sub_name = sub["name"] |
|
125
|
sub_path = sub["abs_path"] |
|
126
|
|
|
127
|
if not Path(sub_path).exists(): |
|
128
|
logger.warning( |
|
129
|
"SubmoduleIngester: submodule %s not found at %s (not initialised?)", |
|
130
|
sub_name, |
|
131
|
sub_path, |
|
132
|
) |
|
133
|
sub_results[sub_name] = {"error": f"path not found: {sub_path}"} |
|
134
|
continue |
|
135
|
|
|
136
|
logger.info("SubmoduleIngester: ingesting submodule %s → %s", sub_name, sub_path) |
|
137
|
try: |
|
138
|
sub_stats = ingester.ingest(str(sub_path), clear=False) |
|
139
|
sub_results[sub_name] = sub_stats |
|
140
|
total_files += sub_stats.get("files", 0) |
|
141
|
except Exception as exc: # noqa: BLE001 |
|
142
|
logger.error("SubmoduleIngester: failed to ingest %s: %s", sub_name, exc) |
|
143
|
sub_results[sub_name] = {"error": str(exc)} |
|
144
|
continue |
|
145
|
|
|
146
|
# Create submodule Repository node |
|
147
|
self.store.create_node( |
|
148
|
NodeLabel.Repository, |
|
149
|
{ |
|
150
|
"name": sub_name, |
|
151
|
"path": str(sub_path), |
|
152
|
"language": "", |
|
153
|
"description": sub.get("url", ""), |
|
154
|
}, |
|
155
|
) |
|
156
|
|
|
157
|
# parent -DEPENDS_ON-> submodule |
|
158
|
self.store.create_edge( |
|
159
|
NodeLabel.Repository, |
|
160
|
{"name": parent_name}, |
|
161
|
EdgeType.DEPENDS_ON, |
|
162
|
NodeLabel.Repository, |
|
163
|
{"name": sub_name}, |
|
164
|
) |
|
165
|
|
|
166
|
return { |
|
167
|
"parent": parent_stats, |
|
168
|
"submodules": sub_results, |
|
169
|
"total_files": total_files, |
|
170
|
} |
|
171
|
|
|
172
|
|
|
173
|
# ── Parser ──────────────────────────────────────────────────────────────────── |
|
174
|
|
|
175
|
|
|
176
|
def _parse_gitmodules(text: str, repo_root: Path) -> list[dict[str, Any]]: |
|
177
|
""" |
|
178
|
Parse a ``.gitmodules`` file into a list of submodule dicts. |
|
179
|
|
|
180
|
``.gitmodules`` format:: |
|
181
|
|
|
182
|
[submodule "name"] |
|
183
|
path = relative/path |
|
184
|
url = https://... |
|
185
|
""" |
|
186
|
submodules: list[dict[str, Any]] = [] |
|
187
|
current: dict[str, str] = {} |
|
188
|
|
|
189
|
for line in text.splitlines(): |
|
190
|
line = line.strip() |
|
191
|
|
|
192
|
header = re.match(r'^\[submodule\s+"([^"]+)"\]$', line) |
|
193
|
if header: |
|
194
|
if current.get("name"): |
|
195
|
submodules.append(_finalise(current, repo_root)) |
|
196
|
current = {"name": header.group(1)} |
|
197
|
continue |
|
198
|
|
|
199
|
kv = re.match(r"^(\w+)\s*=\s*(.+)$", line) |
|
200
|
if kv: |
|
201
|
current[kv.group(1).strip()] = kv.group(2).strip() |
|
202
|
|
|
203
|
if current.get("name"): |
|
204
|
submodules.append(_finalise(current, repo_root)) |
|
205
|
|
|
206
|
return submodules |
|
207
|
|
|
208
|
|
|
209
|
def _finalise(raw: dict[str, str], repo_root: Path) -> dict[str, Any]: |
|
210
|
rel_path = raw.get("path", raw.get("name", "")) |
|
211
|
return { |
|
212
|
"name": raw["name"], |
|
213
|
"path": rel_path, |
|
214
|
"url": raw.get("url", ""), |
|
215
|
"abs_path": str(repo_root / rel_path), |
|
216
|
} |
|
217
|
|