|
1
|
""" |
|
2
|
WikiIngester — pulls pages from a project wiki into the knowledge graph. |
|
3
|
|
|
4
|
Supports: |
|
5
|
- GitHub wiki (cloned as a git repo at <repo>.wiki.git) |
|
6
|
- Local markdown directory (any folder of .md files) |
|
7
|
|
|
8
|
Each page becomes a WikiPage node. Headings and bold terms are scanned for |
|
9
|
names that match existing Concept/Function/Class nodes — matches get a |
|
10
|
DOCUMENTS edge so agents can traverse wiki ↔ code. |
|
11
|
""" |
|
12
|
|
|
13
|
import logging |
|
14
|
import re |
|
15
|
import urllib.request |
|
16
|
from pathlib import Path |
|
17
|
|
|
18
|
from navegador.graph.schema import EdgeType, NodeLabel |
|
19
|
from navegador.graph.store import GraphStore |
|
20
|
|
|
21
|
logger = logging.getLogger(__name__) |
|
22
|
|
|
23
|
_HEADING_RE = re.compile(r"^#{1,6}\s+(.+)", re.MULTILINE) |
|
24
|
_BOLD_RE = re.compile(r"\*\*(.+?)\*\*|__(.+?)__") |
|
25
|
|
|
26
|
|
|
27
|
def _extract_terms(markdown: str) -> list[str]: |
|
28
|
"""Pull heading text and bold terms out of a markdown string.""" |
|
29
|
terms = [m.group(1).strip() for m in _HEADING_RE.finditer(markdown)] |
|
30
|
for m in _BOLD_RE.finditer(markdown): |
|
31
|
term = (m.group(1) or m.group(2) or "").strip() |
|
32
|
if term: |
|
33
|
terms.append(term) |
|
34
|
return list(dict.fromkeys(terms)) # dedupe, preserve order |
|
35
|
|
|
36
|
|
|
37
|
class WikiIngester: |
|
38
|
""" |
|
39
|
Ingests wiki pages into the navegador graph. |
|
40
|
|
|
41
|
Usage: |
|
42
|
ingester = WikiIngester(store) |
|
43
|
|
|
44
|
# From a local directory of markdown files |
|
45
|
ingester.ingest_local("/path/to/wiki-clone") |
|
46
|
|
|
47
|
# From GitHub (clones the wiki repo) |
|
48
|
ingester.ingest_github("owner/repo") |
|
49
|
""" |
|
50
|
|
|
51
|
def __init__(self, store: GraphStore) -> None: |
|
52
|
self.store = store |
|
53
|
|
|
54
|
# ── Entry points ────────────────────────────────────────────────────────── |
|
55
|
|
|
56
|
def ingest_local(self, wiki_dir: str | Path) -> dict[str, int]: |
|
57
|
"""Ingest all .md files in a local directory.""" |
|
58
|
wiki_dir = Path(wiki_dir) |
|
59
|
if not wiki_dir.exists(): |
|
60
|
raise FileNotFoundError(f"Wiki directory not found: {wiki_dir}") |
|
61
|
|
|
62
|
stats = {"pages": 0, "links": 0} |
|
63
|
for md_file in sorted(wiki_dir.rglob("*.md")): |
|
64
|
content = md_file.read_text(encoding="utf-8", errors="replace") |
|
65
|
page_name = md_file.stem.replace("-", " ").replace("_", " ") |
|
66
|
links = self._ingest_page( |
|
67
|
name=page_name, |
|
68
|
content=content, |
|
69
|
source="local", |
|
70
|
url=str(md_file), |
|
71
|
) |
|
72
|
stats["pages"] += 1 |
|
73
|
stats["links"] += links |
|
74
|
|
|
75
|
logger.info("Wiki (local): %d pages, %d links", stats["pages"], stats["links"]) |
|
76
|
return stats |
|
77
|
|
|
78
|
def ingest_github( |
|
79
|
self, |
|
80
|
repo: str, |
|
81
|
token: str = "", |
|
82
|
clone_dir: str | Path | None = None, |
|
83
|
) -> dict[str, int]: |
|
84
|
""" |
|
85
|
Ingest a GitHub wiki by cloning it then processing locally. |
|
86
|
|
|
87
|
Args: |
|
88
|
repo: "owner/repo" — the GitHub repository. |
|
89
|
token: GitHub personal access token (needed for private repos). |
|
90
|
clone_dir: Where to clone. Defaults to a temp directory. |
|
91
|
""" |
|
92
|
import subprocess |
|
93
|
import tempfile |
|
94
|
|
|
95
|
wiki_url = f"https://github.com/{repo}.wiki.git" |
|
96
|
if token: |
|
97
|
wiki_url = f"https://{token}@github.com/{repo}.wiki.git" |
|
98
|
|
|
99
|
if clone_dir is None: |
|
100
|
tmp = tempfile.mkdtemp(prefix="navegador-wiki-") |
|
101
|
clone_dir = Path(tmp) |
|
102
|
else: |
|
103
|
clone_dir = Path(clone_dir) |
|
104
|
|
|
105
|
logger.info("Cloning wiki %s → %s", wiki_url, clone_dir) |
|
106
|
result = subprocess.run( |
|
107
|
["git", "clone", "--depth=1", wiki_url, str(clone_dir)], |
|
108
|
capture_output=True, |
|
109
|
text=True, |
|
110
|
) |
|
111
|
if result.returncode != 0: |
|
112
|
# Wiki may not exist yet — treat as empty |
|
113
|
logger.warning("Wiki clone failed: %s", result.stderr.strip()) |
|
114
|
return {"pages": 0, "links": 0} |
|
115
|
|
|
116
|
return self.ingest_local(clone_dir) |
|
117
|
|
|
118
|
def ingest_github_api(self, repo: str, token: str = "") -> dict[str, int]: |
|
119
|
""" |
|
120
|
Ingest GitHub wiki using the GitHub REST API (no git required). |
|
121
|
Falls back gracefully if the wiki is empty or API rate-limited. |
|
122
|
|
|
123
|
Args: |
|
124
|
repo: "owner/repo" |
|
125
|
token: GitHub personal access token (recommended to avoid rate limits). |
|
126
|
""" |
|
127
|
headers = {"Accept": "application/vnd.github.v3+json"} |
|
128
|
if token: |
|
129
|
headers["Authorization"] = f"token {token}" |
|
130
|
|
|
131
|
owner, name = repo.split("/", 1) |
|
132
|
# GitHub doesn't expose wiki pages via REST — use GraphQL or clone |
|
133
|
# For now we pull the repo's README and docs/ folder as knowledge seeds |
|
134
|
stats = {"pages": 0, "links": 0} |
|
135
|
|
|
136
|
for path in ["README.md", "CONTRIBUTING.md", "ARCHITECTURE.md", "docs/index.md"]: |
|
137
|
url = f"https://api.github.com/repos/{owner}/{name}/contents/{path}" |
|
138
|
try: |
|
139
|
req = urllib.request.Request(url, headers=headers) |
|
140
|
with urllib.request.urlopen(req, timeout=10) as resp: |
|
141
|
import base64 |
|
142
|
import json as _json |
|
143
|
|
|
144
|
data = _json.loads(resp.read().decode()) |
|
145
|
content = base64.b64decode(data["content"]).decode("utf-8", errors="replace") # noqa: E501 |
|
146
|
page_name = Path(path).stem.replace("-", " ").replace("_", " ") |
|
147
|
html_url = data.get("html_url", "") |
|
148
|
links = self._ingest_page(page_name, content, source="github", url=html_url) |
|
149
|
stats["pages"] += 1 |
|
150
|
stats["links"] += links |
|
151
|
except Exception as exc: |
|
152
|
logger.warning("Skipping %s: %s", path, exc) |
|
153
|
|
|
154
|
logger.info("Wiki (GitHub API): %d pages, %d links", stats["pages"], stats["links"]) |
|
155
|
return stats |
|
156
|
|
|
157
|
# ── Internal ────────────────────────────────────────────────────────────── |
|
158
|
|
|
159
|
def _ingest_page( |
|
160
|
self, |
|
161
|
name: str, |
|
162
|
content: str, |
|
163
|
source: str, |
|
164
|
url: str, |
|
165
|
) -> int: |
|
166
|
"""Store one wiki page and return the number of DOCUMENTS links created.""" |
|
167
|
self.store.create_node( |
|
168
|
NodeLabel.WikiPage, |
|
169
|
{ |
|
170
|
"name": name, |
|
171
|
"url": url, |
|
172
|
"source": source, |
|
173
|
"content": content[:4000], # cap stored content |
|
174
|
}, |
|
175
|
) |
|
176
|
|
|
177
|
links = 0 |
|
178
|
for term in _extract_terms(content): |
|
179
|
links += self._try_link(name, term) |
|
180
|
|
|
181
|
return links |
|
182
|
|
|
183
|
def _try_link(self, wiki_page_name: str, term: str) -> int: |
|
184
|
""" |
|
185
|
If term matches an existing graph node by name, create a DOCUMENTS edge. |
|
186
|
Returns 1 if a link was created, 0 otherwise. |
|
187
|
""" |
|
188
|
# Check across all likely node types |
|
189
|
cypher = """ |
|
190
|
MATCH (n) |
|
191
|
WHERE (n:Concept OR n:Class OR n:Function OR n:Method OR n:Rule OR n:Decision) |
|
192
|
AND toLower(n.name) = toLower($term) |
|
193
|
RETURN labels(n)[0] AS label, n.name AS name |
|
194
|
LIMIT 1 |
|
195
|
""" |
|
196
|
result = self.store.query(cypher, {"term": term}) |
|
197
|
rows = result.result_set or [] |
|
198
|
if not rows: |
|
199
|
return 0 |
|
200
|
|
|
201
|
label_str, node_name = rows[0][0], rows[0][1] |
|
202
|
try: |
|
203
|
label = NodeLabel(label_str) |
|
204
|
self.store.create_edge( |
|
205
|
NodeLabel.WikiPage, |
|
206
|
{"name": wiki_page_name}, |
|
207
|
EdgeType.DOCUMENTS, |
|
208
|
label, |
|
209
|
{"name": node_name}, |
|
210
|
) |
|
211
|
return 1 |
|
212
|
except ValueError: |
|
213
|
return 0 |
|
214
|
|