Navegador

navegador / navegador / ingestion / planopticon.py
Blame History Raw 552 lines
1
"""
2
PlanopticonIngester — loads planopticon output into the navegador knowledge graph.
3
4
Planopticon extracts structured knowledge from videos, meetings, and documents:
5
entities, relationships, key points, action items, and diagrams. This ingester
6
maps that output onto navegador's knowledge layer so agents can query business
7
context alongside code.
8
9
Supported input:
10
- manifest.json (single video — primary entry point)
11
- interchange.json (canonical planopticon exchange format)
12
- knowledge_graph.json (raw KG export, no manifest required)
13
14
Mapping:
15
planopticon → navegador
16
─────────────────────────────────────────
17
node type=person → Person
18
node type=concept/technology → Concept
19
node type=organization → Concept (domain = "organization")
20
PlanningEntity type=decision → Decision
21
PlanningEntity type=requirement/constraint → Rule
22
diagram → WikiPage (content = mermaid/description)
23
key_point → Concept (tagged from source)
24
action_item.assignee → Person + ASSIGNED_TO edge
25
relationship.type → mapped EdgeType (see EDGE_MAP)
26
27
Usage:
28
from navegador.ingestion.planopticon import PlanopticonIngester
29
store = GraphStore.sqlite(".navegador/graph.db")
30
ing = PlanopticonIngester(store)
31
32
stats = ing.ingest_manifest("planopticon-output/manifest.json")
33
stats = ing.ingest_kg("planopticon-output/results/knowledge_graph.json")
34
stats = ing.ingest_interchange("planopticon-output/interchange.json")
35
"""
36
37
import json
38
import logging
39
from pathlib import Path
40
from typing import Any
41
42
from navegador.graph.schema import EdgeType, NodeLabel
43
from navegador.graph.store import GraphStore
44
45
logger = logging.getLogger(__name__)
46
47
# ── Relationship type mapping ─────────────────────────────────────────────────
48
49
EDGE_MAP: dict[str, EdgeType] = {
50
"related_to": EdgeType.RELATED_TO,
51
"uses": EdgeType.DEPENDS_ON,
52
"depends_on": EdgeType.DEPENDS_ON,
53
"built_on": EdgeType.DEPENDS_ON,
54
"implements": EdgeType.IMPLEMENTS,
55
"requires": EdgeType.DEPENDS_ON,
56
"blocked_by": EdgeType.DEPENDS_ON,
57
"has_risk": EdgeType.RELATED_TO,
58
"addresses": EdgeType.RELATED_TO,
59
"has_tradeoff": EdgeType.RELATED_TO,
60
"delivers": EdgeType.IMPLEMENTS,
61
"parent_of": EdgeType.CONTAINS,
62
"assigned_to": EdgeType.ASSIGNED_TO,
63
"owned_by": EdgeType.ASSIGNED_TO,
64
"owns": EdgeType.ASSIGNED_TO,
65
"employed_by": EdgeType.ASSIGNED_TO,
66
"works_with": EdgeType.RELATED_TO,
67
"governs": EdgeType.GOVERNS,
68
"documents": EdgeType.DOCUMENTS,
69
}
70
71
# planopticon node type → navegador NodeLabel
72
NODE_TYPE_MAP: dict[str, NodeLabel] = {
73
"concept": NodeLabel.Concept,
74
"technology": NodeLabel.Concept,
75
"organization": NodeLabel.Concept,
76
"diagram": NodeLabel.WikiPage,
77
"time": NodeLabel.Concept,
78
"person": NodeLabel.Person,
79
}
80
81
# planning_type → navegador NodeLabel
82
PLANNING_TYPE_MAP: dict[str, NodeLabel] = {
83
"decision": NodeLabel.Decision,
84
"requirement": NodeLabel.Rule,
85
"constraint": NodeLabel.Rule,
86
"risk": NodeLabel.Rule,
87
"goal": NodeLabel.Concept,
88
"assumption": NodeLabel.Concept,
89
"feature": NodeLabel.Concept,
90
"milestone": NodeLabel.Concept,
91
"task": NodeLabel.Concept,
92
"dependency": NodeLabel.Concept,
93
}
94
95
96
class PlanopticonIngester:
97
"""
98
Reads planopticon output and writes it into a GraphStore.
99
100
All paths may be relative (resolved against the manifest's parent directory)
101
or absolute.
102
"""
103
104
def __init__(self, store: GraphStore, source_tag: str = "") -> None:
105
self.store = store
106
self.source_tag = source_tag # optional label for provenance
107
self._stats: dict[str, int] = {}
108
109
# ── Entry points ──────────────────────────────────────────────────────────
110
111
def ingest_manifest(self, manifest_path: str | Path) -> dict[str, int]:
112
"""
113
Primary entry point — reads manifest.json and ingests everything:
114
knowledge graph, key points, action items, and diagrams.
115
"""
116
manifest_path = Path(manifest_path).resolve()
117
base_dir = manifest_path.parent
118
manifest = self._load_json(manifest_path)
119
120
stats = self._reset_stats()
121
title = manifest.get("video", {}).get("title", manifest_path.parent.name)
122
self.source_tag = self.source_tag or title
123
124
# 1. Knowledge graph (entities + relationships)
125
kg_path = manifest.get("knowledge_graph_json")
126
if kg_path:
127
self.ingest_kg(base_dir / kg_path)
128
129
# 2. Key points → Concept nodes
130
kp_path = manifest.get("key_points_json")
131
key_points = (
132
self._load_json(base_dir / kp_path) if kp_path else manifest.get("key_points", [])
133
)
134
self._ingest_key_points(key_points, title)
135
136
# 3. Action items → Person nodes + ASSIGNED_TO edges
137
ai_path = manifest.get("action_items_json")
138
action_items = (
139
self._load_json(base_dir / ai_path) if ai_path else manifest.get("action_items", [])
140
)
141
self._ingest_action_items(action_items, title)
142
143
# 4. Diagrams → WikiPage nodes
144
for diagram in manifest.get("diagrams", []):
145
self._ingest_diagram(diagram, base_dir, title)
146
147
logger.info(
148
"PlanopticonIngester (%s): nodes=%d edges=%d",
149
title,
150
stats.get("nodes", 0),
151
stats.get("edges", 0),
152
)
153
return stats
154
155
def ingest_kg(self, kg_path: str | Path) -> dict[str, int]:
156
"""
157
Ingest a knowledge_graph.json (KnowledgeGraphData) directly.
158
Can be used standalone without a manifest.
159
"""
160
kg_path = Path(kg_path).resolve()
161
data = self._load_json(kg_path)
162
stats = self._reset_stats()
163
164
for node in data.get("nodes", []):
165
self._ingest_kg_node(node)
166
167
for rel in data.get("relationships", []):
168
self._ingest_kg_relationship(rel)
169
170
# Ingest sources as WikiPage nodes
171
for source in data.get("sources", []):
172
self._ingest_source(source)
173
174
return stats
175
176
def ingest_interchange(self, interchange_path: str | Path) -> dict[str, int]:
177
"""
178
Ingest a planopticon interchange.json (PlanOpticonExchange format).
179
Includes planning taxonomy, artifacts, and full entity graph.
180
"""
181
interchange_path = Path(interchange_path).resolve()
182
data = self._load_json(interchange_path)
183
stats = self._reset_stats()
184
185
project = data.get("project", {})
186
project_name = project.get("name", interchange_path.parent.name)
187
self.source_tag = self.source_tag or project_name
188
189
# Domain from project tags
190
for tag in project.get("tags", []):
191
self.store.create_node(NodeLabel.Domain, {"name": tag, "description": ""})
192
193
# Entities (planning taxonomy takes priority over raw type)
194
for entity in data.get("entities", []):
195
planning_type = entity.get("planning_type")
196
if planning_type and planning_type in PLANNING_TYPE_MAP:
197
self._ingest_planning_entity(entity)
198
else:
199
self._ingest_kg_node(entity)
200
201
# Relationships
202
for rel in data.get("relationships", []):
203
self._ingest_kg_relationship(rel)
204
205
# Artifacts → WikiPage nodes
206
for artifact in data.get("artifacts", []):
207
self._ingest_artifact(artifact, project_name)
208
209
# Sources
210
for source in data.get("sources", []):
211
self._ingest_source(source)
212
213
return stats
214
215
def ingest_batch(self, batch_manifest_path: str | Path) -> dict[str, int]:
216
"""
217
Ingest a batch manifest — processes each video's manifest in turn,
218
then the merged knowledge graph if present.
219
"""
220
batch_manifest_path = Path(batch_manifest_path).resolve()
221
base_dir = batch_manifest_path.parent
222
batch = self._load_json(batch_manifest_path)
223
stats = self._reset_stats()
224
225
# Merged KG supersedes individual ones if present
226
merged = batch.get("merged_knowledge_graph_json")
227
if merged:
228
sub = self.ingest_kg(base_dir / merged)
229
self._merge_stats(sub)
230
else:
231
for video in batch.get("videos", []):
232
if video.get("status") == "completed":
233
mp = video.get("manifest_path")
234
if mp:
235
sub = self.ingest_manifest(base_dir / mp)
236
self._merge_stats(sub)
237
238
return stats
239
240
# ── Node ingestion ────────────────────────────────────────────────────────
241
242
def _ingest_kg_node(self, node: dict[str, Any]) -> None:
243
raw_type = node.get("type", "concept")
244
label = NODE_TYPE_MAP.get(raw_type, NodeLabel.Concept)
245
246
name = (node.get("name") or node.get("id") or "").strip()
247
if not name:
248
return
249
250
descriptions = node.get("descriptions", [])
251
description = descriptions[0] if descriptions else node.get("description", "")
252
253
if label == NodeLabel.Person:
254
self.store.create_node(
255
NodeLabel.Person,
256
{
257
"name": name,
258
"email": "",
259
"role": node.get("role", ""),
260
"team": node.get("organization", ""),
261
},
262
)
263
elif label == NodeLabel.WikiPage:
264
self.store.create_node(
265
NodeLabel.WikiPage,
266
{
267
"name": name,
268
"url": node.get("source", ""),
269
"source": self.source_tag,
270
"content": description[:4000],
271
},
272
)
273
else:
274
domain = "organization" if raw_type == "organization" else node.get("domain", "")
275
self.store.create_node(
276
NodeLabel.Concept,
277
{
278
"name": name,
279
"description": description,
280
"domain": domain,
281
"status": node.get("status", ""),
282
},
283
)
284
if domain:
285
self._ensure_domain(domain)
286
self.store.create_edge(
287
NodeLabel.Concept,
288
{"name": name},
289
EdgeType.BELONGS_TO,
290
NodeLabel.Domain,
291
{"name": domain},
292
)
293
294
self._stats["nodes"] = self._stats.get("nodes", 0) + 1
295
296
# Provenance: link to source WikiPage if present
297
source_id = node.get("source")
298
if source_id:
299
self._lazy_wiki_link(name, label, source_id)
300
301
def _ingest_planning_entity(self, entity: dict[str, Any]) -> None:
302
planning_type = entity.get("planning_type", "concept")
303
label = PLANNING_TYPE_MAP.get(planning_type, NodeLabel.Concept)
304
name = (entity.get("name") or "").strip()
305
if not name:
306
return
307
308
description = entity.get("description", "")
309
domain = entity.get("domain", "")
310
status = entity.get("status", "")
311
priority = entity.get("priority", "")
312
313
if label == NodeLabel.Decision:
314
self.store.create_node(
315
NodeLabel.Decision,
316
{
317
"name": name,
318
"description": description,
319
"domain": domain,
320
"status": status or "accepted",
321
"rationale": entity.get("rationale", ""),
322
},
323
)
324
elif label == NodeLabel.Rule:
325
self.store.create_node(
326
NodeLabel.Rule,
327
{
328
"name": name,
329
"description": description,
330
"domain": domain,
331
"severity": "critical" if priority == "high" else "info",
332
"rationale": entity.get("rationale", ""),
333
},
334
)
335
else:
336
self.store.create_node(
337
NodeLabel.Concept,
338
{
339
"name": name,
340
"description": description,
341
"domain": domain,
342
"status": status,
343
},
344
)
345
346
if domain:
347
self._ensure_domain(domain)
348
self.store.create_edge(
349
label,
350
{"name": name},
351
EdgeType.BELONGS_TO,
352
NodeLabel.Domain,
353
{"name": domain},
354
)
355
356
self._stats["nodes"] = self._stats.get("nodes", 0) + 1
357
358
def _ingest_kg_relationship(self, rel: dict[str, Any]) -> None:
359
src = (rel.get("source") or "").strip()
360
tgt = (rel.get("target") or "").strip()
361
rel_type = (rel.get("type") or "related_to").lower().replace(" ", "_")
362
363
if not src or not tgt:
364
return
365
366
edge_type = EDGE_MAP.get(rel_type, EdgeType.RELATED_TO)
367
368
# We don't know the exact label of each node — use a label-agnostic match
369
cypher = (
370
"""
371
MATCH (a), (b)
372
WHERE a.name = $src AND b.name = $tgt
373
MERGE (a)-[r:"""
374
+ edge_type
375
+ """]->(b)
376
"""
377
)
378
try:
379
self.store.query(cypher, {"src": src, "tgt": tgt})
380
self._stats["edges"] = self._stats.get("edges", 0) + 1
381
except Exception:
382
logger.warning("Could not create edge %s -[%s]-> %s", src, edge_type, tgt)
383
384
def _ingest_key_points(self, key_points: list[dict], source: str) -> None:
385
for kp in key_points:
386
point = (kp.get("point") or "").strip()
387
if not point:
388
continue
389
topic = kp.get("topic") or ""
390
name = point[:120] # use the point text as the concept name
391
self.store.create_node(
392
NodeLabel.Concept,
393
{
394
"name": name,
395
"description": kp.get("details", ""),
396
"domain": topic,
397
"status": "key_point",
398
},
399
)
400
if topic:
401
self._ensure_domain(topic)
402
self.store.create_edge(
403
NodeLabel.Concept,
404
{"name": name},
405
EdgeType.BELONGS_TO,
406
NodeLabel.Domain,
407
{"name": topic},
408
)
409
self._stats["nodes"] = self._stats.get("nodes", 0) + 1
410
411
def _ingest_action_items(self, action_items: list[dict], source: str) -> None:
412
for item in action_items:
413
action = (item.get("action") or "").strip()
414
assignee = (item.get("assignee") or "").strip()
415
if not action:
416
continue
417
418
# Action → Rule (it's a commitment / obligation)
419
self.store.create_node(
420
NodeLabel.Rule,
421
{
422
"name": action[:120],
423
"description": item.get("context", ""),
424
"domain": source,
425
"severity": item.get("priority", "info"),
426
"rationale": f"Action item from {source}",
427
},
428
)
429
self._stats["nodes"] = self._stats.get("nodes", 0) + 1
430
431
if assignee:
432
self.store.create_node(
433
NodeLabel.Person,
434
{
435
"name": assignee,
436
"email": "",
437
"role": "",
438
"team": "",
439
},
440
)
441
self.store.create_edge(
442
NodeLabel.Rule,
443
{"name": action[:120]},
444
EdgeType.ASSIGNED_TO,
445
NodeLabel.Person,
446
{"name": assignee},
447
)
448
self._stats["edges"] = self._stats.get("edges", 0) + 1
449
450
def _ingest_diagram(self, diagram: dict[str, Any], base_dir: Path, source: str) -> None:
451
dtype = diagram.get("diagram_type", "diagram")
452
desc = diagram.get("description") or diagram.get("text_content") or ""
453
mermaid = diagram.get("mermaid", "")
454
ts = diagram.get("timestamp")
455
name = f"{dtype.capitalize()} @ {ts:.0f}s" if ts is not None else f"{dtype.capitalize()}"
456
457
content = mermaid or desc
458
self.store.create_node(
459
NodeLabel.WikiPage,
460
{
461
"name": name,
462
"url": diagram.get("image_path", ""),
463
"source": source,
464
"content": content[:4000],
465
},
466
)
467
self._stats["nodes"] = self._stats.get("nodes", 0) + 1
468
469
# Link diagram elements as concepts
470
for element in diagram.get("elements", []):
471
element = element.strip()
472
if not element:
473
continue
474
self.store.create_node(
475
NodeLabel.Concept,
476
{
477
"name": element,
478
"description": "",
479
"domain": source,
480
"status": "",
481
},
482
)
483
self.store.create_edge(
484
NodeLabel.WikiPage,
485
{"name": name},
486
EdgeType.DOCUMENTS,
487
NodeLabel.Concept,
488
{"name": element},
489
)
490
self._stats["edges"] = self._stats.get("edges", 0) + 1
491
492
def _ingest_source(self, source: dict[str, Any]) -> None:
493
name = (source.get("title") or source.get("source_id") or "").strip()
494
if not name:
495
return
496
self.store.create_node(
497
NodeLabel.WikiPage,
498
{
499
"name": name,
500
"url": source.get("url") or source.get("path") or "",
501
"source": source.get("source_type", ""),
502
"content": "",
503
"updated_at": source.get("ingested_at", ""),
504
},
505
)
506
self._stats["nodes"] = self._stats.get("nodes", 0) + 1
507
508
def _ingest_artifact(self, artifact: dict[str, Any], project_name: str) -> None:
509
name = (artifact.get("name") or "").strip()
510
if not name:
511
return
512
content = artifact.get("content", "")
513
self.store.create_node(
514
NodeLabel.WikiPage,
515
{
516
"name": name,
517
"url": "",
518
"source": project_name,
519
"content": content[:4000],
520
},
521
)
522
self._stats["nodes"] = self._stats.get("nodes", 0) + 1
523
524
# ── Helpers ───────────────────────────────────────────────────────────────
525
526
def _ensure_domain(self, name: str) -> None:
527
self.store.create_node(NodeLabel.Domain, {"name": name, "description": ""})
528
529
def _lazy_wiki_link(self, name: str, label: NodeLabel, source_id: str) -> None:
530
"""Create a DOCUMENTS edge from a WikiPage to this node if the page exists."""
531
try:
532
self.store.create_edge(
533
NodeLabel.WikiPage,
534
{"name": source_id},
535
EdgeType.DOCUMENTS,
536
label,
537
{"name": name},
538
)
539
except Exception:
540
logger.debug("Could not link %s to wiki page %s", name, source_id)
541
542
def _load_json(self, path: Path) -> Any:
543
return json.loads(Path(path).read_text(encoding="utf-8"))
544
545
def _reset_stats(self) -> dict[str, int]:
546
self._stats = {"nodes": 0, "edges": 0}
547
return self._stats
548
549
def _merge_stats(self, other: dict[str, int]) -> None:
550
for k, v in other.items():
551
self._stats[k] = self._stats.get(k, 0) + v
552

Keyboard Shortcuts

Open search /
Next entry (timeline) j
Previous entry (timeline) k
Open focused entry Enter
Show this help ?
Toggle theme Top nav button