Navegador

navegador / navegador / ingestion / planopticon.py
Source Blame History 551 lines
9c8fd06… lmata 1 """
9c8fd06… lmata 2 PlanopticonIngester — loads planopticon output into the navegador knowledge graph.
9c8fd06… lmata 3
9c8fd06… lmata 4 Planopticon extracts structured knowledge from videos, meetings, and documents:
9c8fd06… lmata 5 entities, relationships, key points, action items, and diagrams. This ingester
9c8fd06… lmata 6 maps that output onto navegador's knowledge layer so agents can query business
9c8fd06… lmata 7 context alongside code.
9c8fd06… lmata 8
9c8fd06… lmata 9 Supported input:
9c8fd06… lmata 10 - manifest.json (single video — primary entry point)
9c8fd06… lmata 11 - interchange.json (canonical planopticon exchange format)
9c8fd06… lmata 12 - knowledge_graph.json (raw KG export, no manifest required)
9c8fd06… lmata 13
9c8fd06… lmata 14 Mapping:
9c8fd06… lmata 15 planopticon → navegador
9c8fd06… lmata 16 ─────────────────────────────────────────
9c8fd06… lmata 17 node type=person → Person
9c8fd06… lmata 18 node type=concept/technology → Concept
9c8fd06… lmata 19 node type=organization → Concept (domain = "organization")
9c8fd06… lmata 20 PlanningEntity type=decision → Decision
9c8fd06… lmata 21 PlanningEntity type=requirement/constraint → Rule
9c8fd06… lmata 22 diagram → WikiPage (content = mermaid/description)
9c8fd06… lmata 23 key_point → Concept (tagged from source)
9c8fd06… lmata 24 action_item.assignee → Person + ASSIGNED_TO edge
9c8fd06… lmata 25 relationship.type → mapped EdgeType (see EDGE_MAP)
9c8fd06… lmata 26
9c8fd06… lmata 27 Usage:
9c8fd06… lmata 28 from navegador.ingestion.planopticon import PlanopticonIngester
9c8fd06… lmata 29 store = GraphStore.sqlite(".navegador/graph.db")
9c8fd06… lmata 30 ing = PlanopticonIngester(store)
9c8fd06… lmata 31
9c8fd06… lmata 32 stats = ing.ingest_manifest("planopticon-output/manifest.json")
9c8fd06… lmata 33 stats = ing.ingest_kg("planopticon-output/results/knowledge_graph.json")
9c8fd06… lmata 34 stats = ing.ingest_interchange("planopticon-output/interchange.json")
9c8fd06… lmata 35 """
9c8fd06… lmata 36
9c8fd06… lmata 37 import json
9c8fd06… lmata 38 import logging
9c8fd06… lmata 39 from pathlib import Path
9c8fd06… lmata 40 from typing import Any
9c8fd06… lmata 41
9c8fd06… lmata 42 from navegador.graph.schema import EdgeType, NodeLabel
9c8fd06… lmata 43 from navegador.graph.store import GraphStore
9c8fd06… lmata 44
9c8fd06… lmata 45 logger = logging.getLogger(__name__)
9c8fd06… lmata 46
9c8fd06… lmata 47 # ── Relationship type mapping ─────────────────────────────────────────────────
9c8fd06… lmata 48
9c8fd06… lmata 49 EDGE_MAP: dict[str, EdgeType] = {
758c6b7… lmata 50 "related_to": EdgeType.RELATED_TO,
758c6b7… lmata 51 "uses": EdgeType.DEPENDS_ON,
758c6b7… lmata 52 "depends_on": EdgeType.DEPENDS_ON,
758c6b7… lmata 53 "built_on": EdgeType.DEPENDS_ON,
758c6b7… lmata 54 "implements": EdgeType.IMPLEMENTS,
758c6b7… lmata 55 "requires": EdgeType.DEPENDS_ON,
758c6b7… lmata 56 "blocked_by": EdgeType.DEPENDS_ON,
758c6b7… lmata 57 "has_risk": EdgeType.RELATED_TO,
758c6b7… lmata 58 "addresses": EdgeType.RELATED_TO,
9c8fd06… lmata 59 "has_tradeoff": EdgeType.RELATED_TO,
758c6b7… lmata 60 "delivers": EdgeType.IMPLEMENTS,
758c6b7… lmata 61 "parent_of": EdgeType.CONTAINS,
758c6b7… lmata 62 "assigned_to": EdgeType.ASSIGNED_TO,
758c6b7… lmata 63 "owned_by": EdgeType.ASSIGNED_TO,
758c6b7… lmata 64 "owns": EdgeType.ASSIGNED_TO,
758c6b7… lmata 65 "employed_by": EdgeType.ASSIGNED_TO,
758c6b7… lmata 66 "works_with": EdgeType.RELATED_TO,
758c6b7… lmata 67 "governs": EdgeType.GOVERNS,
758c6b7… lmata 68 "documents": EdgeType.DOCUMENTS,
9c8fd06… lmata 69 }
9c8fd06… lmata 70
9c8fd06… lmata 71 # planopticon node type → navegador NodeLabel
9c8fd06… lmata 72 NODE_TYPE_MAP: dict[str, NodeLabel] = {
758c6b7… lmata 73 "concept": NodeLabel.Concept,
758c6b7… lmata 74 "technology": NodeLabel.Concept,
9c8fd06… lmata 75 "organization": NodeLabel.Concept,
758c6b7… lmata 76 "diagram": NodeLabel.WikiPage,
758c6b7… lmata 77 "time": NodeLabel.Concept,
758c6b7… lmata 78 "person": NodeLabel.Person,
9c8fd06… lmata 79 }
9c8fd06… lmata 80
9c8fd06… lmata 81 # planning_type → navegador NodeLabel
9c8fd06… lmata 82 PLANNING_TYPE_MAP: dict[str, NodeLabel] = {
758c6b7… lmata 83 "decision": NodeLabel.Decision,
9c8fd06… lmata 84 "requirement": NodeLabel.Rule,
758c6b7… lmata 85 "constraint": NodeLabel.Rule,
758c6b7… lmata 86 "risk": NodeLabel.Rule,
758c6b7… lmata 87 "goal": NodeLabel.Concept,
758c6b7… lmata 88 "assumption": NodeLabel.Concept,
758c6b7… lmata 89 "feature": NodeLabel.Concept,
758c6b7… lmata 90 "milestone": NodeLabel.Concept,
758c6b7… lmata 91 "task": NodeLabel.Concept,
758c6b7… lmata 92 "dependency": NodeLabel.Concept,
9c8fd06… lmata 93 }
9c8fd06… lmata 94
9c8fd06… lmata 95
9c8fd06… lmata 96 class PlanopticonIngester:
9c8fd06… lmata 97 """
9c8fd06… lmata 98 Reads planopticon output and writes it into a GraphStore.
9c8fd06… lmata 99
9c8fd06… lmata 100 All paths may be relative (resolved against the manifest's parent directory)
9c8fd06… lmata 101 or absolute.
9c8fd06… lmata 102 """
9c8fd06… lmata 103
9c8fd06… lmata 104 def __init__(self, store: GraphStore, source_tag: str = "") -> None:
9c8fd06… lmata 105 self.store = store
9c8fd06… lmata 106 self.source_tag = source_tag # optional label for provenance
9c8fd06… lmata 107 self._stats: dict[str, int] = {}
9c8fd06… lmata 108
9c8fd06… lmata 109 # ── Entry points ──────────────────────────────────────────────────────────
9c8fd06… lmata 110
9c8fd06… lmata 111 def ingest_manifest(self, manifest_path: str | Path) -> dict[str, int]:
9c8fd06… lmata 112 """
9c8fd06… lmata 113 Primary entry point — reads manifest.json and ingests everything:
9c8fd06… lmata 114 knowledge graph, key points, action items, and diagrams.
9c8fd06… lmata 115 """
9c8fd06… lmata 116 manifest_path = Path(manifest_path).resolve()
9c8fd06… lmata 117 base_dir = manifest_path.parent
9c8fd06… lmata 118 manifest = self._load_json(manifest_path)
9c8fd06… lmata 119
9c8fd06… lmata 120 stats = self._reset_stats()
9c8fd06… lmata 121 title = manifest.get("video", {}).get("title", manifest_path.parent.name)
9c8fd06… lmata 122 self.source_tag = self.source_tag or title
9c8fd06… lmata 123
9c8fd06… lmata 124 # 1. Knowledge graph (entities + relationships)
9c8fd06… lmata 125 kg_path = manifest.get("knowledge_graph_json")
9c8fd06… lmata 126 if kg_path:
9c8fd06… lmata 127 self.ingest_kg(base_dir / kg_path)
9c8fd06… lmata 128
9c8fd06… lmata 129 # 2. Key points → Concept nodes
9c8fd06… lmata 130 kp_path = manifest.get("key_points_json")
9c8fd06… lmata 131 key_points = (
9c8fd06… lmata 132 self._load_json(base_dir / kp_path) if kp_path else manifest.get("key_points", [])
9c8fd06… lmata 133 )
9c8fd06… lmata 134 self._ingest_key_points(key_points, title)
9c8fd06… lmata 135
9c8fd06… lmata 136 # 3. Action items → Person nodes + ASSIGNED_TO edges
9c8fd06… lmata 137 ai_path = manifest.get("action_items_json")
9c8fd06… lmata 138 action_items = (
9c8fd06… lmata 139 self._load_json(base_dir / ai_path) if ai_path else manifest.get("action_items", [])
9c8fd06… lmata 140 )
9c8fd06… lmata 141 self._ingest_action_items(action_items, title)
9c8fd06… lmata 142
9c8fd06… lmata 143 # 4. Diagrams → WikiPage nodes
9c8fd06… lmata 144 for diagram in manifest.get("diagrams", []):
9c8fd06… lmata 145 self._ingest_diagram(diagram, base_dir, title)
9c8fd06… lmata 146
9c8fd06… lmata 147 logger.info(
9c8fd06… lmata 148 "PlanopticonIngester (%s): nodes=%d edges=%d",
758c6b7… lmata 149 title,
758c6b7… lmata 150 stats.get("nodes", 0),
758c6b7… lmata 151 stats.get("edges", 0),
9c8fd06… lmata 152 )
9c8fd06… lmata 153 return stats
9c8fd06… lmata 154
9c8fd06… lmata 155 def ingest_kg(self, kg_path: str | Path) -> dict[str, int]:
9c8fd06… lmata 156 """
9c8fd06… lmata 157 Ingest a knowledge_graph.json (KnowledgeGraphData) directly.
9c8fd06… lmata 158 Can be used standalone without a manifest.
9c8fd06… lmata 159 """
9c8fd06… lmata 160 kg_path = Path(kg_path).resolve()
9c8fd06… lmata 161 data = self._load_json(kg_path)
9c8fd06… lmata 162 stats = self._reset_stats()
9c8fd06… lmata 163
9c8fd06… lmata 164 for node in data.get("nodes", []):
9c8fd06… lmata 165 self._ingest_kg_node(node)
9c8fd06… lmata 166
9c8fd06… lmata 167 for rel in data.get("relationships", []):
9c8fd06… lmata 168 self._ingest_kg_relationship(rel)
9c8fd06… lmata 169
9c8fd06… lmata 170 # Ingest sources as WikiPage nodes
9c8fd06… lmata 171 for source in data.get("sources", []):
9c8fd06… lmata 172 self._ingest_source(source)
9c8fd06… lmata 173
9c8fd06… lmata 174 return stats
9c8fd06… lmata 175
9c8fd06… lmata 176 def ingest_interchange(self, interchange_path: str | Path) -> dict[str, int]:
9c8fd06… lmata 177 """
9c8fd06… lmata 178 Ingest a planopticon interchange.json (PlanOpticonExchange format).
9c8fd06… lmata 179 Includes planning taxonomy, artifacts, and full entity graph.
9c8fd06… lmata 180 """
9c8fd06… lmata 181 interchange_path = Path(interchange_path).resolve()
9c8fd06… lmata 182 data = self._load_json(interchange_path)
9c8fd06… lmata 183 stats = self._reset_stats()
9c8fd06… lmata 184
9c8fd06… lmata 185 project = data.get("project", {})
9c8fd06… lmata 186 project_name = project.get("name", interchange_path.parent.name)
9c8fd06… lmata 187 self.source_tag = self.source_tag or project_name
9c8fd06… lmata 188
9c8fd06… lmata 189 # Domain from project tags
9c8fd06… lmata 190 for tag in project.get("tags", []):
9c8fd06… lmata 191 self.store.create_node(NodeLabel.Domain, {"name": tag, "description": ""})
9c8fd06… lmata 192
9c8fd06… lmata 193 # Entities (planning taxonomy takes priority over raw type)
9c8fd06… lmata 194 for entity in data.get("entities", []):
9c8fd06… lmata 195 planning_type = entity.get("planning_type")
9c8fd06… lmata 196 if planning_type and planning_type in PLANNING_TYPE_MAP:
9c8fd06… lmata 197 self._ingest_planning_entity(entity)
9c8fd06… lmata 198 else:
9c8fd06… lmata 199 self._ingest_kg_node(entity)
9c8fd06… lmata 200
9c8fd06… lmata 201 # Relationships
9c8fd06… lmata 202 for rel in data.get("relationships", []):
9c8fd06… lmata 203 self._ingest_kg_relationship(rel)
9c8fd06… lmata 204
9c8fd06… lmata 205 # Artifacts → WikiPage nodes
9c8fd06… lmata 206 for artifact in data.get("artifacts", []):
9c8fd06… lmata 207 self._ingest_artifact(artifact, project_name)
9c8fd06… lmata 208
9c8fd06… lmata 209 # Sources
9c8fd06… lmata 210 for source in data.get("sources", []):
9c8fd06… lmata 211 self._ingest_source(source)
9c8fd06… lmata 212
9c8fd06… lmata 213 return stats
9c8fd06… lmata 214
9c8fd06… lmata 215 def ingest_batch(self, batch_manifest_path: str | Path) -> dict[str, int]:
9c8fd06… lmata 216 """
9c8fd06… lmata 217 Ingest a batch manifest — processes each video's manifest in turn,
9c8fd06… lmata 218 then the merged knowledge graph if present.
9c8fd06… lmata 219 """
9c8fd06… lmata 220 batch_manifest_path = Path(batch_manifest_path).resolve()
9c8fd06… lmata 221 base_dir = batch_manifest_path.parent
9c8fd06… lmata 222 batch = self._load_json(batch_manifest_path)
9c8fd06… lmata 223 stats = self._reset_stats()
9c8fd06… lmata 224
9c8fd06… lmata 225 # Merged KG supersedes individual ones if present
9c8fd06… lmata 226 merged = batch.get("merged_knowledge_graph_json")
9c8fd06… lmata 227 if merged:
9c8fd06… lmata 228 sub = self.ingest_kg(base_dir / merged)
9c8fd06… lmata 229 self._merge_stats(sub)
9c8fd06… lmata 230 else:
9c8fd06… lmata 231 for video in batch.get("videos", []):
9c8fd06… lmata 232 if video.get("status") == "completed":
9c8fd06… lmata 233 mp = video.get("manifest_path")
9c8fd06… lmata 234 if mp:
9c8fd06… lmata 235 sub = self.ingest_manifest(base_dir / mp)
9c8fd06… lmata 236 self._merge_stats(sub)
9c8fd06… lmata 237
9c8fd06… lmata 238 return stats
9c8fd06… lmata 239
9c8fd06… lmata 240 # ── Node ingestion ────────────────────────────────────────────────────────
9c8fd06… lmata 241
9c8fd06… lmata 242 def _ingest_kg_node(self, node: dict[str, Any]) -> None:
9c8fd06… lmata 243 raw_type = node.get("type", "concept")
9c8fd06… lmata 244 label = NODE_TYPE_MAP.get(raw_type, NodeLabel.Concept)
9c8fd06… lmata 245
9c8fd06… lmata 246 name = (node.get("name") or node.get("id") or "").strip()
9c8fd06… lmata 247 if not name:
9c8fd06… lmata 248 return
9c8fd06… lmata 249
9c8fd06… lmata 250 descriptions = node.get("descriptions", [])
9c8fd06… lmata 251 description = descriptions[0] if descriptions else node.get("description", "")
9c8fd06… lmata 252
9c8fd06… lmata 253 if label == NodeLabel.Person:
758c6b7… lmata 254 self.store.create_node(
758c6b7… lmata 255 NodeLabel.Person,
758c6b7… lmata 256 {
758c6b7… lmata 257 "name": name,
758c6b7… lmata 258 "email": "",
758c6b7… lmata 259 "role": node.get("role", ""),
758c6b7… lmata 260 "team": node.get("organization", ""),
758c6b7… lmata 261 },
758c6b7… lmata 262 )
9c8fd06… lmata 263 elif label == NodeLabel.WikiPage:
758c6b7… lmata 264 self.store.create_node(
758c6b7… lmata 265 NodeLabel.WikiPage,
758c6b7… lmata 266 {
758c6b7… lmata 267 "name": name,
758c6b7… lmata 268 "url": node.get("source", ""),
758c6b7… lmata 269 "source": self.source_tag,
758c6b7… lmata 270 "content": description[:4000],
758c6b7… lmata 271 },
758c6b7… lmata 272 )
9c8fd06… lmata 273 else:
9c8fd06… lmata 274 domain = "organization" if raw_type == "organization" else node.get("domain", "")
758c6b7… lmata 275 self.store.create_node(
758c6b7… lmata 276 NodeLabel.Concept,
758c6b7… lmata 277 {
758c6b7… lmata 278 "name": name,
758c6b7… lmata 279 "description": description,
758c6b7… lmata 280 "domain": domain,
758c6b7… lmata 281 "status": node.get("status", ""),
758c6b7… lmata 282 },
758c6b7… lmata 283 )
9c8fd06… lmata 284 if domain:
9c8fd06… lmata 285 self._ensure_domain(domain)
9c8fd06… lmata 286 self.store.create_edge(
758c6b7… lmata 287 NodeLabel.Concept,
758c6b7… lmata 288 {"name": name},
9c8fd06… lmata 289 EdgeType.BELONGS_TO,
758c6b7… lmata 290 NodeLabel.Domain,
758c6b7… lmata 291 {"name": domain},
9c8fd06… lmata 292 )
9c8fd06… lmata 293
9c8fd06… lmata 294 self._stats["nodes"] = self._stats.get("nodes", 0) + 1
9c8fd06… lmata 295
9c8fd06… lmata 296 # Provenance: link to source WikiPage if present
9c8fd06… lmata 297 source_id = node.get("source")
9c8fd06… lmata 298 if source_id:
9c8fd06… lmata 299 self._lazy_wiki_link(name, label, source_id)
9c8fd06… lmata 300
9c8fd06… lmata 301 def _ingest_planning_entity(self, entity: dict[str, Any]) -> None:
9c8fd06… lmata 302 planning_type = entity.get("planning_type", "concept")
9c8fd06… lmata 303 label = PLANNING_TYPE_MAP.get(planning_type, NodeLabel.Concept)
9c8fd06… lmata 304 name = (entity.get("name") or "").strip()
9c8fd06… lmata 305 if not name:
9c8fd06… lmata 306 return
9c8fd06… lmata 307
9c8fd06… lmata 308 description = entity.get("description", "")
9c8fd06… lmata 309 domain = entity.get("domain", "")
9c8fd06… lmata 310 status = entity.get("status", "")
9c8fd06… lmata 311 priority = entity.get("priority", "")
9c8fd06… lmata 312
9c8fd06… lmata 313 if label == NodeLabel.Decision:
758c6b7… lmata 314 self.store.create_node(
758c6b7… lmata 315 NodeLabel.Decision,
758c6b7… lmata 316 {
758c6b7… lmata 317 "name": name,
758c6b7… lmata 318 "description": description,
758c6b7… lmata 319 "domain": domain,
758c6b7… lmata 320 "status": status or "accepted",
758c6b7… lmata 321 "rationale": entity.get("rationale", ""),
758c6b7… lmata 322 },
758c6b7… lmata 323 )
9c8fd06… lmata 324 elif label == NodeLabel.Rule:
758c6b7… lmata 325 self.store.create_node(
758c6b7… lmata 326 NodeLabel.Rule,
758c6b7… lmata 327 {
758c6b7… lmata 328 "name": name,
758c6b7… lmata 329 "description": description,
758c6b7… lmata 330 "domain": domain,
758c6b7… lmata 331 "severity": "critical" if priority == "high" else "info",
758c6b7… lmata 332 "rationale": entity.get("rationale", ""),
758c6b7… lmata 333 },
758c6b7… lmata 334 )
9c8fd06… lmata 335 else:
758c6b7… lmata 336 self.store.create_node(
758c6b7… lmata 337 NodeLabel.Concept,
758c6b7… lmata 338 {
758c6b7… lmata 339 "name": name,
758c6b7… lmata 340 "description": description,
758c6b7… lmata 341 "domain": domain,
758c6b7… lmata 342 "status": status,
758c6b7… lmata 343 },
758c6b7… lmata 344 )
9c8fd06… lmata 345
9c8fd06… lmata 346 if domain:
9c8fd06… lmata 347 self._ensure_domain(domain)
9c8fd06… lmata 348 self.store.create_edge(
758c6b7… lmata 349 label,
758c6b7… lmata 350 {"name": name},
9c8fd06… lmata 351 EdgeType.BELONGS_TO,
758c6b7… lmata 352 NodeLabel.Domain,
758c6b7… lmata 353 {"name": domain},
9c8fd06… lmata 354 )
9c8fd06… lmata 355
9c8fd06… lmata 356 self._stats["nodes"] = self._stats.get("nodes", 0) + 1
9c8fd06… lmata 357
9c8fd06… lmata 358 def _ingest_kg_relationship(self, rel: dict[str, Any]) -> None:
9c8fd06… lmata 359 src = (rel.get("source") or "").strip()
9c8fd06… lmata 360 tgt = (rel.get("target") or "").strip()
9c8fd06… lmata 361 rel_type = (rel.get("type") or "related_to").lower().replace(" ", "_")
9c8fd06… lmata 362
9c8fd06… lmata 363 if not src or not tgt:
9c8fd06… lmata 364 return
9c8fd06… lmata 365
9c8fd06… lmata 366 edge_type = EDGE_MAP.get(rel_type, EdgeType.RELATED_TO)
9c8fd06… lmata 367
9c8fd06… lmata 368 # We don't know the exact label of each node — use a label-agnostic match
758c6b7… lmata 369 cypher = (
758c6b7… lmata 370 """
9c8fd06… lmata 371 MATCH (a), (b)
9c8fd06… lmata 372 WHERE a.name = $src AND b.name = $tgt
758c6b7… lmata 373 MERGE (a)-[r:"""
758c6b7… lmata 374 + edge_type
758c6b7… lmata 375 + """]->(b)
9c8fd06… lmata 376 """
758c6b7… lmata 377 )
9c8fd06… lmata 378 try:
9c8fd06… lmata 379 self.store.query(cypher, {"src": src, "tgt": tgt})
9c8fd06… lmata 380 self._stats["edges"] = self._stats.get("edges", 0) + 1
9c8fd06… lmata 381 except Exception:
7e708ec… lmata 382 logger.warning("Could not create edge %s -[%s]-> %s", src, edge_type, tgt)
9c8fd06… lmata 383
9c8fd06… lmata 384 def _ingest_key_points(self, key_points: list[dict], source: str) -> None:
9c8fd06… lmata 385 for kp in key_points:
9c8fd06… lmata 386 point = (kp.get("point") or "").strip()
9c8fd06… lmata 387 if not point:
9c8fd06… lmata 388 continue
9c8fd06… lmata 389 topic = kp.get("topic") or ""
9c8fd06… lmata 390 name = point[:120] # use the point text as the concept name
758c6b7… lmata 391 self.store.create_node(
758c6b7… lmata 392 NodeLabel.Concept,
758c6b7… lmata 393 {
758c6b7… lmata 394 "name": name,
758c6b7… lmata 395 "description": kp.get("details", ""),
758c6b7… lmata 396 "domain": topic,
758c6b7… lmata 397 "status": "key_point",
758c6b7… lmata 398 },
758c6b7… lmata 399 )
9c8fd06… lmata 400 if topic:
9c8fd06… lmata 401 self._ensure_domain(topic)
9c8fd06… lmata 402 self.store.create_edge(
758c6b7… lmata 403 NodeLabel.Concept,
758c6b7… lmata 404 {"name": name},
9c8fd06… lmata 405 EdgeType.BELONGS_TO,
758c6b7… lmata 406 NodeLabel.Domain,
758c6b7… lmata 407 {"name": topic},
9c8fd06… lmata 408 )
9c8fd06… lmata 409 self._stats["nodes"] = self._stats.get("nodes", 0) + 1
9c8fd06… lmata 410
9c8fd06… lmata 411 def _ingest_action_items(self, action_items: list[dict], source: str) -> None:
9c8fd06… lmata 412 for item in action_items:
9c8fd06… lmata 413 action = (item.get("action") or "").strip()
9c8fd06… lmata 414 assignee = (item.get("assignee") or "").strip()
9c8fd06… lmata 415 if not action:
9c8fd06… lmata 416 continue
9c8fd06… lmata 417
9c8fd06… lmata 418 # Action → Rule (it's a commitment / obligation)
758c6b7… lmata 419 self.store.create_node(
758c6b7… lmata 420 NodeLabel.Rule,
758c6b7… lmata 421 {
758c6b7… lmata 422 "name": action[:120],
758c6b7… lmata 423 "description": item.get("context", ""),
758c6b7… lmata 424 "domain": source,
758c6b7… lmata 425 "severity": item.get("priority", "info"),
758c6b7… lmata 426 "rationale": f"Action item from {source}",
758c6b7… lmata 427 },
758c6b7… lmata 428 )
9c8fd06… lmata 429 self._stats["nodes"] = self._stats.get("nodes", 0) + 1
9c8fd06… lmata 430
9c8fd06… lmata 431 if assignee:
758c6b7… lmata 432 self.store.create_node(
758c6b7… lmata 433 NodeLabel.Person,
758c6b7… lmata 434 {
758c6b7… lmata 435 "name": assignee,
758c6b7… lmata 436 "email": "",
758c6b7… lmata 437 "role": "",
758c6b7… lmata 438 "team": "",
758c6b7… lmata 439 },
758c6b7… lmata 440 )
9c8fd06… lmata 441 self.store.create_edge(
758c6b7… lmata 442 NodeLabel.Rule,
758c6b7… lmata 443 {"name": action[:120]},
9c8fd06… lmata 444 EdgeType.ASSIGNED_TO,
758c6b7… lmata 445 NodeLabel.Person,
758c6b7… lmata 446 {"name": assignee},
9c8fd06… lmata 447 )
9c8fd06… lmata 448 self._stats["edges"] = self._stats.get("edges", 0) + 1
9c8fd06… lmata 449
9c8fd06… lmata 450 def _ingest_diagram(self, diagram: dict[str, Any], base_dir: Path, source: str) -> None:
9c8fd06… lmata 451 dtype = diagram.get("diagram_type", "diagram")
9c8fd06… lmata 452 desc = diagram.get("description") or diagram.get("text_content") or ""
9c8fd06… lmata 453 mermaid = diagram.get("mermaid", "")
9c8fd06… lmata 454 ts = diagram.get("timestamp")
9c8fd06… lmata 455 name = f"{dtype.capitalize()} @ {ts:.0f}s" if ts is not None else f"{dtype.capitalize()}"
9c8fd06… lmata 456
9c8fd06… lmata 457 content = mermaid or desc
758c6b7… lmata 458 self.store.create_node(
758c6b7… lmata 459 NodeLabel.WikiPage,
758c6b7… lmata 460 {
758c6b7… lmata 461 "name": name,
758c6b7… lmata 462 "url": diagram.get("image_path", ""),
758c6b7… lmata 463 "source": source,
758c6b7… lmata 464 "content": content[:4000],
758c6b7… lmata 465 },
758c6b7… lmata 466 )
9c8fd06… lmata 467 self._stats["nodes"] = self._stats.get("nodes", 0) + 1
9c8fd06… lmata 468
9c8fd06… lmata 469 # Link diagram elements as concepts
9c8fd06… lmata 470 for element in diagram.get("elements", []):
9c8fd06… lmata 471 element = element.strip()
9c8fd06… lmata 472 if not element:
9c8fd06… lmata 473 continue
758c6b7… lmata 474 self.store.create_node(
758c6b7… lmata 475 NodeLabel.Concept,
758c6b7… lmata 476 {
758c6b7… lmata 477 "name": element,
758c6b7… lmata 478 "description": "",
758c6b7… lmata 479 "domain": source,
758c6b7… lmata 480 "status": "",
758c6b7… lmata 481 },
758c6b7… lmata 482 )
9c8fd06… lmata 483 self.store.create_edge(
758c6b7… lmata 484 NodeLabel.WikiPage,
758c6b7… lmata 485 {"name": name},
9c8fd06… lmata 486 EdgeType.DOCUMENTS,
758c6b7… lmata 487 NodeLabel.Concept,
758c6b7… lmata 488 {"name": element},
9c8fd06… lmata 489 )
9c8fd06… lmata 490 self._stats["edges"] = self._stats.get("edges", 0) + 1
9c8fd06… lmata 491
9c8fd06… lmata 492 def _ingest_source(self, source: dict[str, Any]) -> None:
9c8fd06… lmata 493 name = (source.get("title") or source.get("source_id") or "").strip()
9c8fd06… lmata 494 if not name:
9c8fd06… lmata 495 return
758c6b7… lmata 496 self.store.create_node(
758c6b7… lmata 497 NodeLabel.WikiPage,
758c6b7… lmata 498 {
758c6b7… lmata 499 "name": name,
758c6b7… lmata 500 "url": source.get("url") or source.get("path") or "",
758c6b7… lmata 501 "source": source.get("source_type", ""),
758c6b7… lmata 502 "content": "",
758c6b7… lmata 503 "updated_at": source.get("ingested_at", ""),
758c6b7… lmata 504 },
758c6b7… lmata 505 )
9c8fd06… lmata 506 self._stats["nodes"] = self._stats.get("nodes", 0) + 1
9c8fd06… lmata 507
9c8fd06… lmata 508 def _ingest_artifact(self, artifact: dict[str, Any], project_name: str) -> None:
9c8fd06… lmata 509 name = (artifact.get("name") or "").strip()
9c8fd06… lmata 510 if not name:
9c8fd06… lmata 511 return
9c8fd06… lmata 512 content = artifact.get("content", "")
758c6b7… lmata 513 self.store.create_node(
758c6b7… lmata 514 NodeLabel.WikiPage,
758c6b7… lmata 515 {
758c6b7… lmata 516 "name": name,
758c6b7… lmata 517 "url": "",
758c6b7… lmata 518 "source": project_name,
758c6b7… lmata 519 "content": content[:4000],
758c6b7… lmata 520 },
758c6b7… lmata 521 )
9c8fd06… lmata 522 self._stats["nodes"] = self._stats.get("nodes", 0) + 1
9c8fd06… lmata 523
9c8fd06… lmata 524 # ── Helpers ───────────────────────────────────────────────────────────────
9c8fd06… lmata 525
9c8fd06… lmata 526 def _ensure_domain(self, name: str) -> None:
9c8fd06… lmata 527 self.store.create_node(NodeLabel.Domain, {"name": name, "description": ""})
9c8fd06… lmata 528
9c8fd06… lmata 529 def _lazy_wiki_link(self, name: str, label: NodeLabel, source_id: str) -> None:
9c8fd06… lmata 530 """Create a DOCUMENTS edge from a WikiPage to this node if the page exists."""
9c8fd06… lmata 531 try:
9c8fd06… lmata 532 self.store.create_edge(
758c6b7… lmata 533 NodeLabel.WikiPage,
758c6b7… lmata 534 {"name": source_id},
9c8fd06… lmata 535 EdgeType.DOCUMENTS,
758c6b7… lmata 536 label,
758c6b7… lmata 537 {"name": name},
9c8fd06… lmata 538 )
9c8fd06… lmata 539 except Exception:
7e708ec… lmata 540 logger.debug("Could not link %s to wiki page %s", name, source_id)
9c8fd06… lmata 541
9c8fd06… lmata 542 def _load_json(self, path: Path) -> Any:
7e708ec… lmata 543 return json.loads(Path(path).read_text(encoding="utf-8"))
9c8fd06… lmata 544
9c8fd06… lmata 545 def _reset_stats(self) -> dict[str, int]:
9c8fd06… lmata 546 self._stats = {"nodes": 0, "edges": 0}
9c8fd06… lmata 547 return self._stats
9c8fd06… lmata 548
9c8fd06… lmata 549 def _merge_stats(self, other: dict[str, int]) -> None:
9c8fd06… lmata 550 for k, v in other.items():
9c8fd06… lmata 551 self._stats[k] = self._stats.get(k, 0) + v

Keyboard Shortcuts

Open search /
Next entry (timeline) j
Previous entry (timeline) k
Open focused entry Enter
Show this help ?
Toggle theme Top nav button