|
1
|
""" |
|
2
|
WorkPartitioner — divide graph work across N agents. |
|
3
|
|
|
4
|
Splits the set of ingested files into roughly equal partitions so that |
|
5
|
multiple agents can ingest or analyse different parts of a repository |
|
6
|
concurrently without overlap. |
|
7
|
|
|
8
|
Usage: |
|
9
|
from navegador.graph.store import GraphStore |
|
10
|
from navegador.cluster.partitioning import WorkPartitioner |
|
11
|
|
|
12
|
store = GraphStore.sqlite(".navegador/graph.db") |
|
13
|
partitioner = WorkPartitioner(store) |
|
14
|
partitions = partitioner.partition(n_agents=4) |
|
15
|
|
|
16
|
for p in partitions: |
|
17
|
print(p.agent_id, p.file_paths, p.estimated_work) |
|
18
|
""" |
|
19
|
|
|
20
|
from __future__ import annotations |
|
21
|
|
|
22
|
import logging |
|
23
|
import math |
|
24
|
from dataclasses import dataclass |
|
25
|
from typing import Any |
|
26
|
|
|
27
|
logger = logging.getLogger(__name__) |
|
28
|
|
|
29
|
|
|
30
|
@dataclass |
|
31
|
class Partition: |
|
32
|
"""A slice of repository work assigned to one agent.""" |
|
33
|
|
|
34
|
agent_id: str |
|
35
|
file_paths: list[str] |
|
36
|
estimated_work: int # proxy = number of files |
|
37
|
|
|
38
|
def to_dict(self) -> dict[str, Any]: |
|
39
|
return { |
|
40
|
"agent_id": self.agent_id, |
|
41
|
"file_paths": self.file_paths, |
|
42
|
"estimated_work": self.estimated_work, |
|
43
|
} |
|
44
|
|
|
45
|
|
|
46
|
class WorkPartitioner: |
|
47
|
""" |
|
48
|
Partition repository files across N agents. |
|
49
|
|
|
50
|
The current implementation uses a simple round-robin file-count split. |
|
51
|
A future version can replace this with a graph community detection |
|
52
|
algorithm (e.g. Louvain via `networkx`) for tighter semantic cohesion. |
|
53
|
|
|
54
|
Parameters |
|
55
|
---------- |
|
56
|
store: |
|
57
|
A ``GraphStore`` instance to query file paths from. |
|
58
|
""" |
|
59
|
|
|
60
|
def __init__(self, store: Any) -> None: |
|
61
|
self._store = store |
|
62
|
|
|
63
|
# ── Internal ────────────────────────────────────────────────────────────── |
|
64
|
|
|
65
|
def _get_all_file_paths(self) -> list[str]: |
|
66
|
"""Retrieve distinct file paths recorded in the graph.""" |
|
67
|
result = self._store.query( |
|
68
|
"MATCH (n) WHERE n.file_path IS NOT NULL RETURN DISTINCT n.file_path AS fp ORDER BY fp" |
|
69
|
) |
|
70
|
if not result.result_set: |
|
71
|
return [] |
|
72
|
paths: list[str] = [] |
|
73
|
for row in result.result_set: |
|
74
|
fp = row[0] |
|
75
|
if fp and fp not in paths: |
|
76
|
paths.append(fp) |
|
77
|
return paths |
|
78
|
|
|
79
|
@staticmethod |
|
80
|
def _split_evenly(items: list[str], n: int) -> list[list[str]]: |
|
81
|
"""Split *items* into *n* roughly equal-sized buckets.""" |
|
82
|
if n <= 0: |
|
83
|
raise ValueError("n_agents must be >= 1") |
|
84
|
if not items: |
|
85
|
return [[] for _ in range(n)] |
|
86
|
chunk_size = math.ceil(len(items) / n) |
|
87
|
buckets = [] |
|
88
|
for i in range(0, len(items), chunk_size): |
|
89
|
buckets.append(items[i : i + chunk_size]) |
|
90
|
# Pad with empty lists if fewer chunks than agents |
|
91
|
while len(buckets) < n: |
|
92
|
buckets.append([]) |
|
93
|
return buckets[:n] |
|
94
|
|
|
95
|
# ── Public API ──────────────────────────────────────────────────────────── |
|
96
|
|
|
97
|
def partition(self, n_agents: int) -> list[Partition]: |
|
98
|
""" |
|
99
|
Divide all graph file paths into *n_agents* partitions. |
|
100
|
|
|
101
|
Parameters |
|
102
|
---------- |
|
103
|
n_agents: |
|
104
|
Number of agents (partitions) to create. Must be >= 1. |
|
105
|
|
|
106
|
Returns |
|
107
|
------- |
|
108
|
list[Partition] |
|
109
|
One ``Partition`` per agent. Agent IDs are ``"agent-0"``, |
|
110
|
``"agent-1"``, … ``"agent-(n-1)"``. |
|
111
|
""" |
|
112
|
if n_agents < 1: |
|
113
|
raise ValueError("n_agents must be >= 1") |
|
114
|
|
|
115
|
file_paths = self._get_all_file_paths() |
|
116
|
buckets = self._split_evenly(file_paths, n_agents) |
|
117
|
|
|
118
|
partitions = [ |
|
119
|
Partition( |
|
120
|
agent_id=f"agent-{i}", |
|
121
|
file_paths=bucket, |
|
122
|
estimated_work=len(bucket), |
|
123
|
) |
|
124
|
for i, bucket in enumerate(buckets) |
|
125
|
] |
|
126
|
|
|
127
|
logger.info( |
|
128
|
"Partitioned %d files across %d agents", |
|
129
|
len(file_paths), |
|
130
|
n_agents, |
|
131
|
) |
|
132
|
return partitions |
|
133
|
|