Fossil SCM

Extended file handling with main logic to import an archive into fossil. Main parts are determining the various orders for expansion and import, and expanding all revisions in an archive into full texts, iteratively applying the stored deltas.

aku 2007-12-05 07:55 trunk
Commit e100314ec2deed52da780bd226ab583efd59271b
Parent 1a20a81f33681c6…
1 file changed +248
~ tools/cvs2fossil/lib/c2f_file.tcl
M tools/cvs2fossil/lib/c2f_file.tcl
+248
--- tools/cvs2fossil/lib/c2f_file.tcl
+++ tools/cvs2fossil/lib/c2f_file.tcl
@@ -17,14 +17,16 @@
1717
 ## Requirements
1818
1919
 package require Tcl 8.4 ; # Required runtime.
2020
 package require snit ; # OO system.
2121
 package require struct::set ; # Set operations.
22
+package require struct::list ; # Higher order operations.
2223
 package require vc::fossil::import::cvs::file::rev ; # CVS per file revisions.
2324
 package require vc::fossil::import::cvs::file::sym ; # CVS per file symbols.
2425
 package require vc::fossil::import::cvs::state ; # State storage.
2526
 package require vc::fossil::import::cvs::integrity ; # State integrity checks.
27
+package require vc::fossil::import::cvs::gtcore ; # Graph traversal core.
2628
 package require vc::tools::trouble ; # Error reporting.
2729
 package require vc::tools::log ; # User feedback
2830
 package require vc::tools::misc ; # Text formatting
2931
3032
 # # ## ### ##### ######## ############# #####################
@@ -249,10 +251,255 @@
249251
 }
250252
251253
 $self AggregateSymbolData
252254
 return
253255
 }
256
+
257
+ # # ## ### ##### ######## #############
258
+ ## Pass XII (Import).
259
+
260
+ method pushto {repository} {
261
+ set ws [$repository workspace]
262
+ struct::list assign [$self Expand $ws] filemap revmap
263
+ # filemap = dict (path -> uuid)
264
+ # revmap = dict (path -> rid)
265
+
266
+ array set idmap [$repository importfiles $filemap]
267
+
268
+ # Wipe workspace clean of the imported files.
269
+ foreach x [glob -directory $ws r*] { file delete $x }
270
+
271
+ foreach {path rid} $revmap {
272
+ set uuid $idmap($path)
273
+ state run {
274
+ INSERT INTO revuuid (rid, uuid)
275
+ VALUES ($rid, $uuid)
276
+ }
277
+ }
278
+ return
279
+ }
280
+
281
+ method Expand {dir} {
282
+ set ex [struct::graph ex] ; # Expansion graph.
283
+ set zp [struct::graph zp] ; # Zip/Import graph.
284
+
285
+ close [open $dir/r__empty__ w];# Base for detached roots on branches.
286
+
287
+ # Phase I: Pull the revisions from memory and fill the graphs
288
+ # with them...
289
+
290
+ set earcs {} ; # Arcs for expansion graph
291
+ set zarcs {} ; # Arcs for zip graph
292
+ set revmap {} ; # path -> rid map to later merge uuid information
293
+
294
+ foreach {rid revnr parent child coff clen} [state run {
295
+ SELECT R.rid, R.rev, R.parent, R.child, R.coff, R.clen
296
+ FROM revision R
297
+ WHERE R.fid = $myid
298
+ }] {
299
+ lappend revmap r$revnr $rid
300
+
301
+ $zp node insert $rid
302
+ $zp node set $rid revnr $revnr
303
+ $zp node set $rid label <$revnr>
304
+
305
+ if {$child ne ""} {
306
+ lappend zarcs $child $rid
307
+ }
308
+
309
+ $ex node insert $rid
310
+ $ex node set $rid text [list $coff $clen]
311
+ $ex node set $rid revnr $revnr
312
+ $ex node set $rid label <$revnr>
313
+
314
+ if {[rev istrunkrevnr $revnr]} {
315
+ # On the trunk, this revision is a delta based on the
316
+ # child. That makes the child our predecessor.
317
+
318
+ if {$child eq ""} continue
319
+ lappend earcs $child $rid
320
+ } else {
321
+ # On a branch this revision is a delta based on the
322
+ # parent. That makes the parent our predecessor.
323
+
324
+ if {$parent eq ""} {
325
+ # Detached branch root, this is a patch based on
326
+ # the empty string.
327
+ $ex node set $rid __base__ r__empty__
328
+ continue
329
+ }
330
+ lappend earcs $parent $rid
331
+ }
332
+ }
333
+
334
+ # Phase II: Insert the accumulated dependencies
335
+
336
+ foreach {from to} $earcs { $ex arc insert $from $to }
337
+ foreach {from to} $zarcs { $zp arc insert $from $to }
338
+
339
+ # Phase III: Traverse the graphs, expand the file, and
340
+ # generate import instructions.
341
+
342
+ set archive [file join [$myproject fullpath] $mypath]
343
+ set ac [open $archive r]
344
+ fconfigure $ac -translation binary
345
+
346
+ # First traverse the expansion graph, this gives us the
347
+ # revisions in the order we have to expand them, which we do.
348
+
349
+ gtcore datacmd [mymethod ExpandData]
350
+ gtcore formatcmd [mymethod ExpandFormat]
351
+ gtcore sortcmd [mymethod ExpandSort]
352
+ gtcore savecmd [mymethod Expand1 $ac $dir]
353
+
354
+ gtcore traverse $ex ; # The graph is gone after the call
355
+ close $ac
356
+
357
+ # Now traverse the import graph, this builds the instruction
358
+ # map for the fossil deltas.
359
+
360
+ gtcore datacmd [mymethod ExpandData]
361
+ gtcore formatcmd [mymethod ExpandFormat]
362
+ gtcore sortcmd [mymethod ExpandSort]
363
+ gtcore savecmd [mymethod Expand2]
364
+
365
+ set myimport {}
366
+ gtcore traverse $zp ; # The graph is gone after the call
367
+ set filemap $myimport
368
+ unset myimport
369
+
370
+ # And back to import control
371
+
372
+ return [list $filemap $revmap]
373
+ }
374
+
375
+ method ExpandData {graph node} { return [$graph node get $node revnr] }
376
+ method ExpandFormat {graph item} { return <[lindex $item 1]> } ; # revnr
377
+ method ExpandSort {graph candidates} {
378
+ # candidates = list(item), item = list(node revnr)
379
+ # Sort by node and revnr -> Trunk revisions come first.
380
+ return [lsort -index 1 -dict [lsort -index 0 -dict $candidates]]
381
+ }
382
+ method Expand1 {chan dir graph node} {
383
+ set revnr [$graph node get $node revnr]
384
+ set fname r$revnr
385
+ struct::list assign [$graph node get $node text] offset length
386
+
387
+ seek $chan $offset start
388
+ set data [string map {@@ @} [read $chan $length]]
389
+
390
+ if {![$graph node keyexists $node __base__]} {
391
+ # Full text node. Get the data, decode it, and save.
392
+
393
+ log write 2 file {Expanding <$revnr>, full text}
394
+
395
+ fileutil::writeFile -translation binary $dir/$fname $data
396
+ } else {
397
+ # Delta node. __base__ is the name of the file containing
398
+ # the baseline. The patch is at the specified location of
399
+ # the archive file.
400
+
401
+ set fbase [$graph node get $node __base__]
402
+ log write 2 file {Expanding <$revnr>, is delta of <$fbase>}
403
+
404
+ set base [fileutil::cat -translation binary $dir/$fbase]
405
+
406
+ # Writing the patch to disk is just for better
407
+ # debugging. It is not used otherwise.
408
+ fileutil::writeFile $dir/rpatch $data
409
+ fileutil::writeFile -translation binary $dir/$fname \
410
+ [Apply $base $data]
411
+ }
412
+
413
+ # Post to all successors that the just generated file is their
414
+ # baseline.
415
+
416
+ foreach out [$graph nodes -out $node] {
417
+ $graph node set $out __base__ $fname
418
+ }
419
+ return
420
+ }
421
+
422
+ proc Apply {base delta} {
423
+ # base = base text.
424
+ # delta = delta in rcs format.
425
+ #
426
+ # Both strings are unencoded, i.e. things like @@, etc. have
427
+ # already been replaced with their proper characters.
428
+ #
429
+ # Return value is the patched text.
430
+
431
+ set base [split $base \n]
432
+ set blen [llength $base]
433
+ set ooff 0
434
+ set res ""
435
+
436
+ set lines [split $delta \n]
437
+ set nlines [llength $lines]
438
+
439
+ for {set i 0} {$i < $nlines} {} {
440
+ if {![regexp {^([ad])(\d+)\s(\d+)$} [lindex $lines $i] -> cmd sl cn]} {
441
+ trouble internal "Bad ed command '[lindex $lines $i]'"
442
+ }
443
+
444
+ incr i
445
+ set el [expr {$sl + $cn}]
446
+
447
+ switch -exact -- $cmd {
448
+ d {
449
+ incr sl -1
450
+ incr el -1
451
+ if {$sl < $ooff} { trouble internal {Deletion before last edit} }
452
+ if {$sl > $blen} { trouble internal {Deletion past file end} }
453
+ if {$el > $blen} { trouble internal {Deletion beyond file end} }
454
+ foreach x [lrange $base $ooff $sl] { lappend res $x }
455
+ set ooff $el
456
+ }
457
+ a {
458
+ if {$sl < $ooff} { trouble internal {Insert before last edit} }
459
+ if {$sl > $blen} { trouble internal {Insert past file end} }
460
+
461
+ foreach x [lrange $base $ooff $sl] { lappend res $x }
462
+ foreach x [lrange $lines $i [expr {$i + $cn}]] { lappend res $x }
463
+ set ooff $sl
464
+ incr i $cn
465
+ }
466
+ }
467
+ }
468
+ foreach x [lrange $base $ooff end] { lappend res $x }
469
+ return [join $res \n]
470
+ }
471
+
472
+ method Expand2 {graph node} {
473
+ set revnr [$graph node get $node revnr]
474
+
475
+ # First import the file.
476
+ lappend myimport [list A r$revnr {}]
477
+
478
+ if {[$graph node keyexists $node __base__]} {
479
+ # Delta node. __base__ is the name of the file containing
480
+ # the baseline. Generate instruction to make the delta as
481
+ # well.
482
+
483
+ set fbase [$graph node get $node __base__]
484
+ lappend myimport [list D r$revnr r$fbase]
485
+ }
486
+
487
+ # Post to all successors that the just generated file is their
488
+ # baseline. Exception: Those which ave already a baseline set.
489
+ # Together with the sorting of trunk revisions first the trunk
490
+ # should one uninterupted line, with branch roots _not_ delta
491
+ # compressed per their branches.
492
+
493
+ foreach out [$graph nodes -out $node] {
494
+ if {[$graph node keyexists $out __base__]} continue
495
+ $graph node set $out __base__ $revnr
496
+ }
497
+ return
498
+ }
499
+
500
+ variable myimport
254501
255502
 # # ## ### ##### ######## #############
256503
 ## State
257504
258505
 variable myid {} ; # File id in the persistent state.
@@ -1120,13 +1367,14 @@
11201367
 namespace import ::vc::tools::misc::*
11211368
 namespace import ::vc::tools::trouble
11221369
 namespace import ::vc::tools::log
11231370
 namespace import ::vc::fossil::import::cvs::state
11241371
 namespace import ::vc::fossil::import::cvs::integrity
1372
+ namespace import ::vc::fossil::import::cvs::gtcore
11251373
 }
11261374
 }
11271375
11281376
 # # ## ### ##### ######## ############# #####################
11291377
 ## Ready
11301378
11311379
 package provide vc::fossil::import::cvs::file 1.0
11321380
 return
11331381
--- tools/cvs2fossil/lib/c2f_file.tcl
+++ tools/cvs2fossil/lib/c2f_file.tcl
@@ -17,14 +17,16 @@
17 ## Requirements
18
19 package require Tcl 8.4 ; # Required runtime.
20 package require snit ; # OO system.
21 package require struct::set ; # Set operations.
 
22 package require vc::fossil::import::cvs::file::rev ; # CVS per file revisions.
23 package require vc::fossil::import::cvs::file::sym ; # CVS per file symbols.
24 package require vc::fossil::import::cvs::state ; # State storage.
25 package require vc::fossil::import::cvs::integrity ; # State integrity checks.
 
26 package require vc::tools::trouble ; # Error reporting.
27 package require vc::tools::log ; # User feedback
28 package require vc::tools::misc ; # Text formatting
29
30 # # ## ### ##### ######## ############# #####################
@@ -249,10 +251,255 @@
249 }
250
251 $self AggregateSymbolData
252 return
253 }
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
254
255 # # ## ### ##### ######## #############
256 ## State
257
258 variable myid {} ; # File id in the persistent state.
@@ -1120,13 +1367,14 @@
1120 namespace import ::vc::tools::misc::*
1121 namespace import ::vc::tools::trouble
1122 namespace import ::vc::tools::log
1123 namespace import ::vc::fossil::import::cvs::state
1124 namespace import ::vc::fossil::import::cvs::integrity
 
1125 }
1126 }
1127
1128 # # ## ### ##### ######## ############# #####################
1129 ## Ready
1130
1131 package provide vc::fossil::import::cvs::file 1.0
1132 return
1133
--- tools/cvs2fossil/lib/c2f_file.tcl
+++ tools/cvs2fossil/lib/c2f_file.tcl
@@ -17,14 +17,16 @@
17 ## Requirements
18
19 package require Tcl 8.4 ; # Required runtime.
20 package require snit ; # OO system.
21 package require struct::set ; # Set operations.
22 package require struct::list ; # Higher order operations.
23 package require vc::fossil::import::cvs::file::rev ; # CVS per file revisions.
24 package require vc::fossil::import::cvs::file::sym ; # CVS per file symbols.
25 package require vc::fossil::import::cvs::state ; # State storage.
26 package require vc::fossil::import::cvs::integrity ; # State integrity checks.
27 package require vc::fossil::import::cvs::gtcore ; # Graph traversal core.
28 package require vc::tools::trouble ; # Error reporting.
29 package require vc::tools::log ; # User feedback
30 package require vc::tools::misc ; # Text formatting
31
32 # # ## ### ##### ######## ############# #####################
@@ -249,10 +251,255 @@
251 }
252
253 $self AggregateSymbolData
254 return
255 }
256
257 # # ## ### ##### ######## #############
258 ## Pass XII (Import).
259
260 method pushto {repository} {
261 set ws [$repository workspace]
262 struct::list assign [$self Expand $ws] filemap revmap
263 # filemap = dict (path -> uuid)
264 # revmap = dict (path -> rid)
265
266 array set idmap [$repository importfiles $filemap]
267
268 # Wipe workspace clean of the imported files.
269 foreach x [glob -directory $ws r*] { file delete $x }
270
271 foreach {path rid} $revmap {
272 set uuid $idmap($path)
273 state run {
274 INSERT INTO revuuid (rid, uuid)
275 VALUES ($rid, $uuid)
276 }
277 }
278 return
279 }
280
281 method Expand {dir} {
282 set ex [struct::graph ex] ; # Expansion graph.
283 set zp [struct::graph zp] ; # Zip/Import graph.
284
285 close [open $dir/r__empty__ w];# Base for detached roots on branches.
286
287 # Phase I: Pull the revisions from memory and fill the graphs
288 # with them...
289
290 set earcs {} ; # Arcs for expansion graph
291 set zarcs {} ; # Arcs for zip graph
292 set revmap {} ; # path -> rid map to later merge uuid information
293
294 foreach {rid revnr parent child coff clen} [state run {
295 SELECT R.rid, R.rev, R.parent, R.child, R.coff, R.clen
296 FROM revision R
297 WHERE R.fid = $myid
298 }] {
299 lappend revmap r$revnr $rid
300
301 $zp node insert $rid
302 $zp node set $rid revnr $revnr
303 $zp node set $rid label <$revnr>
304
305 if {$child ne ""} {
306 lappend zarcs $child $rid
307 }
308
309 $ex node insert $rid
310 $ex node set $rid text [list $coff $clen]
311 $ex node set $rid revnr $revnr
312 $ex node set $rid label <$revnr>
313
314 if {[rev istrunkrevnr $revnr]} {
315 # On the trunk, this revision is a delta based on the
316 # child. That makes the child our predecessor.
317
318 if {$child eq ""} continue
319 lappend earcs $child $rid
320 } else {
321 # On a branch this revision is a delta based on the
322 # parent. That makes the parent our predecessor.
323
324 if {$parent eq ""} {
325 # Detached branch root, this is a patch based on
326 # the empty string.
327 $ex node set $rid __base__ r__empty__
328 continue
329 }
330 lappend earcs $parent $rid
331 }
332 }
333
334 # Phase II: Insert the accumulated dependencies
335
336 foreach {from to} $earcs { $ex arc insert $from $to }
337 foreach {from to} $zarcs { $zp arc insert $from $to }
338
339 # Phase III: Traverse the graphs, expand the file, and
340 # generate import instructions.
341
342 set archive [file join [$myproject fullpath] $mypath]
343 set ac [open $archive r]
344 fconfigure $ac -translation binary
345
346 # First traverse the expansion graph, this gives us the
347 # revisions in the order we have to expand them, which we do.
348
349 gtcore datacmd [mymethod ExpandData]
350 gtcore formatcmd [mymethod ExpandFormat]
351 gtcore sortcmd [mymethod ExpandSort]
352 gtcore savecmd [mymethod Expand1 $ac $dir]
353
354 gtcore traverse $ex ; # The graph is gone after the call
355 close $ac
356
357 # Now traverse the import graph, this builds the instruction
358 # map for the fossil deltas.
359
360 gtcore datacmd [mymethod ExpandData]
361 gtcore formatcmd [mymethod ExpandFormat]
362 gtcore sortcmd [mymethod ExpandSort]
363 gtcore savecmd [mymethod Expand2]
364
365 set myimport {}
366 gtcore traverse $zp ; # The graph is gone after the call
367 set filemap $myimport
368 unset myimport
369
370 # And back to import control
371
372 return [list $filemap $revmap]
373 }
374
375 method ExpandData {graph node} { return [$graph node get $node revnr] }
376 method ExpandFormat {graph item} { return <[lindex $item 1]> } ; # revnr
377 method ExpandSort {graph candidates} {
378 # candidates = list(item), item = list(node revnr)
379 # Sort by node and revnr -> Trunk revisions come first.
380 return [lsort -index 1 -dict [lsort -index 0 -dict $candidates]]
381 }
382 method Expand1 {chan dir graph node} {
383 set revnr [$graph node get $node revnr]
384 set fname r$revnr
385 struct::list assign [$graph node get $node text] offset length
386
387 seek $chan $offset start
388 set data [string map {@@ @} [read $chan $length]]
389
390 if {![$graph node keyexists $node __base__]} {
391 # Full text node. Get the data, decode it, and save.
392
393 log write 2 file {Expanding <$revnr>, full text}
394
395 fileutil::writeFile -translation binary $dir/$fname $data
396 } else {
397 # Delta node. __base__ is the name of the file containing
398 # the baseline. The patch is at the specified location of
399 # the archive file.
400
401 set fbase [$graph node get $node __base__]
402 log write 2 file {Expanding <$revnr>, is delta of <$fbase>}
403
404 set base [fileutil::cat -translation binary $dir/$fbase]
405
406 # Writing the patch to disk is just for better
407 # debugging. It is not used otherwise.
408 fileutil::writeFile $dir/rpatch $data
409 fileutil::writeFile -translation binary $dir/$fname \
410 [Apply $base $data]
411 }
412
413 # Post to all successors that the just generated file is their
414 # baseline.
415
416 foreach out [$graph nodes -out $node] {
417 $graph node set $out __base__ $fname
418 }
419 return
420 }
421
422 proc Apply {base delta} {
423 # base = base text.
424 # delta = delta in rcs format.
425 #
426 # Both strings are unencoded, i.e. things like @@, etc. have
427 # already been replaced with their proper characters.
428 #
429 # Return value is the patched text.
430
431 set base [split $base \n]
432 set blen [llength $base]
433 set ooff 0
434 set res ""
435
436 set lines [split $delta \n]
437 set nlines [llength $lines]
438
439 for {set i 0} {$i < $nlines} {} {
440 if {![regexp {^([ad])(\d+)\s(\d+)$} [lindex $lines $i] -> cmd sl cn]} {
441 trouble internal "Bad ed command '[lindex $lines $i]'"
442 }
443
444 incr i
445 set el [expr {$sl + $cn}]
446
447 switch -exact -- $cmd {
448 d {
449 incr sl -1
450 incr el -1
451 if {$sl < $ooff} { trouble internal {Deletion before last edit} }
452 if {$sl > $blen} { trouble internal {Deletion past file end} }
453 if {$el > $blen} { trouble internal {Deletion beyond file end} }
454 foreach x [lrange $base $ooff $sl] { lappend res $x }
455 set ooff $el
456 }
457 a {
458 if {$sl < $ooff} { trouble internal {Insert before last edit} }
459 if {$sl > $blen} { trouble internal {Insert past file end} }
460
461 foreach x [lrange $base $ooff $sl] { lappend res $x }
462 foreach x [lrange $lines $i [expr {$i + $cn}]] { lappend res $x }
463 set ooff $sl
464 incr i $cn
465 }
466 }
467 }
468 foreach x [lrange $base $ooff end] { lappend res $x }
469 return [join $res \n]
470 }
471
472 method Expand2 {graph node} {
473 set revnr [$graph node get $node revnr]
474
475 # First import the file.
476 lappend myimport [list A r$revnr {}]
477
478 if {[$graph node keyexists $node __base__]} {
479 # Delta node. __base__ is the name of the file containing
480 # the baseline. Generate instruction to make the delta as
481 # well.
482
483 set fbase [$graph node get $node __base__]
484 lappend myimport [list D r$revnr r$fbase]
485 }
486
487 # Post to all successors that the just generated file is their
488 # baseline. Exception: Those which ave already a baseline set.
489 # Together with the sorting of trunk revisions first the trunk
490 # should one uninterupted line, with branch roots _not_ delta
491 # compressed per their branches.
492
493 foreach out [$graph nodes -out $node] {
494 if {[$graph node keyexists $out __base__]} continue
495 $graph node set $out __base__ $revnr
496 }
497 return
498 }
499
500 variable myimport
501
502 # # ## ### ##### ######## #############
503 ## State
504
505 variable myid {} ; # File id in the persistent state.
@@ -1120,13 +1367,14 @@
1367 namespace import ::vc::tools::misc::*
1368 namespace import ::vc::tools::trouble
1369 namespace import ::vc::tools::log
1370 namespace import ::vc::fossil::import::cvs::state
1371 namespace import ::vc::fossil::import::cvs::integrity
1372 namespace import ::vc::fossil::import::cvs::gtcore
1373 }
1374 }
1375
1376 # # ## ### ##### ######## ############# #####################
1377 ## Ready
1378
1379 package provide vc::fossil::import::cvs::file 1.0
1380 return
1381

Keyboard Shortcuts

Open search /
Next entry (timeline) j
Previous entry (timeline) k
Open focused entry Enter
Show this help ?
Toggle theme Top nav button