Fossil SCM

Added high-level logging for memory tracing to the code breaking the preliminary changesets. First runs indicate that the DEPC array becomes so very large, caused by a high amount of indirect dependencies (several hundred).

aku 2008-02-21 05:13 trunk
Commit c2ad73ed9243ac9497046fa8eae19f904b56bc77
Parent f46458d5bdc2218…
1 file changed +31 -3
~ tools/cvs2fossil/lib/c2f_prev.tcl
M tools/cvs2fossil/lib/c2f_prev.tcl
+31 -3
--- tools/cvs2fossil/lib/c2f_prev.tcl
+++ tools/cvs2fossil/lib/c2f_prev.tcl
@@ -203,11 +203,17 @@
203203
 # Data structures:
204204
 # Map: POS revision id -> position in list.
205205
 # CROSS position in list -> number of dependencies crossing it
206206
 # DEPC dependency -> positions it crosses
207207
 # List: RANGE Of the positions itself.
208
+ # Map: DELTA position in list -> time delta between its revision
209
+ # and the next, if any.
208210
 # A dependency is a single-element map parent -> child
211
+
212
+ # InitializeBreakState initializes their contents after
213
+ # upvar'ing them from this scope. It uses the information in
214
+ # DEPENDENCIES to do so.
209215
210216
 InitializeBreakState $myitems
211217
212218
 set fragments {}
213219
 set new [list $range]
@@ -799,10 +805,13 @@
799805
 dependencies dependencies
800806
801807
 # First we create a map of positions to make it easier to
802808
 # determine whether a dependency crosses a particular index.
803809
810
+ log write 14 csets {IBS: #rev [llength $revisions]}
811
+ log write 14 csets {IBS: pos map, cross counter}
812
+
804813
 array set pos {}
805814
 array set cross {}
806815
 array set depc {}
807816
 set range {}
808817
 set n 0
@@ -810,10 +819,12 @@
810819
 lappend range $n
811820
 set pos($rev) $n
812821
 set cross($n) 0
813822
 incr n
814823
 }
824
+
825
+ log write 14 csets {IBS: pos/[array size pos], cross/[array size cross]}
815826
816827
 # Secondly we count the crossings per position, by iterating
817828
 # over the recorded internal dependencies.
818829
819830
 # Note: If the timestamps are badly out of order it is
@@ -823,10 +834,12 @@
823834
 #
824835
 # Note 2: start == end is not possible. It indicates a
825836
 # self-dependency due to the uniqueness of positions,
826837
 # and that is something we have ruled out already, see
827838
 # 'rev internalsuccessors'.
839
+
840
+ log write 14 csets {IBS: cross counter filling, pos/cross map}
828841
829842
 foreach {rid children} [array get dependencies] {
830843
 foreach child $children {
831844
 set dkey [list $rid $child]
832845
 set start $pos($rid)
@@ -848,11 +861,16 @@
848861
 }
849862
 set depc($dkey) $crosses
850863
 }
851864
 }
852865
866
+ log write 14 csets {IBS: pos/[array size pos], cross/[array size cross], depc/[array size depc] (for [llength $revisions])}
867
+ log write 14 csets {IBS: timestamps, deltas}
868
+
853869
 InitializeDeltas $revisions
870
+
871
+ log write 14 csets {IBS: delta [array size delta]}
854872
 return
855873
 }
856874
857875
 proc InitializeDeltas {revisions} {
858876
 upvar 1 delta delta
@@ -869,10 +887,12 @@
869887
 FROM revision R
870888
 WHERE R.rid IN $theset
871889
 }]] {
872890
 set stamp($rid) $time
873891
 }
892
+
893
+ log write 14 csets {IBS: stamp [array size stamp]}
874894
875895
 set n 0
876896
 foreach rid [lrange $revisions 0 end-1] rnext [lrange $revisions 1 end] {
877897
 set delta($n) [expr {$stamp($rnext) - $stamp($rid)}]
878898
 incr n
@@ -1081,11 +1101,11 @@
10811101
 # var(dv) = dict (revision -> list (revision))
10821102
 typemethod internalsuccessors {dv revisions} {
10831103
 upvar 1 $dv dependencies
10841104
 set theset ('[join $revisions {','}]')
10851105
1086
- log write 14 cset internalsuccessors
1106
+ log write 14 csets internalsuccessors
10871107
10881108
 # See 'successors' below for the main explanation of
10891109
 # the various cases. This piece is special in that it
10901110
 # restricts the successors we look for to the same set of
10911111
 # revisions we start from. Sensible as we are looking for
@@ -1144,19 +1164,22 @@
11441164
11451165
 # We allow revisions to be far apart in time in the same
11461166
 # changeset, but in turn need the pseudo-dependencies to
11471167
 # handle this.
11481168
1149
- log write 14 cset pseudo-internalsuccessors
1169
+ log write 14 csets {internal [array size dep]}
1170
+ log write 14 csets {collected [array size dependencies]}
1171
+ log write 14 csets pseudo-internalsuccessors
11501172
11511173
 array set fids {}
11521174
 foreach {rid fid} [state run [subst -nocommands -nobackslashes {
11531175
 SELECT R.rid, R.fid
11541176
 FROM revision R
11551177
 WHERE R.rid IN $theset
11561178
 }]] { lappend fids($fid) $rid }
11571179
1180
+ set groups {}
11581181
 foreach {fid rids} [array get fids] {
11591182
 if {[llength $rids] < 2} continue
11601183
 foreach a $rids {
11611184
 foreach b $rids {
11621185
 if {$a == $b} continue
@@ -1165,13 +1188,18 @@
11651188
 lappend dependencies($a) $b
11661189
 set dep($a,$b) .
11671190
 set dep($b,$a) .
11681191
 }
11691192
 }
1193
+ set n [llength $rids]
1194
+ lappend groups [list $n [expr {($n*$n-$n)/2}]]
11701195
 }
11711196
1172
- log write 14 cset complete
1197
+ log write 14 csets {pseudo [array size fids] ([lsort -index 0 -decreasing -integer $groups])}
1198
+ log write 14 csets {internal [array size dep]}
1199
+ log write 14 csets {collected [array size dependencies]}
1200
+ log write 14 csets complete
11731201
 return
11741202
 }
11751203
11761204
 # result = 4-list (itemtype itemid nextitemtype nextitemid ...)
11771205
 typemethod loops {revisions} {
11781206
--- tools/cvs2fossil/lib/c2f_prev.tcl
+++ tools/cvs2fossil/lib/c2f_prev.tcl
@@ -203,11 +203,17 @@
203 # Data structures:
204 # Map: POS revision id -> position in list.
205 # CROSS position in list -> number of dependencies crossing it
206 # DEPC dependency -> positions it crosses
207 # List: RANGE Of the positions itself.
 
 
208 # A dependency is a single-element map parent -> child
 
 
 
 
209
210 InitializeBreakState $myitems
211
212 set fragments {}
213 set new [list $range]
@@ -799,10 +805,13 @@
799 dependencies dependencies
800
801 # First we create a map of positions to make it easier to
802 # determine whether a dependency crosses a particular index.
803
 
 
 
804 array set pos {}
805 array set cross {}
806 array set depc {}
807 set range {}
808 set n 0
@@ -810,10 +819,12 @@
810 lappend range $n
811 set pos($rev) $n
812 set cross($n) 0
813 incr n
814 }
 
 
815
816 # Secondly we count the crossings per position, by iterating
817 # over the recorded internal dependencies.
818
819 # Note: If the timestamps are badly out of order it is
@@ -823,10 +834,12 @@
823 #
824 # Note 2: start == end is not possible. It indicates a
825 # self-dependency due to the uniqueness of positions,
826 # and that is something we have ruled out already, see
827 # 'rev internalsuccessors'.
 
 
828
829 foreach {rid children} [array get dependencies] {
830 foreach child $children {
831 set dkey [list $rid $child]
832 set start $pos($rid)
@@ -848,11 +861,16 @@
848 }
849 set depc($dkey) $crosses
850 }
851 }
852
 
 
 
853 InitializeDeltas $revisions
 
 
854 return
855 }
856
857 proc InitializeDeltas {revisions} {
858 upvar 1 delta delta
@@ -869,10 +887,12 @@
869 FROM revision R
870 WHERE R.rid IN $theset
871 }]] {
872 set stamp($rid) $time
873 }
 
 
874
875 set n 0
876 foreach rid [lrange $revisions 0 end-1] rnext [lrange $revisions 1 end] {
877 set delta($n) [expr {$stamp($rnext) - $stamp($rid)}]
878 incr n
@@ -1081,11 +1101,11 @@
1081 # var(dv) = dict (revision -> list (revision))
1082 typemethod internalsuccessors {dv revisions} {
1083 upvar 1 $dv dependencies
1084 set theset ('[join $revisions {','}]')
1085
1086 log write 14 cset internalsuccessors
1087
1088 # See 'successors' below for the main explanation of
1089 # the various cases. This piece is special in that it
1090 # restricts the successors we look for to the same set of
1091 # revisions we start from. Sensible as we are looking for
@@ -1144,19 +1164,22 @@
1144
1145 # We allow revisions to be far apart in time in the same
1146 # changeset, but in turn need the pseudo-dependencies to
1147 # handle this.
1148
1149 log write 14 cset pseudo-internalsuccessors
 
 
1150
1151 array set fids {}
1152 foreach {rid fid} [state run [subst -nocommands -nobackslashes {
1153 SELECT R.rid, R.fid
1154 FROM revision R
1155 WHERE R.rid IN $theset
1156 }]] { lappend fids($fid) $rid }
1157
 
1158 foreach {fid rids} [array get fids] {
1159 if {[llength $rids] < 2} continue
1160 foreach a $rids {
1161 foreach b $rids {
1162 if {$a == $b} continue
@@ -1165,13 +1188,18 @@
1165 lappend dependencies($a) $b
1166 set dep($a,$b) .
1167 set dep($b,$a) .
1168 }
1169 }
 
 
1170 }
1171
1172 log write 14 cset complete
 
 
 
1173 return
1174 }
1175
1176 # result = 4-list (itemtype itemid nextitemtype nextitemid ...)
1177 typemethod loops {revisions} {
1178
--- tools/cvs2fossil/lib/c2f_prev.tcl
+++ tools/cvs2fossil/lib/c2f_prev.tcl
@@ -203,11 +203,17 @@
203 # Data structures:
204 # Map: POS revision id -> position in list.
205 # CROSS position in list -> number of dependencies crossing it
206 # DEPC dependency -> positions it crosses
207 # List: RANGE Of the positions itself.
208 # Map: DELTA position in list -> time delta between its revision
209 # and the next, if any.
210 # A dependency is a single-element map parent -> child
211
212 # InitializeBreakState initializes their contents after
213 # upvar'ing them from this scope. It uses the information in
214 # DEPENDENCIES to do so.
215
216 InitializeBreakState $myitems
217
218 set fragments {}
219 set new [list $range]
@@ -799,10 +805,13 @@
805 dependencies dependencies
806
807 # First we create a map of positions to make it easier to
808 # determine whether a dependency crosses a particular index.
809
810 log write 14 csets {IBS: #rev [llength $revisions]}
811 log write 14 csets {IBS: pos map, cross counter}
812
813 array set pos {}
814 array set cross {}
815 array set depc {}
816 set range {}
817 set n 0
@@ -810,10 +819,12 @@
819 lappend range $n
820 set pos($rev) $n
821 set cross($n) 0
822 incr n
823 }
824
825 log write 14 csets {IBS: pos/[array size pos], cross/[array size cross]}
826
827 # Secondly we count the crossings per position, by iterating
828 # over the recorded internal dependencies.
829
830 # Note: If the timestamps are badly out of order it is
@@ -823,10 +834,12 @@
834 #
835 # Note 2: start == end is not possible. It indicates a
836 # self-dependency due to the uniqueness of positions,
837 # and that is something we have ruled out already, see
838 # 'rev internalsuccessors'.
839
840 log write 14 csets {IBS: cross counter filling, pos/cross map}
841
842 foreach {rid children} [array get dependencies] {
843 foreach child $children {
844 set dkey [list $rid $child]
845 set start $pos($rid)
@@ -848,11 +861,16 @@
861 }
862 set depc($dkey) $crosses
863 }
864 }
865
866 log write 14 csets {IBS: pos/[array size pos], cross/[array size cross], depc/[array size depc] (for [llength $revisions])}
867 log write 14 csets {IBS: timestamps, deltas}
868
869 InitializeDeltas $revisions
870
871 log write 14 csets {IBS: delta [array size delta]}
872 return
873 }
874
875 proc InitializeDeltas {revisions} {
876 upvar 1 delta delta
@@ -869,10 +887,12 @@
887 FROM revision R
888 WHERE R.rid IN $theset
889 }]] {
890 set stamp($rid) $time
891 }
892
893 log write 14 csets {IBS: stamp [array size stamp]}
894
895 set n 0
896 foreach rid [lrange $revisions 0 end-1] rnext [lrange $revisions 1 end] {
897 set delta($n) [expr {$stamp($rnext) - $stamp($rid)}]
898 incr n
@@ -1081,11 +1101,11 @@
1101 # var(dv) = dict (revision -> list (revision))
1102 typemethod internalsuccessors {dv revisions} {
1103 upvar 1 $dv dependencies
1104 set theset ('[join $revisions {','}]')
1105
1106 log write 14 csets internalsuccessors
1107
1108 # See 'successors' below for the main explanation of
1109 # the various cases. This piece is special in that it
1110 # restricts the successors we look for to the same set of
1111 # revisions we start from. Sensible as we are looking for
@@ -1144,19 +1164,22 @@
1164
1165 # We allow revisions to be far apart in time in the same
1166 # changeset, but in turn need the pseudo-dependencies to
1167 # handle this.
1168
1169 log write 14 csets {internal [array size dep]}
1170 log write 14 csets {collected [array size dependencies]}
1171 log write 14 csets pseudo-internalsuccessors
1172
1173 array set fids {}
1174 foreach {rid fid} [state run [subst -nocommands -nobackslashes {
1175 SELECT R.rid, R.fid
1176 FROM revision R
1177 WHERE R.rid IN $theset
1178 }]] { lappend fids($fid) $rid }
1179
1180 set groups {}
1181 foreach {fid rids} [array get fids] {
1182 if {[llength $rids] < 2} continue
1183 foreach a $rids {
1184 foreach b $rids {
1185 if {$a == $b} continue
@@ -1165,13 +1188,18 @@
1188 lappend dependencies($a) $b
1189 set dep($a,$b) .
1190 set dep($b,$a) .
1191 }
1192 }
1193 set n [llength $rids]
1194 lappend groups [list $n [expr {($n*$n-$n)/2}]]
1195 }
1196
1197 log write 14 csets {pseudo [array size fids] ([lsort -index 0 -decreasing -integer $groups])}
1198 log write 14 csets {internal [array size dep]}
1199 log write 14 csets {collected [array size dependencies]}
1200 log write 14 csets complete
1201 return
1202 }
1203
1204 # result = 4-list (itemtype itemid nextitemtype nextitemid ...)
1205 typemethod loops {revisions} {
1206

Keyboard Shortcuts

Open search /
Next entry (timeline) j
Previous entry (timeline) k
Open focused entry Enter
Show this help ?
Toggle theme Top nav button