Fossil SCM

Rebuild faster by caching the last few manifest parses.

drh 2010-10-08 19:55 trunk
Commit 0a55d162275a4179ed4376014a4b95f0843d5062
Parent cfbbad3d480b0e2…
1 file changed +83 -7
~ src/manifest.c
M src/manifest.c
+83 -7
--- src/manifest.c
+++ src/manifest.c
@@ -83,10 +83,22 @@
8383
 char *zValue; /* Value of the field */
8484
 } *aField; /* One for each J card */
8585
 };
8686
 #endif
8787
88
+/*
89
+** A cache of parsed manifests. This reduces the number of
90
+** calls to manifest_parse() when doing a rebuild.
91
+*/
92
+#define MX_MANIFEST_CACHE 4
93
+static struct {
94
+ int nxAge;
95
+ int aRid[MX_MANIFEST_CACHE];
96
+ int aAge[MX_MANIFEST_CACHE];
97
+ Manifest aLine[MX_MANIFEST_CACHE];
98
+} manifestCache;
99
+
88100
89101
 /*
90102
 ** Clear the memory allocated in a manifest object
91103
 */
92104
 void manifest_clear(Manifest *p){
@@ -96,10 +108,64 @@
96108
 free(p->azCChild);
97109
 free(p->aTag);
98110
 free(p->aField);
99111
 memset(p, 0, sizeof(*p));
100112
 }
113
+
114
+/*
115
+** Add an element to the manifest cache using LRU replacement.
116
+*/
117
+void manifest_cache_insert(int rid, Manifest *p){
118
+ int i;
119
+ for(i=0; i<MX_MANIFEST_CACHE; i++){
120
+ if( manifestCache.aRid[i]==0 ) break;
121
+ }
122
+ if( i>=MX_MANIFEST_CACHE ){
123
+ int oldest = 0;
124
+ int oldestAge = manifestCache.aAge[0];
125
+ for(i=1; i<MX_MANIFEST_CACHE; i++){
126
+ if( manifestCache.aAge[i]<oldestAge ){
127
+ oldest = i;
128
+ oldestAge = manifestCache.aAge[i];
129
+ }
130
+ }
131
+ manifest_clear(&manifestCache.aLine[oldest]);
132
+ i = oldest;
133
+ }
134
+ manifestCache.aAge[i] = ++manifestCache.nxAge;
135
+ manifestCache.aRid[i] = rid;
136
+ manifestCache.aLine[i] = *p;
137
+}
138
+
139
+/*
140
+** Try to extract a line from the manifest cache. Return 1 if found.
141
+** Return 0 if not found.
142
+*/
143
+int manifest_cache_find(int rid, Manifest *p){
144
+ int i;
145
+ for(i=0; i<MX_MANIFEST_CACHE; i++){
146
+ if( manifestCache.aRid[i]==rid ){
147
+ *p = manifestCache.aLine[i];
148
+ manifestCache.aRid[i] = 0;
149
+ return 1;
150
+ }
151
+ }
152
+ return 0;
153
+}
154
+
155
+/*
156
+** Clear the manifest cache.
157
+*/
158
+void manifest_cache_clear(void){
159
+ int i;
160
+ for(i=0; i<MX_MANIFEST_CACHE; i++){
161
+ if( manifestCache.aRid[i]>0 ){
162
+ manifest_clear(&manifestCache.aLine[i]);
163
+ }
164
+ }
165
+ memset(&manifestCache, 0, sizeof(manifestCache));
166
+}
101167
102168
 /*
103169
 ** Parse a blob into a Manifest object. The Manifest object
104170
 ** takes over the input blob and will free it when the
105171
 ** Manifest object is freed. Zeros are inserted into the blob
@@ -810,25 +876,29 @@
810876
 ** Edited files have both mlink.pid!=0 and mlink.fid!=0
811877
 */
812878
 static void add_mlink(int pid, Manifest *pParent, int cid, Manifest *pChild){
813879
 Manifest other;
814880
 Blob otherContent;
881
+ int otherRid;
815882
 int i, j;
816883
817884
 if( db_exists("SELECT 1 FROM mlink WHERE mid=%d", cid) ){
818885
 return;
819886
 }
820887
 assert( pParent==0 || pChild==0 );
821888
 if( pParent==0 ){
822889
 pParent = &other;
823
- content_get(pid, &otherContent);
890
+ otherRid = pid;
824891
 }else{
825892
 pChild = &other;
826
- content_get(cid, &otherContent);
893
+ otherRid = cid;
827894
 }
828
- if( blob_size(&otherContent)==0 ) return;
829
- if( manifest_parse(&other, &otherContent)==0 ) return;
895
+ if( manifest_cache_find(otherRid, &other)==0 ){
896
+ content_get(otherRid, &otherContent);
897
+ if( blob_size(&otherContent)==0 ) return;
898
+ if( manifest_parse(&other, &otherContent)==0 ) return;
899
+ }
830900
 content_deltify(pid, cid, 0);
831901
832902
 /* Use the iRename fields to find the cross-linkage between
833903
 ** renamed files. */
834904
 for(j=0; j<pChild->nFile; j++){
@@ -882,11 +952,11 @@
882952
 }else{
883953
 add_one_mlink(cid, 0, pChild->aFile[j].zUuid, pChild->aFile[j].zName,0);
884954
 }
885955
 j++;
886956
 }
887
- manifest_clear(&other);
957
+ manifest_cache_insert(otherRid, &other);
888958
 }
889959
890960
 /*
891961
 ** True if manifest_crosslink_begin() has been called but
892962
 ** manifest_crosslink_end() is still pending.
@@ -1027,11 +1097,13 @@
10271097
 int i;
10281098
 Manifest m;
10291099
 Stmt q;
10301100
 int parentid = 0;
10311101
1032
- if( manifest_parse(&m, pContent)==0 ){
1102
+ if( manifest_cache_find(rid, &m) ){
1103
+ blob_reset(pContent);
1104
+ }else if( manifest_parse(&m, pContent)==0 ){
10331105
 return 0;
10341106
 }
10351107
 if( g.xlinkClusterOnly && m.type!=CFTYPE_CLUSTER ){
10361108
 manifest_clear(&m);
10371109
 return 0;
@@ -1263,11 +1335,15 @@
12631335
 );
12641336
 free(zComment);
12651337
 }
12661338
 }
12671339
 db_end_transaction(0);
1268
- manifest_clear(&m);
1340
+ if( m.type==CFTYPE_MANIFEST ){
1341
+ manifest_cache_insert(rid, &m);
1342
+ }else{
1343
+ manifest_clear(&m);
1344
+ }
12691345
 return 1;
12701346
 }
12711347
12721348
 /*
12731349
 ** Given a checkin name, load and parse the manifest for that checkin.
12741350
--- src/manifest.c
+++ src/manifest.c
@@ -83,10 +83,22 @@
83 char *zValue; /* Value of the field */
84 } *aField; /* One for each J card */
85 };
86 #endif
87
 
 
 
 
 
 
 
 
 
 
 
 
88
89 /*
90 ** Clear the memory allocated in a manifest object
91 */
92 void manifest_clear(Manifest *p){
@@ -96,10 +108,64 @@
96 free(p->azCChild);
97 free(p->aTag);
98 free(p->aField);
99 memset(p, 0, sizeof(*p));
100 }
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
101
102 /*
103 ** Parse a blob into a Manifest object. The Manifest object
104 ** takes over the input blob and will free it when the
105 ** Manifest object is freed. Zeros are inserted into the blob
@@ -810,25 +876,29 @@
810 ** Edited files have both mlink.pid!=0 and mlink.fid!=0
811 */
812 static void add_mlink(int pid, Manifest *pParent, int cid, Manifest *pChild){
813 Manifest other;
814 Blob otherContent;
 
815 int i, j;
816
817 if( db_exists("SELECT 1 FROM mlink WHERE mid=%d", cid) ){
818 return;
819 }
820 assert( pParent==0 || pChild==0 );
821 if( pParent==0 ){
822 pParent = &other;
823 content_get(pid, &otherContent);
824 }else{
825 pChild = &other;
826 content_get(cid, &otherContent);
827 }
828 if( blob_size(&otherContent)==0 ) return;
829 if( manifest_parse(&other, &otherContent)==0 ) return;
 
 
 
830 content_deltify(pid, cid, 0);
831
832 /* Use the iRename fields to find the cross-linkage between
833 ** renamed files. */
834 for(j=0; j<pChild->nFile; j++){
@@ -882,11 +952,11 @@
882 }else{
883 add_one_mlink(cid, 0, pChild->aFile[j].zUuid, pChild->aFile[j].zName,0);
884 }
885 j++;
886 }
887 manifest_clear(&other);
888 }
889
890 /*
891 ** True if manifest_crosslink_begin() has been called but
892 ** manifest_crosslink_end() is still pending.
@@ -1027,11 +1097,13 @@
1027 int i;
1028 Manifest m;
1029 Stmt q;
1030 int parentid = 0;
1031
1032 if( manifest_parse(&m, pContent)==0 ){
 
 
1033 return 0;
1034 }
1035 if( g.xlinkClusterOnly && m.type!=CFTYPE_CLUSTER ){
1036 manifest_clear(&m);
1037 return 0;
@@ -1263,11 +1335,15 @@
1263 );
1264 free(zComment);
1265 }
1266 }
1267 db_end_transaction(0);
1268 manifest_clear(&m);
 
 
 
 
1269 return 1;
1270 }
1271
1272 /*
1273 ** Given a checkin name, load and parse the manifest for that checkin.
1274
--- src/manifest.c
+++ src/manifest.c
@@ -83,10 +83,22 @@
83 char *zValue; /* Value of the field */
84 } *aField; /* One for each J card */
85 };
86 #endif
87
88 /*
89 ** A cache of parsed manifests. This reduces the number of
90 ** calls to manifest_parse() when doing a rebuild.
91 */
92 #define MX_MANIFEST_CACHE 4
93 static struct {
94 int nxAge;
95 int aRid[MX_MANIFEST_CACHE];
96 int aAge[MX_MANIFEST_CACHE];
97 Manifest aLine[MX_MANIFEST_CACHE];
98 } manifestCache;
99
100
101 /*
102 ** Clear the memory allocated in a manifest object
103 */
104 void manifest_clear(Manifest *p){
@@ -96,10 +108,64 @@
108 free(p->azCChild);
109 free(p->aTag);
110 free(p->aField);
111 memset(p, 0, sizeof(*p));
112 }
113
114 /*
115 ** Add an element to the manifest cache using LRU replacement.
116 */
117 void manifest_cache_insert(int rid, Manifest *p){
118 int i;
119 for(i=0; i<MX_MANIFEST_CACHE; i++){
120 if( manifestCache.aRid[i]==0 ) break;
121 }
122 if( i>=MX_MANIFEST_CACHE ){
123 int oldest = 0;
124 int oldestAge = manifestCache.aAge[0];
125 for(i=1; i<MX_MANIFEST_CACHE; i++){
126 if( manifestCache.aAge[i]<oldestAge ){
127 oldest = i;
128 oldestAge = manifestCache.aAge[i];
129 }
130 }
131 manifest_clear(&manifestCache.aLine[oldest]);
132 i = oldest;
133 }
134 manifestCache.aAge[i] = ++manifestCache.nxAge;
135 manifestCache.aRid[i] = rid;
136 manifestCache.aLine[i] = *p;
137 }
138
139 /*
140 ** Try to extract a line from the manifest cache. Return 1 if found.
141 ** Return 0 if not found.
142 */
143 int manifest_cache_find(int rid, Manifest *p){
144 int i;
145 for(i=0; i<MX_MANIFEST_CACHE; i++){
146 if( manifestCache.aRid[i]==rid ){
147 *p = manifestCache.aLine[i];
148 manifestCache.aRid[i] = 0;
149 return 1;
150 }
151 }
152 return 0;
153 }
154
155 /*
156 ** Clear the manifest cache.
157 */
158 void manifest_cache_clear(void){
159 int i;
160 for(i=0; i<MX_MANIFEST_CACHE; i++){
161 if( manifestCache.aRid[i]>0 ){
162 manifest_clear(&manifestCache.aLine[i]);
163 }
164 }
165 memset(&manifestCache, 0, sizeof(manifestCache));
166 }
167
168 /*
169 ** Parse a blob into a Manifest object. The Manifest object
170 ** takes over the input blob and will free it when the
171 ** Manifest object is freed. Zeros are inserted into the blob
@@ -810,25 +876,29 @@
876 ** Edited files have both mlink.pid!=0 and mlink.fid!=0
877 */
878 static void add_mlink(int pid, Manifest *pParent, int cid, Manifest *pChild){
879 Manifest other;
880 Blob otherContent;
881 int otherRid;
882 int i, j;
883
884 if( db_exists("SELECT 1 FROM mlink WHERE mid=%d", cid) ){
885 return;
886 }
887 assert( pParent==0 || pChild==0 );
888 if( pParent==0 ){
889 pParent = &other;
890 otherRid = pid;
891 }else{
892 pChild = &other;
893 otherRid = cid;
894 }
895 if( manifest_cache_find(otherRid, &other)==0 ){
896 content_get(otherRid, &otherContent);
897 if( blob_size(&otherContent)==0 ) return;
898 if( manifest_parse(&other, &otherContent)==0 ) return;
899 }
900 content_deltify(pid, cid, 0);
901
902 /* Use the iRename fields to find the cross-linkage between
903 ** renamed files. */
904 for(j=0; j<pChild->nFile; j++){
@@ -882,11 +952,11 @@
952 }else{
953 add_one_mlink(cid, 0, pChild->aFile[j].zUuid, pChild->aFile[j].zName,0);
954 }
955 j++;
956 }
957 manifest_cache_insert(otherRid, &other);
958 }
959
960 /*
961 ** True if manifest_crosslink_begin() has been called but
962 ** manifest_crosslink_end() is still pending.
@@ -1027,11 +1097,13 @@
1097 int i;
1098 Manifest m;
1099 Stmt q;
1100 int parentid = 0;
1101
1102 if( manifest_cache_find(rid, &m) ){
1103 blob_reset(pContent);
1104 }else if( manifest_parse(&m, pContent)==0 ){
1105 return 0;
1106 }
1107 if( g.xlinkClusterOnly && m.type!=CFTYPE_CLUSTER ){
1108 manifest_clear(&m);
1109 return 0;
@@ -1263,11 +1335,15 @@
1335 );
1336 free(zComment);
1337 }
1338 }
1339 db_end_transaction(0);
1340 if( m.type==CFTYPE_MANIFEST ){
1341 manifest_cache_insert(rid, &m);
1342 }else{
1343 manifest_clear(&m);
1344 }
1345 return 1;
1346 }
1347
1348 /*
1349 ** Given a checkin name, load and parse the manifest for that checkin.
1350

Keyboard Shortcuts

Open search /
Next entry (timeline) j
Previous entry (timeline) k
Open focused entry Enter
Show this help ?
Toggle theme Top nav button