Fossil SCM

Experimental option --fast on the clone command only clones manifest artifacts and related tags. This is an experiment to see how much bandwidth and time are saved by omitting the download of file content until it is actually needed.

drh 2018-06-04 16:27 UTC trunk
Commit 9e8a7d4ca85aaf6b1e19badbf96d802b4f75caf8b78eda41fba378e850811f94
2 files changed +2 +54 -6
--- src/clone.c
+++ src/clone.c
@@ -109,10 +109,11 @@
109109
** admin user. This can be overridden using the -A|--admin-user
110110
** parameter.
111111
**
112112
** Options:
113113
** --admin-user|-A USERNAME Make USERNAME the administrator
114
+** --fast Minimal clone - most content left on server
114115
** --once Don't remember the URI.
115116
** --private Also clone private branches
116117
** --ssl-identity FILENAME Use the SSL identity if requested by the server
117118
** --ssh-command|-c SSH Use SSH as the "ssh" command
118119
** --httpauth|-B USER:PASS Add HTTP Basic Authorization to requests
@@ -129,10 +130,11 @@
129130
int urlFlags = URL_PROMPT_PW | URL_REMEMBER;
130131
int syncFlags = SYNC_CLONE;
131132
132133
/* Also clone private branches */
133134
if( find_option("private",0,0)!=0 ) syncFlags |= SYNC_PRIVATE;
135
+ if( find_option("fast",0,0)!=0 ) syncFlags |= SYNC_ONLY_CKIN;
134136
if( find_option("once",0,0)!=0) urlFlags &= ~URL_REMEMBER;
135137
if( find_option("verbose","v",0)!=0) syncFlags |= SYNC_VERBOSE;
136138
if( find_option("unversioned","u",0)!=0 ) syncFlags |= SYNC_UNVERSIONED;
137139
zHttpAuth = find_option("httpauth","B",1);
138140
zDefaultUser = find_option("admin-user","A",1);
139141
--- src/clone.c
+++ src/clone.c
@@ -109,10 +109,11 @@
109 ** admin user. This can be overridden using the -A|--admin-user
110 ** parameter.
111 **
112 ** Options:
113 ** --admin-user|-A USERNAME Make USERNAME the administrator
 
114 ** --once Don't remember the URI.
115 ** --private Also clone private branches
116 ** --ssl-identity FILENAME Use the SSL identity if requested by the server
117 ** --ssh-command|-c SSH Use SSH as the "ssh" command
118 ** --httpauth|-B USER:PASS Add HTTP Basic Authorization to requests
@@ -129,10 +130,11 @@
129 int urlFlags = URL_PROMPT_PW | URL_REMEMBER;
130 int syncFlags = SYNC_CLONE;
131
132 /* Also clone private branches */
133 if( find_option("private",0,0)!=0 ) syncFlags |= SYNC_PRIVATE;
 
134 if( find_option("once",0,0)!=0) urlFlags &= ~URL_REMEMBER;
135 if( find_option("verbose","v",0)!=0) syncFlags |= SYNC_VERBOSE;
136 if( find_option("unversioned","u",0)!=0 ) syncFlags |= SYNC_UNVERSIONED;
137 zHttpAuth = find_option("httpauth","B",1);
138 zDefaultUser = find_option("admin-user","A",1);
139
--- src/clone.c
+++ src/clone.c
@@ -109,10 +109,11 @@
109 ** admin user. This can be overridden using the -A|--admin-user
110 ** parameter.
111 **
112 ** Options:
113 ** --admin-user|-A USERNAME Make USERNAME the administrator
114 ** --fast Minimal clone - most content left on server
115 ** --once Don't remember the URI.
116 ** --private Also clone private branches
117 ** --ssl-identity FILENAME Use the SSL identity if requested by the server
118 ** --ssh-command|-c SSH Use SSH as the "ssh" command
119 ** --httpauth|-B USER:PASS Add HTTP Basic Authorization to requests
@@ -129,10 +130,11 @@
130 int urlFlags = URL_PROMPT_PW | URL_REMEMBER;
131 int syncFlags = SYNC_CLONE;
132
133 /* Also clone private branches */
134 if( find_option("private",0,0)!=0 ) syncFlags |= SYNC_PRIVATE;
135 if( find_option("fast",0,0)!=0 ) syncFlags |= SYNC_ONLY_CKIN;
136 if( find_option("once",0,0)!=0) urlFlags &= ~URL_REMEMBER;
137 if( find_option("verbose","v",0)!=0) syncFlags |= SYNC_VERBOSE;
138 if( find_option("unversioned","u",0)!=0 ) syncFlags |= SYNC_UNVERSIONED;
139 zHttpAuth = find_option("httpauth","B",1);
140 zDefaultUser = find_option("admin-user","A",1);
141
+54 -6
--- src/xfer.c
+++ src/xfer.c
@@ -49,12 +49,15 @@
4949
int nDanglingFile; /* Number of dangling deltas received */
5050
int mxSend; /* Stop sending "file" when pOut reaches this size */
5151
int resync; /* Send igot cards for all holdings */
5252
u8 syncPrivate; /* True to enable syncing private content */
5353
u8 nextIsPrivate; /* If true, next "file" received is a private */
54
+ u8 ckinOnly; /* Clone only check-ins and related artifacts */
55
+ u8 useOkToSend; /* Omit artifacts found in okToSend */
5456
u32 clientVersion; /* Version of the client software */
5557
time_t maxTime; /* Time when this transfer should be finished */
58
+ Bag okToSend; /* IDs of BLOBs acceptable to send */
5659
};
5760
5861
5962
/*
6063
** The input blob contains an artifact. Convert it into a record ID.
@@ -1352,11 +1355,14 @@
13521355
@ push %s(db_get("server-code", "x")) %s(db_get("project-code", "x"))
13531356
@ error not\sauthorized\sto\sclone
13541357
nErr++;
13551358
break;
13561359
}
1357
- if( db_get_boolean("uv-sync",0) && !uvCatalogSent ){
1360
+ if( db_get_boolean("uv-sync",0)
1361
+ && !uvCatalogSent
1362
+ && !xfer.ckinOnly
1363
+ ){
13581364
@ pragma uv-pull-only
13591365
send_unversioned_catalog(&xfer);
13601366
uvCatalogSent = 1;
13611367
}
13621368
if( xfer.nToken==3
@@ -1369,11 +1375,13 @@
13691375
}
13701376
blob_is_int(&xfer.aToken[2], &seqno);
13711377
max = db_int(0, "SELECT max(rid) FROM blob");
13721378
while( xfer.mxSend>blob_size(xfer.pOut) && seqno<=max){
13731379
if( time(NULL) >= xfer.maxTime ) break;
1374
- if( iVers>=3 ){
1380
+ if( xfer.useOkToSend && bag_find(&xfer.okToSend, seqno)==0 ){
1381
+ /* no-op */
1382
+ }else if( iVers>=3 ){
13751383
send_compressed_file(&xfer, seqno);
13761384
}else{
13771385
send_file(&xfer, seqno, 0, 1);
13781386
}
13791387
seqno++;
@@ -1516,10 +1524,39 @@
15161524
** Send igot cards for all known artifacts.
15171525
*/
15181526
if( blob_eq(&xfer.aToken[1], "send-catalog") ){
15191527
xfer.resync = 0x7fffffff;
15201528
}
1529
+
1530
+ /* pragma ckin-only
1531
+ **
1532
+ ** Only send manifests and related tag artifacts. Omit
1533
+ ** file content, wiki, and ticket artifacts.
1534
+ */
1535
+ if( blob_eq(&xfer.aToken[1], "ckin-only") ){
1536
+ Stmt q;
1537
+ if( !xfer.useOkToSend ){
1538
+ xfer.useOkToSend = 1;
1539
+ bag_init(&xfer.okToSend);
1540
+ }
1541
+ db_prepare(&q, "SELECT objid FROM event WHERE type='ci'");
1542
+ while( db_step(&q)==SQLITE_ROW ){
1543
+ bag_insert(&xfer.okToSend, db_column_int(&q, 0));
1544
+ }
1545
+ db_finalize(&q);
1546
+ db_prepare(&q,
1547
+ "SELECT srcid FROM tagxref"
1548
+ " WHERE srcid!=0"
1549
+ " AND srcid!=rid"
1550
+ " AND rid IN (SELECT objid FROM event WHERE type='ci')"
1551
+ );
1552
+ while( db_step(&q)==SQLITE_ROW ){
1553
+ bag_insert(&xfer.okToSend, db_column_int(&q, 0));
1554
+ }
1555
+ db_finalize(&q);
1556
+ xfer.ckinOnly = 1;
1557
+ }
15211558
15221559
/* pragma client-version VERSION
15231560
**
15241561
** Let the server know what version of Fossil is running on the client.
15251562
*/
@@ -1654,10 +1691,11 @@
16541691
#define SYNC_UNVERSIONED 0x0040 /* Sync unversioned content */
16551692
#define SYNC_UV_REVERT 0x0080 /* Copy server unversioned to client */
16561693
#define SYNC_FROMPARENT 0x0100 /* Pull from the parent project */
16571694
#define SYNC_UV_TRACE 0x0200 /* Describe UV activities */
16581695
#define SYNC_UV_DRYRUN 0x0400 /* Do not actually exchange files */
1696
+#define SYNC_ONLY_CKIN 0x0800 /* Only manifasts & related tags */
16591697
#endif
16601698
16611699
/*
16621700
** Floating-point absolute value
16631701
*/
@@ -1745,11 +1783,16 @@
17451783
origConfigRcvMask = 0;
17461784
17471785
17481786
/* Send the send-private pragma if we are trying to sync private data */
17491787
if( syncFlags & SYNC_PRIVATE ){
1750
- blob_append(&send, "pragma send-private\n", -1);
1788
+ blob_append(&send, "pragma send-private\n", -1); nCardSent++;
1789
+ }
1790
+
1791
+ /* Let the server know if we are only interested in check-ins */
1792
+ if( syncFlags & SYNC_ONLY_CKIN ){
1793
+ blob_appendf(&send, "pragma ckin-only\n"); nCardSent++;
17511794
}
17521795
17531796
/* When syncing unversioned files, create a TEMP table in which to store
17541797
** the names of files that need to be sent from client to server.
17551798
**
@@ -1822,11 +1865,12 @@
18221865
18231866
/* Generate gimme cards for phantoms and leaf cards
18241867
** for all leaves.
18251868
*/
18261869
if( (syncFlags & SYNC_PULL)!=0
1827
- || ((syncFlags & SYNC_CLONE)!=0 && cloneSeqno==1)
1870
+ || ((syncFlags & (SYNC_CLONE|SYNC_ONLY_CKIN))==SYNC_CLONE
1871
+ && cloneSeqno==1)
18281872
){
18291873
request_phantoms(&xfer, mxPhantomReq);
18301874
}
18311875
if( syncFlags & SYNC_PUSH ){
18321876
send_unsent(&xfer);
@@ -2169,12 +2213,16 @@
21692213
){
21702214
if( zPCode==0 ){
21712215
zPCode = mprintf("%b", &xfer.aToken[2]);
21722216
db_set("project-code", zPCode, 0);
21732217
}
2174
- if( cloneSeqno>0 ) blob_appendf(&send, "clone 3 %d\n", cloneSeqno);
2175
- nCardSent++;
2218
+ if( cloneSeqno>0 ){
2219
+ if( syncFlags & SYNC_ONLY_CKIN ){
2220
+ blob_appendf(&send, "pragma ckin-only\n"); nCardSent++;
2221
+ }
2222
+ blob_appendf(&send, "clone 3 %d\n", cloneSeqno); nCardSent++;
2223
+ }
21762224
}else
21772225
21782226
/* config NAME SIZE \n CONTENT
21792227
**
21802228
** Receive a configuration value from the server.
21812229
--- src/xfer.c
+++ src/xfer.c
@@ -49,12 +49,15 @@
49 int nDanglingFile; /* Number of dangling deltas received */
50 int mxSend; /* Stop sending "file" when pOut reaches this size */
51 int resync; /* Send igot cards for all holdings */
52 u8 syncPrivate; /* True to enable syncing private content */
53 u8 nextIsPrivate; /* If true, next "file" received is a private */
 
 
54 u32 clientVersion; /* Version of the client software */
55 time_t maxTime; /* Time when this transfer should be finished */
 
56 };
57
58
59 /*
60 ** The input blob contains an artifact. Convert it into a record ID.
@@ -1352,11 +1355,14 @@
1352 @ push %s(db_get("server-code", "x")) %s(db_get("project-code", "x"))
1353 @ error not\sauthorized\sto\sclone
1354 nErr++;
1355 break;
1356 }
1357 if( db_get_boolean("uv-sync",0) && !uvCatalogSent ){
 
 
 
1358 @ pragma uv-pull-only
1359 send_unversioned_catalog(&xfer);
1360 uvCatalogSent = 1;
1361 }
1362 if( xfer.nToken==3
@@ -1369,11 +1375,13 @@
1369 }
1370 blob_is_int(&xfer.aToken[2], &seqno);
1371 max = db_int(0, "SELECT max(rid) FROM blob");
1372 while( xfer.mxSend>blob_size(xfer.pOut) && seqno<=max){
1373 if( time(NULL) >= xfer.maxTime ) break;
1374 if( iVers>=3 ){
 
 
1375 send_compressed_file(&xfer, seqno);
1376 }else{
1377 send_file(&xfer, seqno, 0, 1);
1378 }
1379 seqno++;
@@ -1516,10 +1524,39 @@
1516 ** Send igot cards for all known artifacts.
1517 */
1518 if( blob_eq(&xfer.aToken[1], "send-catalog") ){
1519 xfer.resync = 0x7fffffff;
1520 }
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1521
1522 /* pragma client-version VERSION
1523 **
1524 ** Let the server know what version of Fossil is running on the client.
1525 */
@@ -1654,10 +1691,11 @@
1654 #define SYNC_UNVERSIONED 0x0040 /* Sync unversioned content */
1655 #define SYNC_UV_REVERT 0x0080 /* Copy server unversioned to client */
1656 #define SYNC_FROMPARENT 0x0100 /* Pull from the parent project */
1657 #define SYNC_UV_TRACE 0x0200 /* Describe UV activities */
1658 #define SYNC_UV_DRYRUN 0x0400 /* Do not actually exchange files */
 
1659 #endif
1660
1661 /*
1662 ** Floating-point absolute value
1663 */
@@ -1745,11 +1783,16 @@
1745 origConfigRcvMask = 0;
1746
1747
1748 /* Send the send-private pragma if we are trying to sync private data */
1749 if( syncFlags & SYNC_PRIVATE ){
1750 blob_append(&send, "pragma send-private\n", -1);
 
 
 
 
 
1751 }
1752
1753 /* When syncing unversioned files, create a TEMP table in which to store
1754 ** the names of files that need to be sent from client to server.
1755 **
@@ -1822,11 +1865,12 @@
1822
1823 /* Generate gimme cards for phantoms and leaf cards
1824 ** for all leaves.
1825 */
1826 if( (syncFlags & SYNC_PULL)!=0
1827 || ((syncFlags & SYNC_CLONE)!=0 && cloneSeqno==1)
 
1828 ){
1829 request_phantoms(&xfer, mxPhantomReq);
1830 }
1831 if( syncFlags & SYNC_PUSH ){
1832 send_unsent(&xfer);
@@ -2169,12 +2213,16 @@
2169 ){
2170 if( zPCode==0 ){
2171 zPCode = mprintf("%b", &xfer.aToken[2]);
2172 db_set("project-code", zPCode, 0);
2173 }
2174 if( cloneSeqno>0 ) blob_appendf(&send, "clone 3 %d\n", cloneSeqno);
2175 nCardSent++;
 
 
 
 
2176 }else
2177
2178 /* config NAME SIZE \n CONTENT
2179 **
2180 ** Receive a configuration value from the server.
2181
--- src/xfer.c
+++ src/xfer.c
@@ -49,12 +49,15 @@
49 int nDanglingFile; /* Number of dangling deltas received */
50 int mxSend; /* Stop sending "file" when pOut reaches this size */
51 int resync; /* Send igot cards for all holdings */
52 u8 syncPrivate; /* True to enable syncing private content */
53 u8 nextIsPrivate; /* If true, next "file" received is a private */
54 u8 ckinOnly; /* Clone only check-ins and related artifacts */
55 u8 useOkToSend; /* Omit artifacts found in okToSend */
56 u32 clientVersion; /* Version of the client software */
57 time_t maxTime; /* Time when this transfer should be finished */
58 Bag okToSend; /* IDs of BLOBs acceptable to send */
59 };
60
61
62 /*
63 ** The input blob contains an artifact. Convert it into a record ID.
@@ -1352,11 +1355,14 @@
1355 @ push %s(db_get("server-code", "x")) %s(db_get("project-code", "x"))
1356 @ error not\sauthorized\sto\sclone
1357 nErr++;
1358 break;
1359 }
1360 if( db_get_boolean("uv-sync",0)
1361 && !uvCatalogSent
1362 && !xfer.ckinOnly
1363 ){
1364 @ pragma uv-pull-only
1365 send_unversioned_catalog(&xfer);
1366 uvCatalogSent = 1;
1367 }
1368 if( xfer.nToken==3
@@ -1369,11 +1375,13 @@
1375 }
1376 blob_is_int(&xfer.aToken[2], &seqno);
1377 max = db_int(0, "SELECT max(rid) FROM blob");
1378 while( xfer.mxSend>blob_size(xfer.pOut) && seqno<=max){
1379 if( time(NULL) >= xfer.maxTime ) break;
1380 if( xfer.useOkToSend && bag_find(&xfer.okToSend, seqno)==0 ){
1381 /* no-op */
1382 }else if( iVers>=3 ){
1383 send_compressed_file(&xfer, seqno);
1384 }else{
1385 send_file(&xfer, seqno, 0, 1);
1386 }
1387 seqno++;
@@ -1516,10 +1524,39 @@
1524 ** Send igot cards for all known artifacts.
1525 */
1526 if( blob_eq(&xfer.aToken[1], "send-catalog") ){
1527 xfer.resync = 0x7fffffff;
1528 }
1529
1530 /* pragma ckin-only
1531 **
1532 ** Only send manifests and related tag artifacts. Omit
1533 ** file content, wiki, and ticket artifacts.
1534 */
1535 if( blob_eq(&xfer.aToken[1], "ckin-only") ){
1536 Stmt q;
1537 if( !xfer.useOkToSend ){
1538 xfer.useOkToSend = 1;
1539 bag_init(&xfer.okToSend);
1540 }
1541 db_prepare(&q, "SELECT objid FROM event WHERE type='ci'");
1542 while( db_step(&q)==SQLITE_ROW ){
1543 bag_insert(&xfer.okToSend, db_column_int(&q, 0));
1544 }
1545 db_finalize(&q);
1546 db_prepare(&q,
1547 "SELECT srcid FROM tagxref"
1548 " WHERE srcid!=0"
1549 " AND srcid!=rid"
1550 " AND rid IN (SELECT objid FROM event WHERE type='ci')"
1551 );
1552 while( db_step(&q)==SQLITE_ROW ){
1553 bag_insert(&xfer.okToSend, db_column_int(&q, 0));
1554 }
1555 db_finalize(&q);
1556 xfer.ckinOnly = 1;
1557 }
1558
1559 /* pragma client-version VERSION
1560 **
1561 ** Let the server know what version of Fossil is running on the client.
1562 */
@@ -1654,10 +1691,11 @@
1691 #define SYNC_UNVERSIONED 0x0040 /* Sync unversioned content */
1692 #define SYNC_UV_REVERT 0x0080 /* Copy server unversioned to client */
1693 #define SYNC_FROMPARENT 0x0100 /* Pull from the parent project */
1694 #define SYNC_UV_TRACE 0x0200 /* Describe UV activities */
1695 #define SYNC_UV_DRYRUN 0x0400 /* Do not actually exchange files */
1696 #define SYNC_ONLY_CKIN 0x0800 /* Only manifasts & related tags */
1697 #endif
1698
1699 /*
1700 ** Floating-point absolute value
1701 */
@@ -1745,11 +1783,16 @@
1783 origConfigRcvMask = 0;
1784
1785
1786 /* Send the send-private pragma if we are trying to sync private data */
1787 if( syncFlags & SYNC_PRIVATE ){
1788 blob_append(&send, "pragma send-private\n", -1); nCardSent++;
1789 }
1790
1791 /* Let the server know if we are only interested in check-ins */
1792 if( syncFlags & SYNC_ONLY_CKIN ){
1793 blob_appendf(&send, "pragma ckin-only\n"); nCardSent++;
1794 }
1795
1796 /* When syncing unversioned files, create a TEMP table in which to store
1797 ** the names of files that need to be sent from client to server.
1798 **
@@ -1822,11 +1865,12 @@
1865
1866 /* Generate gimme cards for phantoms and leaf cards
1867 ** for all leaves.
1868 */
1869 if( (syncFlags & SYNC_PULL)!=0
1870 || ((syncFlags & (SYNC_CLONE|SYNC_ONLY_CKIN))==SYNC_CLONE
1871 && cloneSeqno==1)
1872 ){
1873 request_phantoms(&xfer, mxPhantomReq);
1874 }
1875 if( syncFlags & SYNC_PUSH ){
1876 send_unsent(&xfer);
@@ -2169,12 +2213,16 @@
2213 ){
2214 if( zPCode==0 ){
2215 zPCode = mprintf("%b", &xfer.aToken[2]);
2216 db_set("project-code", zPCode, 0);
2217 }
2218 if( cloneSeqno>0 ){
2219 if( syncFlags & SYNC_ONLY_CKIN ){
2220 blob_appendf(&send, "pragma ckin-only\n"); nCardSent++;
2221 }
2222 blob_appendf(&send, "clone 3 %d\n", cloneSeqno); nCardSent++;
2223 }
2224 }else
2225
2226 /* config NAME SIZE \n CONTENT
2227 **
2228 ** Receive a configuration value from the server.
2229

Keyboard Shortcuts

Open search /
Next entry (timeline) j
Previous entry (timeline) k
Open focused entry Enter
Show this help ?
Toggle theme Top nav button