Fossil SCM

Optimize sync of long cluster chains by sending all cluster artifacts on the third client/server round-trip.

drh 2024-12-20 21:16 trunk
Commit 637fcef83023bad32ecf73db8cd9e55a122649e1373684d7aef0263b202541fc
Parent 9953b169766f74f…
1 file changed +55 -2
~ src/xfer.c
M src/xfer.c
+55 -2
--- src/xfer.c
+++ src/xfer.c
@@ -1041,10 +1041,43 @@
10411041
 }
10421042
 db_finalize(&q);
10431043
 if( cnt==0 ) pXfer->resync = 0;
10441044
 return cnt;
10451045
 }
1046
+
1047
+/*
1048
+** Send an igot message for every cluster artifact that is not a phantom,
1049
+** is not shunned, is not private, and that is not in the UNCLUSTERED table.
1050
+** Return the number of cards sent.
1051
+*/
1052
+static int send_all_clusters(Xfer *pXfer){
1053
+ Stmt q;
1054
+ int cnt = 0;
1055
+ const char *zExtra;
1056
+ if( db_table_exists("temp","onremote") ){
1057
+ zExtra = " AND NOT EXISTS(SELECT 1 FROM onremote WHERE rid=blob.rid)";
1058
+ }else{
1059
+ zExtra = "";
1060
+ }
1061
+ db_prepare(&q,
1062
+ "SELECT uuid"
1063
+ " FROM tagxref JOIN blob ON tagxref.rid=blob.rid AND tagxref.tagid=%d"
1064
+ " WHERE NOT EXISTS(SELECT 1 FROM shun WHERE uuid=blob.uuid)"
1065
+ " AND NOT EXISTS(SELECT 1 FROM phantom WHERE rid=blob.rid)"
1066
+ " AND NOT EXISTS(SELECT 1 FROM unclustered WHERE rid=blob.rid)"
1067
+ " AND NOT EXISTS(SELECT 1 FROM private WHERE rid=blob.rid)%s",
1068
+ TAG_CLUSTER, zExtra /*safe-for-%s*/
1069
+ );
1070
+ while( db_step(&q)==SQLITE_ROW ){
1071
+ if( cnt==0 ) blob_appendf(pXfer->pOut, "# sending-clusters\n");
1072
+ blob_appendf(pXfer->pOut, "igot %s\n", db_column_text(&q, 0));
1073
+ cnt++;
1074
+ }
1075
+ db_finalize(&q);
1076
+ if( cnt ) blob_appendf(pXfer->pOut, "# end-of-clusters\n");
1077
+ return cnt;
1078
+}
10461079
10471080
 /*
10481081
 ** Send an igot message for every artifact.
10491082
 */
10501083
 static void send_all(Xfer *pXfer){
@@ -1781,10 +1814,19 @@
17811814
 ** The client sends this message to the server to ask the server
17821815
 ** to tell it about alternative repositories in the reply.
17831816
 */
17841817
 if( blob_eq(&xfer.aToken[1], "req-links") ){
17851818
 bSendLinks = 1;
1819
+ }else
1820
+
1821
+ /* pragma req-clusters
1822
+ **
1823
+ ** This pragma requests that the server send igot cards for every
1824
+ ** cluster artifact that it knows about.
1825
+ */
1826
+ if( blob_eq(&xfer.aToken[1], "req-clusters") ){
1827
+ send_all_clusters(&xfer);
17861828
 }
17871829
17881830
 }else
17891831
17901832
 /* Unknown message
@@ -1981,11 +2023,11 @@
19812023
 int nCardSent = 0; /* Number of cards sent */
19822024
 int nCardRcvd = 0; /* Number of cards received */
19832025
 int nCycle = 0; /* Number of round trips to the server */
19842026
 int size; /* Size of a config value or uvfile */
19852027
 int origConfigRcvMask; /* Original value of configRcvMask */
1986
- int nFileRecv; /* Number of files received */
2028
+ int nFileRecv = 0; /* Number of files received */
19872029
 int mxPhantomReq = 200; /* Max number of phantoms to request per comm */
19882030
 const char *zCookie; /* Server cookie */
19892031
 i64 nUncSent, nUncRcvd; /* Bytes sent and received (before compression) */
19902032
 i64 nSent, nRcvd; /* Bytes sent and received (after compression) */
19912033
 int cloneSeqno = 1; /* Sequence number for clones */
@@ -2007,10 +2049,11 @@
20072049
 int uvHashSent = 0; /* The "pragma uv-hash" message has been sent */
20082050
 int uvDoPush = 0; /* Generate uvfile messages to send to server */
20092051
 int uvPullOnly = 0; /* 1: pull-only. 2: pull-only warning issued */
20102052
 int nUvGimmeSent = 0; /* Number of uvgimme cards sent on this cycle */
20112053
 int nUvFileRcvd = 0; /* Number of uvfile cards received on this cycle */
2054
+ int nGimmeRcvd = 0; /* Number of gimme cards recevied on the prev cycle */
20122055
 sqlite3_int64 mtime; /* Modification time on a UV file */
20132056
 int autopushFailed = 0; /* Autopush following commit failed if true */
20142057
 const char *zCkinLock; /* Name of check-in to lock. NULL for none */
20152058
 const char *zClientId; /* A unique identifier for this check-out */
20162059
 unsigned int mHttpFlags;/* Flags for the http_exchange() subsystem */
@@ -2181,15 +2224,21 @@
21812224
 */
21822225
 if( (syncFlags & SYNC_PULL)!=0
21832226
 || ((syncFlags & SYNC_CLONE)!=0 && cloneSeqno==1)
21842227
 ){
21852228
 request_phantoms(&xfer, mxPhantomReq);
2229
+ if( xfer.nGimmeSent>0 && nCycle==2 && (syncFlags & SYNC_PULL)!=0 ){
2230
+ blob_appendf(&send, "pragma req-clusters\n");
2231
+ }
21862232
 }
21872233
 if( syncFlags & SYNC_PUSH ){
21882234
 send_unsent(&xfer);
21892235
 nCardSent += send_unclustered(&xfer);
21902236
 if( syncFlags & SYNC_PRIVATE ) send_private(&xfer);
2237
+ if( nGimmeRcvd>0 && nCycle==2 ){
2238
+ send_all_clusters(&xfer);
2239
+ }
21912240
 }
21922241
21932242
 /* Client sends configuration parameter requests. On a clone, delay sending
21942243
 ** this until the second cycle since the login card might fail on
21952244
 ** the first cycle.
@@ -2369,10 +2418,11 @@
23692418
 nCardSent++;
23702419
 }
23712420
 go = 0;
23722421
 nUvGimmeSent = 0;
23732422
 nUvFileRcvd = 0;
2423
+ nGimmeRcvd = 0;
23742424
 nPriorArtifact = nArtifactRcvd;
23752425
23762426
 /* Process the reply that came back from the server */
23772427
 while( blob_line(&recv, &xfer.line) ){
23782428
 if( blob_buffer(&xfer.line)[0]=='#' ){
@@ -2449,11 +2499,14 @@
24492499
 && blob_is_hname(&xfer.aToken[1])
24502500
 ){
24512501
 remote_unk(&xfer.aToken[1]);
24522502
 if( syncFlags & SYNC_PUSH ){
24532503
 int rid = rid_from_uuid(&xfer.aToken[1], 0, 0);
2454
- if( rid ) send_file(&xfer, rid, &xfer.aToken[1], 0);
2504
+ if( rid ){
2505
+ send_file(&xfer, rid, &xfer.aToken[1], 0);
2506
+ nGimmeRcvd++;
2507
+ }
24552508
 }
24562509
 }else
24572510
24582511
 /* igot HASH ?PRIVATEFLAG?
24592512
 **
24602513
--- src/xfer.c
+++ src/xfer.c
@@ -1041,10 +1041,43 @@
1041 }
1042 db_finalize(&q);
1043 if( cnt==0 ) pXfer->resync = 0;
1044 return cnt;
1045 }
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1046
1047 /*
1048 ** Send an igot message for every artifact.
1049 */
1050 static void send_all(Xfer *pXfer){
@@ -1781,10 +1814,19 @@
1781 ** The client sends this message to the server to ask the server
1782 ** to tell it about alternative repositories in the reply.
1783 */
1784 if( blob_eq(&xfer.aToken[1], "req-links") ){
1785 bSendLinks = 1;
 
 
 
 
 
 
 
 
 
1786 }
1787
1788 }else
1789
1790 /* Unknown message
@@ -1981,11 +2023,11 @@
1981 int nCardSent = 0; /* Number of cards sent */
1982 int nCardRcvd = 0; /* Number of cards received */
1983 int nCycle = 0; /* Number of round trips to the server */
1984 int size; /* Size of a config value or uvfile */
1985 int origConfigRcvMask; /* Original value of configRcvMask */
1986 int nFileRecv; /* Number of files received */
1987 int mxPhantomReq = 200; /* Max number of phantoms to request per comm */
1988 const char *zCookie; /* Server cookie */
1989 i64 nUncSent, nUncRcvd; /* Bytes sent and received (before compression) */
1990 i64 nSent, nRcvd; /* Bytes sent and received (after compression) */
1991 int cloneSeqno = 1; /* Sequence number for clones */
@@ -2007,10 +2049,11 @@
2007 int uvHashSent = 0; /* The "pragma uv-hash" message has been sent */
2008 int uvDoPush = 0; /* Generate uvfile messages to send to server */
2009 int uvPullOnly = 0; /* 1: pull-only. 2: pull-only warning issued */
2010 int nUvGimmeSent = 0; /* Number of uvgimme cards sent on this cycle */
2011 int nUvFileRcvd = 0; /* Number of uvfile cards received on this cycle */
 
2012 sqlite3_int64 mtime; /* Modification time on a UV file */
2013 int autopushFailed = 0; /* Autopush following commit failed if true */
2014 const char *zCkinLock; /* Name of check-in to lock. NULL for none */
2015 const char *zClientId; /* A unique identifier for this check-out */
2016 unsigned int mHttpFlags;/* Flags for the http_exchange() subsystem */
@@ -2181,15 +2224,21 @@
2181 */
2182 if( (syncFlags & SYNC_PULL)!=0
2183 || ((syncFlags & SYNC_CLONE)!=0 && cloneSeqno==1)
2184 ){
2185 request_phantoms(&xfer, mxPhantomReq);
 
 
 
2186 }
2187 if( syncFlags & SYNC_PUSH ){
2188 send_unsent(&xfer);
2189 nCardSent += send_unclustered(&xfer);
2190 if( syncFlags & SYNC_PRIVATE ) send_private(&xfer);
 
 
 
2191 }
2192
2193 /* Client sends configuration parameter requests. On a clone, delay sending
2194 ** this until the second cycle since the login card might fail on
2195 ** the first cycle.
@@ -2369,10 +2418,11 @@
2369 nCardSent++;
2370 }
2371 go = 0;
2372 nUvGimmeSent = 0;
2373 nUvFileRcvd = 0;
 
2374 nPriorArtifact = nArtifactRcvd;
2375
2376 /* Process the reply that came back from the server */
2377 while( blob_line(&recv, &xfer.line) ){
2378 if( blob_buffer(&xfer.line)[0]=='#' ){
@@ -2449,11 +2499,14 @@
2449 && blob_is_hname(&xfer.aToken[1])
2450 ){
2451 remote_unk(&xfer.aToken[1]);
2452 if( syncFlags & SYNC_PUSH ){
2453 int rid = rid_from_uuid(&xfer.aToken[1], 0, 0);
2454 if( rid ) send_file(&xfer, rid, &xfer.aToken[1], 0);
 
 
 
2455 }
2456 }else
2457
2458 /* igot HASH ?PRIVATEFLAG?
2459 **
2460
--- src/xfer.c
+++ src/xfer.c
@@ -1041,10 +1041,43 @@
1041 }
1042 db_finalize(&q);
1043 if( cnt==0 ) pXfer->resync = 0;
1044 return cnt;
1045 }
1046
1047 /*
1048 ** Send an igot message for every cluster artifact that is not a phantom,
1049 ** is not shunned, is not private, and that is not in the UNCLUSTERED table.
1050 ** Return the number of cards sent.
1051 */
1052 static int send_all_clusters(Xfer *pXfer){
1053 Stmt q;
1054 int cnt = 0;
1055 const char *zExtra;
1056 if( db_table_exists("temp","onremote") ){
1057 zExtra = " AND NOT EXISTS(SELECT 1 FROM onremote WHERE rid=blob.rid)";
1058 }else{
1059 zExtra = "";
1060 }
1061 db_prepare(&q,
1062 "SELECT uuid"
1063 " FROM tagxref JOIN blob ON tagxref.rid=blob.rid AND tagxref.tagid=%d"
1064 " WHERE NOT EXISTS(SELECT 1 FROM shun WHERE uuid=blob.uuid)"
1065 " AND NOT EXISTS(SELECT 1 FROM phantom WHERE rid=blob.rid)"
1066 " AND NOT EXISTS(SELECT 1 FROM unclustered WHERE rid=blob.rid)"
1067 " AND NOT EXISTS(SELECT 1 FROM private WHERE rid=blob.rid)%s",
1068 TAG_CLUSTER, zExtra /*safe-for-%s*/
1069 );
1070 while( db_step(&q)==SQLITE_ROW ){
1071 if( cnt==0 ) blob_appendf(pXfer->pOut, "# sending-clusters\n");
1072 blob_appendf(pXfer->pOut, "igot %s\n", db_column_text(&q, 0));
1073 cnt++;
1074 }
1075 db_finalize(&q);
1076 if( cnt ) blob_appendf(pXfer->pOut, "# end-of-clusters\n");
1077 return cnt;
1078 }
1079
1080 /*
1081 ** Send an igot message for every artifact.
1082 */
1083 static void send_all(Xfer *pXfer){
@@ -1781,10 +1814,19 @@
1814 ** The client sends this message to the server to ask the server
1815 ** to tell it about alternative repositories in the reply.
1816 */
1817 if( blob_eq(&xfer.aToken[1], "req-links") ){
1818 bSendLinks = 1;
1819 }else
1820
1821 /* pragma req-clusters
1822 **
1823 ** This pragma requests that the server send igot cards for every
1824 ** cluster artifact that it knows about.
1825 */
1826 if( blob_eq(&xfer.aToken[1], "req-clusters") ){
1827 send_all_clusters(&xfer);
1828 }
1829
1830 }else
1831
1832 /* Unknown message
@@ -1981,11 +2023,11 @@
2023 int nCardSent = 0; /* Number of cards sent */
2024 int nCardRcvd = 0; /* Number of cards received */
2025 int nCycle = 0; /* Number of round trips to the server */
2026 int size; /* Size of a config value or uvfile */
2027 int origConfigRcvMask; /* Original value of configRcvMask */
2028 int nFileRecv = 0; /* Number of files received */
2029 int mxPhantomReq = 200; /* Max number of phantoms to request per comm */
2030 const char *zCookie; /* Server cookie */
2031 i64 nUncSent, nUncRcvd; /* Bytes sent and received (before compression) */
2032 i64 nSent, nRcvd; /* Bytes sent and received (after compression) */
2033 int cloneSeqno = 1; /* Sequence number for clones */
@@ -2007,10 +2049,11 @@
2049 int uvHashSent = 0; /* The "pragma uv-hash" message has been sent */
2050 int uvDoPush = 0; /* Generate uvfile messages to send to server */
2051 int uvPullOnly = 0; /* 1: pull-only. 2: pull-only warning issued */
2052 int nUvGimmeSent = 0; /* Number of uvgimme cards sent on this cycle */
2053 int nUvFileRcvd = 0; /* Number of uvfile cards received on this cycle */
2054 int nGimmeRcvd = 0; /* Number of gimme cards recevied on the prev cycle */
2055 sqlite3_int64 mtime; /* Modification time on a UV file */
2056 int autopushFailed = 0; /* Autopush following commit failed if true */
2057 const char *zCkinLock; /* Name of check-in to lock. NULL for none */
2058 const char *zClientId; /* A unique identifier for this check-out */
2059 unsigned int mHttpFlags;/* Flags for the http_exchange() subsystem */
@@ -2181,15 +2224,21 @@
2224 */
2225 if( (syncFlags & SYNC_PULL)!=0
2226 || ((syncFlags & SYNC_CLONE)!=0 && cloneSeqno==1)
2227 ){
2228 request_phantoms(&xfer, mxPhantomReq);
2229 if( xfer.nGimmeSent>0 && nCycle==2 && (syncFlags & SYNC_PULL)!=0 ){
2230 blob_appendf(&send, "pragma req-clusters\n");
2231 }
2232 }
2233 if( syncFlags & SYNC_PUSH ){
2234 send_unsent(&xfer);
2235 nCardSent += send_unclustered(&xfer);
2236 if( syncFlags & SYNC_PRIVATE ) send_private(&xfer);
2237 if( nGimmeRcvd>0 && nCycle==2 ){
2238 send_all_clusters(&xfer);
2239 }
2240 }
2241
2242 /* Client sends configuration parameter requests. On a clone, delay sending
2243 ** this until the second cycle since the login card might fail on
2244 ** the first cycle.
@@ -2369,10 +2418,11 @@
2418 nCardSent++;
2419 }
2420 go = 0;
2421 nUvGimmeSent = 0;
2422 nUvFileRcvd = 0;
2423 nGimmeRcvd = 0;
2424 nPriorArtifact = nArtifactRcvd;
2425
2426 /* Process the reply that came back from the server */
2427 while( blob_line(&recv, &xfer.line) ){
2428 if( blob_buffer(&xfer.line)[0]=='#' ){
@@ -2449,11 +2499,14 @@
2499 && blob_is_hname(&xfer.aToken[1])
2500 ){
2501 remote_unk(&xfer.aToken[1]);
2502 if( syncFlags & SYNC_PUSH ){
2503 int rid = rid_from_uuid(&xfer.aToken[1], 0, 0);
2504 if( rid ){
2505 send_file(&xfer, rid, &xfer.aToken[1], 0);
2506 nGimmeRcvd++;
2507 }
2508 }
2509 }else
2510
2511 /* igot HASH ?PRIVATEFLAG?
2512 **
2513

Keyboard Shortcuts

Open search /
Next entry (timeline) j
Previous entry (timeline) k
Open focused entry Enter
Show this help ?
Toggle theme Top nav button