Fossil SCM
Fix the "fossil import --git" command so that it does delta compression on the fly, and hence uses much less temporary disk space. Also show progress information unless --quiet is used.
Commit
063779a14cb7c4a7da4ed7ce453a213b370b252d
Parent
96101215aefb469…
1 file changed
+17
-7
+17
-7
| --- src/import.c | ||
| +++ src/import.c | ||
| @@ -128,11 +128,16 @@ | ||
| 128 | 128 | ** to the newly inserted artifact. |
| 129 | 129 | ** |
| 130 | 130 | ** If saveUuid is true, then pContent is a commit record. Record its |
| 131 | 131 | ** UUID in gg.zPrevCheckin. |
| 132 | 132 | */ |
| 133 | -static int fast_insert_content(Blob *pContent, const char *zMark, int saveUuid){ | |
| 133 | +static int fast_insert_content( | |
| 134 | + Blob *pContent, /* Content to insert */ | |
| 135 | + const char *zMark, /* Label using this mark, if not NULL */ | |
| 136 | + int saveUuid, /* Save SHA1 hash in gg.zPrevCheckin */ | |
| 137 | + int doParse /* Invoke manifest_crosslink() */ | |
| 138 | +){ | |
| 134 | 139 | Blob hash; |
| 135 | 140 | Blob cmpr; |
| 136 | 141 | int rid; |
| 137 | 142 | |
| 138 | 143 | sha1sum_blob(pContent, &hash); |
| @@ -148,10 +153,13 @@ | ||
| 148 | 153 | db_bind_blob(&ins, ":content", &cmpr); |
| 149 | 154 | db_step(&ins); |
| 150 | 155 | db_reset(&ins); |
| 151 | 156 | blob_reset(&cmpr); |
| 152 | 157 | rid = db_last_insert_rowid(); |
| 158 | + if( doParse ){ | |
| 159 | + manifest_crosslink(rid, pContent, MC_NONE); | |
| 160 | + } | |
| 153 | 161 | } |
| 154 | 162 | if( zMark ){ |
| 155 | 163 | db_multi_exec( |
| 156 | 164 | "INSERT OR IGNORE INTO xmark(tname, trid, tuuid)" |
| 157 | 165 | "VALUES(%Q,%d,%B)", |
| @@ -176,11 +184,11 @@ | ||
| 176 | 184 | ** to the BLOB table. |
| 177 | 185 | */ |
| 178 | 186 | static void finish_blob(void){ |
| 179 | 187 | Blob content; |
| 180 | 188 | blob_init(&content, gg.aData, gg.nData); |
| 181 | - fast_insert_content(&content, gg.zMark, 0); | |
| 189 | + fast_insert_content(&content, gg.zMark, 0, 0); | |
| 182 | 190 | blob_reset(&content); |
| 183 | 191 | import_reset(0); |
| 184 | 192 | } |
| 185 | 193 | |
| 186 | 194 | /* |
| @@ -194,12 +202,11 @@ | ||
| 194 | 202 | blob_appendf(&record, "D %s\n", gg.zDate); |
| 195 | 203 | blob_appendf(&record, "T +%F %s\n", gg.zTag, gg.zFrom); |
| 196 | 204 | blob_appendf(&record, "U %F\n", gg.zUser); |
| 197 | 205 | md5sum_blob(&record, &cksum); |
| 198 | 206 | blob_appendf(&record, "Z %b\n", &cksum); |
| 199 | - fast_insert_content(&record, 0, 0); | |
| 200 | - blob_reset(&record); | |
| 207 | + fast_insert_content(&record, 0, 0, 1); | |
| 201 | 208 | blob_reset(&cksum); |
| 202 | 209 | } |
| 203 | 210 | import_reset(0); |
| 204 | 211 | } |
| 205 | 212 | |
| @@ -238,10 +245,11 @@ | ||
| 238 | 245 | import_prior_files(); |
| 239 | 246 | qsort(gg.aFile, gg.nFile, sizeof(gg.aFile[0]), mfile_cmp); |
| 240 | 247 | blob_zero(&record); |
| 241 | 248 | blob_appendf(&record, "C %F\n", gg.zComment); |
| 242 | 249 | blob_appendf(&record, "D %s\n", gg.zDate); |
| 250 | + if( !g.fQuiet ) fossil_print("%.10s\r", gg.zDate); | |
| 243 | 251 | for(i=0; i<gg.nFile; i++){ |
| 244 | 252 | const char *zUuid = gg.aFile[i].zUuid; |
| 245 | 253 | if( zUuid==0 ) continue; |
| 246 | 254 | blob_appendf(&record, "F %F %s", gg.aFile[i].zName, zUuid); |
| 247 | 255 | if( gg.aFile[i].isExe ){ |
| @@ -290,12 +298,11 @@ | ||
| 290 | 298 | db_multi_exec("INSERT INTO xbranch(tname, brnm) VALUES(%Q,%Q)", |
| 291 | 299 | gg.zMark, gg.zBranch); |
| 292 | 300 | blob_appendf(&record, "U %F\n", gg.zUser); |
| 293 | 301 | md5sum_blob(&record, &cksum); |
| 294 | 302 | blob_appendf(&record, "Z %b\n", &cksum); |
| 295 | - fast_insert_content(&record, gg.zMark, 1); | |
| 296 | - blob_reset(&record); | |
| 303 | + fast_insert_content(&record, gg.zMark, 1, 1); | |
| 297 | 304 | blob_reset(&cksum); |
| 298 | 305 | |
| 299 | 306 | /* The "git fast-export" command might output multiple "commit" lines |
| 300 | 307 | ** that reference a tag using "refs/tags/TAGNAME". The tag should only |
| 301 | 308 | ** be applied to the last commit that is output. The problem is we do not |
| @@ -1503,10 +1510,11 @@ | ||
| 1503 | 1510 | ** --flat The whole dump is a single branch |
| 1504 | 1511 | ** |
| 1505 | 1512 | ** Common Options: |
| 1506 | 1513 | ** -i|--incremental allow importing into an existing repository |
| 1507 | 1514 | ** -f|--force overwrite repository if already exist |
| 1515 | +** -q|--quiet omit progress output | |
| 1508 | 1516 | ** |
| 1509 | 1517 | ** The --incremental option allows an existing repository to be extended |
| 1510 | 1518 | ** with new content. |
| 1511 | 1519 | ** |
| 1512 | 1520 | ** |
| @@ -1638,19 +1646,21 @@ | ||
| 1638 | 1646 | "CREATE TEMP TABLE xmark(tname TEXT UNIQUE, trid INT, tuuid TEXT);" |
| 1639 | 1647 | "CREATE TEMP TABLE xbranch(tname TEXT UNIQUE, brnm TEXT);" |
| 1640 | 1648 | "CREATE TEMP TABLE xtag(tname TEXT UNIQUE, tcontent TEXT);" |
| 1641 | 1649 | ); |
| 1642 | 1650 | |
| 1651 | + manifest_crosslink_begin(); | |
| 1643 | 1652 | git_fast_import(pIn); |
| 1644 | 1653 | db_prepare(&q, "SELECT tcontent FROM xtag"); |
| 1645 | 1654 | while( db_step(&q)==SQLITE_ROW ){ |
| 1646 | 1655 | Blob record; |
| 1647 | 1656 | db_ephemeral_blob(&q, 0, &record); |
| 1648 | - fast_insert_content(&record, 0, 0); | |
| 1657 | + fast_insert_content(&record, 0, 0, 1); | |
| 1649 | 1658 | import_reset(0); |
| 1650 | 1659 | } |
| 1651 | 1660 | db_finalize(&q); |
| 1661 | + manifest_crosslink_end(MC_NONE); | |
| 1652 | 1662 | } |
| 1653 | 1663 | |
| 1654 | 1664 | verify_cancel(); |
| 1655 | 1665 | db_end_transaction(0); |
| 1656 | 1666 | db_begin_transaction(); |
| 1657 | 1667 |
| --- src/import.c | |
| +++ src/import.c | |
| @@ -128,11 +128,16 @@ | |
| 128 | ** to the newly inserted artifact. |
| 129 | ** |
| 130 | ** If saveUuid is true, then pContent is a commit record. Record its |
| 131 | ** UUID in gg.zPrevCheckin. |
| 132 | */ |
| 133 | static int fast_insert_content(Blob *pContent, const char *zMark, int saveUuid){ |
| 134 | Blob hash; |
| 135 | Blob cmpr; |
| 136 | int rid; |
| 137 | |
| 138 | sha1sum_blob(pContent, &hash); |
| @@ -148,10 +153,13 @@ | |
| 148 | db_bind_blob(&ins, ":content", &cmpr); |
| 149 | db_step(&ins); |
| 150 | db_reset(&ins); |
| 151 | blob_reset(&cmpr); |
| 152 | rid = db_last_insert_rowid(); |
| 153 | } |
| 154 | if( zMark ){ |
| 155 | db_multi_exec( |
| 156 | "INSERT OR IGNORE INTO xmark(tname, trid, tuuid)" |
| 157 | "VALUES(%Q,%d,%B)", |
| @@ -176,11 +184,11 @@ | |
| 176 | ** to the BLOB table. |
| 177 | */ |
| 178 | static void finish_blob(void){ |
| 179 | Blob content; |
| 180 | blob_init(&content, gg.aData, gg.nData); |
| 181 | fast_insert_content(&content, gg.zMark, 0); |
| 182 | blob_reset(&content); |
| 183 | import_reset(0); |
| 184 | } |
| 185 | |
| 186 | /* |
| @@ -194,12 +202,11 @@ | |
| 194 | blob_appendf(&record, "D %s\n", gg.zDate); |
| 195 | blob_appendf(&record, "T +%F %s\n", gg.zTag, gg.zFrom); |
| 196 | blob_appendf(&record, "U %F\n", gg.zUser); |
| 197 | md5sum_blob(&record, &cksum); |
| 198 | blob_appendf(&record, "Z %b\n", &cksum); |
| 199 | fast_insert_content(&record, 0, 0); |
| 200 | blob_reset(&record); |
| 201 | blob_reset(&cksum); |
| 202 | } |
| 203 | import_reset(0); |
| 204 | } |
| 205 | |
| @@ -238,10 +245,11 @@ | |
| 238 | import_prior_files(); |
| 239 | qsort(gg.aFile, gg.nFile, sizeof(gg.aFile[0]), mfile_cmp); |
| 240 | blob_zero(&record); |
| 241 | blob_appendf(&record, "C %F\n", gg.zComment); |
| 242 | blob_appendf(&record, "D %s\n", gg.zDate); |
| 243 | for(i=0; i<gg.nFile; i++){ |
| 244 | const char *zUuid = gg.aFile[i].zUuid; |
| 245 | if( zUuid==0 ) continue; |
| 246 | blob_appendf(&record, "F %F %s", gg.aFile[i].zName, zUuid); |
| 247 | if( gg.aFile[i].isExe ){ |
| @@ -290,12 +298,11 @@ | |
| 290 | db_multi_exec("INSERT INTO xbranch(tname, brnm) VALUES(%Q,%Q)", |
| 291 | gg.zMark, gg.zBranch); |
| 292 | blob_appendf(&record, "U %F\n", gg.zUser); |
| 293 | md5sum_blob(&record, &cksum); |
| 294 | blob_appendf(&record, "Z %b\n", &cksum); |
| 295 | fast_insert_content(&record, gg.zMark, 1); |
| 296 | blob_reset(&record); |
| 297 | blob_reset(&cksum); |
| 298 | |
| 299 | /* The "git fast-export" command might output multiple "commit" lines |
| 300 | ** that reference a tag using "refs/tags/TAGNAME". The tag should only |
| 301 | ** be applied to the last commit that is output. The problem is we do not |
| @@ -1503,10 +1510,11 @@ | |
| 1503 | ** --flat The whole dump is a single branch |
| 1504 | ** |
| 1505 | ** Common Options: |
| 1506 | ** -i|--incremental allow importing into an existing repository |
| 1507 | ** -f|--force overwrite repository if already exist |
| 1508 | ** |
| 1509 | ** The --incremental option allows an existing repository to be extended |
| 1510 | ** with new content. |
| 1511 | ** |
| 1512 | ** |
| @@ -1638,19 +1646,21 @@ | |
| 1638 | "CREATE TEMP TABLE xmark(tname TEXT UNIQUE, trid INT, tuuid TEXT);" |
| 1639 | "CREATE TEMP TABLE xbranch(tname TEXT UNIQUE, brnm TEXT);" |
| 1640 | "CREATE TEMP TABLE xtag(tname TEXT UNIQUE, tcontent TEXT);" |
| 1641 | ); |
| 1642 | |
| 1643 | git_fast_import(pIn); |
| 1644 | db_prepare(&q, "SELECT tcontent FROM xtag"); |
| 1645 | while( db_step(&q)==SQLITE_ROW ){ |
| 1646 | Blob record; |
| 1647 | db_ephemeral_blob(&q, 0, &record); |
| 1648 | fast_insert_content(&record, 0, 0); |
| 1649 | import_reset(0); |
| 1650 | } |
| 1651 | db_finalize(&q); |
| 1652 | } |
| 1653 | |
| 1654 | verify_cancel(); |
| 1655 | db_end_transaction(0); |
| 1656 | db_begin_transaction(); |
| 1657 |
| --- src/import.c | |
| +++ src/import.c | |
| @@ -128,11 +128,16 @@ | |
| 128 | ** to the newly inserted artifact. |
| 129 | ** |
| 130 | ** If saveUuid is true, then pContent is a commit record. Record its |
| 131 | ** UUID in gg.zPrevCheckin. |
| 132 | */ |
| 133 | static int fast_insert_content( |
| 134 | Blob *pContent, /* Content to insert */ |
| 135 | const char *zMark, /* Label using this mark, if not NULL */ |
| 136 | int saveUuid, /* Save SHA1 hash in gg.zPrevCheckin */ |
| 137 | int doParse /* Invoke manifest_crosslink() */ |
| 138 | ){ |
| 139 | Blob hash; |
| 140 | Blob cmpr; |
| 141 | int rid; |
| 142 | |
| 143 | sha1sum_blob(pContent, &hash); |
| @@ -148,10 +153,13 @@ | |
| 153 | db_bind_blob(&ins, ":content", &cmpr); |
| 154 | db_step(&ins); |
| 155 | db_reset(&ins); |
| 156 | blob_reset(&cmpr); |
| 157 | rid = db_last_insert_rowid(); |
| 158 | if( doParse ){ |
| 159 | manifest_crosslink(rid, pContent, MC_NONE); |
| 160 | } |
| 161 | } |
| 162 | if( zMark ){ |
| 163 | db_multi_exec( |
| 164 | "INSERT OR IGNORE INTO xmark(tname, trid, tuuid)" |
| 165 | "VALUES(%Q,%d,%B)", |
| @@ -176,11 +184,11 @@ | |
| 184 | ** to the BLOB table. |
| 185 | */ |
| 186 | static void finish_blob(void){ |
| 187 | Blob content; |
| 188 | blob_init(&content, gg.aData, gg.nData); |
| 189 | fast_insert_content(&content, gg.zMark, 0, 0); |
| 190 | blob_reset(&content); |
| 191 | import_reset(0); |
| 192 | } |
| 193 | |
| 194 | /* |
| @@ -194,12 +202,11 @@ | |
| 202 | blob_appendf(&record, "D %s\n", gg.zDate); |
| 203 | blob_appendf(&record, "T +%F %s\n", gg.zTag, gg.zFrom); |
| 204 | blob_appendf(&record, "U %F\n", gg.zUser); |
| 205 | md5sum_blob(&record, &cksum); |
| 206 | blob_appendf(&record, "Z %b\n", &cksum); |
| 207 | fast_insert_content(&record, 0, 0, 1); |
| 208 | blob_reset(&cksum); |
| 209 | } |
| 210 | import_reset(0); |
| 211 | } |
| 212 | |
| @@ -238,10 +245,11 @@ | |
| 245 | import_prior_files(); |
| 246 | qsort(gg.aFile, gg.nFile, sizeof(gg.aFile[0]), mfile_cmp); |
| 247 | blob_zero(&record); |
| 248 | blob_appendf(&record, "C %F\n", gg.zComment); |
| 249 | blob_appendf(&record, "D %s\n", gg.zDate); |
| 250 | if( !g.fQuiet ) fossil_print("%.10s\r", gg.zDate); |
| 251 | for(i=0; i<gg.nFile; i++){ |
| 252 | const char *zUuid = gg.aFile[i].zUuid; |
| 253 | if( zUuid==0 ) continue; |
| 254 | blob_appendf(&record, "F %F %s", gg.aFile[i].zName, zUuid); |
| 255 | if( gg.aFile[i].isExe ){ |
| @@ -290,12 +298,11 @@ | |
| 298 | db_multi_exec("INSERT INTO xbranch(tname, brnm) VALUES(%Q,%Q)", |
| 299 | gg.zMark, gg.zBranch); |
| 300 | blob_appendf(&record, "U %F\n", gg.zUser); |
| 301 | md5sum_blob(&record, &cksum); |
| 302 | blob_appendf(&record, "Z %b\n", &cksum); |
| 303 | fast_insert_content(&record, gg.zMark, 1, 1); |
| 304 | blob_reset(&cksum); |
| 305 | |
| 306 | /* The "git fast-export" command might output multiple "commit" lines |
| 307 | ** that reference a tag using "refs/tags/TAGNAME". The tag should only |
| 308 | ** be applied to the last commit that is output. The problem is we do not |
| @@ -1503,10 +1510,11 @@ | |
| 1510 | ** --flat The whole dump is a single branch |
| 1511 | ** |
| 1512 | ** Common Options: |
| 1513 | ** -i|--incremental allow importing into an existing repository |
| 1514 | ** -f|--force overwrite repository if already exist |
| 1515 | ** -q|--quiet omit progress output |
| 1516 | ** |
| 1517 | ** The --incremental option allows an existing repository to be extended |
| 1518 | ** with new content. |
| 1519 | ** |
| 1520 | ** |
| @@ -1638,19 +1646,21 @@ | |
| 1646 | "CREATE TEMP TABLE xmark(tname TEXT UNIQUE, trid INT, tuuid TEXT);" |
| 1647 | "CREATE TEMP TABLE xbranch(tname TEXT UNIQUE, brnm TEXT);" |
| 1648 | "CREATE TEMP TABLE xtag(tname TEXT UNIQUE, tcontent TEXT);" |
| 1649 | ); |
| 1650 | |
| 1651 | manifest_crosslink_begin(); |
| 1652 | git_fast_import(pIn); |
| 1653 | db_prepare(&q, "SELECT tcontent FROM xtag"); |
| 1654 | while( db_step(&q)==SQLITE_ROW ){ |
| 1655 | Blob record; |
| 1656 | db_ephemeral_blob(&q, 0, &record); |
| 1657 | fast_insert_content(&record, 0, 0, 1); |
| 1658 | import_reset(0); |
| 1659 | } |
| 1660 | db_finalize(&q); |
| 1661 | manifest_crosslink_end(MC_NONE); |
| 1662 | } |
| 1663 | |
| 1664 | verify_cancel(); |
| 1665 | db_end_transaction(0); |
| 1666 | db_begin_transaction(); |
| 1667 |