Fossil SCM

Improvements to the logic in the "fossil all" command that removes redundant entries for repositories in the global_config table. If two or more entries share the same inode, only use the first one. On Windows (which lacks inodes) determine uniqueness by the full, canonical pathname.

drh 2024-11-23 21:27 trunk
Commit 106de276eec32eabb265d2b0f40bed48cb58ce6c7b6d6acb5f2f7f7d4327c603
Parent 4c6e394d1eaf275…
3 files changed +23 -6 +2 +62
~ src/allrepo.c ~ src/db.c ~ src/file.c
M src/allrepo.c
+23 -6
--- src/allrepo.c
+++ src/allrepo.c
@@ -50,11 +50,10 @@
5050
 for(i=iStart; i<g.argc; i++){
5151
 blob_appendf(pExtra, " %s", g.argv[i]);
5252
 }
5353
 }
5454
55
-
5655
 /*
5756
 ** COMMAND: all
5857
 **
5958
 ** Usage: %fossil all SUBCOMMAND ...
6059
 **
@@ -429,44 +428,62 @@
429428
 "add cache changes clean dbstat extras fts-config git ignore "
430429
 "info list ls pull push rebuild remote "
431430
 "server settings sync ui unset whatis");
432431
 }
433432
 verify_all_options();
434
- db_multi_exec("CREATE TEMP TABLE repolist(name,tag);");
433
+ db_multi_exec(
434
+ "CREATE TEMP TABLE repolist(\n"
435
+ " name TEXT, -- Filename\n"
436
+ " tag TEXT, -- Key for the GLOBAL_CONFIG table entry\n"
437
+ " inode TEXT -- Unique identifier for this file\n"
438
+ ");\n"
439
+
440
+ /* The seenFile() table holds inode names for entries that have
441
+ ** already been processed. */
442
+ "CREATE TEMP TABLE seenFile(x TEXT COLLATE nocase);\n"
443
+
444
+ /* The toDel() table holds the "tag" for entries that need to be
445
+ ** deleted because they are redundant or no longer exist */
446
+ "CREATE TEMP TABLE toDel(x TEXT);\n"
447
+ );
448
+ sqlite3_create_function(g.db, "inode", 1, SQLITE_UTF8, 0,
449
+ file_inode_sql_func, 0, 0);
435450
 if( useCheckouts ){
436451
 db_multi_exec(
437452
 "INSERT INTO repolist "
438
- "SELECT DISTINCT substr(name, 7), name COLLATE nocase"
453
+ "SELECT substr(name, 7), name, inode(substr(name,7))"
439454
 " FROM global_config"
440455
 " WHERE substr(name, 1, 6)=='ckout:'"
441456
 " ORDER BY 1"
442457
 );
443458
 }else{
444459
 db_multi_exec(
445460
 "INSERT INTO repolist "
446
- "SELECT DISTINCT substr(name, 6), name COLLATE nocase"
461
+ "SELECT substr(name, 6), name, inode(substr(name,6))"
447462
 " FROM global_config"
448463
 " WHERE substr(name, 1, 5)=='repo:'"
449464
 " ORDER BY 1"
450465
 );
451466
 }
452
- db_multi_exec("CREATE TEMP TABLE toDel(x TEXT)");
453
- db_prepare(&q, "SELECT name, tag FROM repolist ORDER BY 1");
467
+ db_prepare(&q,"SELECT name, tag, inode FROM repolist ORDER BY 1");
454468
 while( db_step(&q)==SQLITE_ROW ){
455469
 int rc;
456470
 const char *zFilename = db_column_text(&q, 0);
471
+ const char *zInode = db_column_text(&q,2);
457472
 #if !USE_SEE
458473
 if( sqlite3_strglob("*.efossil", zFilename)==0 ) continue;
459474
 #endif
460475
 if( file_access(zFilename, F_OK)
461476
 || !file_is_canonical(zFilename)
462477
 || (useCheckouts && file_isdir(zFilename, ExtFILE)!=1)
478
+ || db_exists("SELECT 1 FROM temp.seenFile where x=%Q", zInode)
463479
 ){
464480
 db_multi_exec("INSERT INTO toDel VALUES(%Q)", db_column_text(&q, 1));
465481
 nToDel++;
466482
 continue;
467483
 }
484
+ db_multi_exec("INSERT INTO seenFile(x) VALUES(%Q)", zInode);
468485
 if( zCmd[0]=='l' ){
469486
 fossil_print("%s\n", zFilename);
470487
 continue;
471488
 }else if( showFile ){
472489
 fossil_print("%s: %s\n", useCheckouts ? "check-out" : "repository",
473490
--- src/allrepo.c
+++ src/allrepo.c
@@ -50,11 +50,10 @@
50 for(i=iStart; i<g.argc; i++){
51 blob_appendf(pExtra, " %s", g.argv[i]);
52 }
53 }
54
55
56 /*
57 ** COMMAND: all
58 **
59 ** Usage: %fossil all SUBCOMMAND ...
60 **
@@ -429,44 +428,62 @@
429 "add cache changes clean dbstat extras fts-config git ignore "
430 "info list ls pull push rebuild remote "
431 "server settings sync ui unset whatis");
432 }
433 verify_all_options();
434 db_multi_exec("CREATE TEMP TABLE repolist(name,tag);");
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
435 if( useCheckouts ){
436 db_multi_exec(
437 "INSERT INTO repolist "
438 "SELECT DISTINCT substr(name, 7), name COLLATE nocase"
439 " FROM global_config"
440 " WHERE substr(name, 1, 6)=='ckout:'"
441 " ORDER BY 1"
442 );
443 }else{
444 db_multi_exec(
445 "INSERT INTO repolist "
446 "SELECT DISTINCT substr(name, 6), name COLLATE nocase"
447 " FROM global_config"
448 " WHERE substr(name, 1, 5)=='repo:'"
449 " ORDER BY 1"
450 );
451 }
452 db_multi_exec("CREATE TEMP TABLE toDel(x TEXT)");
453 db_prepare(&q, "SELECT name, tag FROM repolist ORDER BY 1");
454 while( db_step(&q)==SQLITE_ROW ){
455 int rc;
456 const char *zFilename = db_column_text(&q, 0);
 
457 #if !USE_SEE
458 if( sqlite3_strglob("*.efossil", zFilename)==0 ) continue;
459 #endif
460 if( file_access(zFilename, F_OK)
461 || !file_is_canonical(zFilename)
462 || (useCheckouts && file_isdir(zFilename, ExtFILE)!=1)
 
463 ){
464 db_multi_exec("INSERT INTO toDel VALUES(%Q)", db_column_text(&q, 1));
465 nToDel++;
466 continue;
467 }
 
468 if( zCmd[0]=='l' ){
469 fossil_print("%s\n", zFilename);
470 continue;
471 }else if( showFile ){
472 fossil_print("%s: %s\n", useCheckouts ? "check-out" : "repository",
473
--- src/allrepo.c
+++ src/allrepo.c
@@ -50,11 +50,10 @@
50 for(i=iStart; i<g.argc; i++){
51 blob_appendf(pExtra, " %s", g.argv[i]);
52 }
53 }
54
 
55 /*
56 ** COMMAND: all
57 **
58 ** Usage: %fossil all SUBCOMMAND ...
59 **
@@ -429,44 +428,62 @@
428 "add cache changes clean dbstat extras fts-config git ignore "
429 "info list ls pull push rebuild remote "
430 "server settings sync ui unset whatis");
431 }
432 verify_all_options();
433 db_multi_exec(
434 "CREATE TEMP TABLE repolist(\n"
435 " name TEXT, -- Filename\n"
436 " tag TEXT, -- Key for the GLOBAL_CONFIG table entry\n"
437 " inode TEXT -- Unique identifier for this file\n"
438 ");\n"
439
440 /* The seenFile() table holds inode names for entries that have
441 ** already been processed. */
442 "CREATE TEMP TABLE seenFile(x TEXT COLLATE nocase);\n"
443
444 /* The toDel() table holds the "tag" for entries that need to be
445 ** deleted because they are redundant or no longer exist */
446 "CREATE TEMP TABLE toDel(x TEXT);\n"
447 );
448 sqlite3_create_function(g.db, "inode", 1, SQLITE_UTF8, 0,
449 file_inode_sql_func, 0, 0);
450 if( useCheckouts ){
451 db_multi_exec(
452 "INSERT INTO repolist "
453 "SELECT substr(name, 7), name, inode(substr(name,7))"
454 " FROM global_config"
455 " WHERE substr(name, 1, 6)=='ckout:'"
456 " ORDER BY 1"
457 );
458 }else{
459 db_multi_exec(
460 "INSERT INTO repolist "
461 "SELECT substr(name, 6), name, inode(substr(name,6))"
462 " FROM global_config"
463 " WHERE substr(name, 1, 5)=='repo:'"
464 " ORDER BY 1"
465 );
466 }
467 db_prepare(&q,"SELECT name, tag, inode FROM repolist ORDER BY 1");
 
468 while( db_step(&q)==SQLITE_ROW ){
469 int rc;
470 const char *zFilename = db_column_text(&q, 0);
471 const char *zInode = db_column_text(&q,2);
472 #if !USE_SEE
473 if( sqlite3_strglob("*.efossil", zFilename)==0 ) continue;
474 #endif
475 if( file_access(zFilename, F_OK)
476 || !file_is_canonical(zFilename)
477 || (useCheckouts && file_isdir(zFilename, ExtFILE)!=1)
478 || db_exists("SELECT 1 FROM temp.seenFile where x=%Q", zInode)
479 ){
480 db_multi_exec("INSERT INTO toDel VALUES(%Q)", db_column_text(&q, 1));
481 nToDel++;
482 continue;
483 }
484 db_multi_exec("INSERT INTO seenFile(x) VALUES(%Q)", zInode);
485 if( zCmd[0]=='l' ){
486 fossil_print("%s\n", zFilename);
487 continue;
488 }else if( showFile ){
489 fossil_print("%s: %s\n", useCheckouts ? "check-out" : "repository",
490
M src/db.c
+2
--- src/db.c
+++ src/db.c
@@ -1557,10 +1557,12 @@
15571557
 sqlite3_create_function(db, "url_nouser", 1, SQLITE_UTF8, 0,
15581558
 url_nouser_func,0,0);
15591559
 sqlite3_create_function(db, "chat_msg_from_event", 4,
15601560
 SQLITE_UTF8 | SQLITE_INNOCUOUS, 0,
15611561
 chat_msg_from_event, 0, 0);
1562
+ sqlite3_create_function(db, "inode", 1, SQLITE_UTF8, 0,
1563
+ file_inode_sql_func,0,0);
15621564
15631565
 }
15641566
15651567
 #if USE_SEE
15661568
 /*
15671569
--- src/db.c
+++ src/db.c
@@ -1557,10 +1557,12 @@
1557 sqlite3_create_function(db, "url_nouser", 1, SQLITE_UTF8, 0,
1558 url_nouser_func,0,0);
1559 sqlite3_create_function(db, "chat_msg_from_event", 4,
1560 SQLITE_UTF8 | SQLITE_INNOCUOUS, 0,
1561 chat_msg_from_event, 0, 0);
 
 
1562
1563 }
1564
1565 #if USE_SEE
1566 /*
1567
--- src/db.c
+++ src/db.c
@@ -1557,10 +1557,12 @@
1557 sqlite3_create_function(db, "url_nouser", 1, SQLITE_UTF8, 0,
1558 url_nouser_func,0,0);
1559 sqlite3_create_function(db, "chat_msg_from_event", 4,
1560 SQLITE_UTF8 | SQLITE_INNOCUOUS, 0,
1561 chat_msg_from_event, 0, 0);
1562 sqlite3_create_function(db, "inode", 1, SQLITE_UTF8, 0,
1563 file_inode_sql_func,0,0);
1564
1565 }
1566
1567 #if USE_SEE
1568 /*
1569
M src/file.c
+62
--- src/file.c
+++ src/file.c
@@ -1656,10 +1656,12 @@
16561656
 g.allowSymlinks = !is_false(zAllow);
16571657
 }
16581658
 if( zRoot==0 ) zRoot = g.zLocalRoot==0 ? "" : g.zLocalRoot;
16591659
 fossil_print("db_allow_symlinks() = %d\n", db_allow_symlinks());
16601660
 fossil_print("local-root = [%s]\n", zRoot);
1661
+ sqlite3_create_function(g.db, "inode", 1, SQLITE_UTF8, 0,
1662
+ file_inode_sql_func, 0, 0);
16611663
 for(i=2; i<g.argc; i++){
16621664
 char *z;
16631665
 emitFileStat(g.argv[i], slashFlag, resetFlag);
16641666
 z = file_canonical_name_dup(g.argv[i]);
16651667
 fossil_print(" file_canonical_name = %s\n", z);
@@ -1669,10 +1671,13 @@
16691671
 }else{
16701672
 int n = file_nondir_objects_on_path(zRoot, z);
16711673
 fossil_print("%.*s\n", n, z);
16721674
 }
16731675
 fossil_free(z);
1676
+ z = db_text(0, "SELECT inode(%Q)", g.argv[i]);
1677
+ fossil_print(" file_inode_sql_func = \"%s\"\n", z);
1678
+ fossil_free(z);
16741679
 }
16751680
 }
16761681
16771682
 /*
16781683
 ** COMMAND: test-canonical-name
@@ -2980,5 +2985,62 @@
29802985
 }
29812986
 }
29822987
 fossil_free(zCkoutDb);
29832988
 return rc;
29842989
 }
2990
+
2991
+/*
2992
+** This is the implementation of inode(FILENAME) SQL function.
2993
+**
2994
+** dev_inode(FILENAME) returns a string. If FILENAME exists and is
2995
+** a regular file, then the return string is of the form:
2996
+**
2997
+** DEV/INODE
2998
+**
2999
+** Where DEV and INODE are the device number and inode number for
3000
+** the file. Or, on Windows, the return value is the canonical
3001
+** name of the file, because Windows does not have INODEs.
3002
+**
3003
+** If FILENAME does not exist, then the return is an empty string.
3004
+**
3005
+** The value of inode() can be used to eliminate files from a list
3006
+** that have duplicates because they have differing names due to links.
3007
+**
3008
+** Code that wants to use this SQL function needs to first register
3009
+** it using a call such as the following:
3010
+**
3011
+** sqlite3_create_function(g.db, "inode", 1, SQLITE_UTF8, 0,
3012
+** file_inode_sql_func, 0, 0);
3013
+*/
3014
+void file_inode_sql_func(
3015
+ sqlite3_context *context,
3016
+ int argc,
3017
+ sqlite3_value **argv
3018
+){
3019
+ const char *zFilename;
3020
+ assert( argc==1 );
3021
+ zFilename = (const char*)sqlite3_value_text(argv[0]);
3022
+ if( zFilename==0 || zFilename[0]==0 || file_access(zFilename,F_OK) ){
3023
+ sqlite3_result_text(context, "", 0, SQLITE_STATIC);
3024
+ return;
3025
+ }
3026
+#if defined(_WIN32)
3027
+ {
3028
+ const char *zCanonical = file_canonical_name_dup(zFilename);
3029
+ sqlite3_result_text(context, zCanonical, -1, fossil_free);
3030
+ }
3031
+#else
3032
+ {
3033
+ struct stat buf;
3034
+ int rc;
3035
+ memset(&buf, 0, sizeof(buf));
3036
+ rc = stat(zFilename, &buf);
3037
+ if( rc ){
3038
+ sqlite3_result_text(context, "", 0, SQLITE_STATIC);
3039
+ }else{
3040
+ sqlite3_result_text(context,
3041
+ mprintf("%lld/%lld", (i64)buf.st_dev, (i64)buf.st_ino), -1,
3042
+ fossil_free);
3043
+ }
3044
+ }
3045
+#endif
3046
+}
29853047
--- src/file.c
+++ src/file.c
@@ -1656,10 +1656,12 @@
1656 g.allowSymlinks = !is_false(zAllow);
1657 }
1658 if( zRoot==0 ) zRoot = g.zLocalRoot==0 ? "" : g.zLocalRoot;
1659 fossil_print("db_allow_symlinks() = %d\n", db_allow_symlinks());
1660 fossil_print("local-root = [%s]\n", zRoot);
 
 
1661 for(i=2; i<g.argc; i++){
1662 char *z;
1663 emitFileStat(g.argv[i], slashFlag, resetFlag);
1664 z = file_canonical_name_dup(g.argv[i]);
1665 fossil_print(" file_canonical_name = %s\n", z);
@@ -1669,10 +1671,13 @@
1669 }else{
1670 int n = file_nondir_objects_on_path(zRoot, z);
1671 fossil_print("%.*s\n", n, z);
1672 }
1673 fossil_free(z);
 
 
 
1674 }
1675 }
1676
1677 /*
1678 ** COMMAND: test-canonical-name
@@ -2980,5 +2985,62 @@
2980 }
2981 }
2982 fossil_free(zCkoutDb);
2983 return rc;
2984 }
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
2985
--- src/file.c
+++ src/file.c
@@ -1656,10 +1656,12 @@
1656 g.allowSymlinks = !is_false(zAllow);
1657 }
1658 if( zRoot==0 ) zRoot = g.zLocalRoot==0 ? "" : g.zLocalRoot;
1659 fossil_print("db_allow_symlinks() = %d\n", db_allow_symlinks());
1660 fossil_print("local-root = [%s]\n", zRoot);
1661 sqlite3_create_function(g.db, "inode", 1, SQLITE_UTF8, 0,
1662 file_inode_sql_func, 0, 0);
1663 for(i=2; i<g.argc; i++){
1664 char *z;
1665 emitFileStat(g.argv[i], slashFlag, resetFlag);
1666 z = file_canonical_name_dup(g.argv[i]);
1667 fossil_print(" file_canonical_name = %s\n", z);
@@ -1669,10 +1671,13 @@
1671 }else{
1672 int n = file_nondir_objects_on_path(zRoot, z);
1673 fossil_print("%.*s\n", n, z);
1674 }
1675 fossil_free(z);
1676 z = db_text(0, "SELECT inode(%Q)", g.argv[i]);
1677 fossil_print(" file_inode_sql_func = \"%s\"\n", z);
1678 fossil_free(z);
1679 }
1680 }
1681
1682 /*
1683 ** COMMAND: test-canonical-name
@@ -2980,5 +2985,62 @@
2985 }
2986 }
2987 fossil_free(zCkoutDb);
2988 return rc;
2989 }
2990
2991 /*
2992 ** This is the implementation of inode(FILENAME) SQL function.
2993 **
2994 ** dev_inode(FILENAME) returns a string. If FILENAME exists and is
2995 ** a regular file, then the return string is of the form:
2996 **
2997 ** DEV/INODE
2998 **
2999 ** Where DEV and INODE are the device number and inode number for
3000 ** the file. Or, on Windows, the return value is the canonical
3001 ** name of the file, because Windows does not have INODEs.
3002 **
3003 ** If FILENAME does not exist, then the return is an empty string.
3004 **
3005 ** The value of inode() can be used to eliminate files from a list
3006 ** that have duplicates because they have differing names due to links.
3007 **
3008 ** Code that wants to use this SQL function needs to first register
3009 ** it using a call such as the following:
3010 **
3011 ** sqlite3_create_function(g.db, "inode", 1, SQLITE_UTF8, 0,
3012 ** file_inode_sql_func, 0, 0);
3013 */
3014 void file_inode_sql_func(
3015 sqlite3_context *context,
3016 int argc,
3017 sqlite3_value **argv
3018 ){
3019 const char *zFilename;
3020 assert( argc==1 );
3021 zFilename = (const char*)sqlite3_value_text(argv[0]);
3022 if( zFilename==0 || zFilename[0]==0 || file_access(zFilename,F_OK) ){
3023 sqlite3_result_text(context, "", 0, SQLITE_STATIC);
3024 return;
3025 }
3026 #if defined(_WIN32)
3027 {
3028 const char *zCanonical = file_canonical_name_dup(zFilename);
3029 sqlite3_result_text(context, zCanonical, -1, fossil_free);
3030 }
3031 #else
3032 {
3033 struct stat buf;
3034 int rc;
3035 memset(&buf, 0, sizeof(buf));
3036 rc = stat(zFilename, &buf);
3037 if( rc ){
3038 sqlite3_result_text(context, "", 0, SQLITE_STATIC);
3039 }else{
3040 sqlite3_result_text(context,
3041 mprintf("%lld/%lld", (i64)buf.st_dev, (i64)buf.st_ino), -1,
3042 fossil_free);
3043 }
3044 }
3045 #endif
3046 }
3047

Keyboard Shortcuts

Open search /
Next entry (timeline) j
Previous entry (timeline) k
Open focused entry Enter
Show this help ?
Toggle theme Top nav button