Fossil SCM

The search logic now works correctly (I think) regardless of whether the repository uses a legacy FTS4 index or a newer FTS5 index. This allows the Fossil binary to be upgraded on systems without disrupting the search feature and without requiring a search index rebuild. The search index is automatically upgraded to FTS5 the next time the search index is rebuilt.

drh 2023-01-24 14:36 search-fts5
Commit a07e6b87cb306d7b0b359aa18c8d9618a9172d867cd81447bf49ee528d3bd3c3
Parent 8307133b7cc14b3…
2 files changed +36 -4 +4 -3
~ src/search.c ~ src/setup.c
M src/search.c
+36 -4
--- src/search.c
+++ src/search.c
@@ -921,29 +921,38 @@
921921
 unsigned int srchFlags /* What to search over */
922922
 ){
923923
 Blob sql;
924924
 char *zPat = mprintf("%s",zPattern);
925925
 int i;
926
+ static const char *zSnippetCall;
926927
 if( srchFlags==0 ) return;
927928
 sqlite3_create_function(g.db, "rank", 1, SQLITE_UTF8|SQLITE_INNOCUOUS, 0,
928929
 search_rank_sqlfunc, 0, 0);
929930
 for(i=0; zPat[i]; i++){
930931
 if( zPat[i]=='-' || zPat[i]=='"' ) zPat[i] = ' ';
931932
 }
932933
 blob_init(&sql, 0, 0);
934
+ if( search_index_type(0)==4 ){
935
+ /* If this repo is still using the legacy FTS4 search index, then
936
+ ** the snippet() function is slightly different */
937
+ zSnippetCall = "snippet(ftsidx,'<mark>','</mark>',' ... ',-1,35)";
938
+ }else{
939
+ /* This is the common case - Using newer FTS5 search index */
940
+ zSnippetCall = "snippet(ftsidx,-1,'<mark>','</mark>',' ... ',35)";
941
+ }
933942
 blob_appendf(&sql,
934943
 "INSERT INTO x(label,url,score,id,date,snip) "
935944
 " SELECT ftsdocs.label,"
936945
 " ftsdocs.url,"
937946
 " rank(matchinfo(ftsidx,'pcsx')),"
938947
 " ftsdocs.type || ftsdocs.rid,"
939948
 " datetime(ftsdocs.mtime),"
940
- " snippet(ftsidx,-1,'<mark>','</mark>',' ... ',35)"
949
+ " %s"
941950
 " FROM ftsidx CROSS JOIN ftsdocs"
942951
 " WHERE ftsidx MATCH %Q"
943952
 " AND ftsdocs.rowid=ftsidx.rowid",
944
- zPat
953
+ zSnippetCall /*safe-for-%s*/, zPat
945954
 );
946955
 fossil_free(zPat);
947956
 if( srchFlags!=SRCH_ALL ){
948957
 const char *zSep = " AND (";
949958
 static const struct { unsigned m; char c; } aMask[] = {
@@ -1551,18 +1560,41 @@
15511560
 db_multi_exec(zFtsDrop/*works-like:""*/);
15521561
 searchIdxExists = 0;
15531562
 }
15541563
15551564
 /*
1556
-** Return true if the full-text search index exists
1565
+** Return true if the full-text search index exists. See also the
1566
+** search_index_type() function.
15571567
 */
15581568
 int search_index_exists(void){
15591569
 if( searchIdxExists<0 ){
15601570
 searchIdxExists = db_table_exists("repository","ftsdocs");
15611571
 }
15621572
 return searchIdxExists;
15631573
 }
1574
+
1575
+/*
1576
+** Determine which full-text search index is currently being used to
1577
+** add searching. Return values:
1578
+**
1579
+** 0 No search index is available
1580
+** 4 FTS3/4
1581
+** 5 FTS5
1582
+**
1583
+** Results are cached. Make the argument 1 to reset the cache. See
1584
+** also the search_index_exists() routine.
1585
+*/
1586
+int search_index_type(int bReset){
1587
+ static int idxType = -1;
1588
+ if( idxType<0 || bReset ){
1589
+ idxType = db_int(0,
1590
+ "SELECT CASE WHEN sql GLOB '*fts4*' THEN 4 ELSE 5 END"
1591
+ " FROM repository.sqlite_schema WHERE name='ftsidx'"
1592
+ );
1593
+ }
1594
+ return idxType;
1595
+}
15641596
15651597
 /*
15661598
 ** Fill the FTSDOCS table with unindexed entries for everything
15671599
 ** in the repository. This uses INSERT OR IGNORE so entries already
15681600
 ** in FTSDOCS are unchanged.
@@ -1957,11 +1989,11 @@
19571989
 db_get_boolean(aSetng[i].zSetting,0) ? "on" : "off");
19581990
 }
19591991
 fossil_print("%-17s %s\n", "Porter stemmer:",
19601992
 db_get_boolean("search-stemmer",0) ? "on" : "off");
19611993
 if( search_index_exists() ){
1962
- fossil_print("%-17s enabled\n", "full-text index:");
1994
+ fossil_print("%-17s FTS%d\n", "full-text index:", search_index_type(1));
19631995
 fossil_print("%-17s %d\n", "documents:",
19641996
 db_int(0, "SELECT count(*) FROM ftsdocs"));
19651997
 }else{
19661998
 fossil_print("%-17s disabled\n", "full-text index:");
19671999
 }
19682000
--- src/search.c
+++ src/search.c
@@ -921,29 +921,38 @@
921 unsigned int srchFlags /* What to search over */
922 ){
923 Blob sql;
924 char *zPat = mprintf("%s",zPattern);
925 int i;
 
926 if( srchFlags==0 ) return;
927 sqlite3_create_function(g.db, "rank", 1, SQLITE_UTF8|SQLITE_INNOCUOUS, 0,
928 search_rank_sqlfunc, 0, 0);
929 for(i=0; zPat[i]; i++){
930 if( zPat[i]=='-' || zPat[i]=='"' ) zPat[i] = ' ';
931 }
932 blob_init(&sql, 0, 0);
 
 
 
 
 
 
 
 
933 blob_appendf(&sql,
934 "INSERT INTO x(label,url,score,id,date,snip) "
935 " SELECT ftsdocs.label,"
936 " ftsdocs.url,"
937 " rank(matchinfo(ftsidx,'pcsx')),"
938 " ftsdocs.type || ftsdocs.rid,"
939 " datetime(ftsdocs.mtime),"
940 " snippet(ftsidx,-1,'<mark>','</mark>',' ... ',35)"
941 " FROM ftsidx CROSS JOIN ftsdocs"
942 " WHERE ftsidx MATCH %Q"
943 " AND ftsdocs.rowid=ftsidx.rowid",
944 zPat
945 );
946 fossil_free(zPat);
947 if( srchFlags!=SRCH_ALL ){
948 const char *zSep = " AND (";
949 static const struct { unsigned m; char c; } aMask[] = {
@@ -1551,18 +1560,41 @@
1551 db_multi_exec(zFtsDrop/*works-like:""*/);
1552 searchIdxExists = 0;
1553 }
1554
1555 /*
1556 ** Return true if the full-text search index exists
 
1557 */
1558 int search_index_exists(void){
1559 if( searchIdxExists<0 ){
1560 searchIdxExists = db_table_exists("repository","ftsdocs");
1561 }
1562 return searchIdxExists;
1563 }
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1564
1565 /*
1566 ** Fill the FTSDOCS table with unindexed entries for everything
1567 ** in the repository. This uses INSERT OR IGNORE so entries already
1568 ** in FTSDOCS are unchanged.
@@ -1957,11 +1989,11 @@
1957 db_get_boolean(aSetng[i].zSetting,0) ? "on" : "off");
1958 }
1959 fossil_print("%-17s %s\n", "Porter stemmer:",
1960 db_get_boolean("search-stemmer",0) ? "on" : "off");
1961 if( search_index_exists() ){
1962 fossil_print("%-17s enabled\n", "full-text index:");
1963 fossil_print("%-17s %d\n", "documents:",
1964 db_int(0, "SELECT count(*) FROM ftsdocs"));
1965 }else{
1966 fossil_print("%-17s disabled\n", "full-text index:");
1967 }
1968
--- src/search.c
+++ src/search.c
@@ -921,29 +921,38 @@
921 unsigned int srchFlags /* What to search over */
922 ){
923 Blob sql;
924 char *zPat = mprintf("%s",zPattern);
925 int i;
926 static const char *zSnippetCall;
927 if( srchFlags==0 ) return;
928 sqlite3_create_function(g.db, "rank", 1, SQLITE_UTF8|SQLITE_INNOCUOUS, 0,
929 search_rank_sqlfunc, 0, 0);
930 for(i=0; zPat[i]; i++){
931 if( zPat[i]=='-' || zPat[i]=='"' ) zPat[i] = ' ';
932 }
933 blob_init(&sql, 0, 0);
934 if( search_index_type(0)==4 ){
935 /* If this repo is still using the legacy FTS4 search index, then
936 ** the snippet() function is slightly different */
937 zSnippetCall = "snippet(ftsidx,'<mark>','</mark>',' ... ',-1,35)";
938 }else{
939 /* This is the common case - Using newer FTS5 search index */
940 zSnippetCall = "snippet(ftsidx,-1,'<mark>','</mark>',' ... ',35)";
941 }
942 blob_appendf(&sql,
943 "INSERT INTO x(label,url,score,id,date,snip) "
944 " SELECT ftsdocs.label,"
945 " ftsdocs.url,"
946 " rank(matchinfo(ftsidx,'pcsx')),"
947 " ftsdocs.type || ftsdocs.rid,"
948 " datetime(ftsdocs.mtime),"
949 " %s"
950 " FROM ftsidx CROSS JOIN ftsdocs"
951 " WHERE ftsidx MATCH %Q"
952 " AND ftsdocs.rowid=ftsidx.rowid",
953 zSnippetCall /*safe-for-%s*/, zPat
954 );
955 fossil_free(zPat);
956 if( srchFlags!=SRCH_ALL ){
957 const char *zSep = " AND (";
958 static const struct { unsigned m; char c; } aMask[] = {
@@ -1551,18 +1560,41 @@
1560 db_multi_exec(zFtsDrop/*works-like:""*/);
1561 searchIdxExists = 0;
1562 }
1563
1564 /*
1565 ** Return true if the full-text search index exists. See also the
1566 ** search_index_type() function.
1567 */
1568 int search_index_exists(void){
1569 if( searchIdxExists<0 ){
1570 searchIdxExists = db_table_exists("repository","ftsdocs");
1571 }
1572 return searchIdxExists;
1573 }
1574
1575 /*
1576 ** Determine which full-text search index is currently being used to
1577 ** add searching. Return values:
1578 **
1579 ** 0 No search index is available
1580 ** 4 FTS3/4
1581 ** 5 FTS5
1582 **
1583 ** Results are cached. Make the argument 1 to reset the cache. See
1584 ** also the search_index_exists() routine.
1585 */
1586 int search_index_type(int bReset){
1587 static int idxType = -1;
1588 if( idxType<0 || bReset ){
1589 idxType = db_int(0,
1590 "SELECT CASE WHEN sql GLOB '*fts4*' THEN 4 ELSE 5 END"
1591 " FROM repository.sqlite_schema WHERE name='ftsidx'"
1592 );
1593 }
1594 return idxType;
1595 }
1596
1597 /*
1598 ** Fill the FTSDOCS table with unindexed entries for everything
1599 ** in the repository. This uses INSERT OR IGNORE so entries already
1600 ** in FTSDOCS are unchanged.
@@ -1957,11 +1989,11 @@
1989 db_get_boolean(aSetng[i].zSetting,0) ? "on" : "off");
1990 }
1991 fossil_print("%-17s %s\n", "Porter stemmer:",
1992 db_get_boolean("search-stemmer",0) ? "on" : "off");
1993 if( search_index_exists() ){
1994 fossil_print("%-17s FTS%d\n", "full-text index:", search_index_type(1));
1995 fossil_print("%-17s %d\n", "documents:",
1996 db_int(0, "SELECT count(*) FROM ftsdocs"));
1997 }else{
1998 fossil_print("%-17s disabled\n", "full-text index:");
1999 }
2000
M src/setup.c
+4 -3
--- src/setup.c
+++ src/setup.c
@@ -2067,18 +2067,19 @@
20672067
 search_create_index();
20682068
 search_fill_index();
20692069
 search_update_index(search_restrict(SRCH_ALL));
20702070
 }
20712071
 if( search_index_exists() ){
2072
- @ <p>Currently using an SQLite FTS5 search index. This makes search
2073
- @ run faster, especially on large repositories, but takes up space.</p>
2072
+ @ <p>Currently using an SQLite FTS%d(search_index_type(0)) search index.
2073
+ @ The index helps search run faster, especially on large repositories,
2074
+ @ but takes up space.</p>
20742075
 onoff_attribute("Use Porter Stemmer","search-stemmer","ss",0,0);
20752076
 @ <p><input type="submit" name="fts0" value="Delete The Full-Text Index">
20762077
 @ <input type="submit" name="fts1" value="Rebuild The Full-Text Index">
20772078
 style_submenu_element("FTS Index Debugging","%R/test-ftsdocs");
20782079
 }else{
2079
- @ <p>The SQLite FTS5 search index is disabled. All searching will be
2080
+ @ <p>The SQLite search index is disabled. All searching will be
20802081
 @ a full-text scan. This usually works fine, but can be slow for
20812082
 @ larger repositories.</p>
20822083
 onoff_attribute("Use Porter Stemmer","search-stemmer","ss",0,0);
20832084
 @ <p><input type="submit" name="fts1" value="Create A Full-Text Index">
20842085
 }
20852086
--- src/setup.c
+++ src/setup.c
@@ -2067,18 +2067,19 @@
2067 search_create_index();
2068 search_fill_index();
2069 search_update_index(search_restrict(SRCH_ALL));
2070 }
2071 if( search_index_exists() ){
2072 @ <p>Currently using an SQLite FTS5 search index. This makes search
2073 @ run faster, especially on large repositories, but takes up space.</p>
 
2074 onoff_attribute("Use Porter Stemmer","search-stemmer","ss",0,0);
2075 @ <p><input type="submit" name="fts0" value="Delete The Full-Text Index">
2076 @ <input type="submit" name="fts1" value="Rebuild The Full-Text Index">
2077 style_submenu_element("FTS Index Debugging","%R/test-ftsdocs");
2078 }else{
2079 @ <p>The SQLite FTS5 search index is disabled. All searching will be
2080 @ a full-text scan. This usually works fine, but can be slow for
2081 @ larger repositories.</p>
2082 onoff_attribute("Use Porter Stemmer","search-stemmer","ss",0,0);
2083 @ <p><input type="submit" name="fts1" value="Create A Full-Text Index">
2084 }
2085
--- src/setup.c
+++ src/setup.c
@@ -2067,18 +2067,19 @@
2067 search_create_index();
2068 search_fill_index();
2069 search_update_index(search_restrict(SRCH_ALL));
2070 }
2071 if( search_index_exists() ){
2072 @ <p>Currently using an SQLite FTS%d(search_index_type(0)) search index.
2073 @ The index helps search run faster, especially on large repositories,
2074 @ but takes up space.</p>
2075 onoff_attribute("Use Porter Stemmer","search-stemmer","ss",0,0);
2076 @ <p><input type="submit" name="fts0" value="Delete The Full-Text Index">
2077 @ <input type="submit" name="fts1" value="Rebuild The Full-Text Index">
2078 style_submenu_element("FTS Index Debugging","%R/test-ftsdocs");
2079 }else{
2080 @ <p>The SQLite search index is disabled. All searching will be
2081 @ a full-text scan. This usually works fine, but can be slow for
2082 @ larger repositories.</p>
2083 onoff_attribute("Use Porter Stemmer","search-stemmer","ss",0,0);
2084 @ <p><input type="submit" name="fts1" value="Create A Full-Text Index">
2085 }
2086

Keyboard Shortcuts

Open search /
Next entry (timeline) j
Previous entry (timeline) k
Open focused entry Enter
Show this help ?
Toggle theme Top nav button