Fossil SCM

Upgrade FTS search from v4 to v5.

stephan 2023-01-24 21:25 trunk merge
Commit c3c4ef167ef78b14d359c4931e7a76996f5dd7b9a9fea435bc0ce537b1927637
Parent 4d8c30265b3dc76…
5 files changed +1 +1 +429 -20 +4 -3 +1
~ src/db.c ~ src/db.c ~ src/search.c ~ src/setup.c ~ www/changes.wiki
M src/db.c
+1
--- src/db.c
+++ src/db.c
@@ -1802,10 +1802,11 @@
18021802
 if( g.fSqlTrace ) sqlite3_trace_v2(db, SQLITE_TRACE_PROFILE, db_sql_trace, 0);
18031803
 db_add_aux_functions(db);
18041804
 re_add_sql_func(db); /* The REGEXP operator */
18051805
 foci_register(db); /* The "files_of_checkin" virtual table */
18061806
 sqlite3_set_authorizer(db, db_top_authorizer, db);
1807
+ db_register_fts5(db) /* in search.c */;
18071808
 return db;
18081809
 }
18091810
18101811
18111812
 /*
18121813
--- src/db.c
+++ src/db.c
@@ -1802,10 +1802,11 @@
1802 if( g.fSqlTrace ) sqlite3_trace_v2(db, SQLITE_TRACE_PROFILE, db_sql_trace, 0);
1803 db_add_aux_functions(db);
1804 re_add_sql_func(db); /* The REGEXP operator */
1805 foci_register(db); /* The "files_of_checkin" virtual table */
1806 sqlite3_set_authorizer(db, db_top_authorizer, db);
 
1807 return db;
1808 }
1809
1810
1811 /*
1812
--- src/db.c
+++ src/db.c
@@ -1802,10 +1802,11 @@
1802 if( g.fSqlTrace ) sqlite3_trace_v2(db, SQLITE_TRACE_PROFILE, db_sql_trace, 0);
1803 db_add_aux_functions(db);
1804 re_add_sql_func(db); /* The REGEXP operator */
1805 foci_register(db); /* The "files_of_checkin" virtual table */
1806 sqlite3_set_authorizer(db, db_top_authorizer, db);
1807 db_register_fts5(db) /* in search.c */;
1808 return db;
1809 }
1810
1811
1812 /*
1813
M src/db.c
+1
--- src/db.c
+++ src/db.c
@@ -1802,10 +1802,11 @@
18021802
 if( g.fSqlTrace ) sqlite3_trace_v2(db, SQLITE_TRACE_PROFILE, db_sql_trace, 0);
18031803
 db_add_aux_functions(db);
18041804
 re_add_sql_func(db); /* The REGEXP operator */
18051805
 foci_register(db); /* The "files_of_checkin" virtual table */
18061806
 sqlite3_set_authorizer(db, db_top_authorizer, db);
1807
+ db_register_fts5(db) /* in search.c */;
18071808
 return db;
18081809
 }
18091810
18101811
18111812
 /*
18121813
--- src/db.c
+++ src/db.c
@@ -1802,10 +1802,11 @@
1802 if( g.fSqlTrace ) sqlite3_trace_v2(db, SQLITE_TRACE_PROFILE, db_sql_trace, 0);
1803 db_add_aux_functions(db);
1804 re_add_sql_func(db); /* The REGEXP operator */
1805 foci_register(db); /* The "files_of_checkin" virtual table */
1806 sqlite3_set_authorizer(db, db_top_authorizer, db);
 
1807 return db;
1808 }
1809
1810
1811 /*
1812
--- src/db.c
+++ src/db.c
@@ -1802,10 +1802,11 @@
1802 if( g.fSqlTrace ) sqlite3_trace_v2(db, SQLITE_TRACE_PROFILE, db_sql_trace, 0);
1803 db_add_aux_functions(db);
1804 re_add_sql_func(db); /* The REGEXP operator */
1805 foci_register(db); /* The "files_of_checkin" virtual table */
1806 sqlite3_set_authorizer(db, db_top_authorizer, db);
1807 db_register_fts5(db) /* in search.c */;
1808 return db;
1809 }
1810
1811
1812 /*
1813
M src/search.c
+429 -20
--- src/search.c
+++ src/search.c
@@ -921,29 +921,38 @@
921921
 unsigned int srchFlags /* What to search over */
922922
 ){
923923
 Blob sql;
924924
 char *zPat = mprintf("%s",zPattern);
925925
 int i;
926
+ static const char *zSnippetCall;
926927
 if( srchFlags==0 ) return;
927928
 sqlite3_create_function(g.db, "rank", 1, SQLITE_UTF8|SQLITE_INNOCUOUS, 0,
928929
 search_rank_sqlfunc, 0, 0);
929930
 for(i=0; zPat[i]; i++){
930931
 if( zPat[i]=='-' || zPat[i]=='"' ) zPat[i] = ' ';
931932
 }
932933
 blob_init(&sql, 0, 0);
934
+ if( search_index_type(0)==4 ){
935
+ /* If this repo is still using the legacy FTS4 search index, then
936
+ ** the snippet() function is slightly different */
937
+ zSnippetCall = "snippet(ftsidx,'<mark>','</mark>',' ... ',-1,35)";
938
+ }else{
939
+ /* This is the common case - Using newer FTS5 search index */
940
+ zSnippetCall = "snippet(ftsidx,-1,'<mark>','</mark>',' ... ',35)";
941
+ }
933942
 blob_appendf(&sql,
934943
 "INSERT INTO x(label,url,score,id,date,snip) "
935944
 " SELECT ftsdocs.label,"
936945
 " ftsdocs.url,"
937946
 " rank(matchinfo(ftsidx,'pcsx')),"
938947
 " ftsdocs.type || ftsdocs.rid,"
939948
 " datetime(ftsdocs.mtime),"
940
- " snippet(ftsidx,'<mark>','</mark>',' ... ',-1,35)"
949
+ " %s"
941950
 " FROM ftsidx CROSS JOIN ftsdocs"
942951
 " WHERE ftsidx MATCH %Q"
943
- " AND ftsdocs.rowid=ftsidx.docid",
944
- zPat
952
+ " AND ftsdocs.rowid=ftsidx.rowid",
953
+ zSnippetCall /*safe-for-%s*/, zPat
945954
 );
946955
 fossil_free(zPat);
947956
 if( srchFlags!=SRCH_ALL ){
948957
 const char *zSep = " AND (";
949958
 static const struct { unsigned m; char c; } aMask[] = {
@@ -1508,11 +1517,11 @@
15081517
 /* The schema for the full-text index
15091518
 */
15101519
 static const char zFtsSchema[] =
15111520
 @ -- One entry for each possible search result
15121521
 @ CREATE TABLE IF NOT EXISTS repository.ftsdocs(
1513
-@ rowid INTEGER PRIMARY KEY, -- Maps to the ftsidx.docid
1522
+@ rowid INTEGER PRIMARY KEY, -- Maps to the ftsidx.rowid
15141523
 @ type CHAR(1), -- Type of document
15151524
 @ rid INTEGER, -- BLOB.RID or TAG.TAGID for the document
15161525
 @ name TEXT, -- Additional document description
15171526
 @ idxed BOOLEAN, -- True if currently in the index
15181527
 @ label TEXT, -- Label to print on search results
@@ -1526,11 +1535,11 @@
15261535
 @ CREATE VIEW IF NOT EXISTS repository.ftscontent AS
15271536
 @ SELECT rowid, type, rid, name, idxed, label, url, mtime,
15281537
 @ title(type,rid,name) AS 'title', body(type,rid,name) AS 'body'
15291538
 @ FROM ftsdocs;
15301539
 @ CREATE VIRTUAL TABLE IF NOT EXISTS repository.ftsidx
1531
-@ USING fts4(content="ftscontent", title, body%s);
1540
+@ USING fts5(content="ftscontent", title, body%s);
15321541
 ;
15331542
 static const char zFtsDrop[] =
15341543
 @ DROP TABLE IF EXISTS repository.ftsidx;
15351544
 @ DROP VIEW IF EXISTS repository.ftscontent;
15361545
 @ DROP TABLE IF EXISTS repository.ftsdocs;
@@ -1551,18 +1560,41 @@
15511560
 db_multi_exec(zFtsDrop/*works-like:""*/);
15521561
 searchIdxExists = 0;
15531562
 }
15541563
15551564
 /*
1556
-** Return true if the full-text search index exists
1565
+** Return true if the full-text search index exists. See also the
1566
+** search_index_type() function.
15571567
 */
15581568
 int search_index_exists(void){
15591569
 if( searchIdxExists<0 ){
15601570
 searchIdxExists = db_table_exists("repository","ftsdocs");
15611571
 }
15621572
 return searchIdxExists;
15631573
 }
1574
+
1575
+/*
1576
+** Determine which full-text search index is currently being used to
1577
+** add searching. Return values:
1578
+**
1579
+** 0 No search index is available
1580
+** 4 FTS3/4
1581
+** 5 FTS5
1582
+**
1583
+** Results are cached. Make the argument 1 to reset the cache. See
1584
+** also the search_index_exists() routine.
1585
+*/
1586
+int search_index_type(int bReset){
1587
+ static int idxType = -1;
1588
+ if( idxType<0 || bReset ){
1589
+ idxType = db_int(0,
1590
+ "SELECT CASE WHEN sql GLOB '*fts4*' THEN 4 ELSE 5 END"
1591
+ " FROM repository.sqlite_schema WHERE name='ftsidx'"
1592
+ );
1593
+ }
1594
+ return idxType;
1595
+}
15641596
15651597
 /*
15661598
 ** Fill the FTSDOCS table with unindexed entries for everything
15671599
 ** in the repository. This uses INSERT OR IGNORE so entries already
15681600
 ** in FTSDOCS are unchanged.
@@ -1606,11 +1638,11 @@
16061638
 char zType[2];
16071639
 zType[0] = cType;
16081640
 zType[1] = 0;
16091641
 search_sql_setup(g.db);
16101642
 db_multi_exec(
1611
- "DELETE FROM ftsidx WHERE docid IN"
1643
+ "DELETE FROM ftsidx WHERE rowid IN"
16121644
 " (SELECT rowid FROM ftsdocs WHERE type=%Q AND rid=%d AND idxed)",
16131645
 zType, rid
16141646
 );
16151647
 db_multi_exec(
16161648
 "REPLACE INTO ftsdocs(type,rid,name,idxed)"
@@ -1617,11 +1649,11 @@
16171649
 " VALUES(%Q,%d,%Q,0)",
16181650
 zType, rid, zName
16191651
 );
16201652
 if( cType=='w' || cType=='e' ){
16211653
 db_multi_exec(
1622
- "DELETE FROM ftsidx WHERE docid IN"
1654
+ "DELETE FROM ftsidx WHERE rowid IN"
16231655
 " (SELECT rowid FROM ftsdocs WHERE type='%c' AND name=%Q AND idxed)",
16241656
 cType, zName
16251657
 );
16261658
 db_multi_exec(
16271659
 "DELETE FROM ftsdocs WHERE type='%c' AND name=%Q AND rid!=%d",
@@ -1657,11 +1689,11 @@
16571689
 " WHERE foci.checkinID=%d AND blob.uuid=foci.uuid"
16581690
 " AND %z",
16591691
 ckid, glob_expr("foci.filename", db_get("doc-glob",""))
16601692
 );
16611693
 db_multi_exec(
1662
- "DELETE FROM ftsidx WHERE docid IN"
1694
+ "DELETE FROM ftsidx WHERE rowid IN"
16631695
 " (SELECT rowid FROM ftsdocs WHERE type='d'"
16641696
 " AND rid NOT IN (SELECT rid FROM current_docs))"
16651697
 );
16661698
 db_multi_exec(
16671699
 "DELETE FROM ftsdocs WHERE type='d'"
@@ -1676,11 +1708,11 @@
16761708
 " %.17g"
16771709
 " FROM current_docs",
16781710
 zDocBr, rTime
16791711
 );
16801712
 db_multi_exec(
1681
- "INSERT INTO ftsidx(docid,title,body)"
1713
+ "INSERT INTO ftsidx(rowid,title,body)"
16821714
 " SELECT rowid, label, bx FROM ftsdocs WHERE type='d' AND NOT idxed"
16831715
 );
16841716
 db_multi_exec(
16851717
 "UPDATE ftsdocs SET"
16861718
 " idxed=1,"
@@ -1693,11 +1725,11 @@
16931725
 /*
16941726
 ** Deal with all of the unindexed 'c' terms in FTSDOCS
16951727
 */
16961728
 static void search_update_checkin_index(void){
16971729
 db_multi_exec(
1698
- "INSERT INTO ftsidx(docid,title,body)"
1730
+ "INSERT INTO ftsidx(rowid,title,body)"
16991731
 " SELECT rowid, '', body('c',rid,NULL) FROM ftsdocs"
17001732
 " WHERE type='c' AND NOT idxed;"
17011733
 );
17021734
 db_multi_exec(
17031735
 "UPDATE ftsdocs SET idxed=1, name=NULL,"
@@ -1716,11 +1748,11 @@
17161748
 /*
17171749
 ** Deal with all of the unindexed 't' terms in FTSDOCS
17181750
 */
17191751
 static void search_update_ticket_index(void){
17201752
 db_multi_exec(
1721
- "INSERT INTO ftsidx(docid,title,body)"
1753
+ "INSERT INTO ftsidx(rowid,title,body)"
17221754
 " SELECT rowid, title('t',rid,NULL), body('t',rid,NULL) FROM ftsdocs"
17231755
 " WHERE type='t' AND NOT idxed;"
17241756
 );
17251757
 if( db_changes()==0 ) return;
17261758
 db_multi_exec(
@@ -1739,11 +1771,11 @@
17391771
 /*
17401772
 ** Deal with all of the unindexed 'w' terms in FTSDOCS
17411773
 */
17421774
 static void search_update_wiki_index(void){
17431775
 db_multi_exec(
1744
- "INSERT INTO ftsidx(docid,title,body)"
1776
+ "INSERT INTO ftsidx(rowid,title,body)"
17451777
 " SELECT rowid, title('w',rid,NULL),body('w',rid,NULL) FROM ftsdocs"
17461778
 " WHERE type='w' AND NOT idxed;"
17471779
 );
17481780
 if( db_changes()==0 ) return;
17491781
 db_multi_exec(
@@ -1761,11 +1793,11 @@
17611793
 /*
17621794
 ** Deal with all of the unindexed 'f' terms in FTSDOCS
17631795
 */
17641796
 static void search_update_forum_index(void){
17651797
 db_multi_exec(
1766
- "INSERT INTO ftsidx(docid,title,body)"
1798
+ "INSERT INTO ftsidx(rowid,title,body)"
17671799
 " SELECT rowid, title('f',rid,NULL),body('f',rid,NULL) FROM ftsdocs"
17681800
 " WHERE type='f' AND NOT idxed;"
17691801
 );
17701802
 if( db_changes()==0 ) return;
17711803
 db_multi_exec(
@@ -1784,11 +1816,11 @@
17841816
 /*
17851817
 ** Deal with all of the unindexed 'e' terms in FTSDOCS
17861818
 */
17871819
 static void search_update_technote_index(void){
17881820
 db_multi_exec(
1789
- "INSERT INTO ftsidx(docid,title,body)"
1821
+ "INSERT INTO ftsidx(rowid,title,body)"
17901822
 " SELECT rowid, title('e',rid,NULL),body('e',rid,NULL) FROM ftsdocs"
17911823
 " WHERE type='e' AND NOT idxed;"
17921824
 );
17931825
 if( db_changes()==0 ) return;
17941826
 db_multi_exec(
@@ -1869,11 +1901,11 @@
18691901
 **
18701902
 ** The current search settings are displayed after any changes are applied.
18711903
 ** Run this command with no arguments to simply see the settings.
18721904
 */
18731905
 void fts_config_cmd(void){
1874
- static const struct {
1906
+ static const struct {
18751907
 int iCmd;
18761908
 const char *z;
18771909
 } aCmd[] = {
18781910
 { 1, "reindex" },
18791911
 { 2, "index" },
@@ -1957,11 +1989,11 @@
19571989
 db_get_boolean(aSetng[i].zSetting,0) ? "on" : "off");
19581990
 }
19591991
 fossil_print("%-17s %s\n", "Porter stemmer:",
19601992
 db_get_boolean("search-stemmer",0) ? "on" : "off");
19611993
 if( search_index_exists() ){
1962
- fossil_print("%-17s enabled\n", "full-text index:");
1994
+ fossil_print("%-17s FTS%d\n", "full-text index:", search_index_type(1));
19631995
 fossil_print("%-17s %d\n", "documents:",
19641996
 db_int(0, "SELECT count(*) FROM ftsdocs"));
19651997
 }else{
19661998
 fossil_print("%-17s disabled\n", "full-text index:");
19671999
 }
@@ -2002,24 +2034,24 @@
20022034
 const char *zUrl = db_column_text(&q,4);
20032035
 const char *zDocId = db_column_text(&q,0);
20042036
 char *zName;
20052037
 char *z;
20062038
 @ <table border=0>
2007
- @ <tr><td align='right'>docid:<td>&nbsp;&nbsp;<td>%d(id)
2039
+ @ <tr><td align='right'>rowid:<td>&nbsp;&nbsp;<td>%d(id)
20082040
 @ <tr><td align='right'>id:<td><td>%s(zDocId)
20092041
 @ <tr><td align='right'>name:<td><td>%h(db_column_text(&q,1))
20102042
 @ <tr><td align='right'>idxed:<td><td>%d(db_column_int(&q,2))
20112043
 @ <tr><td align='right'>label:<td><td>%h(db_column_text(&q,3))
20122044
 @ <tr><td align='right'>url:<td><td>
20132045
 @ <a href='%R%s(zUrl)'>%h(zUrl)</a>
20142046
 @ <tr><td align='right'>mtime:<td><td>%s(db_column_text(&q,5))
2015
- z = db_text(0, "SELECT title FROM ftsidx WHERE docid=%d",id);
2047
+ z = db_text(0, "SELECT title FROM ftsidx WHERE rowid=%d",id);
20162048
 if( z && z[0] ){
20172049
 @ <tr><td align="right">title:<td><td>%h(z)
20182050
 fossil_free(z);
20192051
 }
2020
- z = db_text(0, "SELECT body FROM ftsidx WHERE docid=%d",id);
2052
+ z = db_text(0, "SELECT body FROM ftsidx WHERE rowid=%d",id);
20212053
 if( z && z[0] ){
20222054
 @ <tr><td align="right" valign="top">body:<td><td>%h(z)
20232055
 fossil_free(z);
20242056
 }
20252057
 @ </table>
@@ -2103,5 +2135,382 @@
21032135
 @ <th align="right">%d(cnt3)
21042136
 @ </tfooter>
21052137
 @ </table>
21062138
 style_finish_page();
21072139
 }
2140
+
2141
+
2142
+/*
2143
+** The Fts5MatchinfoCtx bits were all taken verbatim from:
2144
+**
2145
+** https://sqlite.org/src/finfo?name=ext/fts5/fts5_test_mi.c
2146
+*/
2147
+
2148
+typedef struct Fts5MatchinfoCtx Fts5MatchinfoCtx;
2149
+
2150
+#ifndef SQLITE_AMALGAMATION
2151
+typedef unsigned int u32;
2152
+#endif
2153
+
2154
+struct Fts5MatchinfoCtx {
2155
+ int nCol; /* Number of cols in FTS5 table */
2156
+ int nPhrase; /* Number of phrases in FTS5 query */
2157
+ char *zArg; /* nul-term'd copy of 2nd arg */
2158
+ int nRet; /* Number of elements in aRet[] */
2159
+ u32 *aRet; /* Array of 32-bit unsigned ints to return */
2160
+};
2161
+
2162
+
2163
+/*
2164
+** Return a pointer to the fts5_api pointer for database connection db.
2165
+** If an error occurs, return NULL and leave an error in the database
2166
+** handle (accessible using sqlite3_errcode()/errmsg()).
2167
+*/
2168
+static int fts5_api_from_db(sqlite3 *db, fts5_api **ppApi){
2169
+ sqlite3_stmt *pStmt = 0;
2170
+ int rc;
2171
+
2172
+ *ppApi = 0;
2173
+ rc = sqlite3_prepare(db, "SELECT fts5(?1)", -1, &pStmt, 0);
2174
+ if( rc==SQLITE_OK ){
2175
+ sqlite3_bind_pointer(pStmt, 1, (void*)ppApi, "fts5_api_ptr", 0);
2176
+ (void)sqlite3_step(pStmt);
2177
+ rc = sqlite3_finalize(pStmt);
2178
+ }
2179
+
2180
+ return rc;
2181
+}
2182
+
2183
+
2184
+/*
2185
+** Argument f should be a flag accepted by matchinfo() (a valid character
2186
+** in the string passed as the second argument). If it is not, -1 is
2187
+** returned. Otherwise, if f is a valid matchinfo flag, the value returned
2188
+** is the number of 32-bit integers added to the output array if the
2189
+** table has nCol columns and the query nPhrase phrases.
2190
+*/
2191
+static int fts5MatchinfoFlagsize(int nCol, int nPhrase, char f){
2192
+ int ret = -1;
2193
+ switch( f ){
2194
+ case 'p': ret = 1; break;
2195
+ case 'c': ret = 1; break;
2196
+ case 'x': ret = 3 * nCol * nPhrase; break;
2197
+ case 'y': ret = nCol * nPhrase; break;
2198
+ case 'b': ret = ((nCol + 31) / 32) * nPhrase; break;
2199
+ case 'n': ret = 1; break;
2200
+ case 'a': ret = nCol; break;
2201
+ case 'l': ret = nCol; break;
2202
+ case 's': ret = nCol; break;
2203
+ }
2204
+ return ret;
2205
+}
2206
+
2207
+static int fts5MatchinfoIter(
2208
+ const Fts5ExtensionApi *pApi, /* API offered by current FTS version */
2209
+ Fts5Context *pFts, /* First arg to pass to pApi functions */
2210
+ Fts5MatchinfoCtx *p,
2211
+ int(*x)(const Fts5ExtensionApi*,Fts5Context*,Fts5MatchinfoCtx*,char,u32*)
2212
+){
2213
+ int i;
2214
+ int n = 0;
2215
+ int rc = SQLITE_OK;
2216
+ char f;
2217
+ for(i=0; (f = p->zArg[i]); i++){
2218
+ rc = x(pApi, pFts, p, f, &p->aRet[n]);
2219
+ if( rc!=SQLITE_OK ) break;
2220
+ n += fts5MatchinfoFlagsize(p->nCol, p->nPhrase, f);
2221
+ }
2222
+ return rc;
2223
+}
2224
+
2225
+static int fts5MatchinfoXCb(
2226
+ const Fts5ExtensionApi *pApi,
2227
+ Fts5Context *pFts,
2228
+ void *pUserData
2229
+){
2230
+ Fts5PhraseIter iter;
2231
+ int iCol, iOff;
2232
+ u32 *aOut = (u32*)pUserData;
2233
+ int iPrev = -1;
2234
+
2235
+ for(pApi->xPhraseFirst(pFts, 0, &iter, &iCol, &iOff);
2236
+ iCol>=0;
2237
+ pApi->xPhraseNext(pFts, &iter, &iCol, &iOff)
2238
+ ){
2239
+ aOut[iCol*3+1]++;
2240
+ if( iCol!=iPrev ) aOut[iCol*3 + 2]++;
2241
+ iPrev = iCol;
2242
+ }
2243
+
2244
+ return SQLITE_OK;
2245
+}
2246
+
2247
+static int fts5MatchinfoGlobalCb(
2248
+ const Fts5ExtensionApi *pApi,
2249
+ Fts5Context *pFts,
2250
+ Fts5MatchinfoCtx *p,
2251
+ char f,
2252
+ u32 *aOut
2253
+){
2254
+ int rc = SQLITE_OK;
2255
+ switch( f ){
2256
+ case 'p':
2257
+ aOut[0] = p->nPhrase;
2258
+ break;
2259
+
2260
+ case 'c':
2261
+ aOut[0] = p->nCol;
2262
+ break;
2263
+
2264
+ case 'x': {
2265
+ int i;
2266
+ for(i=0; i<p->nPhrase && rc==SQLITE_OK; i++){
2267
+ void *pPtr = (void*)&aOut[i * p->nCol * 3];
2268
+ rc = pApi->xQueryPhrase(pFts, i, pPtr, fts5MatchinfoXCb);
2269
+ }
2270
+ break;
2271
+ }
2272
+
2273
+ case 'n': {
2274
+ sqlite3_int64 nRow;
2275
+ rc = pApi->xRowCount(pFts, &nRow);
2276
+ aOut[0] = (u32)nRow;
2277
+ break;
2278
+ }
2279
+
2280
+ case 'a': {
2281
+ sqlite3_int64 nRow = 0;
2282
+ rc = pApi->xRowCount(pFts, &nRow);
2283
+ if( nRow==0 ){
2284
+ memset(aOut, 0, sizeof(u32) * p->nCol);
2285
+ }else{
2286
+ int i;
2287
+ for(i=0; rc==SQLITE_OK && i<p->nCol; i++){
2288
+ sqlite3_int64 nToken;
2289
+ rc = pApi->xColumnTotalSize(pFts, i, &nToken);
2290
+ if( rc==SQLITE_OK){
2291
+ aOut[i] = (u32)((2*nToken + nRow) / (2*nRow));
2292
+ }
2293
+ }
2294
+ }
2295
+ break;
2296
+ }
2297
+
2298
+ }
2299
+ return rc;
2300
+}
2301
+
2302
+static int fts5MatchinfoLocalCb(
2303
+ const Fts5ExtensionApi *pApi,
2304
+ Fts5Context *pFts,
2305
+ Fts5MatchinfoCtx *p,
2306
+ char f,
2307
+ u32 *aOut
2308
+){
2309
+ int i;
2310
+ int rc = SQLITE_OK;
2311
+
2312
+ switch( f ){
2313
+ case 'b': {
2314
+ int iPhrase;
2315
+ int nInt = ((p->nCol + 31) / 32) * p->nPhrase;
2316
+ for(i=0; i<nInt; i++) aOut[i] = 0;
2317
+
2318
+ for(iPhrase=0; iPhrase<p->nPhrase; iPhrase++){
2319
+ Fts5PhraseIter iter;
2320
+ int iCol;
2321
+ for(pApi->xPhraseFirstColumn(pFts, iPhrase, &iter, &iCol);
2322
+ iCol>=0;
2323
+ pApi->xPhraseNextColumn(pFts, &iter, &iCol)
2324
+ ){
2325
+ aOut[iPhrase * ((p->nCol+31)/32) + iCol/32] |= ((u32)1 << iCol%32);
2326
+ }
2327
+ }
2328
+
2329
+ break;
2330
+ }
2331
+
2332
+ case 'x':
2333
+ case 'y': {
2334
+ int nMul = (f=='x' ? 3 : 1);
2335
+ int iPhrase;
2336
+
2337
+ for(i=0; i<(p->nCol*p->nPhrase); i++) aOut[i*nMul] = 0;
2338
+
2339
+ for(iPhrase=0; iPhrase<p->nPhrase; iPhrase++){
2340
+ Fts5PhraseIter iter;
2341
+ int iOff, iCol;
2342
+ for(pApi->xPhraseFirst(pFts, iPhrase, &iter, &iCol, &iOff);
2343
+ iOff>=0;
2344
+ pApi->xPhraseNext(pFts, &iter, &iCol, &iOff)
2345
+ ){
2346
+ aOut[nMul * (iCol + iPhrase * p->nCol)]++;
2347
+ }
2348
+ }
2349
+
2350
+ break;
2351
+ }
2352
+
2353
+ case 'l': {
2354
+ for(i=0; rc==SQLITE_OK && i<p->nCol; i++){
2355
+ int nToken;
2356
+ rc = pApi->xColumnSize(pFts, i, &nToken);
2357
+ aOut[i] = (u32)nToken;
2358
+ }
2359
+ break;
2360
+ }
2361
+
2362
+ case 's': {
2363
+ int nInst;
2364
+
2365
+ memset(aOut, 0, sizeof(u32) * p->nCol);
2366
+
2367
+ rc = pApi->xInstCount(pFts, &nInst);
2368
+ for(i=0; rc==SQLITE_OK && i<nInst; i++){
2369
+ int iPhrase, iOff, iCol = 0;
2370
+ int iNextPhrase;
2371
+ int iNextOff;
2372
+ u32 nSeq = 1;
2373
+ int j;
2374
+
2375
+ rc = pApi->xInst(pFts, i, &iPhrase, &iCol, &iOff);
2376
+ iNextPhrase = iPhrase+1;
2377
+ iNextOff = iOff+pApi->xPhraseSize(pFts, 0);
2378
+ for(j=i+1; rc==SQLITE_OK && j<nInst; j++){
2379
+ int ip, ic, io;
2380
+ rc = pApi->xInst(pFts, j, &ip, &ic, &io);
2381
+ if( ic!=iCol || io>iNextOff ) break;
2382
+ if( ip==iNextPhrase && io==iNextOff ){
2383
+ nSeq++;
2384
+ iNextPhrase = ip+1;
2385
+ iNextOff = io + pApi->xPhraseSize(pFts, ip);
2386
+ }
2387
+ }
2388
+
2389
+ if( nSeq>aOut[iCol] ) aOut[iCol] = nSeq;
2390
+ }
2391
+
2392
+ break;
2393
+ }
2394
+ }
2395
+ return rc;
2396
+}
2397
+
2398
+static Fts5MatchinfoCtx *fts5MatchinfoNew(
2399
+ const Fts5ExtensionApi *pApi, /* API offered by current FTS version */
2400
+ Fts5Context *pFts, /* First arg to pass to pApi functions */
2401
+ sqlite3_context *pCtx, /* Context for returning error message */
2402
+ const char *zArg /* Matchinfo flag string */
2403
+){
2404
+ Fts5MatchinfoCtx *p;
2405
+ int nCol;
2406
+ int nPhrase;
2407
+ int i;
2408
+ int nInt;
2409
+ sqlite3_int64 nByte;
2410
+ int rc;
2411
+
2412
+ nCol = pApi->xColumnCount(pFts);
2413
+ nPhrase = pApi->xPhraseCount(pFts);
2414
+
2415
+ nInt = 0;
2416
+ for(i=0; zArg[i]; i++){
2417
+ int n = fts5MatchinfoFlagsize(nCol, nPhrase, zArg[i]);
2418
+ if( n<0 ){
2419
+ char *zErr = sqlite3_mprintf("unrecognized matchinfo flag: %c", zArg[i]);
2420
+ sqlite3_result_error(pCtx, zErr, -1);
2421
+ sqlite3_free(zErr);
2422
+ return 0;
2423
+ }
2424
+ nInt += n;
2425
+ }
2426
+
2427
+ nByte = sizeof(Fts5MatchinfoCtx) /* The struct itself */
2428
+ + sizeof(u32) * nInt /* The p->aRet[] array */
2429
+ + (i+1); /* The p->zArg string */
2430
+ p = (Fts5MatchinfoCtx*)sqlite3_malloc64(nByte);
2431
+ if( p==0 ){
2432
+ sqlite3_result_error_nomem(pCtx);
2433
+ return 0;
2434
+ }
2435
+ memset(p, 0, nByte);
2436
+
2437
+ p->nCol = nCol;
2438
+ p->nPhrase = nPhrase;
2439
+ p->aRet = (u32*)&p[1];
2440
+ p->nRet = nInt;
2441
+ p->zArg = (char*)&p->aRet[nInt];
2442
+ memcpy(p->zArg, zArg, i);
2443
+
2444
+ rc = fts5MatchinfoIter(pApi, pFts, p, fts5MatchinfoGlobalCb);
2445
+ if( rc!=SQLITE_OK ){
2446
+ sqlite3_result_error_code(pCtx, rc);
2447
+ sqlite3_free(p);
2448
+ p = 0;
2449
+ }
2450
+
2451
+ return p;
2452
+}
2453
+
2454
+static void fts5MatchinfoFunc(
2455
+ const Fts5ExtensionApi *pApi, /* API offered by current FTS version */
2456
+ Fts5Context *pFts, /* First arg to pass to pApi functions */
2457
+ sqlite3_context *pCtx, /* Context for returning result/error */
2458
+ int nVal, /* Number of values in apVal[] array */
2459
+ sqlite3_value **apVal /* Array of trailing arguments */
2460
+){
2461
+ const char *zArg;
2462
+ Fts5MatchinfoCtx *p;
2463
+ int rc = SQLITE_OK;
2464
+
2465
+ if( nVal>0 ){
2466
+ zArg = (const char*)sqlite3_value_text(apVal[0]);
2467
+ }else{
2468
+ zArg = "pcx";
2469
+ }
2470
+
2471
+ p = (Fts5MatchinfoCtx*)pApi->xGetAuxdata(pFts, 0);
2472
+ if( p==0 || sqlite3_stricmp(zArg, p->zArg) ){
2473
+ p = fts5MatchinfoNew(pApi, pFts, pCtx, zArg);
2474
+ if( p==0 ){
2475
+ rc = SQLITE_NOMEM;
2476
+ }else{
2477
+ rc = pApi->xSetAuxdata(pFts, p, sqlite3_free);
2478
+ }
2479
+ }
2480
+
2481
+ if( rc==SQLITE_OK ){
2482
+ rc = fts5MatchinfoIter(pApi, pFts, p, fts5MatchinfoLocalCb);
2483
+ }
2484
+ if( rc!=SQLITE_OK ){
2485
+ sqlite3_result_error_code(pCtx, rc);
2486
+ }else{
2487
+ /* No errors has occured, so return a copy of the array of integers. */
2488
+ int nByte = p->nRet * sizeof(u32);
2489
+ sqlite3_result_blob(pCtx, (void*)p->aRet, nByte, SQLITE_TRANSIENT);
2490
+ }
2491
+}
2492
+
2493
+int db_register_fts5(sqlite3 *db){
2494
+ int rc; /* Return code */
2495
+ fts5_api *pApi; /* FTS5 API functions */
2496
+
2497
+ /* Extract the FTS5 API pointer from the database handle. The
2498
+ ** fts5_api_from_db() function above is copied verbatim from the
2499
+ ** FTS5 documentation. Refer there for details. */
2500
+ rc = fts5_api_from_db(db, &pApi);
2501
+ if( rc!=SQLITE_OK ) return rc;
2502
+
2503
+ /* If fts5_api_from_db() returns NULL, then either FTS5 is not registered
2504
+ ** with this database handle, or an error (OOM perhaps?) has occurred.
2505
+ **
2506
+ ** Also check that the fts5_api object is version 2 or newer.
2507
+ */
2508
+ if( pApi==0 || pApi->iVersion<2 ){
2509
+ return SQLITE_ERROR;
2510
+ }
2511
+
2512
+ /* Register the implementation of matchinfo() */
2513
+ rc = pApi->xCreateFunction(pApi, "matchinfo", 0, fts5MatchinfoFunc, 0);
2514
+
2515
+ return rc;
2516
+}
21082517
--- src/search.c
+++ src/search.c
@@ -921,29 +921,38 @@
921 unsigned int srchFlags /* What to search over */
922 ){
923 Blob sql;
924 char *zPat = mprintf("%s",zPattern);
925 int i;
 
926 if( srchFlags==0 ) return;
927 sqlite3_create_function(g.db, "rank", 1, SQLITE_UTF8|SQLITE_INNOCUOUS, 0,
928 search_rank_sqlfunc, 0, 0);
929 for(i=0; zPat[i]; i++){
930 if( zPat[i]=='-' || zPat[i]=='"' ) zPat[i] = ' ';
931 }
932 blob_init(&sql, 0, 0);
 
 
 
 
 
 
 
 
933 blob_appendf(&sql,
934 "INSERT INTO x(label,url,score,id,date,snip) "
935 " SELECT ftsdocs.label,"
936 " ftsdocs.url,"
937 " rank(matchinfo(ftsidx,'pcsx')),"
938 " ftsdocs.type || ftsdocs.rid,"
939 " datetime(ftsdocs.mtime),"
940 " snippet(ftsidx,'<mark>','</mark>',' ... ',-1,35)"
941 " FROM ftsidx CROSS JOIN ftsdocs"
942 " WHERE ftsidx MATCH %Q"
943 " AND ftsdocs.rowid=ftsidx.docid",
944 zPat
945 );
946 fossil_free(zPat);
947 if( srchFlags!=SRCH_ALL ){
948 const char *zSep = " AND (";
949 static const struct { unsigned m; char c; } aMask[] = {
@@ -1508,11 +1517,11 @@
1508 /* The schema for the full-text index
1509 */
1510 static const char zFtsSchema[] =
1511 @ -- One entry for each possible search result
1512 @ CREATE TABLE IF NOT EXISTS repository.ftsdocs(
1513 @ rowid INTEGER PRIMARY KEY, -- Maps to the ftsidx.docid
1514 @ type CHAR(1), -- Type of document
1515 @ rid INTEGER, -- BLOB.RID or TAG.TAGID for the document
1516 @ name TEXT, -- Additional document description
1517 @ idxed BOOLEAN, -- True if currently in the index
1518 @ label TEXT, -- Label to print on search results
@@ -1526,11 +1535,11 @@
1526 @ CREATE VIEW IF NOT EXISTS repository.ftscontent AS
1527 @ SELECT rowid, type, rid, name, idxed, label, url, mtime,
1528 @ title(type,rid,name) AS 'title', body(type,rid,name) AS 'body'
1529 @ FROM ftsdocs;
1530 @ CREATE VIRTUAL TABLE IF NOT EXISTS repository.ftsidx
1531 @ USING fts4(content="ftscontent", title, body%s);
1532 ;
1533 static const char zFtsDrop[] =
1534 @ DROP TABLE IF EXISTS repository.ftsidx;
1535 @ DROP VIEW IF EXISTS repository.ftscontent;
1536 @ DROP TABLE IF EXISTS repository.ftsdocs;
@@ -1551,18 +1560,41 @@
1551 db_multi_exec(zFtsDrop/*works-like:""*/);
1552 searchIdxExists = 0;
1553 }
1554
1555 /*
1556 ** Return true if the full-text search index exists
 
1557 */
1558 int search_index_exists(void){
1559 if( searchIdxExists<0 ){
1560 searchIdxExists = db_table_exists("repository","ftsdocs");
1561 }
1562 return searchIdxExists;
1563 }
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1564
1565 /*
1566 ** Fill the FTSDOCS table with unindexed entries for everything
1567 ** in the repository. This uses INSERT OR IGNORE so entries already
1568 ** in FTSDOCS are unchanged.
@@ -1606,11 +1638,11 @@
1606 char zType[2];
1607 zType[0] = cType;
1608 zType[1] = 0;
1609 search_sql_setup(g.db);
1610 db_multi_exec(
1611 "DELETE FROM ftsidx WHERE docid IN"
1612 " (SELECT rowid FROM ftsdocs WHERE type=%Q AND rid=%d AND idxed)",
1613 zType, rid
1614 );
1615 db_multi_exec(
1616 "REPLACE INTO ftsdocs(type,rid,name,idxed)"
@@ -1617,11 +1649,11 @@
1617 " VALUES(%Q,%d,%Q,0)",
1618 zType, rid, zName
1619 );
1620 if( cType=='w' || cType=='e' ){
1621 db_multi_exec(
1622 "DELETE FROM ftsidx WHERE docid IN"
1623 " (SELECT rowid FROM ftsdocs WHERE type='%c' AND name=%Q AND idxed)",
1624 cType, zName
1625 );
1626 db_multi_exec(
1627 "DELETE FROM ftsdocs WHERE type='%c' AND name=%Q AND rid!=%d",
@@ -1657,11 +1689,11 @@
1657 " WHERE foci.checkinID=%d AND blob.uuid=foci.uuid"
1658 " AND %z",
1659 ckid, glob_expr("foci.filename", db_get("doc-glob",""))
1660 );
1661 db_multi_exec(
1662 "DELETE FROM ftsidx WHERE docid IN"
1663 " (SELECT rowid FROM ftsdocs WHERE type='d'"
1664 " AND rid NOT IN (SELECT rid FROM current_docs))"
1665 );
1666 db_multi_exec(
1667 "DELETE FROM ftsdocs WHERE type='d'"
@@ -1676,11 +1708,11 @@
1676 " %.17g"
1677 " FROM current_docs",
1678 zDocBr, rTime
1679 );
1680 db_multi_exec(
1681 "INSERT INTO ftsidx(docid,title,body)"
1682 " SELECT rowid, label, bx FROM ftsdocs WHERE type='d' AND NOT idxed"
1683 );
1684 db_multi_exec(
1685 "UPDATE ftsdocs SET"
1686 " idxed=1,"
@@ -1693,11 +1725,11 @@
1693 /*
1694 ** Deal with all of the unindexed 'c' terms in FTSDOCS
1695 */
1696 static void search_update_checkin_index(void){
1697 db_multi_exec(
1698 "INSERT INTO ftsidx(docid,title,body)"
1699 " SELECT rowid, '', body('c',rid,NULL) FROM ftsdocs"
1700 " WHERE type='c' AND NOT idxed;"
1701 );
1702 db_multi_exec(
1703 "UPDATE ftsdocs SET idxed=1, name=NULL,"
@@ -1716,11 +1748,11 @@
1716 /*
1717 ** Deal with all of the unindexed 't' terms in FTSDOCS
1718 */
1719 static void search_update_ticket_index(void){
1720 db_multi_exec(
1721 "INSERT INTO ftsidx(docid,title,body)"
1722 " SELECT rowid, title('t',rid,NULL), body('t',rid,NULL) FROM ftsdocs"
1723 " WHERE type='t' AND NOT idxed;"
1724 );
1725 if( db_changes()==0 ) return;
1726 db_multi_exec(
@@ -1739,11 +1771,11 @@
1739 /*
1740 ** Deal with all of the unindexed 'w' terms in FTSDOCS
1741 */
1742 static void search_update_wiki_index(void){
1743 db_multi_exec(
1744 "INSERT INTO ftsidx(docid,title,body)"
1745 " SELECT rowid, title('w',rid,NULL),body('w',rid,NULL) FROM ftsdocs"
1746 " WHERE type='w' AND NOT idxed;"
1747 );
1748 if( db_changes()==0 ) return;
1749 db_multi_exec(
@@ -1761,11 +1793,11 @@
1761 /*
1762 ** Deal with all of the unindexed 'f' terms in FTSDOCS
1763 */
1764 static void search_update_forum_index(void){
1765 db_multi_exec(
1766 "INSERT INTO ftsidx(docid,title,body)"
1767 " SELECT rowid, title('f',rid,NULL),body('f',rid,NULL) FROM ftsdocs"
1768 " WHERE type='f' AND NOT idxed;"
1769 );
1770 if( db_changes()==0 ) return;
1771 db_multi_exec(
@@ -1784,11 +1816,11 @@
1784 /*
1785 ** Deal with all of the unindexed 'e' terms in FTSDOCS
1786 */
1787 static void search_update_technote_index(void){
1788 db_multi_exec(
1789 "INSERT INTO ftsidx(docid,title,body)"
1790 " SELECT rowid, title('e',rid,NULL),body('e',rid,NULL) FROM ftsdocs"
1791 " WHERE type='e' AND NOT idxed;"
1792 );
1793 if( db_changes()==0 ) return;
1794 db_multi_exec(
@@ -1869,11 +1901,11 @@
1869 **
1870 ** The current search settings are displayed after any changes are applied.
1871 ** Run this command with no arguments to simply see the settings.
1872 */
1873 void fts_config_cmd(void){
1874 static const struct {
1875 int iCmd;
1876 const char *z;
1877 } aCmd[] = {
1878 { 1, "reindex" },
1879 { 2, "index" },
@@ -1957,11 +1989,11 @@
1957 db_get_boolean(aSetng[i].zSetting,0) ? "on" : "off");
1958 }
1959 fossil_print("%-17s %s\n", "Porter stemmer:",
1960 db_get_boolean("search-stemmer",0) ? "on" : "off");
1961 if( search_index_exists() ){
1962 fossil_print("%-17s enabled\n", "full-text index:");
1963 fossil_print("%-17s %d\n", "documents:",
1964 db_int(0, "SELECT count(*) FROM ftsdocs"));
1965 }else{
1966 fossil_print("%-17s disabled\n", "full-text index:");
1967 }
@@ -2002,24 +2034,24 @@
2002 const char *zUrl = db_column_text(&q,4);
2003 const char *zDocId = db_column_text(&q,0);
2004 char *zName;
2005 char *z;
2006 @ <table border=0>
2007 @ <tr><td align='right'>docid:<td>&nbsp;&nbsp;<td>%d(id)
2008 @ <tr><td align='right'>id:<td><td>%s(zDocId)
2009 @ <tr><td align='right'>name:<td><td>%h(db_column_text(&q,1))
2010 @ <tr><td align='right'>idxed:<td><td>%d(db_column_int(&q,2))
2011 @ <tr><td align='right'>label:<td><td>%h(db_column_text(&q,3))
2012 @ <tr><td align='right'>url:<td><td>
2013 @ <a href='%R%s(zUrl)'>%h(zUrl)</a>
2014 @ <tr><td align='right'>mtime:<td><td>%s(db_column_text(&q,5))
2015 z = db_text(0, "SELECT title FROM ftsidx WHERE docid=%d",id);
2016 if( z && z[0] ){
2017 @ <tr><td align="right">title:<td><td>%h(z)
2018 fossil_free(z);
2019 }
2020 z = db_text(0, "SELECT body FROM ftsidx WHERE docid=%d",id);
2021 if( z && z[0] ){
2022 @ <tr><td align="right" valign="top">body:<td><td>%h(z)
2023 fossil_free(z);
2024 }
2025 @ </table>
@@ -2103,5 +2135,382 @@
2103 @ <th align="right">%d(cnt3)
2104 @ </tfooter>
2105 @ </table>
2106 style_finish_page();
2107 }
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
2108
--- src/search.c
+++ src/search.c
@@ -921,29 +921,38 @@
921 unsigned int srchFlags /* What to search over */
922 ){
923 Blob sql;
924 char *zPat = mprintf("%s",zPattern);
925 int i;
926 static const char *zSnippetCall;
927 if( srchFlags==0 ) return;
928 sqlite3_create_function(g.db, "rank", 1, SQLITE_UTF8|SQLITE_INNOCUOUS, 0,
929 search_rank_sqlfunc, 0, 0);
930 for(i=0; zPat[i]; i++){
931 if( zPat[i]=='-' || zPat[i]=='"' ) zPat[i] = ' ';
932 }
933 blob_init(&sql, 0, 0);
934 if( search_index_type(0)==4 ){
935 /* If this repo is still using the legacy FTS4 search index, then
936 ** the snippet() function is slightly different */
937 zSnippetCall = "snippet(ftsidx,'<mark>','</mark>',' ... ',-1,35)";
938 }else{
939 /* This is the common case - Using newer FTS5 search index */
940 zSnippetCall = "snippet(ftsidx,-1,'<mark>','</mark>',' ... ',35)";
941 }
942 blob_appendf(&sql,
943 "INSERT INTO x(label,url,score,id,date,snip) "
944 " SELECT ftsdocs.label,"
945 " ftsdocs.url,"
946 " rank(matchinfo(ftsidx,'pcsx')),"
947 " ftsdocs.type || ftsdocs.rid,"
948 " datetime(ftsdocs.mtime),"
949 " %s"
950 " FROM ftsidx CROSS JOIN ftsdocs"
951 " WHERE ftsidx MATCH %Q"
952 " AND ftsdocs.rowid=ftsidx.rowid",
953 zSnippetCall /*safe-for-%s*/, zPat
954 );
955 fossil_free(zPat);
956 if( srchFlags!=SRCH_ALL ){
957 const char *zSep = " AND (";
958 static const struct { unsigned m; char c; } aMask[] = {
@@ -1508,11 +1517,11 @@
1517 /* The schema for the full-text index
1518 */
1519 static const char zFtsSchema[] =
1520 @ -- One entry for each possible search result
1521 @ CREATE TABLE IF NOT EXISTS repository.ftsdocs(
1522 @ rowid INTEGER PRIMARY KEY, -- Maps to the ftsidx.rowid
1523 @ type CHAR(1), -- Type of document
1524 @ rid INTEGER, -- BLOB.RID or TAG.TAGID for the document
1525 @ name TEXT, -- Additional document description
1526 @ idxed BOOLEAN, -- True if currently in the index
1527 @ label TEXT, -- Label to print on search results
@@ -1526,11 +1535,11 @@
1535 @ CREATE VIEW IF NOT EXISTS repository.ftscontent AS
1536 @ SELECT rowid, type, rid, name, idxed, label, url, mtime,
1537 @ title(type,rid,name) AS 'title', body(type,rid,name) AS 'body'
1538 @ FROM ftsdocs;
1539 @ CREATE VIRTUAL TABLE IF NOT EXISTS repository.ftsidx
1540 @ USING fts5(content="ftscontent", title, body%s);
1541 ;
1542 static const char zFtsDrop[] =
1543 @ DROP TABLE IF EXISTS repository.ftsidx;
1544 @ DROP VIEW IF EXISTS repository.ftscontent;
1545 @ DROP TABLE IF EXISTS repository.ftsdocs;
@@ -1551,18 +1560,41 @@
1560 db_multi_exec(zFtsDrop/*works-like:""*/);
1561 searchIdxExists = 0;
1562 }
1563
1564 /*
1565 ** Return true if the full-text search index exists. See also the
1566 ** search_index_type() function.
1567 */
1568 int search_index_exists(void){
1569 if( searchIdxExists<0 ){
1570 searchIdxExists = db_table_exists("repository","ftsdocs");
1571 }
1572 return searchIdxExists;
1573 }
1574
1575 /*
1576 ** Determine which full-text search index is currently being used to
1577 ** add searching. Return values:
1578 **
1579 ** 0 No search index is available
1580 ** 4 FTS3/4
1581 ** 5 FTS5
1582 **
1583 ** Results are cached. Make the argument 1 to reset the cache. See
1584 ** also the search_index_exists() routine.
1585 */
1586 int search_index_type(int bReset){
1587 static int idxType = -1;
1588 if( idxType<0 || bReset ){
1589 idxType = db_int(0,
1590 "SELECT CASE WHEN sql GLOB '*fts4*' THEN 4 ELSE 5 END"
1591 " FROM repository.sqlite_schema WHERE name='ftsidx'"
1592 );
1593 }
1594 return idxType;
1595 }
1596
1597 /*
1598 ** Fill the FTSDOCS table with unindexed entries for everything
1599 ** in the repository. This uses INSERT OR IGNORE so entries already
1600 ** in FTSDOCS are unchanged.
@@ -1606,11 +1638,11 @@
1638 char zType[2];
1639 zType[0] = cType;
1640 zType[1] = 0;
1641 search_sql_setup(g.db);
1642 db_multi_exec(
1643 "DELETE FROM ftsidx WHERE rowid IN"
1644 " (SELECT rowid FROM ftsdocs WHERE type=%Q AND rid=%d AND idxed)",
1645 zType, rid
1646 );
1647 db_multi_exec(
1648 "REPLACE INTO ftsdocs(type,rid,name,idxed)"
@@ -1617,11 +1649,11 @@
1649 " VALUES(%Q,%d,%Q,0)",
1650 zType, rid, zName
1651 );
1652 if( cType=='w' || cType=='e' ){
1653 db_multi_exec(
1654 "DELETE FROM ftsidx WHERE rowid IN"
1655 " (SELECT rowid FROM ftsdocs WHERE type='%c' AND name=%Q AND idxed)",
1656 cType, zName
1657 );
1658 db_multi_exec(
1659 "DELETE FROM ftsdocs WHERE type='%c' AND name=%Q AND rid!=%d",
@@ -1657,11 +1689,11 @@
1689 " WHERE foci.checkinID=%d AND blob.uuid=foci.uuid"
1690 " AND %z",
1691 ckid, glob_expr("foci.filename", db_get("doc-glob",""))
1692 );
1693 db_multi_exec(
1694 "DELETE FROM ftsidx WHERE rowid IN"
1695 " (SELECT rowid FROM ftsdocs WHERE type='d'"
1696 " AND rid NOT IN (SELECT rid FROM current_docs))"
1697 );
1698 db_multi_exec(
1699 "DELETE FROM ftsdocs WHERE type='d'"
@@ -1676,11 +1708,11 @@
1708 " %.17g"
1709 " FROM current_docs",
1710 zDocBr, rTime
1711 );
1712 db_multi_exec(
1713 "INSERT INTO ftsidx(rowid,title,body)"
1714 " SELECT rowid, label, bx FROM ftsdocs WHERE type='d' AND NOT idxed"
1715 );
1716 db_multi_exec(
1717 "UPDATE ftsdocs SET"
1718 " idxed=1,"
@@ -1693,11 +1725,11 @@
1725 /*
1726 ** Deal with all of the unindexed 'c' terms in FTSDOCS
1727 */
1728 static void search_update_checkin_index(void){
1729 db_multi_exec(
1730 "INSERT INTO ftsidx(rowid,title,body)"
1731 " SELECT rowid, '', body('c',rid,NULL) FROM ftsdocs"
1732 " WHERE type='c' AND NOT idxed;"
1733 );
1734 db_multi_exec(
1735 "UPDATE ftsdocs SET idxed=1, name=NULL,"
@@ -1716,11 +1748,11 @@
1748 /*
1749 ** Deal with all of the unindexed 't' terms in FTSDOCS
1750 */
1751 static void search_update_ticket_index(void){
1752 db_multi_exec(
1753 "INSERT INTO ftsidx(rowid,title,body)"
1754 " SELECT rowid, title('t',rid,NULL), body('t',rid,NULL) FROM ftsdocs"
1755 " WHERE type='t' AND NOT idxed;"
1756 );
1757 if( db_changes()==0 ) return;
1758 db_multi_exec(
@@ -1739,11 +1771,11 @@
1771 /*
1772 ** Deal with all of the unindexed 'w' terms in FTSDOCS
1773 */
1774 static void search_update_wiki_index(void){
1775 db_multi_exec(
1776 "INSERT INTO ftsidx(rowid,title,body)"
1777 " SELECT rowid, title('w',rid,NULL),body('w',rid,NULL) FROM ftsdocs"
1778 " WHERE type='w' AND NOT idxed;"
1779 );
1780 if( db_changes()==0 ) return;
1781 db_multi_exec(
@@ -1761,11 +1793,11 @@
1793 /*
1794 ** Deal with all of the unindexed 'f' terms in FTSDOCS
1795 */
1796 static void search_update_forum_index(void){
1797 db_multi_exec(
1798 "INSERT INTO ftsidx(rowid,title,body)"
1799 " SELECT rowid, title('f',rid,NULL),body('f',rid,NULL) FROM ftsdocs"
1800 " WHERE type='f' AND NOT idxed;"
1801 );
1802 if( db_changes()==0 ) return;
1803 db_multi_exec(
@@ -1784,11 +1816,11 @@
1816 /*
1817 ** Deal with all of the unindexed 'e' terms in FTSDOCS
1818 */
1819 static void search_update_technote_index(void){
1820 db_multi_exec(
1821 "INSERT INTO ftsidx(rowid,title,body)"
1822 " SELECT rowid, title('e',rid,NULL),body('e',rid,NULL) FROM ftsdocs"
1823 " WHERE type='e' AND NOT idxed;"
1824 );
1825 if( db_changes()==0 ) return;
1826 db_multi_exec(
@@ -1869,11 +1901,11 @@
1901 **
1902 ** The current search settings are displayed after any changes are applied.
1903 ** Run this command with no arguments to simply see the settings.
1904 */
1905 void fts_config_cmd(void){
1906 static const struct {
1907 int iCmd;
1908 const char *z;
1909 } aCmd[] = {
1910 { 1, "reindex" },
1911 { 2, "index" },
@@ -1957,11 +1989,11 @@
1989 db_get_boolean(aSetng[i].zSetting,0) ? "on" : "off");
1990 }
1991 fossil_print("%-17s %s\n", "Porter stemmer:",
1992 db_get_boolean("search-stemmer",0) ? "on" : "off");
1993 if( search_index_exists() ){
1994 fossil_print("%-17s FTS%d\n", "full-text index:", search_index_type(1));
1995 fossil_print("%-17s %d\n", "documents:",
1996 db_int(0, "SELECT count(*) FROM ftsdocs"));
1997 }else{
1998 fossil_print("%-17s disabled\n", "full-text index:");
1999 }
@@ -2002,24 +2034,24 @@
2034 const char *zUrl = db_column_text(&q,4);
2035 const char *zDocId = db_column_text(&q,0);
2036 char *zName;
2037 char *z;
2038 @ <table border=0>
2039 @ <tr><td align='right'>rowid:<td>&nbsp;&nbsp;<td>%d(id)
2040 @ <tr><td align='right'>id:<td><td>%s(zDocId)
2041 @ <tr><td align='right'>name:<td><td>%h(db_column_text(&q,1))
2042 @ <tr><td align='right'>idxed:<td><td>%d(db_column_int(&q,2))
2043 @ <tr><td align='right'>label:<td><td>%h(db_column_text(&q,3))
2044 @ <tr><td align='right'>url:<td><td>
2045 @ <a href='%R%s(zUrl)'>%h(zUrl)</a>
2046 @ <tr><td align='right'>mtime:<td><td>%s(db_column_text(&q,5))
2047 z = db_text(0, "SELECT title FROM ftsidx WHERE rowid=%d",id);
2048 if( z && z[0] ){
2049 @ <tr><td align="right">title:<td><td>%h(z)
2050 fossil_free(z);
2051 }
2052 z = db_text(0, "SELECT body FROM ftsidx WHERE rowid=%d",id);
2053 if( z && z[0] ){
2054 @ <tr><td align="right" valign="top">body:<td><td>%h(z)
2055 fossil_free(z);
2056 }
2057 @ </table>
@@ -2103,5 +2135,382 @@
2135 @ <th align="right">%d(cnt3)
2136 @ </tfooter>
2137 @ </table>
2138 style_finish_page();
2139 }
2140
2141
2142 /*
2143 ** The Fts5MatchinfoCtx bits were all taken verbatim from:
2144 **
2145 ** https://sqlite.org/src/finfo?name=ext/fts5/fts5_test_mi.c
2146 */
2147
2148 typedef struct Fts5MatchinfoCtx Fts5MatchinfoCtx;
2149
2150 #ifndef SQLITE_AMALGAMATION
2151 typedef unsigned int u32;
2152 #endif
2153
2154 struct Fts5MatchinfoCtx {
2155 int nCol; /* Number of cols in FTS5 table */
2156 int nPhrase; /* Number of phrases in FTS5 query */
2157 char *zArg; /* nul-term'd copy of 2nd arg */
2158 int nRet; /* Number of elements in aRet[] */
2159 u32 *aRet; /* Array of 32-bit unsigned ints to return */
2160 };
2161
2162
2163 /*
2164 ** Return a pointer to the fts5_api pointer for database connection db.
2165 ** If an error occurs, return NULL and leave an error in the database
2166 ** handle (accessible using sqlite3_errcode()/errmsg()).
2167 */
2168 static int fts5_api_from_db(sqlite3 *db, fts5_api **ppApi){
2169 sqlite3_stmt *pStmt = 0;
2170 int rc;
2171
2172 *ppApi = 0;
2173 rc = sqlite3_prepare(db, "SELECT fts5(?1)", -1, &pStmt, 0);
2174 if( rc==SQLITE_OK ){
2175 sqlite3_bind_pointer(pStmt, 1, (void*)ppApi, "fts5_api_ptr", 0);
2176 (void)sqlite3_step(pStmt);
2177 rc = sqlite3_finalize(pStmt);
2178 }
2179
2180 return rc;
2181 }
2182
2183
2184 /*
2185 ** Argument f should be a flag accepted by matchinfo() (a valid character
2186 ** in the string passed as the second argument). If it is not, -1 is
2187 ** returned. Otherwise, if f is a valid matchinfo flag, the value returned
2188 ** is the number of 32-bit integers added to the output array if the
2189 ** table has nCol columns and the query nPhrase phrases.
2190 */
2191 static int fts5MatchinfoFlagsize(int nCol, int nPhrase, char f){
2192 int ret = -1;
2193 switch( f ){
2194 case 'p': ret = 1; break;
2195 case 'c': ret = 1; break;
2196 case 'x': ret = 3 * nCol * nPhrase; break;
2197 case 'y': ret = nCol * nPhrase; break;
2198 case 'b': ret = ((nCol + 31) / 32) * nPhrase; break;
2199 case 'n': ret = 1; break;
2200 case 'a': ret = nCol; break;
2201 case 'l': ret = nCol; break;
2202 case 's': ret = nCol; break;
2203 }
2204 return ret;
2205 }
2206
2207 static int fts5MatchinfoIter(
2208 const Fts5ExtensionApi *pApi, /* API offered by current FTS version */
2209 Fts5Context *pFts, /* First arg to pass to pApi functions */
2210 Fts5MatchinfoCtx *p,
2211 int(*x)(const Fts5ExtensionApi*,Fts5Context*,Fts5MatchinfoCtx*,char,u32*)
2212 ){
2213 int i;
2214 int n = 0;
2215 int rc = SQLITE_OK;
2216 char f;
2217 for(i=0; (f = p->zArg[i]); i++){
2218 rc = x(pApi, pFts, p, f, &p->aRet[n]);
2219 if( rc!=SQLITE_OK ) break;
2220 n += fts5MatchinfoFlagsize(p->nCol, p->nPhrase, f);
2221 }
2222 return rc;
2223 }
2224
2225 static int fts5MatchinfoXCb(
2226 const Fts5ExtensionApi *pApi,
2227 Fts5Context *pFts,
2228 void *pUserData
2229 ){
2230 Fts5PhraseIter iter;
2231 int iCol, iOff;
2232 u32 *aOut = (u32*)pUserData;
2233 int iPrev = -1;
2234
2235 for(pApi->xPhraseFirst(pFts, 0, &iter, &iCol, &iOff);
2236 iCol>=0;
2237 pApi->xPhraseNext(pFts, &iter, &iCol, &iOff)
2238 ){
2239 aOut[iCol*3+1]++;
2240 if( iCol!=iPrev ) aOut[iCol*3 + 2]++;
2241 iPrev = iCol;
2242 }
2243
2244 return SQLITE_OK;
2245 }
2246
2247 static int fts5MatchinfoGlobalCb(
2248 const Fts5ExtensionApi *pApi,
2249 Fts5Context *pFts,
2250 Fts5MatchinfoCtx *p,
2251 char f,
2252 u32 *aOut
2253 ){
2254 int rc = SQLITE_OK;
2255 switch( f ){
2256 case 'p':
2257 aOut[0] = p->nPhrase;
2258 break;
2259
2260 case 'c':
2261 aOut[0] = p->nCol;
2262 break;
2263
2264 case 'x': {
2265 int i;
2266 for(i=0; i<p->nPhrase && rc==SQLITE_OK; i++){
2267 void *pPtr = (void*)&aOut[i * p->nCol * 3];
2268 rc = pApi->xQueryPhrase(pFts, i, pPtr, fts5MatchinfoXCb);
2269 }
2270 break;
2271 }
2272
2273 case 'n': {
2274 sqlite3_int64 nRow;
2275 rc = pApi->xRowCount(pFts, &nRow);
2276 aOut[0] = (u32)nRow;
2277 break;
2278 }
2279
2280 case 'a': {
2281 sqlite3_int64 nRow = 0;
2282 rc = pApi->xRowCount(pFts, &nRow);
2283 if( nRow==0 ){
2284 memset(aOut, 0, sizeof(u32) * p->nCol);
2285 }else{
2286 int i;
2287 for(i=0; rc==SQLITE_OK && i<p->nCol; i++){
2288 sqlite3_int64 nToken;
2289 rc = pApi->xColumnTotalSize(pFts, i, &nToken);
2290 if( rc==SQLITE_OK){
2291 aOut[i] = (u32)((2*nToken + nRow) / (2*nRow));
2292 }
2293 }
2294 }
2295 break;
2296 }
2297
2298 }
2299 return rc;
2300 }
2301
2302 static int fts5MatchinfoLocalCb(
2303 const Fts5ExtensionApi *pApi,
2304 Fts5Context *pFts,
2305 Fts5MatchinfoCtx *p,
2306 char f,
2307 u32 *aOut
2308 ){
2309 int i;
2310 int rc = SQLITE_OK;
2311
2312 switch( f ){
2313 case 'b': {
2314 int iPhrase;
2315 int nInt = ((p->nCol + 31) / 32) * p->nPhrase;
2316 for(i=0; i<nInt; i++) aOut[i] = 0;
2317
2318 for(iPhrase=0; iPhrase<p->nPhrase; iPhrase++){
2319 Fts5PhraseIter iter;
2320 int iCol;
2321 for(pApi->xPhraseFirstColumn(pFts, iPhrase, &iter, &iCol);
2322 iCol>=0;
2323 pApi->xPhraseNextColumn(pFts, &iter, &iCol)
2324 ){
2325 aOut[iPhrase * ((p->nCol+31)/32) + iCol/32] |= ((u32)1 << iCol%32);
2326 }
2327 }
2328
2329 break;
2330 }
2331
2332 case 'x':
2333 case 'y': {
2334 int nMul = (f=='x' ? 3 : 1);
2335 int iPhrase;
2336
2337 for(i=0; i<(p->nCol*p->nPhrase); i++) aOut[i*nMul] = 0;
2338
2339 for(iPhrase=0; iPhrase<p->nPhrase; iPhrase++){
2340 Fts5PhraseIter iter;
2341 int iOff, iCol;
2342 for(pApi->xPhraseFirst(pFts, iPhrase, &iter, &iCol, &iOff);
2343 iOff>=0;
2344 pApi->xPhraseNext(pFts, &iter, &iCol, &iOff)
2345 ){
2346 aOut[nMul * (iCol + iPhrase * p->nCol)]++;
2347 }
2348 }
2349
2350 break;
2351 }
2352
2353 case 'l': {
2354 for(i=0; rc==SQLITE_OK && i<p->nCol; i++){
2355 int nToken;
2356 rc = pApi->xColumnSize(pFts, i, &nToken);
2357 aOut[i] = (u32)nToken;
2358 }
2359 break;
2360 }
2361
2362 case 's': {
2363 int nInst;
2364
2365 memset(aOut, 0, sizeof(u32) * p->nCol);
2366
2367 rc = pApi->xInstCount(pFts, &nInst);
2368 for(i=0; rc==SQLITE_OK && i<nInst; i++){
2369 int iPhrase, iOff, iCol = 0;
2370 int iNextPhrase;
2371 int iNextOff;
2372 u32 nSeq = 1;
2373 int j;
2374
2375 rc = pApi->xInst(pFts, i, &iPhrase, &iCol, &iOff);
2376 iNextPhrase = iPhrase+1;
2377 iNextOff = iOff+pApi->xPhraseSize(pFts, 0);
2378 for(j=i+1; rc==SQLITE_OK && j<nInst; j++){
2379 int ip, ic, io;
2380 rc = pApi->xInst(pFts, j, &ip, &ic, &io);
2381 if( ic!=iCol || io>iNextOff ) break;
2382 if( ip==iNextPhrase && io==iNextOff ){
2383 nSeq++;
2384 iNextPhrase = ip+1;
2385 iNextOff = io + pApi->xPhraseSize(pFts, ip);
2386 }
2387 }
2388
2389 if( nSeq>aOut[iCol] ) aOut[iCol] = nSeq;
2390 }
2391
2392 break;
2393 }
2394 }
2395 return rc;
2396 }
2397
2398 static Fts5MatchinfoCtx *fts5MatchinfoNew(
2399 const Fts5ExtensionApi *pApi, /* API offered by current FTS version */
2400 Fts5Context *pFts, /* First arg to pass to pApi functions */
2401 sqlite3_context *pCtx, /* Context for returning error message */
2402 const char *zArg /* Matchinfo flag string */
2403 ){
2404 Fts5MatchinfoCtx *p;
2405 int nCol;
2406 int nPhrase;
2407 int i;
2408 int nInt;
2409 sqlite3_int64 nByte;
2410 int rc;
2411
2412 nCol = pApi->xColumnCount(pFts);
2413 nPhrase = pApi->xPhraseCount(pFts);
2414
2415 nInt = 0;
2416 for(i=0; zArg[i]; i++){
2417 int n = fts5MatchinfoFlagsize(nCol, nPhrase, zArg[i]);
2418 if( n<0 ){
2419 char *zErr = sqlite3_mprintf("unrecognized matchinfo flag: %c", zArg[i]);
2420 sqlite3_result_error(pCtx, zErr, -1);
2421 sqlite3_free(zErr);
2422 return 0;
2423 }
2424 nInt += n;
2425 }
2426
2427 nByte = sizeof(Fts5MatchinfoCtx) /* The struct itself */
2428 + sizeof(u32) * nInt /* The p->aRet[] array */
2429 + (i+1); /* The p->zArg string */
2430 p = (Fts5MatchinfoCtx*)sqlite3_malloc64(nByte);
2431 if( p==0 ){
2432 sqlite3_result_error_nomem(pCtx);
2433 return 0;
2434 }
2435 memset(p, 0, nByte);
2436
2437 p->nCol = nCol;
2438 p->nPhrase = nPhrase;
2439 p->aRet = (u32*)&p[1];
2440 p->nRet = nInt;
2441 p->zArg = (char*)&p->aRet[nInt];
2442 memcpy(p->zArg, zArg, i);
2443
2444 rc = fts5MatchinfoIter(pApi, pFts, p, fts5MatchinfoGlobalCb);
2445 if( rc!=SQLITE_OK ){
2446 sqlite3_result_error_code(pCtx, rc);
2447 sqlite3_free(p);
2448 p = 0;
2449 }
2450
2451 return p;
2452 }
2453
2454 static void fts5MatchinfoFunc(
2455 const Fts5ExtensionApi *pApi, /* API offered by current FTS version */
2456 Fts5Context *pFts, /* First arg to pass to pApi functions */
2457 sqlite3_context *pCtx, /* Context for returning result/error */
2458 int nVal, /* Number of values in apVal[] array */
2459 sqlite3_value **apVal /* Array of trailing arguments */
2460 ){
2461 const char *zArg;
2462 Fts5MatchinfoCtx *p;
2463 int rc = SQLITE_OK;
2464
2465 if( nVal>0 ){
2466 zArg = (const char*)sqlite3_value_text(apVal[0]);
2467 }else{
2468 zArg = "pcx";
2469 }
2470
2471 p = (Fts5MatchinfoCtx*)pApi->xGetAuxdata(pFts, 0);
2472 if( p==0 || sqlite3_stricmp(zArg, p->zArg) ){
2473 p = fts5MatchinfoNew(pApi, pFts, pCtx, zArg);
2474 if( p==0 ){
2475 rc = SQLITE_NOMEM;
2476 }else{
2477 rc = pApi->xSetAuxdata(pFts, p, sqlite3_free);
2478 }
2479 }
2480
2481 if( rc==SQLITE_OK ){
2482 rc = fts5MatchinfoIter(pApi, pFts, p, fts5MatchinfoLocalCb);
2483 }
2484 if( rc!=SQLITE_OK ){
2485 sqlite3_result_error_code(pCtx, rc);
2486 }else{
2487 /* No errors has occured, so return a copy of the array of integers. */
2488 int nByte = p->nRet * sizeof(u32);
2489 sqlite3_result_blob(pCtx, (void*)p->aRet, nByte, SQLITE_TRANSIENT);
2490 }
2491 }
2492
2493 int db_register_fts5(sqlite3 *db){
2494 int rc; /* Return code */
2495 fts5_api *pApi; /* FTS5 API functions */
2496
2497 /* Extract the FTS5 API pointer from the database handle. The
2498 ** fts5_api_from_db() function above is copied verbatim from the
2499 ** FTS5 documentation. Refer there for details. */
2500 rc = fts5_api_from_db(db, &pApi);
2501 if( rc!=SQLITE_OK ) return rc;
2502
2503 /* If fts5_api_from_db() returns NULL, then either FTS5 is not registered
2504 ** with this database handle, or an error (OOM perhaps?) has occurred.
2505 **
2506 ** Also check that the fts5_api object is version 2 or newer.
2507 */
2508 if( pApi==0 || pApi->iVersion<2 ){
2509 return SQLITE_ERROR;
2510 }
2511
2512 /* Register the implementation of matchinfo() */
2513 rc = pApi->xCreateFunction(pApi, "matchinfo", 0, fts5MatchinfoFunc, 0);
2514
2515 return rc;
2516 }
2517
M src/setup.c
+4 -3
--- src/setup.c
+++ src/setup.c
@@ -2067,18 +2067,19 @@
20672067
 search_create_index();
20682068
 search_fill_index();
20692069
 search_update_index(search_restrict(SRCH_ALL));
20702070
 }
20712071
 if( search_index_exists() ){
2072
- @ <p>Currently using an SQLite FTS4 search index. This makes search
2073
- @ run faster, especially on large repositories, but takes up space.</p>
2072
+ @ <p>Currently using an SQLite FTS%d(search_index_type(0)) search index.
2073
+ @ The index helps search run faster, especially on large repositories,
2074
+ @ but takes up space.</p>
20742075
 onoff_attribute("Use Porter Stemmer","search-stemmer","ss",0,0);
20752076
 @ <p><input type="submit" name="fts0" value="Delete The Full-Text Index">
20762077
 @ <input type="submit" name="fts1" value="Rebuild The Full-Text Index">
20772078
 style_submenu_element("FTS Index Debugging","%R/test-ftsdocs");
20782079
 }else{
2079
- @ <p>The SQLite FTS4 search index is disabled. All searching will be
2080
+ @ <p>The SQLite search index is disabled. All searching will be
20802081
 @ a full-text scan. This usually works fine, but can be slow for
20812082
 @ larger repositories.</p>
20822083
 onoff_attribute("Use Porter Stemmer","search-stemmer","ss",0,0);
20832084
 @ <p><input type="submit" name="fts1" value="Create A Full-Text Index">
20842085
 }
20852086
--- src/setup.c
+++ src/setup.c
@@ -2067,18 +2067,19 @@
2067 search_create_index();
2068 search_fill_index();
2069 search_update_index(search_restrict(SRCH_ALL));
2070 }
2071 if( search_index_exists() ){
2072 @ <p>Currently using an SQLite FTS4 search index. This makes search
2073 @ run faster, especially on large repositories, but takes up space.</p>
 
2074 onoff_attribute("Use Porter Stemmer","search-stemmer","ss",0,0);
2075 @ <p><input type="submit" name="fts0" value="Delete The Full-Text Index">
2076 @ <input type="submit" name="fts1" value="Rebuild The Full-Text Index">
2077 style_submenu_element("FTS Index Debugging","%R/test-ftsdocs");
2078 }else{
2079 @ <p>The SQLite FTS4 search index is disabled. All searching will be
2080 @ a full-text scan. This usually works fine, but can be slow for
2081 @ larger repositories.</p>
2082 onoff_attribute("Use Porter Stemmer","search-stemmer","ss",0,0);
2083 @ <p><input type="submit" name="fts1" value="Create A Full-Text Index">
2084 }
2085
--- src/setup.c
+++ src/setup.c
@@ -2067,18 +2067,19 @@
2067 search_create_index();
2068 search_fill_index();
2069 search_update_index(search_restrict(SRCH_ALL));
2070 }
2071 if( search_index_exists() ){
2072 @ <p>Currently using an SQLite FTS%d(search_index_type(0)) search index.
2073 @ The index helps search run faster, especially on large repositories,
2074 @ but takes up space.</p>
2075 onoff_attribute("Use Porter Stemmer","search-stemmer","ss",0,0);
2076 @ <p><input type="submit" name="fts0" value="Delete The Full-Text Index">
2077 @ <input type="submit" name="fts1" value="Rebuild The Full-Text Index">
2078 style_submenu_element("FTS Index Debugging","%R/test-ftsdocs");
2079 }else{
2080 @ <p>The SQLite search index is disabled. All searching will be
2081 @ a full-text scan. This usually works fine, but can be slow for
2082 @ larger repositories.</p>
2083 onoff_attribute("Use Porter Stemmer","search-stemmer","ss",0,0);
2084 @ <p><input type="submit" name="fts1" value="Create A Full-Text Index">
2085 }
2086
M www/changes.wiki
+1
--- www/changes.wiki
+++ www/changes.wiki
@@ -15,10 +15,11 @@
1515
 * Passwords for remembered remote repositories are now stored as irreversible
1616
 hashes rather than obscured clear-text, for improved security.
1717
 * Writes to the database are disabled by default if the HTTP request
1818
 does not come from the same origin. This enhancement is for defense in depth.
1919
 There where no known attacks prior to this enhancement.
20
+ * Update search infrastructure from FTS4 to FTS5.
2021
2122
 <h2 id='v2_20'>Changes for version 2.20 (2022-11-16)</h2>
2223
 * Added the [/help?cmd=chat-timeline-user|chat-timeline-user setting]. If
2324
 it is not an empty string, then any changes that would appear on the timeline
2425
 are announced in [./chat.md|the chat room].
2526
--- www/changes.wiki
+++ www/changes.wiki
@@ -15,10 +15,11 @@
15 * Passwords for remembered remote repositories are now stored as irreversible
16 hashes rather than obscured clear-text, for improved security.
17 * Writes to the database are disabled by default if the HTTP request
18 does not come from the same origin. This enhancement is for defense in depth.
19 There where no known attacks prior to this enhancement.
 
20
21 <h2 id='v2_20'>Changes for version 2.20 (2022-11-16)</h2>
22 * Added the [/help?cmd=chat-timeline-user|chat-timeline-user setting]. If
23 it is not an empty string, then any changes that would appear on the timeline
24 are announced in [./chat.md|the chat room].
25
--- www/changes.wiki
+++ www/changes.wiki
@@ -15,10 +15,11 @@
15 * Passwords for remembered remote repositories are now stored as irreversible
16 hashes rather than obscured clear-text, for improved security.
17 * Writes to the database are disabled by default if the HTTP request
18 does not come from the same origin. This enhancement is for defense in depth.
19 There where no known attacks prior to this enhancement.
20 * Update search infrastructure from FTS4 to FTS5.
21
22 <h2 id='v2_20'>Changes for version 2.20 (2022-11-16)</h2>
23 * Added the [/help?cmd=chat-timeline-user|chat-timeline-user setting]. If
24 it is not an empty string, then any changes that would appear on the timeline
25 are announced in [./chat.md|the chat room].
26

Keyboard Shortcuts

Open search /
Next entry (timeline) j
Previous entry (timeline) k
Open focused entry Enter
Show this help ?
Toggle theme Top nav button