Fossil SCM

Replace FTS4 with FTS5, per [forum:d05b1855aa|forum post d05b1855aa]. This has been only lightly tested and might require a repo rebuild (or that we rename the associated tables/views rather than recycle them).

stephan 2023-01-24 02:57 trunk
Commit c1933caf03a036b1e07fc08c1b9954e12712e161841f2ea96c79fd88dbfc13a5
Parent a80f27485a0aab5…
4 files changed +1 +394 -17 +2 -2 +1
~ src/db.c ~ src/search.c ~ src/setup.c ~ www/changes.wiki
M src/db.c
+1
--- src/db.c
+++ src/db.c
@@ -1802,10 +1802,11 @@
18021802
 if( g.fSqlTrace ) sqlite3_trace_v2(db, SQLITE_TRACE_PROFILE, db_sql_trace, 0);
18031803
 db_add_aux_functions(db);
18041804
 re_add_sql_func(db); /* The REGEXP operator */
18051805
 foci_register(db); /* The "files_of_checkin" virtual table */
18061806
 sqlite3_set_authorizer(db, db_top_authorizer, db);
1807
+ db_register_fts5(db) /* in search.c */;
18071808
 return db;
18081809
 }
18091810
18101811
18111812
 /*
18121813
--- src/db.c
+++ src/db.c
@@ -1802,10 +1802,11 @@
1802 if( g.fSqlTrace ) sqlite3_trace_v2(db, SQLITE_TRACE_PROFILE, db_sql_trace, 0);
1803 db_add_aux_functions(db);
1804 re_add_sql_func(db); /* The REGEXP operator */
1805 foci_register(db); /* The "files_of_checkin" virtual table */
1806 sqlite3_set_authorizer(db, db_top_authorizer, db);
 
1807 return db;
1808 }
1809
1810
1811 /*
1812
--- src/db.c
+++ src/db.c
@@ -1802,10 +1802,11 @@
1802 if( g.fSqlTrace ) sqlite3_trace_v2(db, SQLITE_TRACE_PROFILE, db_sql_trace, 0);
1803 db_add_aux_functions(db);
1804 re_add_sql_func(db); /* The REGEXP operator */
1805 foci_register(db); /* The "files_of_checkin" virtual table */
1806 sqlite3_set_authorizer(db, db_top_authorizer, db);
1807 db_register_fts5(db) /* in search.c */;
1808 return db;
1809 }
1810
1811
1812 /*
1813
M src/search.c
+394 -17
--- src/search.c
+++ src/search.c
@@ -935,14 +935,14 @@
935935
 " SELECT ftsdocs.label,"
936936
 " ftsdocs.url,"
937937
 " rank(matchinfo(ftsidx,'pcsx')),"
938938
 " ftsdocs.type || ftsdocs.rid,"
939939
 " datetime(ftsdocs.mtime),"
940
- " snippet(ftsidx,'<mark>','</mark>',' ... ',-1,35)"
940
+ " snippet(ftsidx,-1,'<mark>','</mark>',' ... ',35)"
941941
 " FROM ftsidx CROSS JOIN ftsdocs"
942942
 " WHERE ftsidx MATCH %Q"
943
- " AND ftsdocs.rowid=ftsidx.docid",
943
+ " AND ftsdocs.rowid=ftsidx.rowid",
944944
 zPat
945945
 );
946946
 fossil_free(zPat);
947947
 if( srchFlags!=SRCH_ALL ){
948948
 const char *zSep = " AND (";
@@ -1508,11 +1508,11 @@
15081508
 /* The schema for the full-text index
15091509
 */
15101510
 static const char zFtsSchema[] =
15111511
 @ -- One entry for each possible search result
15121512
 @ CREATE TABLE IF NOT EXISTS repository.ftsdocs(
1513
-@ rowid INTEGER PRIMARY KEY, -- Maps to the ftsidx.docid
1513
+@ rowid INTEGER PRIMARY KEY, -- Maps to the ftsidx.rowid
15141514
 @ type CHAR(1), -- Type of document
15151515
 @ rid INTEGER, -- BLOB.RID or TAG.TAGID for the document
15161516
 @ name TEXT, -- Additional document description
15171517
 @ idxed BOOLEAN, -- True if currently in the index
15181518
 @ label TEXT, -- Label to print on search results
@@ -1526,11 +1526,11 @@
15261526
 @ CREATE VIEW IF NOT EXISTS repository.ftscontent AS
15271527
 @ SELECT rowid, type, rid, name, idxed, label, url, mtime,
15281528
 @ title(type,rid,name) AS 'title', body(type,rid,name) AS 'body'
15291529
 @ FROM ftsdocs;
15301530
 @ CREATE VIRTUAL TABLE IF NOT EXISTS repository.ftsidx
1531
-@ USING fts4(content="ftscontent", title, body%s);
1531
+@ USING fts5(content="ftscontent", title, body%s);
15321532
 ;
15331533
 static const char zFtsDrop[] =
15341534
 @ DROP TABLE IF EXISTS repository.ftsidx;
15351535
 @ DROP VIEW IF EXISTS repository.ftscontent;
15361536
 @ DROP TABLE IF EXISTS repository.ftsdocs;
@@ -1606,11 +1606,11 @@
16061606
 char zType[2];
16071607
 zType[0] = cType;
16081608
 zType[1] = 0;
16091609
 search_sql_setup(g.db);
16101610
 db_multi_exec(
1611
- "DELETE FROM ftsidx WHERE docid IN"
1611
+ "DELETE FROM ftsidx WHERE rowid IN"
16121612
 " (SELECT rowid FROM ftsdocs WHERE type=%Q AND rid=%d AND idxed)",
16131613
 zType, rid
16141614
 );
16151615
 db_multi_exec(
16161616
 "REPLACE INTO ftsdocs(type,rid,name,idxed)"
@@ -1617,11 +1617,11 @@
16171617
 " VALUES(%Q,%d,%Q,0)",
16181618
 zType, rid, zName
16191619
 );
16201620
 if( cType=='w' || cType=='e' ){
16211621
 db_multi_exec(
1622
- "DELETE FROM ftsidx WHERE docid IN"
1622
+ "DELETE FROM ftsidx WHERE rowid IN"
16231623
 " (SELECT rowid FROM ftsdocs WHERE type='%c' AND name=%Q AND idxed)",
16241624
 cType, zName
16251625
 );
16261626
 db_multi_exec(
16271627
 "DELETE FROM ftsdocs WHERE type='%c' AND name=%Q AND rid!=%d",
@@ -1657,11 +1657,11 @@
16571657
 " WHERE foci.checkinID=%d AND blob.uuid=foci.uuid"
16581658
 " AND %z",
16591659
 ckid, glob_expr("foci.filename", db_get("doc-glob",""))
16601660
 );
16611661
 db_multi_exec(
1662
- "DELETE FROM ftsidx WHERE docid IN"
1662
+ "DELETE FROM ftsidx WHERE rowid IN"
16631663
 " (SELECT rowid FROM ftsdocs WHERE type='d'"
16641664
 " AND rid NOT IN (SELECT rid FROM current_docs))"
16651665
 );
16661666
 db_multi_exec(
16671667
 "DELETE FROM ftsdocs WHERE type='d'"
@@ -1676,11 +1676,11 @@
16761676
 " %.17g"
16771677
 " FROM current_docs",
16781678
 zDocBr, rTime
16791679
 );
16801680
 db_multi_exec(
1681
- "INSERT INTO ftsidx(docid,title,body)"
1681
+ "INSERT INTO ftsidx(rowid,title,body)"
16821682
 " SELECT rowid, label, bx FROM ftsdocs WHERE type='d' AND NOT idxed"
16831683
 );
16841684
 db_multi_exec(
16851685
 "UPDATE ftsdocs SET"
16861686
 " idxed=1,"
@@ -1693,11 +1693,11 @@
16931693
 /*
16941694
 ** Deal with all of the unindexed 'c' terms in FTSDOCS
16951695
 */
16961696
 static void search_update_checkin_index(void){
16971697
 db_multi_exec(
1698
- "INSERT INTO ftsidx(docid,title,body)"
1698
+ "INSERT INTO ftsidx(rowid,title,body)"
16991699
 " SELECT rowid, '', body('c',rid,NULL) FROM ftsdocs"
17001700
 " WHERE type='c' AND NOT idxed;"
17011701
 );
17021702
 db_multi_exec(
17031703
 "UPDATE ftsdocs SET idxed=1, name=NULL,"
@@ -1716,11 +1716,11 @@
17161716
 /*
17171717
 ** Deal with all of the unindexed 't' terms in FTSDOCS
17181718
 */
17191719
 static void search_update_ticket_index(void){
17201720
 db_multi_exec(
1721
- "INSERT INTO ftsidx(docid,title,body)"
1721
+ "INSERT INTO ftsidx(rowid,title,body)"
17221722
 " SELECT rowid, title('t',rid,NULL), body('t',rid,NULL) FROM ftsdocs"
17231723
 " WHERE type='t' AND NOT idxed;"
17241724
 );
17251725
 if( db_changes()==0 ) return;
17261726
 db_multi_exec(
@@ -1739,11 +1739,11 @@
17391739
 /*
17401740
 ** Deal with all of the unindexed 'w' terms in FTSDOCS
17411741
 */
17421742
 static void search_update_wiki_index(void){
17431743
 db_multi_exec(
1744
- "INSERT INTO ftsidx(docid,title,body)"
1744
+ "INSERT INTO ftsidx(rowid,title,body)"
17451745
 " SELECT rowid, title('w',rid,NULL),body('w',rid,NULL) FROM ftsdocs"
17461746
 " WHERE type='w' AND NOT idxed;"
17471747
 );
17481748
 if( db_changes()==0 ) return;
17491749
 db_multi_exec(
@@ -1761,11 +1761,11 @@
17611761
 /*
17621762
 ** Deal with all of the unindexed 'f' terms in FTSDOCS
17631763
 */
17641764
 static void search_update_forum_index(void){
17651765
 db_multi_exec(
1766
- "INSERT INTO ftsidx(docid,title,body)"
1766
+ "INSERT INTO ftsidx(rowid,title,body)"
17671767
 " SELECT rowid, title('f',rid,NULL),body('f',rid,NULL) FROM ftsdocs"
17681768
 " WHERE type='f' AND NOT idxed;"
17691769
 );
17701770
 if( db_changes()==0 ) return;
17711771
 db_multi_exec(
@@ -1784,11 +1784,11 @@
17841784
 /*
17851785
 ** Deal with all of the unindexed 'e' terms in FTSDOCS
17861786
 */
17871787
 static void search_update_technote_index(void){
17881788
 db_multi_exec(
1789
- "INSERT INTO ftsidx(docid,title,body)"
1789
+ "INSERT INTO ftsidx(rowid,title,body)"
17901790
 " SELECT rowid, title('e',rid,NULL),body('e',rid,NULL) FROM ftsdocs"
17911791
 " WHERE type='e' AND NOT idxed;"
17921792
 );
17931793
 if( db_changes()==0 ) return;
17941794
 db_multi_exec(
@@ -1869,11 +1869,11 @@
18691869
 **
18701870
 ** The current search settings are displayed after any changes are applied.
18711871
 ** Run this command with no arguments to simply see the settings.
18721872
 */
18731873
 void fts_config_cmd(void){
1874
- static const struct {
1874
+ static const struct {
18751875
 int iCmd;
18761876
 const char *z;
18771877
 } aCmd[] = {
18781878
 { 1, "reindex" },
18791879
 { 2, "index" },
@@ -2002,24 +2002,24 @@
20022002
 const char *zUrl = db_column_text(&q,4);
20032003
 const char *zDocId = db_column_text(&q,0);
20042004
 char *zName;
20052005
 char *z;
20062006
 @ <table border=0>
2007
- @ <tr><td align='right'>docid:<td>&nbsp;&nbsp;<td>%d(id)
2007
+ @ <tr><td align='right'>rowid:<td>&nbsp;&nbsp;<td>%d(id)
20082008
 @ <tr><td align='right'>id:<td><td>%s(zDocId)
20092009
 @ <tr><td align='right'>name:<td><td>%h(db_column_text(&q,1))
20102010
 @ <tr><td align='right'>idxed:<td><td>%d(db_column_int(&q,2))
20112011
 @ <tr><td align='right'>label:<td><td>%h(db_column_text(&q,3))
20122012
 @ <tr><td align='right'>url:<td><td>
20132013
 @ <a href='%R%s(zUrl)'>%h(zUrl)</a>
20142014
 @ <tr><td align='right'>mtime:<td><td>%s(db_column_text(&q,5))
2015
- z = db_text(0, "SELECT title FROM ftsidx WHERE docid=%d",id);
2015
+ z = db_text(0, "SELECT title FROM ftsidx WHERE rowid=%d",id);
20162016
 if( z && z[0] ){
20172017
 @ <tr><td align="right">title:<td><td>%h(z)
20182018
 fossil_free(z);
20192019
 }
2020
- z = db_text(0, "SELECT body FROM ftsidx WHERE docid=%d",id);
2020
+ z = db_text(0, "SELECT body FROM ftsidx WHERE rowid=%d",id);
20212021
 if( z && z[0] ){
20222022
 @ <tr><td align="right" valign="top">body:<td><td>%h(z)
20232023
 fossil_free(z);
20242024
 }
20252025
 @ </table>
@@ -2103,5 +2103,382 @@
21032103
 @ <th align="right">%d(cnt3)
21042104
 @ </tfooter>
21052105
 @ </table>
21062106
 style_finish_page();
21072107
 }
2108
+
2109
+
2110
+/*
2111
+** The Fts5MatchinfoCtx bits were all taken verbatim from:
2112
+**
2113
+** https://sqlite.org/src/finfo?name=ext/fts5/fts5_test_mi.c
2114
+*/
2115
+
2116
+typedef struct Fts5MatchinfoCtx Fts5MatchinfoCtx;
2117
+
2118
+#ifndef SQLITE_AMALGAMATION
2119
+typedef unsigned int u32;
2120
+#endif
2121
+
2122
+struct Fts5MatchinfoCtx {
2123
+ int nCol; /* Number of cols in FTS5 table */
2124
+ int nPhrase; /* Number of phrases in FTS5 query */
2125
+ char *zArg; /* nul-term'd copy of 2nd arg */
2126
+ int nRet; /* Number of elements in aRet[] */
2127
+ u32 *aRet; /* Array of 32-bit unsigned ints to return */
2128
+};
2129
+
2130
+
2131
+/*
2132
+** Return a pointer to the fts5_api pointer for database connection db.
2133
+** If an error occurs, return NULL and leave an error in the database
2134
+** handle (accessible using sqlite3_errcode()/errmsg()).
2135
+*/
2136
+static int fts5_api_from_db(sqlite3 *db, fts5_api **ppApi){
2137
+ sqlite3_stmt *pStmt = 0;
2138
+ int rc;
2139
+
2140
+ *ppApi = 0;
2141
+ rc = sqlite3_prepare(db, "SELECT fts5(?1)", -1, &pStmt, 0);
2142
+ if( rc==SQLITE_OK ){
2143
+ sqlite3_bind_pointer(pStmt, 1, (void*)ppApi, "fts5_api_ptr", 0);
2144
+ (void)sqlite3_step(pStmt);
2145
+ rc = sqlite3_finalize(pStmt);
2146
+ }
2147
+
2148
+ return rc;
2149
+}
2150
+
2151
+
2152
+/*
2153
+** Argument f should be a flag accepted by matchinfo() (a valid character
2154
+** in the string passed as the second argument). If it is not, -1 is
2155
+** returned. Otherwise, if f is a valid matchinfo flag, the value returned
2156
+** is the number of 32-bit integers added to the output array if the
2157
+** table has nCol columns and the query nPhrase phrases.
2158
+*/
2159
+static int fts5MatchinfoFlagsize(int nCol, int nPhrase, char f){
2160
+ int ret = -1;
2161
+ switch( f ){
2162
+ case 'p': ret = 1; break;
2163
+ case 'c': ret = 1; break;
2164
+ case 'x': ret = 3 * nCol * nPhrase; break;
2165
+ case 'y': ret = nCol * nPhrase; break;
2166
+ case 'b': ret = ((nCol + 31) / 32) * nPhrase; break;
2167
+ case 'n': ret = 1; break;
2168
+ case 'a': ret = nCol; break;
2169
+ case 'l': ret = nCol; break;
2170
+ case 's': ret = nCol; break;
2171
+ }
2172
+ return ret;
2173
+}
2174
+
2175
+static int fts5MatchinfoIter(
2176
+ const Fts5ExtensionApi *pApi, /* API offered by current FTS version */
2177
+ Fts5Context *pFts, /* First arg to pass to pApi functions */
2178
+ Fts5MatchinfoCtx *p,
2179
+ int(*x)(const Fts5ExtensionApi*,Fts5Context*,Fts5MatchinfoCtx*,char,u32*)
2180
+){
2181
+ int i;
2182
+ int n = 0;
2183
+ int rc = SQLITE_OK;
2184
+ char f;
2185
+ for(i=0; (f = p->zArg[i]); i++){
2186
+ rc = x(pApi, pFts, p, f, &p->aRet[n]);
2187
+ if( rc!=SQLITE_OK ) break;
2188
+ n += fts5MatchinfoFlagsize(p->nCol, p->nPhrase, f);
2189
+ }
2190
+ return rc;
2191
+}
2192
+
2193
+static int fts5MatchinfoXCb(
2194
+ const Fts5ExtensionApi *pApi,
2195
+ Fts5Context *pFts,
2196
+ void *pUserData
2197
+){
2198
+ Fts5PhraseIter iter;
2199
+ int iCol, iOff;
2200
+ u32 *aOut = (u32*)pUserData;
2201
+ int iPrev = -1;
2202
+
2203
+ for(pApi->xPhraseFirst(pFts, 0, &iter, &iCol, &iOff);
2204
+ iCol>=0;
2205
+ pApi->xPhraseNext(pFts, &iter, &iCol, &iOff)
2206
+ ){
2207
+ aOut[iCol*3+1]++;
2208
+ if( iCol!=iPrev ) aOut[iCol*3 + 2]++;
2209
+ iPrev = iCol;
2210
+ }
2211
+
2212
+ return SQLITE_OK;
2213
+}
2214
+
2215
+static int fts5MatchinfoGlobalCb(
2216
+ const Fts5ExtensionApi *pApi,
2217
+ Fts5Context *pFts,
2218
+ Fts5MatchinfoCtx *p,
2219
+ char f,
2220
+ u32 *aOut
2221
+){
2222
+ int rc = SQLITE_OK;
2223
+ switch( f ){
2224
+ case 'p':
2225
+ aOut[0] = p->nPhrase;
2226
+ break;
2227
+
2228
+ case 'c':
2229
+ aOut[0] = p->nCol;
2230
+ break;
2231
+
2232
+ case 'x': {
2233
+ int i;
2234
+ for(i=0; i<p->nPhrase && rc==SQLITE_OK; i++){
2235
+ void *pPtr = (void*)&aOut[i * p->nCol * 3];
2236
+ rc = pApi->xQueryPhrase(pFts, i, pPtr, fts5MatchinfoXCb);
2237
+ }
2238
+ break;
2239
+ }
2240
+
2241
+ case 'n': {
2242
+ sqlite3_int64 nRow;
2243
+ rc = pApi->xRowCount(pFts, &nRow);
2244
+ aOut[0] = (u32)nRow;
2245
+ break;
2246
+ }
2247
+
2248
+ case 'a': {
2249
+ sqlite3_int64 nRow = 0;
2250
+ rc = pApi->xRowCount(pFts, &nRow);
2251
+ if( nRow==0 ){
2252
+ memset(aOut, 0, sizeof(u32) * p->nCol);
2253
+ }else{
2254
+ int i;
2255
+ for(i=0; rc==SQLITE_OK && i<p->nCol; i++){
2256
+ sqlite3_int64 nToken;
2257
+ rc = pApi->xColumnTotalSize(pFts, i, &nToken);
2258
+ if( rc==SQLITE_OK){
2259
+ aOut[i] = (u32)((2*nToken + nRow) / (2*nRow));
2260
+ }
2261
+ }
2262
+ }
2263
+ break;
2264
+ }
2265
+
2266
+ }
2267
+ return rc;
2268
+}
2269
+
2270
+static int fts5MatchinfoLocalCb(
2271
+ const Fts5ExtensionApi *pApi,
2272
+ Fts5Context *pFts,
2273
+ Fts5MatchinfoCtx *p,
2274
+ char f,
2275
+ u32 *aOut
2276
+){
2277
+ int i;
2278
+ int rc = SQLITE_OK;
2279
+
2280
+ switch( f ){
2281
+ case 'b': {
2282
+ int iPhrase;
2283
+ int nInt = ((p->nCol + 31) / 32) * p->nPhrase;
2284
+ for(i=0; i<nInt; i++) aOut[i] = 0;
2285
+
2286
+ for(iPhrase=0; iPhrase<p->nPhrase; iPhrase++){
2287
+ Fts5PhraseIter iter;
2288
+ int iCol;
2289
+ for(pApi->xPhraseFirstColumn(pFts, iPhrase, &iter, &iCol);
2290
+ iCol>=0;
2291
+ pApi->xPhraseNextColumn(pFts, &iter, &iCol)
2292
+ ){
2293
+ aOut[iPhrase * ((p->nCol+31)/32) + iCol/32] |= ((u32)1 << iCol%32);
2294
+ }
2295
+ }
2296
+
2297
+ break;
2298
+ }
2299
+
2300
+ case 'x':
2301
+ case 'y': {
2302
+ int nMul = (f=='x' ? 3 : 1);
2303
+ int iPhrase;
2304
+
2305
+ for(i=0; i<(p->nCol*p->nPhrase); i++) aOut[i*nMul] = 0;
2306
+
2307
+ for(iPhrase=0; iPhrase<p->nPhrase; iPhrase++){
2308
+ Fts5PhraseIter iter;
2309
+ int iOff, iCol;
2310
+ for(pApi->xPhraseFirst(pFts, iPhrase, &iter, &iCol, &iOff);
2311
+ iOff>=0;
2312
+ pApi->xPhraseNext(pFts, &iter, &iCol, &iOff)
2313
+ ){
2314
+ aOut[nMul * (iCol + iPhrase * p->nCol)]++;
2315
+ }
2316
+ }
2317
+
2318
+ break;
2319
+ }
2320
+
2321
+ case 'l': {
2322
+ for(i=0; rc==SQLITE_OK && i<p->nCol; i++){
2323
+ int nToken;
2324
+ rc = pApi->xColumnSize(pFts, i, &nToken);
2325
+ aOut[i] = (u32)nToken;
2326
+ }
2327
+ break;
2328
+ }
2329
+
2330
+ case 's': {
2331
+ int nInst;
2332
+
2333
+ memset(aOut, 0, sizeof(u32) * p->nCol);
2334
+
2335
+ rc = pApi->xInstCount(pFts, &nInst);
2336
+ for(i=0; rc==SQLITE_OK && i<nInst; i++){
2337
+ int iPhrase, iOff, iCol = 0;
2338
+ int iNextPhrase;
2339
+ int iNextOff;
2340
+ u32 nSeq = 1;
2341
+ int j;
2342
+
2343
+ rc = pApi->xInst(pFts, i, &iPhrase, &iCol, &iOff);
2344
+ iNextPhrase = iPhrase+1;
2345
+ iNextOff = iOff+pApi->xPhraseSize(pFts, 0);
2346
+ for(j=i+1; rc==SQLITE_OK && j<nInst; j++){
2347
+ int ip, ic, io;
2348
+ rc = pApi->xInst(pFts, j, &ip, &ic, &io);
2349
+ if( ic!=iCol || io>iNextOff ) break;
2350
+ if( ip==iNextPhrase && io==iNextOff ){
2351
+ nSeq++;
2352
+ iNextPhrase = ip+1;
2353
+ iNextOff = io + pApi->xPhraseSize(pFts, ip);
2354
+ }
2355
+ }
2356
+
2357
+ if( nSeq>aOut[iCol] ) aOut[iCol] = nSeq;
2358
+ }
2359
+
2360
+ break;
2361
+ }
2362
+ }
2363
+ return rc;
2364
+}
2365
+
2366
+static Fts5MatchinfoCtx *fts5MatchinfoNew(
2367
+ const Fts5ExtensionApi *pApi, /* API offered by current FTS version */
2368
+ Fts5Context *pFts, /* First arg to pass to pApi functions */
2369
+ sqlite3_context *pCtx, /* Context for returning error message */
2370
+ const char *zArg /* Matchinfo flag string */
2371
+){
2372
+ Fts5MatchinfoCtx *p;
2373
+ int nCol;
2374
+ int nPhrase;
2375
+ int i;
2376
+ int nInt;
2377
+ sqlite3_int64 nByte;
2378
+ int rc;
2379
+
2380
+ nCol = pApi->xColumnCount(pFts);
2381
+ nPhrase = pApi->xPhraseCount(pFts);
2382
+
2383
+ nInt = 0;
2384
+ for(i=0; zArg[i]; i++){
2385
+ int n = fts5MatchinfoFlagsize(nCol, nPhrase, zArg[i]);
2386
+ if( n<0 ){
2387
+ char *zErr = sqlite3_mprintf("unrecognized matchinfo flag: %c", zArg[i]);
2388
+ sqlite3_result_error(pCtx, zErr, -1);
2389
+ sqlite3_free(zErr);
2390
+ return 0;
2391
+ }
2392
+ nInt += n;
2393
+ }
2394
+
2395
+ nByte = sizeof(Fts5MatchinfoCtx) /* The struct itself */
2396
+ + sizeof(u32) * nInt /* The p->aRet[] array */
2397
+ + (i+1); /* The p->zArg string */
2398
+ p = (Fts5MatchinfoCtx*)sqlite3_malloc64(nByte);
2399
+ if( p==0 ){
2400
+ sqlite3_result_error_nomem(pCtx);
2401
+ return 0;
2402
+ }
2403
+ memset(p, 0, nByte);
2404
+
2405
+ p->nCol = nCol;
2406
+ p->nPhrase = nPhrase;
2407
+ p->aRet = (u32*)&p[1];
2408
+ p->nRet = nInt;
2409
+ p->zArg = (char*)&p->aRet[nInt];
2410
+ memcpy(p->zArg, zArg, i);
2411
+
2412
+ rc = fts5MatchinfoIter(pApi, pFts, p, fts5MatchinfoGlobalCb);
2413
+ if( rc!=SQLITE_OK ){
2414
+ sqlite3_result_error_code(pCtx, rc);
2415
+ sqlite3_free(p);
2416
+ p = 0;
2417
+ }
2418
+
2419
+ return p;
2420
+}
2421
+
2422
+static void fts5MatchinfoFunc(
2423
+ const Fts5ExtensionApi *pApi, /* API offered by current FTS version */
2424
+ Fts5Context *pFts, /* First arg to pass to pApi functions */
2425
+ sqlite3_context *pCtx, /* Context for returning result/error */
2426
+ int nVal, /* Number of values in apVal[] array */
2427
+ sqlite3_value **apVal /* Array of trailing arguments */
2428
+){
2429
+ const char *zArg;
2430
+ Fts5MatchinfoCtx *p;
2431
+ int rc = SQLITE_OK;
2432
+
2433
+ if( nVal>0 ){
2434
+ zArg = (const char*)sqlite3_value_text(apVal[0]);
2435
+ }else{
2436
+ zArg = "pcx";
2437
+ }
2438
+
2439
+ p = (Fts5MatchinfoCtx*)pApi->xGetAuxdata(pFts, 0);
2440
+ if( p==0 || sqlite3_stricmp(zArg, p->zArg) ){
2441
+ p = fts5MatchinfoNew(pApi, pFts, pCtx, zArg);
2442
+ if( p==0 ){
2443
+ rc = SQLITE_NOMEM;
2444
+ }else{
2445
+ rc = pApi->xSetAuxdata(pFts, p, sqlite3_free);
2446
+ }
2447
+ }
2448
+
2449
+ if( rc==SQLITE_OK ){
2450
+ rc = fts5MatchinfoIter(pApi, pFts, p, fts5MatchinfoLocalCb);
2451
+ }
2452
+ if( rc!=SQLITE_OK ){
2453
+ sqlite3_result_error_code(pCtx, rc);
2454
+ }else{
2455
+ /* No errors has occured, so return a copy of the array of integers. */
2456
+ int nByte = p->nRet * sizeof(u32);
2457
+ sqlite3_result_blob(pCtx, (void*)p->aRet, nByte, SQLITE_TRANSIENT);
2458
+ }
2459
+}
2460
+
2461
+int db_register_fts5(sqlite3 *db){
2462
+ int rc; /* Return code */
2463
+ fts5_api *pApi; /* FTS5 API functions */
2464
+
2465
+ /* Extract the FTS5 API pointer from the database handle. The
2466
+ ** fts5_api_from_db() function above is copied verbatim from the
2467
+ ** FTS5 documentation. Refer there for details. */
2468
+ rc = fts5_api_from_db(db, &pApi);
2469
+ if( rc!=SQLITE_OK ) return rc;
2470
+
2471
+ /* If fts5_api_from_db() returns NULL, then either FTS5 is not registered
2472
+ ** with this database handle, or an error (OOM perhaps?) has occurred.
2473
+ **
2474
+ ** Also check that the fts5_api object is version 2 or newer.
2475
+ */
2476
+ if( pApi==0 || pApi->iVersion<2 ){
2477
+ return SQLITE_ERROR;
2478
+ }
2479
+
2480
+ /* Register the implementation of matchinfo() */
2481
+ rc = pApi->xCreateFunction(pApi, "matchinfo", 0, fts5MatchinfoFunc, 0);
2482
+
2483
+ return rc;
2484
+}
21082485
--- src/search.c
+++ src/search.c
@@ -935,14 +935,14 @@
935 " SELECT ftsdocs.label,"
936 " ftsdocs.url,"
937 " rank(matchinfo(ftsidx,'pcsx')),"
938 " ftsdocs.type || ftsdocs.rid,"
939 " datetime(ftsdocs.mtime),"
940 " snippet(ftsidx,'<mark>','</mark>',' ... ',-1,35)"
941 " FROM ftsidx CROSS JOIN ftsdocs"
942 " WHERE ftsidx MATCH %Q"
943 " AND ftsdocs.rowid=ftsidx.docid",
944 zPat
945 );
946 fossil_free(zPat);
947 if( srchFlags!=SRCH_ALL ){
948 const char *zSep = " AND (";
@@ -1508,11 +1508,11 @@
1508 /* The schema for the full-text index
1509 */
1510 static const char zFtsSchema[] =
1511 @ -- One entry for each possible search result
1512 @ CREATE TABLE IF NOT EXISTS repository.ftsdocs(
1513 @ rowid INTEGER PRIMARY KEY, -- Maps to the ftsidx.docid
1514 @ type CHAR(1), -- Type of document
1515 @ rid INTEGER, -- BLOB.RID or TAG.TAGID for the document
1516 @ name TEXT, -- Additional document description
1517 @ idxed BOOLEAN, -- True if currently in the index
1518 @ label TEXT, -- Label to print on search results
@@ -1526,11 +1526,11 @@
1526 @ CREATE VIEW IF NOT EXISTS repository.ftscontent AS
1527 @ SELECT rowid, type, rid, name, idxed, label, url, mtime,
1528 @ title(type,rid,name) AS 'title', body(type,rid,name) AS 'body'
1529 @ FROM ftsdocs;
1530 @ CREATE VIRTUAL TABLE IF NOT EXISTS repository.ftsidx
1531 @ USING fts4(content="ftscontent", title, body%s);
1532 ;
1533 static const char zFtsDrop[] =
1534 @ DROP TABLE IF EXISTS repository.ftsidx;
1535 @ DROP VIEW IF EXISTS repository.ftscontent;
1536 @ DROP TABLE IF EXISTS repository.ftsdocs;
@@ -1606,11 +1606,11 @@
1606 char zType[2];
1607 zType[0] = cType;
1608 zType[1] = 0;
1609 search_sql_setup(g.db);
1610 db_multi_exec(
1611 "DELETE FROM ftsidx WHERE docid IN"
1612 " (SELECT rowid FROM ftsdocs WHERE type=%Q AND rid=%d AND idxed)",
1613 zType, rid
1614 );
1615 db_multi_exec(
1616 "REPLACE INTO ftsdocs(type,rid,name,idxed)"
@@ -1617,11 +1617,11 @@
1617 " VALUES(%Q,%d,%Q,0)",
1618 zType, rid, zName
1619 );
1620 if( cType=='w' || cType=='e' ){
1621 db_multi_exec(
1622 "DELETE FROM ftsidx WHERE docid IN"
1623 " (SELECT rowid FROM ftsdocs WHERE type='%c' AND name=%Q AND idxed)",
1624 cType, zName
1625 );
1626 db_multi_exec(
1627 "DELETE FROM ftsdocs WHERE type='%c' AND name=%Q AND rid!=%d",
@@ -1657,11 +1657,11 @@
1657 " WHERE foci.checkinID=%d AND blob.uuid=foci.uuid"
1658 " AND %z",
1659 ckid, glob_expr("foci.filename", db_get("doc-glob",""))
1660 );
1661 db_multi_exec(
1662 "DELETE FROM ftsidx WHERE docid IN"
1663 " (SELECT rowid FROM ftsdocs WHERE type='d'"
1664 " AND rid NOT IN (SELECT rid FROM current_docs))"
1665 );
1666 db_multi_exec(
1667 "DELETE FROM ftsdocs WHERE type='d'"
@@ -1676,11 +1676,11 @@
1676 " %.17g"
1677 " FROM current_docs",
1678 zDocBr, rTime
1679 );
1680 db_multi_exec(
1681 "INSERT INTO ftsidx(docid,title,body)"
1682 " SELECT rowid, label, bx FROM ftsdocs WHERE type='d' AND NOT idxed"
1683 );
1684 db_multi_exec(
1685 "UPDATE ftsdocs SET"
1686 " idxed=1,"
@@ -1693,11 +1693,11 @@
1693 /*
1694 ** Deal with all of the unindexed 'c' terms in FTSDOCS
1695 */
1696 static void search_update_checkin_index(void){
1697 db_multi_exec(
1698 "INSERT INTO ftsidx(docid,title,body)"
1699 " SELECT rowid, '', body('c',rid,NULL) FROM ftsdocs"
1700 " WHERE type='c' AND NOT idxed;"
1701 );
1702 db_multi_exec(
1703 "UPDATE ftsdocs SET idxed=1, name=NULL,"
@@ -1716,11 +1716,11 @@
1716 /*
1717 ** Deal with all of the unindexed 't' terms in FTSDOCS
1718 */
1719 static void search_update_ticket_index(void){
1720 db_multi_exec(
1721 "INSERT INTO ftsidx(docid,title,body)"
1722 " SELECT rowid, title('t',rid,NULL), body('t',rid,NULL) FROM ftsdocs"
1723 " WHERE type='t' AND NOT idxed;"
1724 );
1725 if( db_changes()==0 ) return;
1726 db_multi_exec(
@@ -1739,11 +1739,11 @@
1739 /*
1740 ** Deal with all of the unindexed 'w' terms in FTSDOCS
1741 */
1742 static void search_update_wiki_index(void){
1743 db_multi_exec(
1744 "INSERT INTO ftsidx(docid,title,body)"
1745 " SELECT rowid, title('w',rid,NULL),body('w',rid,NULL) FROM ftsdocs"
1746 " WHERE type='w' AND NOT idxed;"
1747 );
1748 if( db_changes()==0 ) return;
1749 db_multi_exec(
@@ -1761,11 +1761,11 @@
1761 /*
1762 ** Deal with all of the unindexed 'f' terms in FTSDOCS
1763 */
1764 static void search_update_forum_index(void){
1765 db_multi_exec(
1766 "INSERT INTO ftsidx(docid,title,body)"
1767 " SELECT rowid, title('f',rid,NULL),body('f',rid,NULL) FROM ftsdocs"
1768 " WHERE type='f' AND NOT idxed;"
1769 );
1770 if( db_changes()==0 ) return;
1771 db_multi_exec(
@@ -1784,11 +1784,11 @@
1784 /*
1785 ** Deal with all of the unindexed 'e' terms in FTSDOCS
1786 */
1787 static void search_update_technote_index(void){
1788 db_multi_exec(
1789 "INSERT INTO ftsidx(docid,title,body)"
1790 " SELECT rowid, title('e',rid,NULL),body('e',rid,NULL) FROM ftsdocs"
1791 " WHERE type='e' AND NOT idxed;"
1792 );
1793 if( db_changes()==0 ) return;
1794 db_multi_exec(
@@ -1869,11 +1869,11 @@
1869 **
1870 ** The current search settings are displayed after any changes are applied.
1871 ** Run this command with no arguments to simply see the settings.
1872 */
1873 void fts_config_cmd(void){
1874 static const struct {
1875 int iCmd;
1876 const char *z;
1877 } aCmd[] = {
1878 { 1, "reindex" },
1879 { 2, "index" },
@@ -2002,24 +2002,24 @@
2002 const char *zUrl = db_column_text(&q,4);
2003 const char *zDocId = db_column_text(&q,0);
2004 char *zName;
2005 char *z;
2006 @ <table border=0>
2007 @ <tr><td align='right'>docid:<td>&nbsp;&nbsp;<td>%d(id)
2008 @ <tr><td align='right'>id:<td><td>%s(zDocId)
2009 @ <tr><td align='right'>name:<td><td>%h(db_column_text(&q,1))
2010 @ <tr><td align='right'>idxed:<td><td>%d(db_column_int(&q,2))
2011 @ <tr><td align='right'>label:<td><td>%h(db_column_text(&q,3))
2012 @ <tr><td align='right'>url:<td><td>
2013 @ <a href='%R%s(zUrl)'>%h(zUrl)</a>
2014 @ <tr><td align='right'>mtime:<td><td>%s(db_column_text(&q,5))
2015 z = db_text(0, "SELECT title FROM ftsidx WHERE docid=%d",id);
2016 if( z && z[0] ){
2017 @ <tr><td align="right">title:<td><td>%h(z)
2018 fossil_free(z);
2019 }
2020 z = db_text(0, "SELECT body FROM ftsidx WHERE docid=%d",id);
2021 if( z && z[0] ){
2022 @ <tr><td align="right" valign="top">body:<td><td>%h(z)
2023 fossil_free(z);
2024 }
2025 @ </table>
@@ -2103,5 +2103,382 @@
2103 @ <th align="right">%d(cnt3)
2104 @ </tfooter>
2105 @ </table>
2106 style_finish_page();
2107 }
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
2108
--- src/search.c
+++ src/search.c
@@ -935,14 +935,14 @@
935 " SELECT ftsdocs.label,"
936 " ftsdocs.url,"
937 " rank(matchinfo(ftsidx,'pcsx')),"
938 " ftsdocs.type || ftsdocs.rid,"
939 " datetime(ftsdocs.mtime),"
940 " snippet(ftsidx,-1,'<mark>','</mark>',' ... ',35)"
941 " FROM ftsidx CROSS JOIN ftsdocs"
942 " WHERE ftsidx MATCH %Q"
943 " AND ftsdocs.rowid=ftsidx.rowid",
944 zPat
945 );
946 fossil_free(zPat);
947 if( srchFlags!=SRCH_ALL ){
948 const char *zSep = " AND (";
@@ -1508,11 +1508,11 @@
1508 /* The schema for the full-text index
1509 */
1510 static const char zFtsSchema[] =
1511 @ -- One entry for each possible search result
1512 @ CREATE TABLE IF NOT EXISTS repository.ftsdocs(
1513 @ rowid INTEGER PRIMARY KEY, -- Maps to the ftsidx.rowid
1514 @ type CHAR(1), -- Type of document
1515 @ rid INTEGER, -- BLOB.RID or TAG.TAGID for the document
1516 @ name TEXT, -- Additional document description
1517 @ idxed BOOLEAN, -- True if currently in the index
1518 @ label TEXT, -- Label to print on search results
@@ -1526,11 +1526,11 @@
1526 @ CREATE VIEW IF NOT EXISTS repository.ftscontent AS
1527 @ SELECT rowid, type, rid, name, idxed, label, url, mtime,
1528 @ title(type,rid,name) AS 'title', body(type,rid,name) AS 'body'
1529 @ FROM ftsdocs;
1530 @ CREATE VIRTUAL TABLE IF NOT EXISTS repository.ftsidx
1531 @ USING fts5(content="ftscontent", title, body%s);
1532 ;
1533 static const char zFtsDrop[] =
1534 @ DROP TABLE IF EXISTS repository.ftsidx;
1535 @ DROP VIEW IF EXISTS repository.ftscontent;
1536 @ DROP TABLE IF EXISTS repository.ftsdocs;
@@ -1606,11 +1606,11 @@
1606 char zType[2];
1607 zType[0] = cType;
1608 zType[1] = 0;
1609 search_sql_setup(g.db);
1610 db_multi_exec(
1611 "DELETE FROM ftsidx WHERE rowid IN"
1612 " (SELECT rowid FROM ftsdocs WHERE type=%Q AND rid=%d AND idxed)",
1613 zType, rid
1614 );
1615 db_multi_exec(
1616 "REPLACE INTO ftsdocs(type,rid,name,idxed)"
@@ -1617,11 +1617,11 @@
1617 " VALUES(%Q,%d,%Q,0)",
1618 zType, rid, zName
1619 );
1620 if( cType=='w' || cType=='e' ){
1621 db_multi_exec(
1622 "DELETE FROM ftsidx WHERE rowid IN"
1623 " (SELECT rowid FROM ftsdocs WHERE type='%c' AND name=%Q AND idxed)",
1624 cType, zName
1625 );
1626 db_multi_exec(
1627 "DELETE FROM ftsdocs WHERE type='%c' AND name=%Q AND rid!=%d",
@@ -1657,11 +1657,11 @@
1657 " WHERE foci.checkinID=%d AND blob.uuid=foci.uuid"
1658 " AND %z",
1659 ckid, glob_expr("foci.filename", db_get("doc-glob",""))
1660 );
1661 db_multi_exec(
1662 "DELETE FROM ftsidx WHERE rowid IN"
1663 " (SELECT rowid FROM ftsdocs WHERE type='d'"
1664 " AND rid NOT IN (SELECT rid FROM current_docs))"
1665 );
1666 db_multi_exec(
1667 "DELETE FROM ftsdocs WHERE type='d'"
@@ -1676,11 +1676,11 @@
1676 " %.17g"
1677 " FROM current_docs",
1678 zDocBr, rTime
1679 );
1680 db_multi_exec(
1681 "INSERT INTO ftsidx(rowid,title,body)"
1682 " SELECT rowid, label, bx FROM ftsdocs WHERE type='d' AND NOT idxed"
1683 );
1684 db_multi_exec(
1685 "UPDATE ftsdocs SET"
1686 " idxed=1,"
@@ -1693,11 +1693,11 @@
1693 /*
1694 ** Deal with all of the unindexed 'c' terms in FTSDOCS
1695 */
1696 static void search_update_checkin_index(void){
1697 db_multi_exec(
1698 "INSERT INTO ftsidx(rowid,title,body)"
1699 " SELECT rowid, '', body('c',rid,NULL) FROM ftsdocs"
1700 " WHERE type='c' AND NOT idxed;"
1701 );
1702 db_multi_exec(
1703 "UPDATE ftsdocs SET idxed=1, name=NULL,"
@@ -1716,11 +1716,11 @@
1716 /*
1717 ** Deal with all of the unindexed 't' terms in FTSDOCS
1718 */
1719 static void search_update_ticket_index(void){
1720 db_multi_exec(
1721 "INSERT INTO ftsidx(rowid,title,body)"
1722 " SELECT rowid, title('t',rid,NULL), body('t',rid,NULL) FROM ftsdocs"
1723 " WHERE type='t' AND NOT idxed;"
1724 );
1725 if( db_changes()==0 ) return;
1726 db_multi_exec(
@@ -1739,11 +1739,11 @@
1739 /*
1740 ** Deal with all of the unindexed 'w' terms in FTSDOCS
1741 */
1742 static void search_update_wiki_index(void){
1743 db_multi_exec(
1744 "INSERT INTO ftsidx(rowid,title,body)"
1745 " SELECT rowid, title('w',rid,NULL),body('w',rid,NULL) FROM ftsdocs"
1746 " WHERE type='w' AND NOT idxed;"
1747 );
1748 if( db_changes()==0 ) return;
1749 db_multi_exec(
@@ -1761,11 +1761,11 @@
1761 /*
1762 ** Deal with all of the unindexed 'f' terms in FTSDOCS
1763 */
1764 static void search_update_forum_index(void){
1765 db_multi_exec(
1766 "INSERT INTO ftsidx(rowid,title,body)"
1767 " SELECT rowid, title('f',rid,NULL),body('f',rid,NULL) FROM ftsdocs"
1768 " WHERE type='f' AND NOT idxed;"
1769 );
1770 if( db_changes()==0 ) return;
1771 db_multi_exec(
@@ -1784,11 +1784,11 @@
1784 /*
1785 ** Deal with all of the unindexed 'e' terms in FTSDOCS
1786 */
1787 static void search_update_technote_index(void){
1788 db_multi_exec(
1789 "INSERT INTO ftsidx(rowid,title,body)"
1790 " SELECT rowid, title('e',rid,NULL),body('e',rid,NULL) FROM ftsdocs"
1791 " WHERE type='e' AND NOT idxed;"
1792 );
1793 if( db_changes()==0 ) return;
1794 db_multi_exec(
@@ -1869,11 +1869,11 @@
1869 **
1870 ** The current search settings are displayed after any changes are applied.
1871 ** Run this command with no arguments to simply see the settings.
1872 */
1873 void fts_config_cmd(void){
1874 static const struct {
1875 int iCmd;
1876 const char *z;
1877 } aCmd[] = {
1878 { 1, "reindex" },
1879 { 2, "index" },
@@ -2002,24 +2002,24 @@
2002 const char *zUrl = db_column_text(&q,4);
2003 const char *zDocId = db_column_text(&q,0);
2004 char *zName;
2005 char *z;
2006 @ <table border=0>
2007 @ <tr><td align='right'>rowid:<td>&nbsp;&nbsp;<td>%d(id)
2008 @ <tr><td align='right'>id:<td><td>%s(zDocId)
2009 @ <tr><td align='right'>name:<td><td>%h(db_column_text(&q,1))
2010 @ <tr><td align='right'>idxed:<td><td>%d(db_column_int(&q,2))
2011 @ <tr><td align='right'>label:<td><td>%h(db_column_text(&q,3))
2012 @ <tr><td align='right'>url:<td><td>
2013 @ <a href='%R%s(zUrl)'>%h(zUrl)</a>
2014 @ <tr><td align='right'>mtime:<td><td>%s(db_column_text(&q,5))
2015 z = db_text(0, "SELECT title FROM ftsidx WHERE rowid=%d",id);
2016 if( z && z[0] ){
2017 @ <tr><td align="right">title:<td><td>%h(z)
2018 fossil_free(z);
2019 }
2020 z = db_text(0, "SELECT body FROM ftsidx WHERE rowid=%d",id);
2021 if( z && z[0] ){
2022 @ <tr><td align="right" valign="top">body:<td><td>%h(z)
2023 fossil_free(z);
2024 }
2025 @ </table>
@@ -2103,5 +2103,382 @@
2103 @ <th align="right">%d(cnt3)
2104 @ </tfooter>
2105 @ </table>
2106 style_finish_page();
2107 }
2108
2109
2110 /*
2111 ** The Fts5MatchinfoCtx bits were all taken verbatim from:
2112 **
2113 ** https://sqlite.org/src/finfo?name=ext/fts5/fts5_test_mi.c
2114 */
2115
2116 typedef struct Fts5MatchinfoCtx Fts5MatchinfoCtx;
2117
2118 #ifndef SQLITE_AMALGAMATION
2119 typedef unsigned int u32;
2120 #endif
2121
2122 struct Fts5MatchinfoCtx {
2123 int nCol; /* Number of cols in FTS5 table */
2124 int nPhrase; /* Number of phrases in FTS5 query */
2125 char *zArg; /* nul-term'd copy of 2nd arg */
2126 int nRet; /* Number of elements in aRet[] */
2127 u32 *aRet; /* Array of 32-bit unsigned ints to return */
2128 };
2129
2130
2131 /*
2132 ** Return a pointer to the fts5_api pointer for database connection db.
2133 ** If an error occurs, return NULL and leave an error in the database
2134 ** handle (accessible using sqlite3_errcode()/errmsg()).
2135 */
2136 static int fts5_api_from_db(sqlite3 *db, fts5_api **ppApi){
2137 sqlite3_stmt *pStmt = 0;
2138 int rc;
2139
2140 *ppApi = 0;
2141 rc = sqlite3_prepare(db, "SELECT fts5(?1)", -1, &pStmt, 0);
2142 if( rc==SQLITE_OK ){
2143 sqlite3_bind_pointer(pStmt, 1, (void*)ppApi, "fts5_api_ptr", 0);
2144 (void)sqlite3_step(pStmt);
2145 rc = sqlite3_finalize(pStmt);
2146 }
2147
2148 return rc;
2149 }
2150
2151
2152 /*
2153 ** Argument f should be a flag accepted by matchinfo() (a valid character
2154 ** in the string passed as the second argument). If it is not, -1 is
2155 ** returned. Otherwise, if f is a valid matchinfo flag, the value returned
2156 ** is the number of 32-bit integers added to the output array if the
2157 ** table has nCol columns and the query nPhrase phrases.
2158 */
2159 static int fts5MatchinfoFlagsize(int nCol, int nPhrase, char f){
2160 int ret = -1;
2161 switch( f ){
2162 case 'p': ret = 1; break;
2163 case 'c': ret = 1; break;
2164 case 'x': ret = 3 * nCol * nPhrase; break;
2165 case 'y': ret = nCol * nPhrase; break;
2166 case 'b': ret = ((nCol + 31) / 32) * nPhrase; break;
2167 case 'n': ret = 1; break;
2168 case 'a': ret = nCol; break;
2169 case 'l': ret = nCol; break;
2170 case 's': ret = nCol; break;
2171 }
2172 return ret;
2173 }
2174
2175 static int fts5MatchinfoIter(
2176 const Fts5ExtensionApi *pApi, /* API offered by current FTS version */
2177 Fts5Context *pFts, /* First arg to pass to pApi functions */
2178 Fts5MatchinfoCtx *p,
2179 int(*x)(const Fts5ExtensionApi*,Fts5Context*,Fts5MatchinfoCtx*,char,u32*)
2180 ){
2181 int i;
2182 int n = 0;
2183 int rc = SQLITE_OK;
2184 char f;
2185 for(i=0; (f = p->zArg[i]); i++){
2186 rc = x(pApi, pFts, p, f, &p->aRet[n]);
2187 if( rc!=SQLITE_OK ) break;
2188 n += fts5MatchinfoFlagsize(p->nCol, p->nPhrase, f);
2189 }
2190 return rc;
2191 }
2192
2193 static int fts5MatchinfoXCb(
2194 const Fts5ExtensionApi *pApi,
2195 Fts5Context *pFts,
2196 void *pUserData
2197 ){
2198 Fts5PhraseIter iter;
2199 int iCol, iOff;
2200 u32 *aOut = (u32*)pUserData;
2201 int iPrev = -1;
2202
2203 for(pApi->xPhraseFirst(pFts, 0, &iter, &iCol, &iOff);
2204 iCol>=0;
2205 pApi->xPhraseNext(pFts, &iter, &iCol, &iOff)
2206 ){
2207 aOut[iCol*3+1]++;
2208 if( iCol!=iPrev ) aOut[iCol*3 + 2]++;
2209 iPrev = iCol;
2210 }
2211
2212 return SQLITE_OK;
2213 }
2214
2215 static int fts5MatchinfoGlobalCb(
2216 const Fts5ExtensionApi *pApi,
2217 Fts5Context *pFts,
2218 Fts5MatchinfoCtx *p,
2219 char f,
2220 u32 *aOut
2221 ){
2222 int rc = SQLITE_OK;
2223 switch( f ){
2224 case 'p':
2225 aOut[0] = p->nPhrase;
2226 break;
2227
2228 case 'c':
2229 aOut[0] = p->nCol;
2230 break;
2231
2232 case 'x': {
2233 int i;
2234 for(i=0; i<p->nPhrase && rc==SQLITE_OK; i++){
2235 void *pPtr = (void*)&aOut[i * p->nCol * 3];
2236 rc = pApi->xQueryPhrase(pFts, i, pPtr, fts5MatchinfoXCb);
2237 }
2238 break;
2239 }
2240
2241 case 'n': {
2242 sqlite3_int64 nRow;
2243 rc = pApi->xRowCount(pFts, &nRow);
2244 aOut[0] = (u32)nRow;
2245 break;
2246 }
2247
2248 case 'a': {
2249 sqlite3_int64 nRow = 0;
2250 rc = pApi->xRowCount(pFts, &nRow);
2251 if( nRow==0 ){
2252 memset(aOut, 0, sizeof(u32) * p->nCol);
2253 }else{
2254 int i;
2255 for(i=0; rc==SQLITE_OK && i<p->nCol; i++){
2256 sqlite3_int64 nToken;
2257 rc = pApi->xColumnTotalSize(pFts, i, &nToken);
2258 if( rc==SQLITE_OK){
2259 aOut[i] = (u32)((2*nToken + nRow) / (2*nRow));
2260 }
2261 }
2262 }
2263 break;
2264 }
2265
2266 }
2267 return rc;
2268 }
2269
2270 static int fts5MatchinfoLocalCb(
2271 const Fts5ExtensionApi *pApi,
2272 Fts5Context *pFts,
2273 Fts5MatchinfoCtx *p,
2274 char f,
2275 u32 *aOut
2276 ){
2277 int i;
2278 int rc = SQLITE_OK;
2279
2280 switch( f ){
2281 case 'b': {
2282 int iPhrase;
2283 int nInt = ((p->nCol + 31) / 32) * p->nPhrase;
2284 for(i=0; i<nInt; i++) aOut[i] = 0;
2285
2286 for(iPhrase=0; iPhrase<p->nPhrase; iPhrase++){
2287 Fts5PhraseIter iter;
2288 int iCol;
2289 for(pApi->xPhraseFirstColumn(pFts, iPhrase, &iter, &iCol);
2290 iCol>=0;
2291 pApi->xPhraseNextColumn(pFts, &iter, &iCol)
2292 ){
2293 aOut[iPhrase * ((p->nCol+31)/32) + iCol/32] |= ((u32)1 << iCol%32);
2294 }
2295 }
2296
2297 break;
2298 }
2299
2300 case 'x':
2301 case 'y': {
2302 int nMul = (f=='x' ? 3 : 1);
2303 int iPhrase;
2304
2305 for(i=0; i<(p->nCol*p->nPhrase); i++) aOut[i*nMul] = 0;
2306
2307 for(iPhrase=0; iPhrase<p->nPhrase; iPhrase++){
2308 Fts5PhraseIter iter;
2309 int iOff, iCol;
2310 for(pApi->xPhraseFirst(pFts, iPhrase, &iter, &iCol, &iOff);
2311 iOff>=0;
2312 pApi->xPhraseNext(pFts, &iter, &iCol, &iOff)
2313 ){
2314 aOut[nMul * (iCol + iPhrase * p->nCol)]++;
2315 }
2316 }
2317
2318 break;
2319 }
2320
2321 case 'l': {
2322 for(i=0; rc==SQLITE_OK && i<p->nCol; i++){
2323 int nToken;
2324 rc = pApi->xColumnSize(pFts, i, &nToken);
2325 aOut[i] = (u32)nToken;
2326 }
2327 break;
2328 }
2329
2330 case 's': {
2331 int nInst;
2332
2333 memset(aOut, 0, sizeof(u32) * p->nCol);
2334
2335 rc = pApi->xInstCount(pFts, &nInst);
2336 for(i=0; rc==SQLITE_OK && i<nInst; i++){
2337 int iPhrase, iOff, iCol = 0;
2338 int iNextPhrase;
2339 int iNextOff;
2340 u32 nSeq = 1;
2341 int j;
2342
2343 rc = pApi->xInst(pFts, i, &iPhrase, &iCol, &iOff);
2344 iNextPhrase = iPhrase+1;
2345 iNextOff = iOff+pApi->xPhraseSize(pFts, 0);
2346 for(j=i+1; rc==SQLITE_OK && j<nInst; j++){
2347 int ip, ic, io;
2348 rc = pApi->xInst(pFts, j, &ip, &ic, &io);
2349 if( ic!=iCol || io>iNextOff ) break;
2350 if( ip==iNextPhrase && io==iNextOff ){
2351 nSeq++;
2352 iNextPhrase = ip+1;
2353 iNextOff = io + pApi->xPhraseSize(pFts, ip);
2354 }
2355 }
2356
2357 if( nSeq>aOut[iCol] ) aOut[iCol] = nSeq;
2358 }
2359
2360 break;
2361 }
2362 }
2363 return rc;
2364 }
2365
2366 static Fts5MatchinfoCtx *fts5MatchinfoNew(
2367 const Fts5ExtensionApi *pApi, /* API offered by current FTS version */
2368 Fts5Context *pFts, /* First arg to pass to pApi functions */
2369 sqlite3_context *pCtx, /* Context for returning error message */
2370 const char *zArg /* Matchinfo flag string */
2371 ){
2372 Fts5MatchinfoCtx *p;
2373 int nCol;
2374 int nPhrase;
2375 int i;
2376 int nInt;
2377 sqlite3_int64 nByte;
2378 int rc;
2379
2380 nCol = pApi->xColumnCount(pFts);
2381 nPhrase = pApi->xPhraseCount(pFts);
2382
2383 nInt = 0;
2384 for(i=0; zArg[i]; i++){
2385 int n = fts5MatchinfoFlagsize(nCol, nPhrase, zArg[i]);
2386 if( n<0 ){
2387 char *zErr = sqlite3_mprintf("unrecognized matchinfo flag: %c", zArg[i]);
2388 sqlite3_result_error(pCtx, zErr, -1);
2389 sqlite3_free(zErr);
2390 return 0;
2391 }
2392 nInt += n;
2393 }
2394
2395 nByte = sizeof(Fts5MatchinfoCtx) /* The struct itself */
2396 + sizeof(u32) * nInt /* The p->aRet[] array */
2397 + (i+1); /* The p->zArg string */
2398 p = (Fts5MatchinfoCtx*)sqlite3_malloc64(nByte);
2399 if( p==0 ){
2400 sqlite3_result_error_nomem(pCtx);
2401 return 0;
2402 }
2403 memset(p, 0, nByte);
2404
2405 p->nCol = nCol;
2406 p->nPhrase = nPhrase;
2407 p->aRet = (u32*)&p[1];
2408 p->nRet = nInt;
2409 p->zArg = (char*)&p->aRet[nInt];
2410 memcpy(p->zArg, zArg, i);
2411
2412 rc = fts5MatchinfoIter(pApi, pFts, p, fts5MatchinfoGlobalCb);
2413 if( rc!=SQLITE_OK ){
2414 sqlite3_result_error_code(pCtx, rc);
2415 sqlite3_free(p);
2416 p = 0;
2417 }
2418
2419 return p;
2420 }
2421
2422 static void fts5MatchinfoFunc(
2423 const Fts5ExtensionApi *pApi, /* API offered by current FTS version */
2424 Fts5Context *pFts, /* First arg to pass to pApi functions */
2425 sqlite3_context *pCtx, /* Context for returning result/error */
2426 int nVal, /* Number of values in apVal[] array */
2427 sqlite3_value **apVal /* Array of trailing arguments */
2428 ){
2429 const char *zArg;
2430 Fts5MatchinfoCtx *p;
2431 int rc = SQLITE_OK;
2432
2433 if( nVal>0 ){
2434 zArg = (const char*)sqlite3_value_text(apVal[0]);
2435 }else{
2436 zArg = "pcx";
2437 }
2438
2439 p = (Fts5MatchinfoCtx*)pApi->xGetAuxdata(pFts, 0);
2440 if( p==0 || sqlite3_stricmp(zArg, p->zArg) ){
2441 p = fts5MatchinfoNew(pApi, pFts, pCtx, zArg);
2442 if( p==0 ){
2443 rc = SQLITE_NOMEM;
2444 }else{
2445 rc = pApi->xSetAuxdata(pFts, p, sqlite3_free);
2446 }
2447 }
2448
2449 if( rc==SQLITE_OK ){
2450 rc = fts5MatchinfoIter(pApi, pFts, p, fts5MatchinfoLocalCb);
2451 }
2452 if( rc!=SQLITE_OK ){
2453 sqlite3_result_error_code(pCtx, rc);
2454 }else{
2455 /* No errors has occured, so return a copy of the array of integers. */
2456 int nByte = p->nRet * sizeof(u32);
2457 sqlite3_result_blob(pCtx, (void*)p->aRet, nByte, SQLITE_TRANSIENT);
2458 }
2459 }
2460
2461 int db_register_fts5(sqlite3 *db){
2462 int rc; /* Return code */
2463 fts5_api *pApi; /* FTS5 API functions */
2464
2465 /* Extract the FTS5 API pointer from the database handle. The
2466 ** fts5_api_from_db() function above is copied verbatim from the
2467 ** FTS5 documentation. Refer there for details. */
2468 rc = fts5_api_from_db(db, &pApi);
2469 if( rc!=SQLITE_OK ) return rc;
2470
2471 /* If fts5_api_from_db() returns NULL, then either FTS5 is not registered
2472 ** with this database handle, or an error (OOM perhaps?) has occurred.
2473 **
2474 ** Also check that the fts5_api object is version 2 or newer.
2475 */
2476 if( pApi==0 || pApi->iVersion<2 ){
2477 return SQLITE_ERROR;
2478 }
2479
2480 /* Register the implementation of matchinfo() */
2481 rc = pApi->xCreateFunction(pApi, "matchinfo", 0, fts5MatchinfoFunc, 0);
2482
2483 return rc;
2484 }
2485
M src/setup.c
+2 -2
--- src/setup.c
+++ src/setup.c
@@ -2067,18 +2067,18 @@
20672067
 search_create_index();
20682068
 search_fill_index();
20692069
 search_update_index(search_restrict(SRCH_ALL));
20702070
 }
20712071
 if( search_index_exists() ){
2072
- @ <p>Currently using an SQLite FTS4 search index. This makes search
2072
+ @ <p>Currently using an SQLite FTS5 search index. This makes search
20732073
 @ run faster, especially on large repositories, but takes up space.</p>
20742074
 onoff_attribute("Use Porter Stemmer","search-stemmer","ss",0,0);
20752075
 @ <p><input type="submit" name="fts0" value="Delete The Full-Text Index">
20762076
 @ <input type="submit" name="fts1" value="Rebuild The Full-Text Index">
20772077
 style_submenu_element("FTS Index Debugging","%R/test-ftsdocs");
20782078
 }else{
2079
- @ <p>The SQLite FTS4 search index is disabled. All searching will be
2079
+ @ <p>The SQLite FTS5 search index is disabled. All searching will be
20802080
 @ a full-text scan. This usually works fine, but can be slow for
20812081
 @ larger repositories.</p>
20822082
 onoff_attribute("Use Porter Stemmer","search-stemmer","ss",0,0);
20832083
 @ <p><input type="submit" name="fts1" value="Create A Full-Text Index">
20842084
 }
20852085
--- src/setup.c
+++ src/setup.c
@@ -2067,18 +2067,18 @@
2067 search_create_index();
2068 search_fill_index();
2069 search_update_index(search_restrict(SRCH_ALL));
2070 }
2071 if( search_index_exists() ){
2072 @ <p>Currently using an SQLite FTS4 search index. This makes search
2073 @ run faster, especially on large repositories, but takes up space.</p>
2074 onoff_attribute("Use Porter Stemmer","search-stemmer","ss",0,0);
2075 @ <p><input type="submit" name="fts0" value="Delete The Full-Text Index">
2076 @ <input type="submit" name="fts1" value="Rebuild The Full-Text Index">
2077 style_submenu_element("FTS Index Debugging","%R/test-ftsdocs");
2078 }else{
2079 @ <p>The SQLite FTS4 search index is disabled. All searching will be
2080 @ a full-text scan. This usually works fine, but can be slow for
2081 @ larger repositories.</p>
2082 onoff_attribute("Use Porter Stemmer","search-stemmer","ss",0,0);
2083 @ <p><input type="submit" name="fts1" value="Create A Full-Text Index">
2084 }
2085
--- src/setup.c
+++ src/setup.c
@@ -2067,18 +2067,18 @@
2067 search_create_index();
2068 search_fill_index();
2069 search_update_index(search_restrict(SRCH_ALL));
2070 }
2071 if( search_index_exists() ){
2072 @ <p>Currently using an SQLite FTS5 search index. This makes search
2073 @ run faster, especially on large repositories, but takes up space.</p>
2074 onoff_attribute("Use Porter Stemmer","search-stemmer","ss",0,0);
2075 @ <p><input type="submit" name="fts0" value="Delete The Full-Text Index">
2076 @ <input type="submit" name="fts1" value="Rebuild The Full-Text Index">
2077 style_submenu_element("FTS Index Debugging","%R/test-ftsdocs");
2078 }else{
2079 @ <p>The SQLite FTS5 search index is disabled. All searching will be
2080 @ a full-text scan. This usually works fine, but can be slow for
2081 @ larger repositories.</p>
2082 onoff_attribute("Use Porter Stemmer","search-stemmer","ss",0,0);
2083 @ <p><input type="submit" name="fts1" value="Create A Full-Text Index">
2084 }
2085
M www/changes.wiki
+1
--- www/changes.wiki
+++ www/changes.wiki
@@ -15,10 +15,11 @@
1515
 * Passwords for remembered remote repositories are now stored as irreversible
1616
 hashes rather than obscured clear-text, for improved security.
1717
 * Writes to the database are disabled by default if the HTTP request
1818
 does not come from the same origin. This enhancement is for defense in depth.
1919
 There where no known attacks prior to this enhancement.
20
+ * Update search infrastructure from FTS4 to FTS5.
2021
2122
 <h2 id='v2_20'>Changes for version 2.20 (2022-11-16)</h2>
2223
 * Added the [/help?cmd=chat-timeline-user|chat-timeline-user setting]. If
2324
 it is not an empty string, then any changes that would appear on the timeline
2425
 are announced in [./chat.md|the chat room].
2526
--- www/changes.wiki
+++ www/changes.wiki
@@ -15,10 +15,11 @@
15 * Passwords for remembered remote repositories are now stored as irreversible
16 hashes rather than obscured clear-text, for improved security.
17 * Writes to the database are disabled by default if the HTTP request
18 does not come from the same origin. This enhancement is for defense in depth.
19 There where no known attacks prior to this enhancement.
 
20
21 <h2 id='v2_20'>Changes for version 2.20 (2022-11-16)</h2>
22 * Added the [/help?cmd=chat-timeline-user|chat-timeline-user setting]. If
23 it is not an empty string, then any changes that would appear on the timeline
24 are announced in [./chat.md|the chat room].
25
--- www/changes.wiki
+++ www/changes.wiki
@@ -15,10 +15,11 @@
15 * Passwords for remembered remote repositories are now stored as irreversible
16 hashes rather than obscured clear-text, for improved security.
17 * Writes to the database are disabled by default if the HTTP request
18 does not come from the same origin. This enhancement is for defense in depth.
19 There where no known attacks prior to this enhancement.
20 * Update search infrastructure from FTS4 to FTS5.
21
22 <h2 id='v2_20'>Changes for version 2.20 (2022-11-16)</h2>
23 * Added the [/help?cmd=chat-timeline-user|chat-timeline-user setting]. If
24 it is not an empty string, then any changes that would appear on the timeline
25 are announced in [./chat.md|the chat room].
26

Keyboard Shortcuts

Open search /
Next entry (timeline) j
Previous entry (timeline) k
Open focused entry Enter
Show this help ?
Toggle theme Top nav button