Fossil SCM

Incremental check-in for work on the indexed full-text search.

drh 2015-02-02 20:53 UTC indexed-fts
Commit ec0e590191b10f23c8b3b975b1f1ca3d6149d02e
Parent 32d904e9cff95cf…
2 files changed +7 -6 +193 -43
~ src/db.c ~ src/search.c
M src/db.c
+7 -6
--- src/db.c
+++ src/db.c
@@ -1380,15 +1380,10 @@
13801380
 while( db.pAllStmt ){
13811381
 db_finalize(db.pAllStmt);
13821382
 }
13831383
 db_end_transaction(1);
13841384
 pStmt = 0;
1385
- if( reportErrors ){
1386
- while( (pStmt = sqlite3_next_stmt(g.db, pStmt))!=0 ){
1387
- fossil_warning("unfinalized SQL statement: [%s]", sqlite3_sql(pStmt));
1388
- }
1389
- }
13901385
 db_close_config();
13911386
13921387
 /* If the localdb (the check-out database) is open and if it has
13931388
 ** a lot of unused free space, then VACUUM it as we shut down.
13941389
 */
@@ -1399,12 +1394,18 @@
13991394
 db_multi_exec("VACUUM;");
14001395
 }
14011396
 }
14021397
14031398
 if( g.db ){
1399
+ int rc;
14041400
 sqlite3_wal_checkpoint(g.db, 0);
1405
- sqlite3_close(g.db);
1401
+ rc = sqlite3_close(g.db);
1402
+ if( rc==SQLITE_BUSY && reportErrors ){
1403
+ while( (pStmt = sqlite3_next_stmt(g.db, pStmt))!=0 ){
1404
+ fossil_warning("unfinalized SQL statement: [%s]", sqlite3_sql(pStmt));
1405
+ }
1406
+ }
14061407
 g.db = 0;
14071408
 g.zMainDbType = 0;
14081409
 }
14091410
 g.repositoryOpen = 0;
14101411
 g.localOpen = 0;
14111412
--- src/db.c
+++ src/db.c
@@ -1380,15 +1380,10 @@
1380 while( db.pAllStmt ){
1381 db_finalize(db.pAllStmt);
1382 }
1383 db_end_transaction(1);
1384 pStmt = 0;
1385 if( reportErrors ){
1386 while( (pStmt = sqlite3_next_stmt(g.db, pStmt))!=0 ){
1387 fossil_warning("unfinalized SQL statement: [%s]", sqlite3_sql(pStmt));
1388 }
1389 }
1390 db_close_config();
1391
1392 /* If the localdb (the check-out database) is open and if it has
1393 ** a lot of unused free space, then VACUUM it as we shut down.
1394 */
@@ -1399,12 +1394,18 @@
1399 db_multi_exec("VACUUM;");
1400 }
1401 }
1402
1403 if( g.db ){
 
1404 sqlite3_wal_checkpoint(g.db, 0);
1405 sqlite3_close(g.db);
 
 
 
 
 
1406 g.db = 0;
1407 g.zMainDbType = 0;
1408 }
1409 g.repositoryOpen = 0;
1410 g.localOpen = 0;
1411
--- src/db.c
+++ src/db.c
@@ -1380,15 +1380,10 @@
1380 while( db.pAllStmt ){
1381 db_finalize(db.pAllStmt);
1382 }
1383 db_end_transaction(1);
1384 pStmt = 0;
 
 
 
 
 
1385 db_close_config();
1386
1387 /* If the localdb (the check-out database) is open and if it has
1388 ** a lot of unused free space, then VACUUM it as we shut down.
1389 */
@@ -1399,12 +1394,18 @@
1394 db_multi_exec("VACUUM;");
1395 }
1396 }
1397
1398 if( g.db ){
1399 int rc;
1400 sqlite3_wal_checkpoint(g.db, 0);
1401 rc = sqlite3_close(g.db);
1402 if( rc==SQLITE_BUSY && reportErrors ){
1403 while( (pStmt = sqlite3_next_stmt(g.db, pStmt))!=0 ){
1404 fossil_warning("unfinalized SQL statement: [%s]", sqlite3_sql(pStmt));
1405 }
1406 }
1407 g.db = 0;
1408 g.zMainDbType = 0;
1409 }
1410 g.repositoryOpen = 0;
1411 g.localOpen = 0;
1412
M src/search.c
+193 -43
--- src/search.c
+++ src/search.c
@@ -453,11 +453,11 @@
453453
 void search_sql_setup(sqlite3 *db){
454454
 static int once = 0;
455455
 if( once++ ) return;
456456
 sqlite3_create_function(db, "score", -1, SQLITE_UTF8, 0,
457457
 search_score_sqlfunc, 0, 0);
458
- sqlite3_create_function(db, "snippet", -1, SQLITE_UTF8, &gSearch,
458
+ sqlite3_create_function(db, "fsnippet", -1, SQLITE_UTF8, &gSearch,
459459
 search_score_sqlfunc, 0, 0);
460460
 sqlite3_create_function(db, "search_init", -1, SQLITE_UTF8, 0,
461461
 search_init_sqlfunc, 0, 0);
462462
 sqlite3_create_function(db, "stext", 3, SQLITE_UTF8, 0,
463463
 search_stext_sqlfunc, 0, 0);
@@ -553,64 +553,59 @@
553553
 /*
554554
 ** Remove bits from srchFlags which are disallowed by either the
555555
 ** current server configuration or by user permissions.
556556
 */
557557
 unsigned int search_restrict(unsigned int srchFlags){
558
- if( (srchFlags & SRCH_CKIN)!=0
559
- && (g.perm.Read==0 || db_get_boolean("search-ci",0)==0) ){
558
+ if( g.perm.Read==0 ) srchFlags &= ~(SRCH_CKIN|SRCH_DOC);
559
+ if( g.perm.RdTkt==0 ) srchFlags &= ~(SRCH_TKT);
560
+ if( g.perm.RdWiki==0 ) srchFlags &= ~(SRCH_WIKI);
561
+ if( search_index_exists() ) return srchFlags;
562
+ if( (srchFlags & SRCH_CKIN)!=0 && db_get_boolean("search-ci",0)==0 ){
560563
 srchFlags &= ~SRCH_CKIN;
561564
 }
562
- if( (srchFlags & SRCH_DOC)!=0
563
- && (g.perm.Read==0 || db_get_boolean("search-doc",0)==0) ){
565
+ if( (srchFlags & SRCH_DOC)!=0 && db_get_boolean("search-doc",0)==0 ){
564566
 srchFlags &= ~SRCH_DOC;
565567
 }
566
- if( (srchFlags & SRCH_TKT)!=0
567
- && (g.perm.RdTkt==0 || db_get_boolean("search-tkt",0)==0) ){
568
+ if( (srchFlags & SRCH_TKT)!=0 && db_get_boolean("search-tkt",0)==0 ){
568569
 srchFlags &= ~SRCH_TKT;
569570
 }
570
- if( (srchFlags & SRCH_WIKI)!=0
571
- && (g.perm.RdWiki==0 || db_get_boolean("search-wiki",0)==0) ){
571
+ if( (srchFlags & SRCH_WIKI)!=0 && db_get_boolean("search-wiki",0)==0 ){
572572
 srchFlags &= ~SRCH_WIKI;
573573
 }
574574
 return srchFlags;
575575
 }
576576
577577
 /*
578
-** This routine generates web-page output for a search operation.
579
-** Other web-pages can invoke this routine to add search results
580
-** in the middle of the page.
578
+** When this routine is called, there already exists a table
579
+**
580
+** x(label,url,score,date,snip).
581
+**
582
+** And the srchFlags parameter has been validated. This routine
583
+** fills the X table with search results using a full-text scan.
581584
 **
582
-** Return the number of rows.
585
+** The companion indexed scan routine is search_indexed().
583586
 */
584
-int search_run_and_output(
587
+static void search_fullscan(
585588
 const char *zPattern, /* The query pattern */
586589
 unsigned int srchFlags /* What to search over */
587590
 ){
588
- Stmt q;
589
- int nRow = 0;
590
-
591
- srchFlags = search_restrict(srchFlags);
592
- if( srchFlags==0 ) return 0;
593
- search_sql_setup(g.db);
594
- add_content_sql_commands(g.db);
595591
 search_init(zPattern, "<b>", "</b>", " ... ",
596592
 SRCHFLG_STATIC|SRCHFLG_HTML|SRCHFLG_SCORE);
597
- db_multi_exec(
598
- "CREATE VIRTUAL TABLE IF NOT EXISTS temp.foci USING files_of_checkin;"
599
- "CREATE TEMP TABLE x(label TEXT,url TEXT,date TEXT,snip TEXT);"
600
- );
601593
 if( (srchFlags & SRCH_DOC)!=0 ){
602594
 char *zDocGlob = db_get("doc-glob","");
603595
 char *zDocBr = db_get("doc-branch","trunk");
604596
 if( zDocGlob && zDocGlob[0] && zDocBr && zDocBr[0] ){
597
+ db_multi_exec(
598
+ "CREATE VIRTUAL TABLE IF NOT EXISTS temp.foci USING files_of_checkin;"
599
+ );
605600
 db_multi_exec(
606601
 "INSERT INTO x(label,url,date,snip)"
607602
 " SELECT printf('Document: %%s',foci.filename),"
608603
 " printf('%R/doc/%T/%%s',foci.filename),"
609604
 " (SELECT datetime(event.mtime) FROM event"
610605
 " WHERE objid=symbolic_name_to_rid('trunk')),"
611
- " snippet(stext('d',blob.rid,foci.filename))"
606
+ " fsnippet(stext('d',blob.rid,foci.filename))"
612607
 " FROM foci CROSS JOIN blob"
613608
 " WHERE checkinID=symbolic_name_to_rid('trunk')"
614609
 " AND blob.uuid=foci.uuid"
615610
 " AND %z",
616611
 zDocBr, glob_expr("foci.filename", zDocGlob)
@@ -628,11 +623,11 @@
628623
 ")"
629624
 "INSERT INTO x(label,url,date,snip)"
630625
 " SELECT printf('Wiki: %%s',name),"
631626
 " printf('%R/wiki?name=%%s',urlencode(name)),"
632627
 " datetime(mtime),"
633
- " snippet(stext('w',rid,name))"
628
+ " fsnippet(stext('w',rid,name))"
634629
 " FROM wiki;"
635630
 );
636631
 }
637632
 if( (srchFlags & SRCH_CKIN)!=0 ){
638633
 db_multi_exec(
@@ -644,11 +639,11 @@
644639
 ")"
645640
 "INSERT INTO x(label,url,date,snip)"
646641
 " SELECT printf('Check-in [%%.10s] on %%s',uuid,datetime(mtime)),"
647642
 " printf('%R/timeline?c=%%s&n=8&y=ci',uuid),"
648643
 " datetime(mtime),"
649
- " snippet(stext('c',rid,NULL))"
644
+ " fsnippet(stext('c',rid,NULL))"
650645
 " FROM ckin;"
651646
 );
652647
 }
653648
 if( (srchFlags & SRCH_TKT)!=0 ){
654649
 db_multi_exec(
@@ -655,17 +650,77 @@
655650
 "INSERT INTO x(label,url,date,snip)"
656651
 " SELECT printf('Ticket [%%.17s] on %%s',"
657652
 "tkt_uuid,datetime(tkt_mtime)),"
658653
 " printf('%R/tktview/%%.20s',tkt_uuid),"
659654
 " datetime(tkt_mtime),"
660
- " snippet(stext('t',tkt_id,NULL))"
655
+ " fsnippet(stext('t',tkt_id,NULL))"
661656
 " FROM ticket;"
662657
 );
663658
 }
664
- db_prepare(&q, "SELECT url, substr(snip,9), label"
665
- " FROM x WHERE snip IS NOT NULL"
666
- " ORDER BY substr(snip,1,8) DESC, date DESC;");
659
+ db_multi_exec(
660
+ "UPDATE x SET score=substr(snip,1,8), snip=substr(snip,9)"
661
+ );
662
+}
663
+
664
+/*
665
+** When this routine is called, there already exists a table
666
+**
667
+** x(label,url,score,date,snip).
668
+**
669
+** And the srchFlags parameter has been validated. This routine
670
+** fills the X table with search results using a index scan.
671
+**
672
+** The companion full-text scan routine is search_fullscan().
673
+*/
674
+static void search_indexed(
675
+ const char *zPattern, /* The query pattern */
676
+ unsigned int srchFlags /* What to search over */
677
+){
678
+ db_multi_exec(
679
+ "INSERT INTO x(label,url,score,date,snip) "
680
+ " SELECT ftsdocs.label,"
681
+ " ftsdocs.url,"
682
+ " 1," /*FIX ME*/
683
+ " datetime(ftsdocs.mtime),"
684
+ " fsnippet(ftsidx,'<b>','</b>',' ... ')"
685
+ " FROM ftsidx, ftsdocs"
686
+ " WHERE ftsidx MATCH %Q"
687
+ " AND ftsdocs.id=ftsidx.docid",
688
+ zPattern
689
+ );
690
+}
691
+
692
+
693
+/*
694
+** This routine generates web-page output for a search operation.
695
+** Other web-pages can invoke this routine to add search results
696
+** in the middle of the page.
697
+**
698
+** Return the number of rows.
699
+*/
700
+int search_run_and_output(
701
+ const char *zPattern, /* The query pattern */
702
+ unsigned int srchFlags /* What to search over */
703
+){
704
+ Stmt q;
705
+ int nRow = 0;
706
+
707
+ srchFlags = search_restrict(srchFlags);
708
+ if( srchFlags==0 ) return 0;
709
+ search_sql_setup(g.db);
710
+ add_content_sql_commands(g.db);
711
+ db_multi_exec(
712
+ "CREATE TEMP TABLE x(label,url,score,date,snip);"
713
+ );
714
+ if( !search_index_exists() ){
715
+ search_fullscan(zPattern, srchFlags);
716
+ }else{
717
+ search_indexed(zPattern, srchFlags);
718
+ }
719
+ db_prepare(&q, "SELECT url, snip, label"
720
+ " FROM x"
721
+ " ORDER BY score DESC, date DESC;");
667722
 while( db_step(&q)==SQLITE_ROW ){
668723
 const char *zUrl = db_column_text(&q, 0);
669724
 const char *zSnippet = db_column_text(&q, 1);
670725
 const char *zLabel = db_column_text(&q, 2);
671726
 if( nRow==0 ){
@@ -926,21 +981,24 @@
926981
 /* The schema for the full-text index
927982
 */
928983
 static const char zFtsSchema[] =
929984
 @ -- One entry for each possible search result
930985
 @ CREATE TABLE IF NOT EXISTS "%w".ftsdocs(
931
-@ id INTEGER PRIMARY KEY, -- Maps to the ftsidx.rowid
986
+@ id INTEGER PRIMARY KEY, -- Maps to the ftsidx.docid
932987
 @ type CHAR(1), -- Type of document
933988
 @ rid INTEGER, -- BLOB.RID or TAG.TAGID for the document
934989
 @ name TEXT, -- Additional document description
935990
 @ idxed BOOLEAN, -- True if currently in the index
991
+@ label TEXT, -- Label to print on search results
992
+@ url TEXT, -- URL to access this document
993
+@ mtime DATE, -- Date when document created
936994
 @ UNIQUE(type,rid)
937995
 @ );
938996
 @ CREATE INDEX "%w".ftsdocIdxed ON ftsdocs(type,rid,name) WHERE idxed==0;
939997
 @ CREATE VIEW IF NOT EXISTS "%w".ftscontent AS
940
-@ SELECT id, type, rid, name,
941
-@ stext(type,rid,name) AS stext
998
+@ SELECT id, type, rid, name, idxed, label, url, mtime,fr 5
999
+@ stext(type,rid,name) AS 'stext'
9421000
 @ FROM ftsdocs;
9431001
 @ CREATE VIRTUAL TABLE IF NOT EXISTS "%w".ftsidx
9441002
 @ USING fts4(content="ftscontent", stext);
9451003
 ;
9461004
 static const char zFtsDrop[] =
@@ -1007,20 +1065,87 @@
10071065
 ** to the queue of documents that need to be indexed or reindexed.
10081066
 */
10091067
 void search_doc_touch(char cType, int rid, const char *zName){
10101068
 if( search_index_exists() ){
10111069
 db_multi_exec(
1012
- "DELETE FROM ftsidx WHERE rowid IN"
1070
+ "DELETE FROM ftsidx WHERE docid IN"
10131071
 " (SELECT id FROM ftsdocs WHERE type=%Q AND rid=%d AND idxed)",
10141072
 cType, rid
10151073
 );
10161074
 db_multi_exec(
10171075
 "REPLACE INTO ftsdocs(type,rid,name,idxed)"
10181076
 " VALUES(%Q,%d,%Q,0)",
10191077
 cType, rid, zName
10201078
 );
10211079
 }
1080
+}
1081
+
1082
+/*
1083
+** If the doc-glob and doc-br settings are valid for document search
1084
+** and if the latest check-in on doc-br is in the unindexed set of
1085
+** check-ins, then update all 'd' entries in FTSDOCS that have
1086
+** changed.
1087
+*/
1088
+static void search_update_doc_index(void){
1089
+ const char *zDocBr = db_get("doc-branch","trunk");
1090
+ int ckid = zDocBr ? symbolic_name_to_rid(zDocBr,"ci") : 0;
1091
+ double rTime;
1092
+ char *zBrUuid;
1093
+ if( ckid==0 ) return;
1094
+ if( !db_exists("SELECT 1 FROM ftsdocs WHERE type='c' AND rid=%d"
1095
+ " AND NOT idxed", ckid) ) return;
1096
+
1097
+ /* If we get this far, it means that changes to 'd' entries are
1098
+ ** required. */
1099
+ rTime = db_double(0.0, "SELECT mtime FROM event WHERE objid=%d", ckid);
1100
+ zBrUuid = db_text("","SELECT substr(uuid,1,20) FROM blob WHERE rid=%d",ckid);
1101
+ db_multi_exec(
1102
+ "CREATE TEMP TABLE current_docs(rid INTEGER PRIMARY KEY, name);"
1103
+ "CREATE VIRTUAL TABLE IF NOT EXISTS temp.foci USING files_of_checkin;"
1104
+ "INSERT OR IGNORE INTO current_docs(rid, name)"
1105
+ " SELECT blob.rid, foci.filename FROM foci, blob"
1106
+ " WHERE foci.checkinID=%d AND blob.uuid=foci.uuid"
1107
+ " AND %z",
1108
+ ckid, glob_expr("foci.filename", db_get("doc-glob",""))
1109
+ );
1110
+ db_multi_exec(
1111
+ "DELETE FROM ftsidx WHERE docid IN"
1112
+ " (SELECT id FROM ftsdocs WHERE type='d'"
1113
+ " AND rid NOT IN (SELECT rid FROM current_docs))"
1114
+ );
1115
+ db_multi_exec(
1116
+ "DELETE FROM ftsdocs WHERE type='d'"
1117
+ " AND rid NOT IN (SELECT rid FROM current_docs)"
1118
+ );
1119
+ db_multi_exec(
1120
+ "INSERT OR IGNORE INTO ftsdocs(type,rid,name,idxed,label,url,mtime)"
1121
+ " SELECT 'd', rid, name, 0,"
1122
+ " printf('Document: %%s',name),"
1123
+ " printf('/doc/%q/%%s',urlencode(name)),"
1124
+ " %.17g"
1125
+ " FROM current_docs",
1126
+ zBrUuid, rTime
1127
+ );
1128
+ db_multi_exec(
1129
+ "INSERT INTO ftsidx(docid,stext)"
1130
+ " SELECT id, stext FROM ftscontent WHERE type='d' AND NOT idxed"
1131
+ );
1132
+ db_multi_exec(
1133
+ "UPDATE ftsdocs SET idxed=1 WHERE type='d' AND NOT idxed"
1134
+ );
1135
+}
1136
+
1137
+/*
1138
+** Deal with all of the unindexed entries in the FTSDOCS table - that
1139
+** is to say, all the entries with FTSDOCS.IDXED=0. Add them to the
1140
+** index.
1141
+*/
1142
+void search_update_index(void){
1143
+ if( !search_index_exists() ) return;
1144
+ search_sql_setup(g.db);
1145
+ search_update_doc_index();
1146
+
10221147
 }
10231148
10241149
 /*
10251150
 ** COMMAND: test-fts
10261151
 */
@@ -1030,12 +1155,14 @@
10301155
 static const struct { int iCmd; const char *z; } aCmd[] = {
10311156
 { 1, "create" },
10321157
 { 2, "drop" },
10331158
 { 3, "exists" },
10341159
 { 4, "fill" },
1160
+ { 8, "refill" },
10351161
 { 5, "pending" },
10361162
 { 6, "all" },
1163
+ { 7, "update" },
10371164
 };
10381165
 db_find_and_open_repository(0, 0);
10391166
 if( g.argc<3 ) usage("SUBCMD ...");
10401167
 zSubCmd = g.argv[2];
10411168
 n = (int)strlen(zSubCmd);
@@ -1064,23 +1191,41 @@
10641191
 break;
10651192
 }
10661193
 case 4: { assert( fossil_strncmp(zSubCmd, "fill", n)==0 );
10671194
 search_fill_index();
10681195
 break;
1196
+ }
1197
+ case 8: { assert( fossil_strncmp(zSubCmd, "refill", n)==0 );
1198
+ search_drop_index();
1199
+ search_create_index();
1200
+ search_fill_index();
1201
+ break;
10691202
 }
10701203
 case 5: { assert( fossil_strncmp(zSubCmd, "pending", n)==0 );
10711204
 Stmt q;
10721205
 if( !search_index_exists() ) break;
1073
- db_prepare(&q, "SELECT id, type, rid, quote(name) FROM ftsdocs"
1206
+ db_prepare(&q, "SELECT id, type, rid, quote(label), url, date(mtime)"
1207
+ " FROM ftsdocs"
10741208
 " WHERE NOT idxed");
10751209
 while( db_step(&q)==SQLITE_ROW ){
1076
- fossil_print("%6d: %s %6d %s\n",
1077
- db_column_int(&q, 0),
1078
- db_column_text(&q, 1),
1079
- db_column_int(&q, 2),
1080
- db_column_text(&q, 3)
1081
- );
1210
+ const char *zUrl = db_column_text(&q,4);
1211
+ if( zUrl && zUrl[0] ){
1212
+ fossil_print("%6d: %s %6d %s %s\n %s\n",
1213
+ db_column_int(&q, 0),
1214
+ db_column_text(&q, 1),
1215
+ db_column_int(&q, 2),
1216
+ db_column_text(&q, 5),
1217
+ db_column_text(&q, 3),
1218
+ zUrl);
1219
+ }else{
1220
+ fossil_print("%6d: %s %6d %s %s\n",
1221
+ db_column_int(&q, 0),
1222
+ db_column_text(&q, 1),
1223
+ db_column_int(&q, 2),
1224
+ db_column_text(&q, 5),
1225
+ db_column_text(&q, 3));
1226
+ }
10821227
 }
10831228
 db_finalize(&q);
10841229
 break;
10851230
 }
10861231
 case 6: { assert( fossil_strncmp(zSubCmd, "all", n)==0 );
@@ -1097,8 +1242,13 @@
10971242
 );
10981243
 }
10991244
 db_finalize(&q);
11001245
 break;
11011246
 }
1247
+ case 7: { assert( fossil_strncmp(zSubCmd, "update", n)==0 );
1248
+ search_update_index();
1249
+ break;
1250
+ }
1251
+
11021252
 }
11031253
 db_end_transaction(0);
11041254
 }
11051255
--- src/search.c
+++ src/search.c
@@ -453,11 +453,11 @@
453 void search_sql_setup(sqlite3 *db){
454 static int once = 0;
455 if( once++ ) return;
456 sqlite3_create_function(db, "score", -1, SQLITE_UTF8, 0,
457 search_score_sqlfunc, 0, 0);
458 sqlite3_create_function(db, "snippet", -1, SQLITE_UTF8, &gSearch,
459 search_score_sqlfunc, 0, 0);
460 sqlite3_create_function(db, "search_init", -1, SQLITE_UTF8, 0,
461 search_init_sqlfunc, 0, 0);
462 sqlite3_create_function(db, "stext", 3, SQLITE_UTF8, 0,
463 search_stext_sqlfunc, 0, 0);
@@ -553,64 +553,59 @@
553 /*
554 ** Remove bits from srchFlags which are disallowed by either the
555 ** current server configuration or by user permissions.
556 */
557 unsigned int search_restrict(unsigned int srchFlags){
558 if( (srchFlags & SRCH_CKIN)!=0
559 && (g.perm.Read==0 || db_get_boolean("search-ci",0)==0) ){
 
 
 
560 srchFlags &= ~SRCH_CKIN;
561 }
562 if( (srchFlags & SRCH_DOC)!=0
563 && (g.perm.Read==0 || db_get_boolean("search-doc",0)==0) ){
564 srchFlags &= ~SRCH_DOC;
565 }
566 if( (srchFlags & SRCH_TKT)!=0
567 && (g.perm.RdTkt==0 || db_get_boolean("search-tkt",0)==0) ){
568 srchFlags &= ~SRCH_TKT;
569 }
570 if( (srchFlags & SRCH_WIKI)!=0
571 && (g.perm.RdWiki==0 || db_get_boolean("search-wiki",0)==0) ){
572 srchFlags &= ~SRCH_WIKI;
573 }
574 return srchFlags;
575 }
576
577 /*
578 ** This routine generates web-page output for a search operation.
579 ** Other web-pages can invoke this routine to add search results
580 ** in the middle of the page.
 
 
 
581 **
582 ** Return the number of rows.
583 */
584 int search_run_and_output(
585 const char *zPattern, /* The query pattern */
586 unsigned int srchFlags /* What to search over */
587 ){
588 Stmt q;
589 int nRow = 0;
590
591 srchFlags = search_restrict(srchFlags);
592 if( srchFlags==0 ) return 0;
593 search_sql_setup(g.db);
594 add_content_sql_commands(g.db);
595 search_init(zPattern, "<b>", "</b>", " ... ",
596 SRCHFLG_STATIC|SRCHFLG_HTML|SRCHFLG_SCORE);
597 db_multi_exec(
598 "CREATE VIRTUAL TABLE IF NOT EXISTS temp.foci USING files_of_checkin;"
599 "CREATE TEMP TABLE x(label TEXT,url TEXT,date TEXT,snip TEXT);"
600 );
601 if( (srchFlags & SRCH_DOC)!=0 ){
602 char *zDocGlob = db_get("doc-glob","");
603 char *zDocBr = db_get("doc-branch","trunk");
604 if( zDocGlob && zDocGlob[0] && zDocBr && zDocBr[0] ){
 
 
 
605 db_multi_exec(
606 "INSERT INTO x(label,url,date,snip)"
607 " SELECT printf('Document: %%s',foci.filename),"
608 " printf('%R/doc/%T/%%s',foci.filename),"
609 " (SELECT datetime(event.mtime) FROM event"
610 " WHERE objid=symbolic_name_to_rid('trunk')),"
611 " snippet(stext('d',blob.rid,foci.filename))"
612 " FROM foci CROSS JOIN blob"
613 " WHERE checkinID=symbolic_name_to_rid('trunk')"
614 " AND blob.uuid=foci.uuid"
615 " AND %z",
616 zDocBr, glob_expr("foci.filename", zDocGlob)
@@ -628,11 +623,11 @@
628 ")"
629 "INSERT INTO x(label,url,date,snip)"
630 " SELECT printf('Wiki: %%s',name),"
631 " printf('%R/wiki?name=%%s',urlencode(name)),"
632 " datetime(mtime),"
633 " snippet(stext('w',rid,name))"
634 " FROM wiki;"
635 );
636 }
637 if( (srchFlags & SRCH_CKIN)!=0 ){
638 db_multi_exec(
@@ -644,11 +639,11 @@
644 ")"
645 "INSERT INTO x(label,url,date,snip)"
646 " SELECT printf('Check-in [%%.10s] on %%s',uuid,datetime(mtime)),"
647 " printf('%R/timeline?c=%%s&n=8&y=ci',uuid),"
648 " datetime(mtime),"
649 " snippet(stext('c',rid,NULL))"
650 " FROM ckin;"
651 );
652 }
653 if( (srchFlags & SRCH_TKT)!=0 ){
654 db_multi_exec(
@@ -655,17 +650,77 @@
655 "INSERT INTO x(label,url,date,snip)"
656 " SELECT printf('Ticket [%%.17s] on %%s',"
657 "tkt_uuid,datetime(tkt_mtime)),"
658 " printf('%R/tktview/%%.20s',tkt_uuid),"
659 " datetime(tkt_mtime),"
660 " snippet(stext('t',tkt_id,NULL))"
661 " FROM ticket;"
662 );
663 }
664 db_prepare(&q, "SELECT url, substr(snip,9), label"
665 " FROM x WHERE snip IS NOT NULL"
666 " ORDER BY substr(snip,1,8) DESC, date DESC;");
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
667 while( db_step(&q)==SQLITE_ROW ){
668 const char *zUrl = db_column_text(&q, 0);
669 const char *zSnippet = db_column_text(&q, 1);
670 const char *zLabel = db_column_text(&q, 2);
671 if( nRow==0 ){
@@ -926,21 +981,24 @@
926 /* The schema for the full-text index
927 */
928 static const char zFtsSchema[] =
929 @ -- One entry for each possible search result
930 @ CREATE TABLE IF NOT EXISTS "%w".ftsdocs(
931 @ id INTEGER PRIMARY KEY, -- Maps to the ftsidx.rowid
932 @ type CHAR(1), -- Type of document
933 @ rid INTEGER, -- BLOB.RID or TAG.TAGID for the document
934 @ name TEXT, -- Additional document description
935 @ idxed BOOLEAN, -- True if currently in the index
 
 
 
936 @ UNIQUE(type,rid)
937 @ );
938 @ CREATE INDEX "%w".ftsdocIdxed ON ftsdocs(type,rid,name) WHERE idxed==0;
939 @ CREATE VIEW IF NOT EXISTS "%w".ftscontent AS
940 @ SELECT id, type, rid, name,
941 @ stext(type,rid,name) AS stext
942 @ FROM ftsdocs;
943 @ CREATE VIRTUAL TABLE IF NOT EXISTS "%w".ftsidx
944 @ USING fts4(content="ftscontent", stext);
945 ;
946 static const char zFtsDrop[] =
@@ -1007,20 +1065,87 @@
1007 ** to the queue of documents that need to be indexed or reindexed.
1008 */
1009 void search_doc_touch(char cType, int rid, const char *zName){
1010 if( search_index_exists() ){
1011 db_multi_exec(
1012 "DELETE FROM ftsidx WHERE rowid IN"
1013 " (SELECT id FROM ftsdocs WHERE type=%Q AND rid=%d AND idxed)",
1014 cType, rid
1015 );
1016 db_multi_exec(
1017 "REPLACE INTO ftsdocs(type,rid,name,idxed)"
1018 " VALUES(%Q,%d,%Q,0)",
1019 cType, rid, zName
1020 );
1021 }
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1022 }
1023
1024 /*
1025 ** COMMAND: test-fts
1026 */
@@ -1030,12 +1155,14 @@
1030 static const struct { int iCmd; const char *z; } aCmd[] = {
1031 { 1, "create" },
1032 { 2, "drop" },
1033 { 3, "exists" },
1034 { 4, "fill" },
 
1035 { 5, "pending" },
1036 { 6, "all" },
 
1037 };
1038 db_find_and_open_repository(0, 0);
1039 if( g.argc<3 ) usage("SUBCMD ...");
1040 zSubCmd = g.argv[2];
1041 n = (int)strlen(zSubCmd);
@@ -1064,23 +1191,41 @@
1064 break;
1065 }
1066 case 4: { assert( fossil_strncmp(zSubCmd, "fill", n)==0 );
1067 search_fill_index();
1068 break;
 
 
 
 
 
 
1069 }
1070 case 5: { assert( fossil_strncmp(zSubCmd, "pending", n)==0 );
1071 Stmt q;
1072 if( !search_index_exists() ) break;
1073 db_prepare(&q, "SELECT id, type, rid, quote(name) FROM ftsdocs"
 
1074 " WHERE NOT idxed");
1075 while( db_step(&q)==SQLITE_ROW ){
1076 fossil_print("%6d: %s %6d %s\n",
1077 db_column_int(&q, 0),
1078 db_column_text(&q, 1),
1079 db_column_int(&q, 2),
1080 db_column_text(&q, 3)
1081 );
 
 
 
 
 
 
 
 
 
 
 
1082 }
1083 db_finalize(&q);
1084 break;
1085 }
1086 case 6: { assert( fossil_strncmp(zSubCmd, "all", n)==0 );
@@ -1097,8 +1242,13 @@
1097 );
1098 }
1099 db_finalize(&q);
1100 break;
1101 }
 
 
 
 
 
1102 }
1103 db_end_transaction(0);
1104 }
1105
--- src/search.c
+++ src/search.c
@@ -453,11 +453,11 @@
453 void search_sql_setup(sqlite3 *db){
454 static int once = 0;
455 if( once++ ) return;
456 sqlite3_create_function(db, "score", -1, SQLITE_UTF8, 0,
457 search_score_sqlfunc, 0, 0);
458 sqlite3_create_function(db, "fsnippet", -1, SQLITE_UTF8, &gSearch,
459 search_score_sqlfunc, 0, 0);
460 sqlite3_create_function(db, "search_init", -1, SQLITE_UTF8, 0,
461 search_init_sqlfunc, 0, 0);
462 sqlite3_create_function(db, "stext", 3, SQLITE_UTF8, 0,
463 search_stext_sqlfunc, 0, 0);
@@ -553,64 +553,59 @@
553 /*
554 ** Remove bits from srchFlags which are disallowed by either the
555 ** current server configuration or by user permissions.
556 */
557 unsigned int search_restrict(unsigned int srchFlags){
558 if( g.perm.Read==0 ) srchFlags &= ~(SRCH_CKIN|SRCH_DOC);
559 if( g.perm.RdTkt==0 ) srchFlags &= ~(SRCH_TKT);
560 if( g.perm.RdWiki==0 ) srchFlags &= ~(SRCH_WIKI);
561 if( search_index_exists() ) return srchFlags;
562 if( (srchFlags & SRCH_CKIN)!=0 && db_get_boolean("search-ci",0)==0 ){
563 srchFlags &= ~SRCH_CKIN;
564 }
565 if( (srchFlags & SRCH_DOC)!=0 && db_get_boolean("search-doc",0)==0 ){
 
566 srchFlags &= ~SRCH_DOC;
567 }
568 if( (srchFlags & SRCH_TKT)!=0 && db_get_boolean("search-tkt",0)==0 ){
 
569 srchFlags &= ~SRCH_TKT;
570 }
571 if( (srchFlags & SRCH_WIKI)!=0 && db_get_boolean("search-wiki",0)==0 ){
 
572 srchFlags &= ~SRCH_WIKI;
573 }
574 return srchFlags;
575 }
576
577 /*
578 ** When this routine is called, there already exists a table
579 **
580 ** x(label,url,score,date,snip).
581 **
582 ** And the srchFlags parameter has been validated. This routine
583 ** fills the X table with search results using a full-text scan.
584 **
585 ** The companion indexed scan routine is search_indexed().
586 */
587 static void search_fullscan(
588 const char *zPattern, /* The query pattern */
589 unsigned int srchFlags /* What to search over */
590 ){
 
 
 
 
 
 
 
591 search_init(zPattern, "<b>", "</b>", " ... ",
592 SRCHFLG_STATIC|SRCHFLG_HTML|SRCHFLG_SCORE);
 
 
 
 
593 if( (srchFlags & SRCH_DOC)!=0 ){
594 char *zDocGlob = db_get("doc-glob","");
595 char *zDocBr = db_get("doc-branch","trunk");
596 if( zDocGlob && zDocGlob[0] && zDocBr && zDocBr[0] ){
597 db_multi_exec(
598 "CREATE VIRTUAL TABLE IF NOT EXISTS temp.foci USING files_of_checkin;"
599 );
600 db_multi_exec(
601 "INSERT INTO x(label,url,date,snip)"
602 " SELECT printf('Document: %%s',foci.filename),"
603 " printf('%R/doc/%T/%%s',foci.filename),"
604 " (SELECT datetime(event.mtime) FROM event"
605 " WHERE objid=symbolic_name_to_rid('trunk')),"
606 " fsnippet(stext('d',blob.rid,foci.filename))"
607 " FROM foci CROSS JOIN blob"
608 " WHERE checkinID=symbolic_name_to_rid('trunk')"
609 " AND blob.uuid=foci.uuid"
610 " AND %z",
611 zDocBr, glob_expr("foci.filename", zDocGlob)
@@ -628,11 +623,11 @@
623 ")"
624 "INSERT INTO x(label,url,date,snip)"
625 " SELECT printf('Wiki: %%s',name),"
626 " printf('%R/wiki?name=%%s',urlencode(name)),"
627 " datetime(mtime),"
628 " fsnippet(stext('w',rid,name))"
629 " FROM wiki;"
630 );
631 }
632 if( (srchFlags & SRCH_CKIN)!=0 ){
633 db_multi_exec(
@@ -644,11 +639,11 @@
639 ")"
640 "INSERT INTO x(label,url,date,snip)"
641 " SELECT printf('Check-in [%%.10s] on %%s',uuid,datetime(mtime)),"
642 " printf('%R/timeline?c=%%s&n=8&y=ci',uuid),"
643 " datetime(mtime),"
644 " fsnippet(stext('c',rid,NULL))"
645 " FROM ckin;"
646 );
647 }
648 if( (srchFlags & SRCH_TKT)!=0 ){
649 db_multi_exec(
@@ -655,17 +650,77 @@
650 "INSERT INTO x(label,url,date,snip)"
651 " SELECT printf('Ticket [%%.17s] on %%s',"
652 "tkt_uuid,datetime(tkt_mtime)),"
653 " printf('%R/tktview/%%.20s',tkt_uuid),"
654 " datetime(tkt_mtime),"
655 " fsnippet(stext('t',tkt_id,NULL))"
656 " FROM ticket;"
657 );
658 }
659 db_multi_exec(
660 "UPDATE x SET score=substr(snip,1,8), snip=substr(snip,9)"
661 );
662 }
663
664 /*
665 ** When this routine is called, there already exists a table
666 **
667 ** x(label,url,score,date,snip).
668 **
669 ** And the srchFlags parameter has been validated. This routine
670 ** fills the X table with search results using a index scan.
671 **
672 ** The companion full-text scan routine is search_fullscan().
673 */
674 static void search_indexed(
675 const char *zPattern, /* The query pattern */
676 unsigned int srchFlags /* What to search over */
677 ){
678 db_multi_exec(
679 "INSERT INTO x(label,url,score,date,snip) "
680 " SELECT ftsdocs.label,"
681 " ftsdocs.url,"
682 " 1," /*FIX ME*/
683 " datetime(ftsdocs.mtime),"
684 " fsnippet(ftsidx,'<b>','</b>',' ... ')"
685 " FROM ftsidx, ftsdocs"
686 " WHERE ftsidx MATCH %Q"
687 " AND ftsdocs.id=ftsidx.docid",
688 zPattern
689 );
690 }
691
692
693 /*
694 ** This routine generates web-page output for a search operation.
695 ** Other web-pages can invoke this routine to add search results
696 ** in the middle of the page.
697 **
698 ** Return the number of rows.
699 */
700 int search_run_and_output(
701 const char *zPattern, /* The query pattern */
702 unsigned int srchFlags /* What to search over */
703 ){
704 Stmt q;
705 int nRow = 0;
706
707 srchFlags = search_restrict(srchFlags);
708 if( srchFlags==0 ) return 0;
709 search_sql_setup(g.db);
710 add_content_sql_commands(g.db);
711 db_multi_exec(
712 "CREATE TEMP TABLE x(label,url,score,date,snip);"
713 );
714 if( !search_index_exists() ){
715 search_fullscan(zPattern, srchFlags);
716 }else{
717 search_indexed(zPattern, srchFlags);
718 }
719 db_prepare(&q, "SELECT url, snip, label"
720 " FROM x"
721 " ORDER BY score DESC, date DESC;");
722 while( db_step(&q)==SQLITE_ROW ){
723 const char *zUrl = db_column_text(&q, 0);
724 const char *zSnippet = db_column_text(&q, 1);
725 const char *zLabel = db_column_text(&q, 2);
726 if( nRow==0 ){
@@ -926,21 +981,24 @@
981 /* The schema for the full-text index
982 */
983 static const char zFtsSchema[] =
984 @ -- One entry for each possible search result
985 @ CREATE TABLE IF NOT EXISTS "%w".ftsdocs(
986 @ id INTEGER PRIMARY KEY, -- Maps to the ftsidx.docid
987 @ type CHAR(1), -- Type of document
988 @ rid INTEGER, -- BLOB.RID or TAG.TAGID for the document
989 @ name TEXT, -- Additional document description
990 @ idxed BOOLEAN, -- True if currently in the index
991 @ label TEXT, -- Label to print on search results
992 @ url TEXT, -- URL to access this document
993 @ mtime DATE, -- Date when document created
994 @ UNIQUE(type,rid)
995 @ );
996 @ CREATE INDEX "%w".ftsdocIdxed ON ftsdocs(type,rid,name) WHERE idxed==0;
997 @ CREATE VIEW IF NOT EXISTS "%w".ftscontent AS
998 @ SELECT id, type, rid, name, idxed, label, url, mtime,fr 5
999 @ stext(type,rid,name) AS 'stext'
1000 @ FROM ftsdocs;
1001 @ CREATE VIRTUAL TABLE IF NOT EXISTS "%w".ftsidx
1002 @ USING fts4(content="ftscontent", stext);
1003 ;
1004 static const char zFtsDrop[] =
@@ -1007,20 +1065,87 @@
1065 ** to the queue of documents that need to be indexed or reindexed.
1066 */
1067 void search_doc_touch(char cType, int rid, const char *zName){
1068 if( search_index_exists() ){
1069 db_multi_exec(
1070 "DELETE FROM ftsidx WHERE docid IN"
1071 " (SELECT id FROM ftsdocs WHERE type=%Q AND rid=%d AND idxed)",
1072 cType, rid
1073 );
1074 db_multi_exec(
1075 "REPLACE INTO ftsdocs(type,rid,name,idxed)"
1076 " VALUES(%Q,%d,%Q,0)",
1077 cType, rid, zName
1078 );
1079 }
1080 }
1081
1082 /*
1083 ** If the doc-glob and doc-br settings are valid for document search
1084 ** and if the latest check-in on doc-br is in the unindexed set of
1085 ** check-ins, then update all 'd' entries in FTSDOCS that have
1086 ** changed.
1087 */
1088 static void search_update_doc_index(void){
1089 const char *zDocBr = db_get("doc-branch","trunk");
1090 int ckid = zDocBr ? symbolic_name_to_rid(zDocBr,"ci") : 0;
1091 double rTime;
1092 char *zBrUuid;
1093 if( ckid==0 ) return;
1094 if( !db_exists("SELECT 1 FROM ftsdocs WHERE type='c' AND rid=%d"
1095 " AND NOT idxed", ckid) ) return;
1096
1097 /* If we get this far, it means that changes to 'd' entries are
1098 ** required. */
1099 rTime = db_double(0.0, "SELECT mtime FROM event WHERE objid=%d", ckid);
1100 zBrUuid = db_text("","SELECT substr(uuid,1,20) FROM blob WHERE rid=%d",ckid);
1101 db_multi_exec(
1102 "CREATE TEMP TABLE current_docs(rid INTEGER PRIMARY KEY, name);"
1103 "CREATE VIRTUAL TABLE IF NOT EXISTS temp.foci USING files_of_checkin;"
1104 "INSERT OR IGNORE INTO current_docs(rid, name)"
1105 " SELECT blob.rid, foci.filename FROM foci, blob"
1106 " WHERE foci.checkinID=%d AND blob.uuid=foci.uuid"
1107 " AND %z",
1108 ckid, glob_expr("foci.filename", db_get("doc-glob",""))
1109 );
1110 db_multi_exec(
1111 "DELETE FROM ftsidx WHERE docid IN"
1112 " (SELECT id FROM ftsdocs WHERE type='d'"
1113 " AND rid NOT IN (SELECT rid FROM current_docs))"
1114 );
1115 db_multi_exec(
1116 "DELETE FROM ftsdocs WHERE type='d'"
1117 " AND rid NOT IN (SELECT rid FROM current_docs)"
1118 );
1119 db_multi_exec(
1120 "INSERT OR IGNORE INTO ftsdocs(type,rid,name,idxed,label,url,mtime)"
1121 " SELECT 'd', rid, name, 0,"
1122 " printf('Document: %%s',name),"
1123 " printf('/doc/%q/%%s',urlencode(name)),"
1124 " %.17g"
1125 " FROM current_docs",
1126 zBrUuid, rTime
1127 );
1128 db_multi_exec(
1129 "INSERT INTO ftsidx(docid,stext)"
1130 " SELECT id, stext FROM ftscontent WHERE type='d' AND NOT idxed"
1131 );
1132 db_multi_exec(
1133 "UPDATE ftsdocs SET idxed=1 WHERE type='d' AND NOT idxed"
1134 );
1135 }
1136
1137 /*
1138 ** Deal with all of the unindexed entries in the FTSDOCS table - that
1139 ** is to say, all the entries with FTSDOCS.IDXED=0. Add them to the
1140 ** index.
1141 */
1142 void search_update_index(void){
1143 if( !search_index_exists() ) return;
1144 search_sql_setup(g.db);
1145 search_update_doc_index();
1146
1147 }
1148
1149 /*
1150 ** COMMAND: test-fts
1151 */
@@ -1030,12 +1155,14 @@
1155 static const struct { int iCmd; const char *z; } aCmd[] = {
1156 { 1, "create" },
1157 { 2, "drop" },
1158 { 3, "exists" },
1159 { 4, "fill" },
1160 { 8, "refill" },
1161 { 5, "pending" },
1162 { 6, "all" },
1163 { 7, "update" },
1164 };
1165 db_find_and_open_repository(0, 0);
1166 if( g.argc<3 ) usage("SUBCMD ...");
1167 zSubCmd = g.argv[2];
1168 n = (int)strlen(zSubCmd);
@@ -1064,23 +1191,41 @@
1191 break;
1192 }
1193 case 4: { assert( fossil_strncmp(zSubCmd, "fill", n)==0 );
1194 search_fill_index();
1195 break;
1196 }
1197 case 8: { assert( fossil_strncmp(zSubCmd, "refill", n)==0 );
1198 search_drop_index();
1199 search_create_index();
1200 search_fill_index();
1201 break;
1202 }
1203 case 5: { assert( fossil_strncmp(zSubCmd, "pending", n)==0 );
1204 Stmt q;
1205 if( !search_index_exists() ) break;
1206 db_prepare(&q, "SELECT id, type, rid, quote(label), url, date(mtime)"
1207 " FROM ftsdocs"
1208 " WHERE NOT idxed");
1209 while( db_step(&q)==SQLITE_ROW ){
1210 const char *zUrl = db_column_text(&q,4);
1211 if( zUrl && zUrl[0] ){
1212 fossil_print("%6d: %s %6d %s %s\n %s\n",
1213 db_column_int(&q, 0),
1214 db_column_text(&q, 1),
1215 db_column_int(&q, 2),
1216 db_column_text(&q, 5),
1217 db_column_text(&q, 3),
1218 zUrl);
1219 }else{
1220 fossil_print("%6d: %s %6d %s %s\n",
1221 db_column_int(&q, 0),
1222 db_column_text(&q, 1),
1223 db_column_int(&q, 2),
1224 db_column_text(&q, 5),
1225 db_column_text(&q, 3));
1226 }
1227 }
1228 db_finalize(&q);
1229 break;
1230 }
1231 case 6: { assert( fossil_strncmp(zSubCmd, "all", n)==0 );
@@ -1097,8 +1242,13 @@
1242 );
1243 }
1244 db_finalize(&q);
1245 break;
1246 }
1247 case 7: { assert( fossil_strncmp(zSubCmd, "update", n)==0 );
1248 search_update_index();
1249 break;
1250 }
1251
1252 }
1253 db_end_transaction(0);
1254 }
1255

Keyboard Shortcuts

Open search /
Next entry (timeline) j
Previous entry (timeline) k
Open focused entry Enter
Show this help ?
Toggle theme Top nav button