Fossil SCM

Improvements to the ranking function. Add the undocumented "debug" query parameter to /search.

drh 2015-02-14 02:12 UTC search-enhancements
Commit 9f67861aed7d59fbfd53836d140f22799010dbec
Parent 71295a98b7bce92…
1 file changed +61 -25
~ src/search.c
M src/search.c
+61 -25
--- src/search.c
+++ src/search.c
@@ -651,14 +651,15 @@
651651
 if( zDocGlob && zDocGlob[0] && zDocBr && zDocBr[0] ){
652652
 db_multi_exec(
653653
 "CREATE VIRTUAL TABLE IF NOT EXISTS temp.foci USING files_of_checkin;"
654654
 );
655655
 db_multi_exec(
656
- "INSERT INTO x(label,url,score,date,snip)"
656
+ "INSERT INTO x(label,url,score,id,date,snip)"
657657
 " SELECT printf('Document: %%s',title('d',blob.rid,foci.filename)),"
658658
 " printf('/doc/%T/%%s',foci.filename),"
659659
 " search_score(),"
660
+ " 'd'||blob.rid,"
660661
 " (SELECT datetime(event.mtime) FROM event"
661662
 " WHERE objid=symbolic_name_to_rid('trunk')),"
662663
 " search_snippet()"
663664
 " FROM foci CROSS JOIN blob"
664665
 " WHERE checkinID=symbolic_name_to_rid('trunk')"
@@ -677,14 +678,15 @@
677678
 " FROM tag, tagxref"
678679
 " WHERE tag.tagname GLOB 'wiki-*'"
679680
 " AND tagxref.tagid=tag.tagid"
680681
 " GROUP BY 1"
681682
 ")"
682
- "INSERT INTO x(label,url,score,date,snip)"
683
+ "INSERT INTO x(label,url,score,id,date,snip)"
683684
 " SELECT printf('Wiki: %%s',name),"
684685
 " printf('/wiki?name=%%s',urlencode(name)),"
685686
 " search_score(),"
687
+ " 'w'||rid,"
686688
 " datetime(mtime),"
687689
 " search_snippet()"
688690
 " FROM wiki"
689691
 " WHERE search_match(title('w',rid,name),body('w',rid,name));"
690692
 );
@@ -695,34 +697,45 @@
695697
 " SELECT blob.uuid, event.objid, event.mtime"
696698
 " FROM event, blob"
697699
 " WHERE event.type='ci'"
698700
 " AND blob.rid=event.objid"
699701
 ")"
700
- "INSERT INTO x(label,url,score,date,snip)"
702
+ "INSERT INTO x(label,url,score,id,date,snip)"
701703
 " SELECT printf('Check-in [%%.10s] on %%s',uuid,datetime(mtime)),"
702704
 " printf('/timeline?c=%%s&n=8&y=ci',uuid),"
703705
 " search_score(),"
706
+ " 'c'||rid,"
704707
 " datetime(mtime),"
705708
 " search_snippet()"
706709
 " FROM ckin"
707710
 " WHERE search_match('',body('c',rid,NULL));"
708711
 );
709712
 }
710713
 if( (srchFlags & SRCH_TKT)!=0 ){
711714
 db_multi_exec(
712
- "INSERT INTO x(label,url,score, date,snip)"
715
+ "INSERT INTO x(label,url,score,id,date,snip)"
713716
 " SELECT printf('Ticket: %%s (%%s)',title('t',tkt_id,NULL),"
714717
 "datetime(tkt_mtime)),"
715718
 " printf('/tktview/%%.20s',tkt_uuid),"
716719
 " search_score(),"
720
+ " 't'||tkt_id,"
717721
 " datetime(tkt_mtime),"
718722
 " search_snippet()"
719723
 " FROM ticket"
720724
 " WHERE search_match(title('t',tkt_id,NULL),body('t',tkt_id,NULL));"
721725
 );
722726
 }
723727
 }
728
+
729
+/*
730
+** Number of significant bits in a u32
731
+*/
732
+static int nbits(u32 x){
733
+ int n = 0;
734
+ while( x ){ n++; x >>= 1; }
735
+ return n;
736
+}
724737
725738
 /*
726739
 ** Implemenation of the rank() function used with rank(matchinfo(*,'pcsx')).
727740
 */
728741
 static void search_rank_sqlfunc(
@@ -733,29 +746,42 @@
733746
 const unsigned *aVal = (unsigned int*)sqlite3_value_blob(argv[0]);
734747
 int nVal = sqlite3_value_bytes(argv[0])/4;
735748
 int nCol; /* Number of columns in the index */
736749
 int nTerm; /* Number of search terms in the query */
737750
 int i, j; /* Loop counter */
738
- double r = 1.0; /* Score */
751
+ double r = 0.0; /* Score */
739752
 const unsigned *aX, *aS;
740753
741754
 if( nVal<2 ) return;
742755
 nTerm = aVal[0];
743756
 nCol = aVal[1];
744
- if( nVal<2+3*nCol*nTerm+4*nCol ) return;
757
+ if( nVal<2+3*nCol*nTerm+nCol ) return;
745758
 aS = aVal+2;
746759
 aX = aS+nCol;
747760
 for(j=0; j<nCol; j++){
748
- r *= 1<<((30*(aS[j]-1))/nTerm);
749
- for(i=0; i<nTerm; i++){
750
- int hits_this_row = aX[j + i*nCol];
751
- int hits_all_rows = aX[j + i*nCol + 1];
752
- int rows_with_hit = aX[j + i*nCol + 2];
753
- double avg_hits_per_row = (double)hits_all_rows/(double)rows_with_hit;
754
- r *= hits_this_row/avg_hits_per_row;
755
- }
756
- r *= 2.0;
761
+ double x;
762
+ if( aS[j]>0 ){
763
+ x = 0.0;
764
+ for(i=0; i<nTerm; i++){
765
+ int hits_this_row;
766
+ int hits_all_rows;
767
+ int rows_with_hit;
768
+ double avg_hits_per_row;
769
+
770
+ hits_this_row = aX[j + i*nCol*3];
771
+ if( hits_this_row==0 )continue;
772
+ hits_all_rows = aX[j + i*nCol*3 + 1];
773
+ rows_with_hit = aX[j + i*nCol*3 + 2];
774
+ if( rows_with_hit==0 ) continue;
775
+ avg_hits_per_row = hits_all_rows/(double)rows_with_hit;
776
+ x += hits_this_row/(avg_hits_per_row*nbits(rows_with_hit));
777
+ }
778
+ x *= (1<<((30*(aS[j]-1))/nTerm));
779
+ }else{
780
+ x = 0.0;
781
+ }
782
+ r = r*10.0 + x;
757783
 }
758784
 #define SEARCH_DEBUG_RANK 0
759785
 #if SEARCH_DEBUG_RANK
760786
 {
761787
 Blob x;
@@ -790,14 +816,15 @@
790816
 if( srchFlags==0 ) return;
791817
 sqlite3_create_function(g.db, "rank", 1, SQLITE_UTF8, 0,
792818
 search_rank_sqlfunc, 0, 0);
793819
 blob_init(&sql, 0, 0);
794820
 blob_appendf(&sql,
795
- "INSERT INTO x(label,url,score,date,snip) "
821
+ "INSERT INTO x(label,url,score,id,date,snip) "
796822
 " SELECT ftsdocs.label,"
797823
 " ftsdocs.url,"
798824
 " rank(matchinfo(ftsidx,'pcsx')),"
825
+ " ftsdocs.type || ftsdocs.rid,"
799826
 " datetime(ftsdocs.mtime),"
800827
 " snippet(ftsidx,'<mark>','</mark>',' ... ',-1,35)"
801828
 " FROM ftsidx CROSS JOIN ftsdocs"
802829
 " WHERE ftsidx MATCH %Q"
803830
 " AND ftsdocs.rowid=ftsidx.docid",
@@ -882,29 +909,30 @@
882909
 **
883910
 ** Return the number of rows.
884911
 */
885912
 int search_run_and_output(
886913
 const char *zPattern, /* The query pattern */
887
- unsigned int srchFlags /* What to search over */
914
+ unsigned int srchFlags, /* What to search over */
915
+ int fDebug /* Extra debugging output */
888916
 ){
889917
 Stmt q;
890918
 int nRow = 0;
891919
892920
 srchFlags = search_restrict(srchFlags);
893921
 if( srchFlags==0 ) return 0;
894922
 search_sql_setup(g.db);
895923
 add_content_sql_commands(g.db);
896924
 db_multi_exec(
897
- "CREATE TEMP TABLE x(label,url,score,date,snip);"
925
+ "CREATE TEMP TABLE x(label,url,score,id,date,snip);"
898926
 );
899927
 if( !search_index_exists() ){
900928
 search_fullscan(zPattern, srchFlags);
901929
 }else{
902930
 search_update_index(srchFlags);
903931
 search_indexed(zPattern, srchFlags);
904932
 }
905
- db_prepare(&q, "SELECT url, snip, label"
933
+ db_prepare(&q, "SELECT url, snip, label, score, id"
906934
 " FROM x"
907935
 " ORDER BY score DESC, date DESC;");
908936
 while( db_step(&q)==SQLITE_ROW ){
909937
 const char *zUrl = db_column_text(&q, 0);
910938
 const char *zSnippet = db_column_text(&q, 1);
@@ -911,12 +939,15 @@
911939
 const char *zLabel = db_column_text(&q, 2);
912940
 if( nRow==0 ){
913941
 @ <ol>
914942
 }
915943
 nRow++;
916
- @ <li><p><a href='%R%s(zUrl)'>%h(zLabel)</a><br>
917
- @ <span class='snippet'>%z(cleanSnippet(zSnippet))</span></li>
944
+ @ <li><p><a href='%R%s(zUrl)'>%h(zLabel)</a>
945
+ if( fDebug ){
946
+ @ (%e(db_column_double(&q,3)), %s(db_column_text(&q,4)))
947
+ }
948
+ @ <br><span class='snippet'>%z(cleanSnippet(zSnippet))</span></li>
918949
 }
919950
 db_finalize(&q);
920951
 if( nRow ){
921952
 @ </ol>
922953
 }
@@ -944,10 +975,11 @@
944975
 const char *zType = 0;
945976
 const char *zClass = 0;
946977
 const char *zDisable1;
947978
 const char *zDisable2;
948979
 const char *zPattern;
980
+ int fDebug = PB("debug");
949981
 srchFlags = search_restrict(srchFlags);
950982
 switch( srchFlags ){
951983
 case SRCH_CKIN: zType = " Check-ins"; zClass = "Ckin"; break;
952984
 case SRCH_DOC: zType = " Docs"; zClass = "Doc"; break;
953985
 case SRCH_TKT: zType = " Tickets"; zClass = "Tkt"; break;
@@ -991,10 +1023,13 @@
9911023
 cgi_printf(">%s</option>\n", aY[i].zNm);
9921024
 }
9931025
 @ </select>
9941026
 srchFlags = newFlags;
9951027
 }
1028
+ if( fDebug ){
1029
+ @ <input type="hidden" name="debug" value="1">
1030
+ }
9961031
 @ <input type="submit" value="Search%s(zType)"%s(zDisable2)>
9971032
 if( srchFlags==0 ){
9981033
 @ <p class="generalError">Search is disabled</p>
9991034
 }
10001035
 @ </div></form>
@@ -1003,11 +1038,11 @@
10031038
 if( zClass ){
10041039
 @ <div class='searchResult searchResult%s(zClass)'>
10051040
 }else{
10061041
 @ <div class='searchResult'>
10071042
 }
1008
- if( search_run_and_output(zPattern, srchFlags)==0 ){
1043
+ if( search_run_and_output(zPattern, srchFlags, fDebug)==0 ){
10091044
 @ <p class='searchEmpty'>No matches for: <span>%h(zPattern)</span></p>
10101045
 }
10111046
 @ </div>
10121047
 }
10131048
 }
@@ -1454,25 +1489,26 @@
14541489
 " AND rid NOT IN (SELECT rid FROM current_docs)"
14551490
 );
14561491
 db_multi_exec(
14571492
 "INSERT OR IGNORE INTO ftsdocs(type,rid,name,idxed,label,bx,url,mtime)"
14581493
 " SELECT 'd', rid, name, 0,"
1459
- " 'Document: '||title('d',rid,name),"
1494
+ " title('d',rid,name),"
14601495
 " body('d',rid,name),"
14611496
 " printf('/doc/%q/%%s',urlencode(name)),"
14621497
 " %.17g"
14631498
 " FROM current_docs",
14641499
 zBrUuid, rTime
14651500
 );
14661501
 db_multi_exec(
14671502
 "INSERT INTO ftsidx(docid,title,body)"
1468
- " SELECT rowid, name, bx FROM ftsdocs WHERE type='d' AND NOT idxed"
1503
+ " SELECT rowid, label, bx FROM ftsdocs WHERE type='d' AND NOT idxed"
14691504
 );
14701505
 db_multi_exec(
14711506
 "UPDATE ftsdocs SET"
14721507
 " idxed=1,"
1473
- " bx=NULL"
1508
+ " bx=NULL,"
1509
+ " label='Document: '||label"
14741510
 " WHERE type='d' AND NOT idxed"
14751511
 );
14761512
 }
14771513
14781514
 /*
14791515
--- src/search.c
+++ src/search.c
@@ -651,14 +651,15 @@
651 if( zDocGlob && zDocGlob[0] && zDocBr && zDocBr[0] ){
652 db_multi_exec(
653 "CREATE VIRTUAL TABLE IF NOT EXISTS temp.foci USING files_of_checkin;"
654 );
655 db_multi_exec(
656 "INSERT INTO x(label,url,score,date,snip)"
657 " SELECT printf('Document: %%s',title('d',blob.rid,foci.filename)),"
658 " printf('/doc/%T/%%s',foci.filename),"
659 " search_score(),"
 
660 " (SELECT datetime(event.mtime) FROM event"
661 " WHERE objid=symbolic_name_to_rid('trunk')),"
662 " search_snippet()"
663 " FROM foci CROSS JOIN blob"
664 " WHERE checkinID=symbolic_name_to_rid('trunk')"
@@ -677,14 +678,15 @@
677 " FROM tag, tagxref"
678 " WHERE tag.tagname GLOB 'wiki-*'"
679 " AND tagxref.tagid=tag.tagid"
680 " GROUP BY 1"
681 ")"
682 "INSERT INTO x(label,url,score,date,snip)"
683 " SELECT printf('Wiki: %%s',name),"
684 " printf('/wiki?name=%%s',urlencode(name)),"
685 " search_score(),"
 
686 " datetime(mtime),"
687 " search_snippet()"
688 " FROM wiki"
689 " WHERE search_match(title('w',rid,name),body('w',rid,name));"
690 );
@@ -695,34 +697,45 @@
695 " SELECT blob.uuid, event.objid, event.mtime"
696 " FROM event, blob"
697 " WHERE event.type='ci'"
698 " AND blob.rid=event.objid"
699 ")"
700 "INSERT INTO x(label,url,score,date,snip)"
701 " SELECT printf('Check-in [%%.10s] on %%s',uuid,datetime(mtime)),"
702 " printf('/timeline?c=%%s&n=8&y=ci',uuid),"
703 " search_score(),"
 
704 " datetime(mtime),"
705 " search_snippet()"
706 " FROM ckin"
707 " WHERE search_match('',body('c',rid,NULL));"
708 );
709 }
710 if( (srchFlags & SRCH_TKT)!=0 ){
711 db_multi_exec(
712 "INSERT INTO x(label,url,score, date,snip)"
713 " SELECT printf('Ticket: %%s (%%s)',title('t',tkt_id,NULL),"
714 "datetime(tkt_mtime)),"
715 " printf('/tktview/%%.20s',tkt_uuid),"
716 " search_score(),"
 
717 " datetime(tkt_mtime),"
718 " search_snippet()"
719 " FROM ticket"
720 " WHERE search_match(title('t',tkt_id,NULL),body('t',tkt_id,NULL));"
721 );
722 }
723 }
 
 
 
 
 
 
 
 
 
724
725 /*
726 ** Implemenation of the rank() function used with rank(matchinfo(*,'pcsx')).
727 */
728 static void search_rank_sqlfunc(
@@ -733,29 +746,42 @@
733 const unsigned *aVal = (unsigned int*)sqlite3_value_blob(argv[0]);
734 int nVal = sqlite3_value_bytes(argv[0])/4;
735 int nCol; /* Number of columns in the index */
736 int nTerm; /* Number of search terms in the query */
737 int i, j; /* Loop counter */
738 double r = 1.0; /* Score */
739 const unsigned *aX, *aS;
740
741 if( nVal<2 ) return;
742 nTerm = aVal[0];
743 nCol = aVal[1];
744 if( nVal<2+3*nCol*nTerm+4*nCol ) return;
745 aS = aVal+2;
746 aX = aS+nCol;
747 for(j=0; j<nCol; j++){
748 r *= 1<<((30*(aS[j]-1))/nTerm);
749 for(i=0; i<nTerm; i++){
750 int hits_this_row = aX[j + i*nCol];
751 int hits_all_rows = aX[j + i*nCol + 1];
752 int rows_with_hit = aX[j + i*nCol + 2];
753 double avg_hits_per_row = (double)hits_all_rows/(double)rows_with_hit;
754 r *= hits_this_row/avg_hits_per_row;
755 }
756 r *= 2.0;
 
 
 
 
 
 
 
 
 
 
 
 
 
757 }
758 #define SEARCH_DEBUG_RANK 0
759 #if SEARCH_DEBUG_RANK
760 {
761 Blob x;
@@ -790,14 +816,15 @@
790 if( srchFlags==0 ) return;
791 sqlite3_create_function(g.db, "rank", 1, SQLITE_UTF8, 0,
792 search_rank_sqlfunc, 0, 0);
793 blob_init(&sql, 0, 0);
794 blob_appendf(&sql,
795 "INSERT INTO x(label,url,score,date,snip) "
796 " SELECT ftsdocs.label,"
797 " ftsdocs.url,"
798 " rank(matchinfo(ftsidx,'pcsx')),"
 
799 " datetime(ftsdocs.mtime),"
800 " snippet(ftsidx,'<mark>','</mark>',' ... ',-1,35)"
801 " FROM ftsidx CROSS JOIN ftsdocs"
802 " WHERE ftsidx MATCH %Q"
803 " AND ftsdocs.rowid=ftsidx.docid",
@@ -882,29 +909,30 @@
882 **
883 ** Return the number of rows.
884 */
885 int search_run_and_output(
886 const char *zPattern, /* The query pattern */
887 unsigned int srchFlags /* What to search over */
 
888 ){
889 Stmt q;
890 int nRow = 0;
891
892 srchFlags = search_restrict(srchFlags);
893 if( srchFlags==0 ) return 0;
894 search_sql_setup(g.db);
895 add_content_sql_commands(g.db);
896 db_multi_exec(
897 "CREATE TEMP TABLE x(label,url,score,date,snip);"
898 );
899 if( !search_index_exists() ){
900 search_fullscan(zPattern, srchFlags);
901 }else{
902 search_update_index(srchFlags);
903 search_indexed(zPattern, srchFlags);
904 }
905 db_prepare(&q, "SELECT url, snip, label"
906 " FROM x"
907 " ORDER BY score DESC, date DESC;");
908 while( db_step(&q)==SQLITE_ROW ){
909 const char *zUrl = db_column_text(&q, 0);
910 const char *zSnippet = db_column_text(&q, 1);
@@ -911,12 +939,15 @@
911 const char *zLabel = db_column_text(&q, 2);
912 if( nRow==0 ){
913 @ <ol>
914 }
915 nRow++;
916 @ <li><p><a href='%R%s(zUrl)'>%h(zLabel)</a><br>
917 @ <span class='snippet'>%z(cleanSnippet(zSnippet))</span></li>
 
 
 
918 }
919 db_finalize(&q);
920 if( nRow ){
921 @ </ol>
922 }
@@ -944,10 +975,11 @@
944 const char *zType = 0;
945 const char *zClass = 0;
946 const char *zDisable1;
947 const char *zDisable2;
948 const char *zPattern;
 
949 srchFlags = search_restrict(srchFlags);
950 switch( srchFlags ){
951 case SRCH_CKIN: zType = " Check-ins"; zClass = "Ckin"; break;
952 case SRCH_DOC: zType = " Docs"; zClass = "Doc"; break;
953 case SRCH_TKT: zType = " Tickets"; zClass = "Tkt"; break;
@@ -991,10 +1023,13 @@
991 cgi_printf(">%s</option>\n", aY[i].zNm);
992 }
993 @ </select>
994 srchFlags = newFlags;
995 }
 
 
 
996 @ <input type="submit" value="Search%s(zType)"%s(zDisable2)>
997 if( srchFlags==0 ){
998 @ <p class="generalError">Search is disabled</p>
999 }
1000 @ </div></form>
@@ -1003,11 +1038,11 @@
1003 if( zClass ){
1004 @ <div class='searchResult searchResult%s(zClass)'>
1005 }else{
1006 @ <div class='searchResult'>
1007 }
1008 if( search_run_and_output(zPattern, srchFlags)==0 ){
1009 @ <p class='searchEmpty'>No matches for: <span>%h(zPattern)</span></p>
1010 }
1011 @ </div>
1012 }
1013 }
@@ -1454,25 +1489,26 @@
1454 " AND rid NOT IN (SELECT rid FROM current_docs)"
1455 );
1456 db_multi_exec(
1457 "INSERT OR IGNORE INTO ftsdocs(type,rid,name,idxed,label,bx,url,mtime)"
1458 " SELECT 'd', rid, name, 0,"
1459 " 'Document: '||title('d',rid,name),"
1460 " body('d',rid,name),"
1461 " printf('/doc/%q/%%s',urlencode(name)),"
1462 " %.17g"
1463 " FROM current_docs",
1464 zBrUuid, rTime
1465 );
1466 db_multi_exec(
1467 "INSERT INTO ftsidx(docid,title,body)"
1468 " SELECT rowid, name, bx FROM ftsdocs WHERE type='d' AND NOT idxed"
1469 );
1470 db_multi_exec(
1471 "UPDATE ftsdocs SET"
1472 " idxed=1,"
1473 " bx=NULL"
 
1474 " WHERE type='d' AND NOT idxed"
1475 );
1476 }
1477
1478 /*
1479
--- src/search.c
+++ src/search.c
@@ -651,14 +651,15 @@
651 if( zDocGlob && zDocGlob[0] && zDocBr && zDocBr[0] ){
652 db_multi_exec(
653 "CREATE VIRTUAL TABLE IF NOT EXISTS temp.foci USING files_of_checkin;"
654 );
655 db_multi_exec(
656 "INSERT INTO x(label,url,score,id,date,snip)"
657 " SELECT printf('Document: %%s',title('d',blob.rid,foci.filename)),"
658 " printf('/doc/%T/%%s',foci.filename),"
659 " search_score(),"
660 " 'd'||blob.rid,"
661 " (SELECT datetime(event.mtime) FROM event"
662 " WHERE objid=symbolic_name_to_rid('trunk')),"
663 " search_snippet()"
664 " FROM foci CROSS JOIN blob"
665 " WHERE checkinID=symbolic_name_to_rid('trunk')"
@@ -677,14 +678,15 @@
678 " FROM tag, tagxref"
679 " WHERE tag.tagname GLOB 'wiki-*'"
680 " AND tagxref.tagid=tag.tagid"
681 " GROUP BY 1"
682 ")"
683 "INSERT INTO x(label,url,score,id,date,snip)"
684 " SELECT printf('Wiki: %%s',name),"
685 " printf('/wiki?name=%%s',urlencode(name)),"
686 " search_score(),"
687 " 'w'||rid,"
688 " datetime(mtime),"
689 " search_snippet()"
690 " FROM wiki"
691 " WHERE search_match(title('w',rid,name),body('w',rid,name));"
692 );
@@ -695,34 +697,45 @@
697 " SELECT blob.uuid, event.objid, event.mtime"
698 " FROM event, blob"
699 " WHERE event.type='ci'"
700 " AND blob.rid=event.objid"
701 ")"
702 "INSERT INTO x(label,url,score,id,date,snip)"
703 " SELECT printf('Check-in [%%.10s] on %%s',uuid,datetime(mtime)),"
704 " printf('/timeline?c=%%s&n=8&y=ci',uuid),"
705 " search_score(),"
706 " 'c'||rid,"
707 " datetime(mtime),"
708 " search_snippet()"
709 " FROM ckin"
710 " WHERE search_match('',body('c',rid,NULL));"
711 );
712 }
713 if( (srchFlags & SRCH_TKT)!=0 ){
714 db_multi_exec(
715 "INSERT INTO x(label,url,score,id,date,snip)"
716 " SELECT printf('Ticket: %%s (%%s)',title('t',tkt_id,NULL),"
717 "datetime(tkt_mtime)),"
718 " printf('/tktview/%%.20s',tkt_uuid),"
719 " search_score(),"
720 " 't'||tkt_id,"
721 " datetime(tkt_mtime),"
722 " search_snippet()"
723 " FROM ticket"
724 " WHERE search_match(title('t',tkt_id,NULL),body('t',tkt_id,NULL));"
725 );
726 }
727 }
728
729 /*
730 ** Number of significant bits in a u32
731 */
732 static int nbits(u32 x){
733 int n = 0;
734 while( x ){ n++; x >>= 1; }
735 return n;
736 }
737
738 /*
739 ** Implemenation of the rank() function used with rank(matchinfo(*,'pcsx')).
740 */
741 static void search_rank_sqlfunc(
@@ -733,29 +746,42 @@
746 const unsigned *aVal = (unsigned int*)sqlite3_value_blob(argv[0]);
747 int nVal = sqlite3_value_bytes(argv[0])/4;
748 int nCol; /* Number of columns in the index */
749 int nTerm; /* Number of search terms in the query */
750 int i, j; /* Loop counter */
751 double r = 0.0; /* Score */
752 const unsigned *aX, *aS;
753
754 if( nVal<2 ) return;
755 nTerm = aVal[0];
756 nCol = aVal[1];
757 if( nVal<2+3*nCol*nTerm+nCol ) return;
758 aS = aVal+2;
759 aX = aS+nCol;
760 for(j=0; j<nCol; j++){
761 double x;
762 if( aS[j]>0 ){
763 x = 0.0;
764 for(i=0; i<nTerm; i++){
765 int hits_this_row;
766 int hits_all_rows;
767 int rows_with_hit;
768 double avg_hits_per_row;
769
770 hits_this_row = aX[j + i*nCol*3];
771 if( hits_this_row==0 )continue;
772 hits_all_rows = aX[j + i*nCol*3 + 1];
773 rows_with_hit = aX[j + i*nCol*3 + 2];
774 if( rows_with_hit==0 ) continue;
775 avg_hits_per_row = hits_all_rows/(double)rows_with_hit;
776 x += hits_this_row/(avg_hits_per_row*nbits(rows_with_hit));
777 }
778 x *= (1<<((30*(aS[j]-1))/nTerm));
779 }else{
780 x = 0.0;
781 }
782 r = r*10.0 + x;
783 }
784 #define SEARCH_DEBUG_RANK 0
785 #if SEARCH_DEBUG_RANK
786 {
787 Blob x;
@@ -790,14 +816,15 @@
816 if( srchFlags==0 ) return;
817 sqlite3_create_function(g.db, "rank", 1, SQLITE_UTF8, 0,
818 search_rank_sqlfunc, 0, 0);
819 blob_init(&sql, 0, 0);
820 blob_appendf(&sql,
821 "INSERT INTO x(label,url,score,id,date,snip) "
822 " SELECT ftsdocs.label,"
823 " ftsdocs.url,"
824 " rank(matchinfo(ftsidx,'pcsx')),"
825 " ftsdocs.type || ftsdocs.rid,"
826 " datetime(ftsdocs.mtime),"
827 " snippet(ftsidx,'<mark>','</mark>',' ... ',-1,35)"
828 " FROM ftsidx CROSS JOIN ftsdocs"
829 " WHERE ftsidx MATCH %Q"
830 " AND ftsdocs.rowid=ftsidx.docid",
@@ -882,29 +909,30 @@
909 **
910 ** Return the number of rows.
911 */
912 int search_run_and_output(
913 const char *zPattern, /* The query pattern */
914 unsigned int srchFlags, /* What to search over */
915 int fDebug /* Extra debugging output */
916 ){
917 Stmt q;
918 int nRow = 0;
919
920 srchFlags = search_restrict(srchFlags);
921 if( srchFlags==0 ) return 0;
922 search_sql_setup(g.db);
923 add_content_sql_commands(g.db);
924 db_multi_exec(
925 "CREATE TEMP TABLE x(label,url,score,id,date,snip);"
926 );
927 if( !search_index_exists() ){
928 search_fullscan(zPattern, srchFlags);
929 }else{
930 search_update_index(srchFlags);
931 search_indexed(zPattern, srchFlags);
932 }
933 db_prepare(&q, "SELECT url, snip, label, score, id"
934 " FROM x"
935 " ORDER BY score DESC, date DESC;");
936 while( db_step(&q)==SQLITE_ROW ){
937 const char *zUrl = db_column_text(&q, 0);
938 const char *zSnippet = db_column_text(&q, 1);
@@ -911,12 +939,15 @@
939 const char *zLabel = db_column_text(&q, 2);
940 if( nRow==0 ){
941 @ <ol>
942 }
943 nRow++;
944 @ <li><p><a href='%R%s(zUrl)'>%h(zLabel)</a>
945 if( fDebug ){
946 @ (%e(db_column_double(&q,3)), %s(db_column_text(&q,4)))
947 }
948 @ <br><span class='snippet'>%z(cleanSnippet(zSnippet))</span></li>
949 }
950 db_finalize(&q);
951 if( nRow ){
952 @ </ol>
953 }
@@ -944,10 +975,11 @@
975 const char *zType = 0;
976 const char *zClass = 0;
977 const char *zDisable1;
978 const char *zDisable2;
979 const char *zPattern;
980 int fDebug = PB("debug");
981 srchFlags = search_restrict(srchFlags);
982 switch( srchFlags ){
983 case SRCH_CKIN: zType = " Check-ins"; zClass = "Ckin"; break;
984 case SRCH_DOC: zType = " Docs"; zClass = "Doc"; break;
985 case SRCH_TKT: zType = " Tickets"; zClass = "Tkt"; break;
@@ -991,10 +1023,13 @@
1023 cgi_printf(">%s</option>\n", aY[i].zNm);
1024 }
1025 @ </select>
1026 srchFlags = newFlags;
1027 }
1028 if( fDebug ){
1029 @ <input type="hidden" name="debug" value="1">
1030 }
1031 @ <input type="submit" value="Search%s(zType)"%s(zDisable2)>
1032 if( srchFlags==0 ){
1033 @ <p class="generalError">Search is disabled</p>
1034 }
1035 @ </div></form>
@@ -1003,11 +1038,11 @@
1038 if( zClass ){
1039 @ <div class='searchResult searchResult%s(zClass)'>
1040 }else{
1041 @ <div class='searchResult'>
1042 }
1043 if( search_run_and_output(zPattern, srchFlags, fDebug)==0 ){
1044 @ <p class='searchEmpty'>No matches for: <span>%h(zPattern)</span></p>
1045 }
1046 @ </div>
1047 }
1048 }
@@ -1454,25 +1489,26 @@
1489 " AND rid NOT IN (SELECT rid FROM current_docs)"
1490 );
1491 db_multi_exec(
1492 "INSERT OR IGNORE INTO ftsdocs(type,rid,name,idxed,label,bx,url,mtime)"
1493 " SELECT 'd', rid, name, 0,"
1494 " title('d',rid,name),"
1495 " body('d',rid,name),"
1496 " printf('/doc/%q/%%s',urlencode(name)),"
1497 " %.17g"
1498 " FROM current_docs",
1499 zBrUuid, rTime
1500 );
1501 db_multi_exec(
1502 "INSERT INTO ftsidx(docid,title,body)"
1503 " SELECT rowid, label, bx FROM ftsdocs WHERE type='d' AND NOT idxed"
1504 );
1505 db_multi_exec(
1506 "UPDATE ftsdocs SET"
1507 " idxed=1,"
1508 " bx=NULL,"
1509 " label='Document: '||label"
1510 " WHERE type='d' AND NOT idxed"
1511 );
1512 }
1513
1514 /*
1515

Keyboard Shortcuts

Open search /
Next entry (timeline) j
Previous entry (timeline) k
Open focused entry Enter
Show this help ?
Toggle theme Top nav button