Fossil SCM

Enhance /search to distinguish between the title and the body of a document and provide support for the Porter stemmer for indexed search. Improved scoring and snippet presentation. NB: Run "fossil fts-config reindex" when upgrading through this change.

drh 2015-02-14 15:17 trunk merge

Commit 0f96ffb9c2a0a05deaea2803edd477e927c02c53

Parent 653dd402eafe152…

6 files changed +2 -7 +1 -1 +291 -79 +2 +2 +63 -8

~ src/db.c ~ src/main.mk ~ src/search.c ~ src/setup.c ~ src/setup.c ~ src/wikiformat.c

M src/db.c

+2 -7

		--- src/db.c
		+++ src/db.c
		@@ -65,14 +65,10 @@
65	65	*/
66	66	static void db_err(const char *zFormat, ...){
67	67	va_list ap;
68	68	char *z;
69	69	int rc = 1;
70		- static const char zRebuildMsg[] =
71		- "If you have recently updated your fossil executable, you might\n"
72		- "need to run \"fossil all rebuild\" to bring the repository\n"
73		- "schemas up to date.\n";
74	70	va_start(ap, zFormat);
75	71	z = vmprintf(zFormat, ap);
76	72	va_end(ap);
77	73	#ifdef FOSSIL_ENABLE_JSON
78	74	if( g.json.isJsonMode ){
		@@ -88,15 +84,14 @@
88	84	@ error Database\serror:\s%F(z)
89	85	cgi_reply();
90	86	}
91	87	else if( g.cgiOutput ){
92	88	g.cgiOutput = 0;
93		- cgi_printf("<h1>Database Error</h1>\n"
94		- "<pre>%h</pre>\n<p>%s</p>\n", z, zRebuildMsg);
	89	+ cgi_printf("<h1>Database Error</h1>\n<p>%h</p>\n", z);
95	90	cgi_reply();
96	91	}else{
97		- fprintf(stderr, "%s: %s\n\n%s", g.argv[0], z, zRebuildMsg);
	92	+ fprintf(stderr, "%s: %s\n", g.argv[0], z);
98	93	}
99	94	free(z);
100	95	db_force_rollback();
101	96	fossil_exit(rc);
102	97	}
103	98

	--- src/db.c
	+++ src/db.c
	@@ -65,14 +65,10 @@
65	*/
66	static void db_err(const char *zFormat, ...){
67	va_list ap;
68	char *z;
69	int rc = 1;
70	static const char zRebuildMsg[] =
71	"If you have recently updated your fossil executable, you might\n"
72	"need to run \"fossil all rebuild\" to bring the repository\n"
73	"schemas up to date.\n";
74	va_start(ap, zFormat);
75	z = vmprintf(zFormat, ap);
76	va_end(ap);
77	#ifdef FOSSIL_ENABLE_JSON
78	if( g.json.isJsonMode ){
	@@ -88,15 +84,14 @@
88	@ error Database\serror:\s%F(z)
89	cgi_reply();
90	}
91	else if( g.cgiOutput ){
92	g.cgiOutput = 0;
93	cgi_printf("<h1>Database Error</h1>\n"
94	"<pre>%h</pre>\n<p>%s</p>\n", z, zRebuildMsg);
95	cgi_reply();
96	}else{
97	fprintf(stderr, "%s: %s\n\n%s", g.argv[0], z, zRebuildMsg);
98	}
99	free(z);
100	db_force_rollback();
101	fossil_exit(rc);
102	}
103

	--- src/db.c
	+++ src/db.c
	@@ -65,14 +65,10 @@
65	*/
66	static void db_err(const char *zFormat, ...){
67	va_list ap;
68	char *z;
69	int rc = 1;




70	va_start(ap, zFormat);
71	z = vmprintf(zFormat, ap);
72	va_end(ap);
73	#ifdef FOSSIL_ENABLE_JSON
74	if( g.json.isJsonMode ){
	@@ -88,15 +84,14 @@
84	@ error Database\serror:\s%F(z)
85	cgi_reply();
86	}
87	else if( g.cgiOutput ){
88	g.cgiOutput = 0;
89	cgi_printf("<h1>Database Error</h1>\n<p>%h</p>\n", z);

90	cgi_reply();
91	}else{
92	fprintf(stderr, "%s: %s\n", g.argv[0], z);
93	}
94	free(z);
95	db_force_rollback();
96	fossil_exit(rc);
97	}
98

M src/main.mk

+1 -1

		--- src/main.mk
		+++ src/main.mk
		@@ -491,11 +491,11 @@
491	491	$(OBJDIR)/cson_amalgamation.o
492	492
493	493
494	494	$(APPNAME): $(OBJDIR)/headers $(OBJDIR)/codecheck1 $(OBJ) $(EXTRAOBJ)
495	495	$(OBJDIR)/codecheck1 $(TRANS_SRC)
496		- $(TCC) -o $(APPNAME) $(OBJ) $(EXTRAOBJ) $(LIB)
	496	+ $(TCC) $(CFLAGS) -o $(APPNAME) $(OBJ) $(EXTRAOBJ) $(LIB)
497	497
498	498	# This rule prevents make from using its default rules to try build
499	499	# an executable named "manifest" out of the file named "manifest.c"
500	500	#
501	501	$(SRCDIR)/../manifest:
502	502

	--- src/main.mk
	+++ src/main.mk
	@@ -491,11 +491,11 @@
491	$(OBJDIR)/cson_amalgamation.o
492
493
494	$(APPNAME): $(OBJDIR)/headers $(OBJDIR)/codecheck1 $(OBJ) $(EXTRAOBJ)
495	$(OBJDIR)/codecheck1 $(TRANS_SRC)
496	$(TCC) -o $(APPNAME) $(OBJ) $(EXTRAOBJ) $(LIB)
497
498	# This rule prevents make from using its default rules to try build
499	# an executable named "manifest" out of the file named "manifest.c"
500	#
501	$(SRCDIR)/../manifest:
502

	--- src/main.mk
	+++ src/main.mk
	@@ -491,11 +491,11 @@
491	$(OBJDIR)/cson_amalgamation.o
492
493
494	$(APPNAME): $(OBJDIR)/headers $(OBJDIR)/codecheck1 $(OBJ) $(EXTRAOBJ)
495	$(OBJDIR)/codecheck1 $(TRANS_SRC)
496	$(TCC) $(CFLAGS) -o $(APPNAME) $(OBJ) $(EXTRAOBJ) $(LIB)
497
498	# This rule prevents make from using its default rules to try build
499	# an executable named "manifest" out of the file named "manifest.c"
500	#
501	$(SRCDIR)/../manifest:
502

M src/search.c

+291 -79

		--- src/search.c
		+++ src/search.c
		@@ -213,11 +213,11 @@
213	213	aiLastDoc[j] = iDoc;
214	214	aiLastOfst[j] = i;
215	215	for(k=1; j-k>=0 && anMatch[j-k] && aiWordIdx[j-k]==iWord-k; k++){}
216	216	for(ii=0; ii<k; ii++){
217	217	if( anMatch[j-ii]<k ){
218		- anMatch[j-ii] = k;
	218	+ anMatch[j-ii] = k*(nDoc-iDoc);
219	219	aiBestDoc[j-ii] = aiLastDoc[j-ii];
220	220	aiBestOfst[j-ii] = aiLastOfst[j-ii];
221	221	}
222	222	}
223	223	break;
		@@ -396,14 +396,18 @@
396	396	static void search_match_sqlfunc(
397	397	sqlite3_context *context,
398	398	int argc,
399	399	sqlite3_value **argv
400	400	){
401		- const char zSText = (const char)sqlite3_value_text(argv[0]);
	401	+ const char *azDoc[5];
	402	+ int nDoc;
402	403	int rc;
403		- if( zSText==0 ) return;
404		- rc = search_match(&gSearch, 1, &zSText);
	404	+ for(nDoc=0; nDoc<ArraySize(azDoc) && nDoc<argc; nDoc++){
	405	+ azDoc[nDoc] = (const char*)sqlite3_value_text(argv[nDoc]);
	406	+ if( azDoc[nDoc]==0 ) azDoc[nDoc] = "";
	407	+ }
	408	+ rc = search_match(&gSearch, nDoc, azDoc);
405	409	sqlite3_result_int(context, rc);
406	410	}
407	411
408	412	/*
409	413	** These SQL functions return the results of the last
		@@ -435,16 +439,43 @@
435	439	static void search_stext_sqlfunc(
436	440	sqlite3_context *context,
437	441	int argc,
438	442	sqlite3_value **argv
439	443	){
440		- Blob txt;
	444	+ const char zType = (const char)sqlite3_value_text(argv[0]);
	445	+ int rid = sqlite3_value_int(argv[1]);
	446	+ const char zName = (const char)sqlite3_value_text(argv[2]);
	447	+ sqlite3_result_text(context, search_stext_cached(zType[0],rid,zName,0), -1,
	448	+ SQLITE_TRANSIENT);
	449	+}
	450	+static void search_title_sqlfunc(
	451	+ sqlite3_context *context,
	452	+ int argc,
	453	+ sqlite3_value **argv
	454	+){
	455	+ const char zType = (const char)sqlite3_value_text(argv[0]);
	456	+ int rid = sqlite3_value_int(argv[1]);
	457	+ const char zName = (const char)sqlite3_value_text(argv[2]);
	458	+ int nHdr;
	459	+ char *z = search_stext_cached(zType[0], rid, zName, &nHdr);
	460	+ if( nHdr \|\| zType[0]!='d' ){
	461	+ sqlite3_result_text(context, z, nHdr, SQLITE_TRANSIENT);
	462	+ }else{
	463	+ sqlite3_result_value(context, argv[2]);
	464	+ }
	465	+}
	466	+static void search_body_sqlfunc(
	467	+ sqlite3_context *context,
	468	+ int argc,
	469	+ sqlite3_value **argv
	470	+){
441	471	const char zType = (const char)sqlite3_value_text(argv[0]);
442	472	int rid = sqlite3_value_int(argv[1]);
443	473	const char zName = (const char)sqlite3_value_text(argv[2]);
444		- search_stext(zType[0], rid, zName, &txt);
445		- sqlite3_result_text(context, blob_materialize(&txt), -1, fossil_free);
	474	+ int nHdr;
	475	+ char *z = search_stext_cached(zType[0], rid, zName, &nHdr);
	476	+ sqlite3_result_text(context, z+nHdr+1, -1, SQLITE_TRANSIENT);
446	477	}
447	478
448	479	/*
449	480	** Encode a string for use as a query parameter in a URL
450	481	*/
		@@ -463,20 +494,24 @@
463	494	** do not delete the Search object.
464	495	*/
465	496	void search_sql_setup(sqlite3 *db){
466	497	static int once = 0;
467	498	if( once++ ) return;
468		- sqlite3_create_function(db, "search_match", 1, SQLITE_UTF8, 0,
	499	+ sqlite3_create_function(db, "search_match", -1, SQLITE_UTF8, 0,
469	500	search_match_sqlfunc, 0, 0);
470	501	sqlite3_create_function(db, "search_score", 0, SQLITE_UTF8, 0,
471	502	search_score_sqlfunc, 0, 0);
472	503	sqlite3_create_function(db, "search_snippet", 0, SQLITE_UTF8, 0,
473	504	search_snippet_sqlfunc, 0, 0);
474	505	sqlite3_create_function(db, "search_init", -1, SQLITE_UTF8, 0,
475	506	search_init_sqlfunc, 0, 0);
476	507	sqlite3_create_function(db, "stext", 3, SQLITE_UTF8, 0,
477	508	search_stext_sqlfunc, 0, 0);
	509	+ sqlite3_create_function(db, "title", 3, SQLITE_UTF8, 0,
	510	+ search_title_sqlfunc, 0, 0);
	511	+ sqlite3_create_function(db, "body", 3, SQLITE_UTF8, 0,
	512	+ search_body_sqlfunc, 0, 0);
478	513	sqlite3_create_function(db, "urlencode", 1, SQLITE_UTF8, 0,
479	514	search_urlencode_sqlfunc, 0, 0);
480	515	}
481	516
482	517	/*
		@@ -616,21 +651,23 @@
616	651	if( zDocGlob && zDocGlob[0] && zDocBr && zDocBr[0] ){
617	652	db_multi_exec(
618	653	"CREATE VIRTUAL TABLE IF NOT EXISTS temp.foci USING files_of_checkin;"
619	654	);
620	655	db_multi_exec(
621		- "INSERT INTO x(label,url,score,date,snip)"
622		- " SELECT printf('Document: %%s',foci.filename),"
	656	+ "INSERT INTO x(label,url,score,id,date,snip)"
	657	+ " SELECT printf('Document: %%s',title('d',blob.rid,foci.filename)),"
623	658	" printf('/doc/%T/%%s',foci.filename),"
624	659	" search_score(),"
	660	+ " 'd'\|\|blob.rid,"
625	661	" (SELECT datetime(event.mtime) FROM event"
626	662	" WHERE objid=symbolic_name_to_rid('trunk')),"
627	663	" search_snippet()"
628	664	" FROM foci CROSS JOIN blob"
629	665	" WHERE checkinID=symbolic_name_to_rid('trunk')"
630	666	" AND blob.uuid=foci.uuid"
631		- " AND search_match(stext('d',blob.rid,foci.filename))"
	667	+ " AND search_match(title('d',blob.rid,foci.filename),"
	668	+ " body('d',blob.rid,foci.filename))"
632	669	" AND %z",
633	670	zDocBr, glob_expr("foci.filename", zDocGlob)
634	671	);
635	672	}
636	673	}
		@@ -641,18 +678,19 @@
641	678	" FROM tag, tagxref"
642	679	" WHERE tag.tagname GLOB 'wiki-*'"
643	680	" AND tagxref.tagid=tag.tagid"
644	681	" GROUP BY 1"
645	682	")"
646		- "INSERT INTO x(label,url,score,date,snip)"
	683	+ "INSERT INTO x(label,url,score,id,date,snip)"
647	684	" SELECT printf('Wiki: %%s',name),"
648	685	" printf('/wiki?name=%%s',urlencode(name)),"
649	686	" search_score(),"
	687	+ " 'w'\|\|rid,"
650	688	" datetime(mtime),"
651	689	" search_snippet()"
652	690	" FROM wiki"
653		- " WHERE search_match(stext('w',rid,name));"
	691	+ " WHERE search_match(title('w',rid,name),body('w',rid,name));"
654	692	);
655	693	}
656	694	if( (srchFlags & SRCH_CKIN)!=0 ){
657	695	db_multi_exec(
658	696	"WITH ckin(uuid,rid,mtime) AS ("
		@@ -659,34 +697,45 @@
659	697	" SELECT blob.uuid, event.objid, event.mtime"
660	698	" FROM event, blob"
661	699	" WHERE event.type='ci'"
662	700	" AND blob.rid=event.objid"
663	701	")"
664		- "INSERT INTO x(label,url,score,date,snip)"
	702	+ "INSERT INTO x(label,url,score,id,date,snip)"
665	703	" SELECT printf('Check-in [%%.10s] on %%s',uuid,datetime(mtime)),"
666	704	" printf('/timeline?c=%%s&n=8&y=ci',uuid),"
667	705	" search_score(),"
	706	+ " 'c'\|\|rid,"
668	707	" datetime(mtime),"
669	708	" search_snippet()"
670	709	" FROM ckin"
671		- " WHERE search_match(stext('c',rid,NULL));"
	710	+ " WHERE search_match('',body('c',rid,NULL));"
672	711	);
673	712	}
674	713	if( (srchFlags & SRCH_TKT)!=0 ){
675	714	db_multi_exec(
676		- "INSERT INTO x(label,url,score, date,snip)"
677		- " SELECT printf('Ticket [%%.17s] on %%s',"
678		- "tkt_uuid,datetime(tkt_mtime)),"
	715	+ "INSERT INTO x(label,url,score,id,date,snip)"
	716	+ " SELECT printf('Ticket: %%s (%%s)',title('t',tkt_id,NULL),"
	717	+ "datetime(tkt_mtime)),"
679	718	" printf('/tktview/%%.20s',tkt_uuid),"
680	719	" search_score(),"
	720	+ " 't'\|\|tkt_id,"
681	721	" datetime(tkt_mtime),"
682	722	" search_snippet()"
683	723	" FROM ticket"
684		- " WHERE search_match(stext('t',tkt_id,NULL));"
	724	+ " WHERE search_match(title('t',tkt_id,NULL),body('t',tkt_id,NULL));"
685	725	);
686	726	}
687	727	}
	728	+
	729	+/*
	730	+** Number of significant bits in a u32
	731	+*/
	732	+static int nbits(u32 x){
	733	+ int n = 0;
	734	+ while( x ){ n++; x >>= 1; }
	735	+ return n;
	736	+}
688	737
689	738	/*
690	739	** Implemenation of the rank() function used with rank(matchinfo(*,'pcsx')).
691	740	*/
692	741	static void search_rank_sqlfunc(
		@@ -694,24 +743,45 @@
694	743	int argc,
695	744	sqlite3_value **argv
696	745	){
697	746	const unsigned aVal = (unsigned int)sqlite3_value_blob(argv[0]);
698	747	int nVal = sqlite3_value_bytes(argv[0])/4;
	748	+ int nCol; /* Number of columns in the index */
699	749	int nTerm; /* Number of search terms in the query */
700		- int i; /* Loop counter */
701		- double r = 1.0; /* Score */
	750	+ int i, j; /* Loop counter */
	751	+ double r = 0.0; /* Score */
	752	+ const unsigned aX, aS;
702	753
703		- if( nVal<6 ) return;
704		- if( aVal[1]!=1 ) return;
	754	+ if( nVal<2 ) return;
705	755	nTerm = aVal[0];
706		- r = 1<<((30(aVal[2]-1))/nTerm);
707		- for(i=1; i<=nTerm; i++){
708		- int hits_this_row = aVal[3*i];
709		- int hits_all_rows = aVal[3*i+1];
710		- int rows_with_hit = aVal[3*i+2];
711		- double avg_hits_per_row = (double)hits_all_rows/(double)rows_with_hit;
712		- r *= hits_this_row/avg_hits_per_row;
	756	+ nCol = aVal[1];
	757	+ if( nVal<2+3nColnTerm+nCol ) return;
	758	+ aS = aVal+2;
	759	+ aX = aS+nCol;
	760	+ for(j=0; j<nCol; j++){
	761	+ double x;
	762	+ if( aS[j]>0 ){
	763	+ x = 0.0;
	764	+ for(i=0; i<nTerm; i++){
	765	+ int hits_this_row;
	766	+ int hits_all_rows;
	767	+ int rows_with_hit;
	768	+ double avg_hits_per_row;
	769	+
	770	+ hits_this_row = aX[j + inCol3];
	771	+ if( hits_this_row==0 )continue;
	772	+ hits_all_rows = aX[j + inCol3 + 1];
	773	+ rows_with_hit = aX[j + inCol3 + 2];
	774	+ if( rows_with_hit==0 ) continue;
	775	+ avg_hits_per_row = hits_all_rows/(double)rows_with_hit;
	776	+ x += hits_this_row/(avg_hits_per_row*nbits(rows_with_hit));
	777	+ }
	778	+ x = (1<<((30(aS[j]-1))/nTerm));
	779	+ }else{
	780	+ x = 0.0;
	781	+ }
	782	+ r = r*10.0 + x;
713	783	}
714	784	#define SEARCH_DEBUG_RANK 0
715	785	#if SEARCH_DEBUG_RANK
716	786	{
717	787	Blob x;
		@@ -746,14 +816,15 @@
746	816	if( srchFlags==0 ) return;
747	817	sqlite3_create_function(g.db, "rank", 1, SQLITE_UTF8, 0,
748	818	search_rank_sqlfunc, 0, 0);
749	819	blob_init(&sql, 0, 0);
750	820	blob_appendf(&sql,
751		- "INSERT INTO x(label,url,score,date,snip) "
	821	+ "INSERT INTO x(label,url,score,id,date,snip) "
752	822	" SELECT ftsdocs.label,"
753	823	" ftsdocs.url,"
754	824	" rank(matchinfo(ftsidx,'pcsx')),"
	825	+ " ftsdocs.type \|\| ftsdocs.rid,"
755	826	" datetime(ftsdocs.mtime),"
756	827	" snippet(ftsidx,'<mark>','</mark>',' ... ',-1,35)"
757	828	" FROM ftsidx CROSS JOIN ftsdocs"
758	829	" WHERE ftsidx MATCH %Q"
759	830	" AND ftsdocs.rowid=ftsidx.docid",
		@@ -838,29 +909,30 @@
838	909	**
839	910	** Return the number of rows.
840	911	*/
841	912	int search_run_and_output(
842	913	const char zPattern, / The query pattern */
843		- unsigned int srchFlags /* What to search over */
	914	+ unsigned int srchFlags, /* What to search over */
	915	+ int fDebug /* Extra debugging output */
844	916	){
845	917	Stmt q;
846	918	int nRow = 0;
847	919
848	920	srchFlags = search_restrict(srchFlags);
849	921	if( srchFlags==0 ) return 0;
850	922	search_sql_setup(g.db);
851	923	add_content_sql_commands(g.db);
852	924	db_multi_exec(
853		- "CREATE TEMP TABLE x(label,url,score,date,snip);"
	925	+ "CREATE TEMP TABLE x(label,url,score,id,date,snip);"
854	926	);
855	927	if( !search_index_exists() ){
856	928	search_fullscan(zPattern, srchFlags);
857	929	}else{
858	930	search_update_index(srchFlags);
859	931	search_indexed(zPattern, srchFlags);
860	932	}
861		- db_prepare(&q, "SELECT url, snip, label"
	933	+ db_prepare(&q, "SELECT url, snip, label, score, id"
862	934	" FROM x"
863	935	" ORDER BY score DESC, date DESC;");
864	936	while( db_step(&q)==SQLITE_ROW ){
865	937	const char *zUrl = db_column_text(&q, 0);
866	938	const char *zSnippet = db_column_text(&q, 1);
		@@ -867,12 +939,15 @@
867	939	const char *zLabel = db_column_text(&q, 2);
868	940	if( nRow==0 ){
869	941	@ <ol>
870	942	}
871	943	nRow++;
872		- @ <li><p><a href='%R%s(zUrl)'>%h(zLabel)</a><br>
873		- @ <span class='snippet'>%z(cleanSnippet(zSnippet))</span></li>
	944	+ @ <li><p><a href='%R%s(zUrl)'>%h(zLabel)</a>
	945	+ if( fDebug ){
	946	+ @ (%e(db_column_double(&q,3)), %s(db_column_text(&q,4)))
	947	+ }
	948	+ @ <br><span class='snippet'>%z(cleanSnippet(zSnippet))</span></li>
874	949	}
875	950	db_finalize(&q);
876	951	if( nRow ){
877	952	@ </ol>
878	953	}
		@@ -900,10 +975,11 @@
900	975	const char *zType = 0;
901	976	const char *zClass = 0;
902	977	const char *zDisable1;
903	978	const char *zDisable2;
904	979	const char *zPattern;
	980	+ int fDebug = PB("debug");
905	981	srchFlags = search_restrict(srchFlags);
906	982	switch( srchFlags ){
907	983	case SRCH_CKIN: zType = " Check-ins"; zClass = "Ckin"; break;
908	984	case SRCH_DOC: zType = " Docs"; zClass = "Doc"; break;
909	985	case SRCH_TKT: zType = " Tickets"; zClass = "Tkt"; break;
		@@ -947,10 +1023,13 @@
947	1023	cgi_printf(">%s</option>\n", aY[i].zNm);
948	1024	}
949	1025	@ </select>
950	1026	srchFlags = newFlags;
951	1027	}
	1028	+ if( fDebug ){
	1029	+ @ <input type="hidden" name="debug" value="1">
	1030	+ }
952	1031	@ <input type="submit" value="Search%s(zType)"%s(zDisable2)>
953	1032	if( srchFlags==0 ){
954	1033	@ <p class="generalError">Search is disabled</p>
955	1034	}
956	1035	@ </div></form>
		@@ -959,11 +1038,11 @@
959	1038	if( zClass ){
960	1039	@ <div class='searchResult searchResult%s(zClass)'>
961	1040	}else{
962	1041	@ <div class='searchResult'>
963	1042	}
964		- if( search_run_and_output(zPattern, srchFlags)==0 ){
	1043	+ if( search_run_and_output(zPattern, srchFlags, fDebug)==0 ){
965	1044	@ <p class='searchEmpty'>No matches for: <span>%h(zPattern)</span></p>
966	1045	}
967	1046	@ </div>
968	1047	}
969	1048	}
		@@ -983,10 +1062,14 @@
983	1062
984	1063
985	1064	/*
986	1065	** This is a helper function for search_stext(). Writing into pOut
987	1066	** the search text obtained from pIn according to zMimetype.
	1067	+**
	1068	+** The title of the document is the first line of text. All subsequent
	1069	+** lines are the body. If the document has no title, the first line
	1070	+** is blank.
988	1071	*/
989	1072	static void get_stext_by_mimetype(
990	1073	Blob *pIn,
991	1074	const char *zMimetype,
992	1075	Blob *pOut
		@@ -994,41 +1077,74 @@
994	1077	Blob html, title;
995	1078	blob_init(&html, 0, 0);
996	1079	blob_init(&title, 0, 0);
997	1080	if( zMimetype==0 ) zMimetype = "text/plain";
998	1081	if( fossil_strcmp(zMimetype,"text/x-fossil-wiki")==0 ){
999		- wiki_convert(pIn, &html, 0);
	1082	+ Blob tail;
	1083	+ blob_init(&tail, 0, 0);
	1084	+ if( wiki_find_title(pIn, &title, &tail) ){
	1085	+ blob_appendf(pOut, "%s\n", blob_str(&title));
	1086	+ wiki_convert(&tail, &html, 0);
	1087	+ blob_reset(&tail);
	1088	+ }else{
	1089	+ blob_append(pOut, "\n", 1);
	1090	+ wiki_convert(pIn, &html, 0);
	1091	+ }
1000	1092	html_to_plaintext(blob_str(&html), pOut);
1001	1093	}else if( fossil_strcmp(zMimetype,"text/x-markdown")==0 ){
1002	1094	markdown_to_html(pIn, &title, &html);
	1095	+ if( blob_size(&title) ){
	1096	+ blob_appendf(pOut, "%s\n", blob_str(&title));
	1097	+ }else{
	1098	+ blob_append(pOut, "\n", 1);
	1099	+ }
1003	1100	html_to_plaintext(blob_str(&html), pOut);
1004	1101	}else if( fossil_strcmp(zMimetype,"text/html")==0 ){
	1102	+ if( doc_is_embedded_html(pIn, &title) ){
	1103	+ blob_appendf(pOut, "%s\n", blob_str(&title));
	1104	+ }
1005	1105	html_to_plaintext(blob_str(pIn), pOut);
1006	1106	}else{
1007		- pOut = pIn;
1008		- blob_init(pIn, 0, 0);
	1107	+ blob_append(pOut, blob_buffer(pIn), blob_size(pIn));
1009	1108	}
1010	1109	blob_reset(&html);
1011	1110	blob_reset(&title);
1012	1111	}
1013	1112
1014	1113	/*
1015	1114	** Query pQuery is pointing at a single row of output. Append a text
1016	1115	** representation of every text-compatible column to pAccum.
1017	1116	*/
1018		-static void append_all_ticket_fields(Blob pAccum, Stmt pQuery){
	1117	+static void append_all_ticket_fields(Blob pAccum, Stmt pQuery, int iTitle){
1019	1118	int n = db_column_count(pQuery);
1020	1119	int i;
	1120	+ const char *zMime = 0;
	1121	+ if( iTitle>=0 && iTitle<n ){
	1122	+ if( db_column_type(pQuery,iTitle)==SQLITE_TEXT ){
	1123	+ blob_append(pAccum, db_column_text(pQuery,iTitle), -1);
	1124	+ }
	1125	+ blob_append(pAccum, "\n", 1);
	1126	+ }
1021	1127	for(i=0; i<n; i++){
1022	1128	const char *zColName = db_column_name(pQuery,i);
	1129	+ int eType = db_column_type(pQuery,i);
	1130	+ if( i==iTitle ) continue;
1023	1131	if( fossil_strnicmp(zColName,"tkt_",4)==0 ) continue;
1024		- if( fossil_stricmp(zColName,"mimetype")==0 ) continue;
1025		- switch( db_column_type(pQuery,i) ){
1026		- case SQLITE_INTEGER:
1027		- case SQLITE_FLOAT:
1028		- case SQLITE_TEXT:
1029		- blob_appendf(pAccum, "%s: %s \|\n", zColName, db_column_text(pQuery,i));
	1132	+ if( fossil_strnicmp(zColName,"private_",8)==0 ) continue;
	1133	+ if( eType==SQLITE_BLOB \|\| eType==SQLITE_NULL ) continue;
	1134	+ if( fossil_stricmp(zColName,"mimetype")==0 ){
	1135	+ zMime = db_column_text(pQuery,i);
	1136	+ if( fossil_strcmp(zMime,"text/plain")==0 ) zMime = 0;
	1137	+ }else if( zMime==0 \|\| eType!=SQLITE_TEXT ){
	1138	+ blob_appendf(pAccum, "%s: %s \|\n", zColName, db_column_text(pQuery,i));
	1139	+ }else{
	1140	+ Blob txt;
	1141	+ blob_init(&txt, db_column_text(pQuery,i), -1);
	1142	+ blob_appendf(pAccum, "%s: ", zColName);
	1143	+ get_stext_by_mimetype(&txt, zMime, pAccum);
	1144	+ blob_append(pAccum, " \|", 2);
	1145	+ blob_reset(&txt);
1030	1146	}
1031	1147	}
1032	1148	}
1033	1149
1034	1150
		@@ -1054,11 +1170,11 @@
1054	1170	){
1055	1171	blob_init(pOut, 0, 0);
1056	1172	switch( cType ){
1057	1173	case 'd': { /* Documents */
1058	1174	Blob doc;
1059		- content_get(rid, &doc);
	1175	+ content_get(rid, &doc);
1060	1176	blob_to_utf8_no_bom(&doc, 0);
1061	1177	get_stext_by_mimetype(&doc, mimetype_from_name(zName), pOut);
1062	1178	blob_reset(&doc);
1063	1179	break;
1064	1180	}
		@@ -1073,10 +1189,11 @@
1073	1189	manifest_destroy(pWiki);
1074	1190	break;
1075	1191	}
1076	1192	case 'c': { /* Check-in Comments */
1077	1193	static Stmt q;
	1194	+ static int isPlainText = -1;
1078	1195	db_static_prepare(&q,
1079	1196	"SELECT coalesce(ecomment,comment)"
1080	1197	" \|\|' (user: '\|\|coalesce(euser,user,'?')"
1081	1198	" \|\|', tags: '\|\|"
1082	1199	" (SELECT group_concat(substr(tag.tagname,5),',')"
		@@ -1083,44 +1200,99 @@
1083	1200	" FROM tag, tagxref"
1084	1201	" WHERE tagname GLOB 'sym-*' AND tag.tagid=tagxref.tagid"
1085	1202	" AND tagxref.rid=event.objid AND tagxref.tagtype>0)"
1086	1203	" \|\|')'"
1087	1204	" FROM event WHERE objid=:x AND type='ci'");
	1205	+ if( isPlainText<0 ){
	1206	+ isPlainText = db_get_boolean("timeline-plaintext",0);
	1207	+ }
1088	1208	db_bind_int(&q, ":x", rid);
1089	1209	if( db_step(&q)==SQLITE_ROW ){
1090		- db_column_blob(&q, 0, pOut);
1091	1210	blob_append(pOut, "\n", 1);
	1211	+ if( isPlainText ){
	1212	+ db_column_blob(&q, 0, pOut);
	1213	+ }else{
	1214	+ Blob x;
	1215	+ blob_init(&x,0,0);
	1216	+ db_column_blob(&q, 0, &x);
	1217	+ get_stext_by_mimetype(&x, "text/x-fossil-wiki", pOut);
	1218	+ blob_reset(&x);
	1219	+ }
1092	1220	}
1093	1221	db_reset(&q);
1094	1222	break;
1095	1223	}
1096	1224	case 't': { /* Tickets */
1097	1225	static Stmt q1;
1098		- Blob raw;
	1226	+ static int iTitle = -1;
1099	1227	db_static_prepare(&q1, "SELECT * FROM ticket WHERE tkt_id=:rid");
1100		- blob_init(&raw,0,0);
1101	1228	db_bind_int(&q1, ":rid", rid);
1102	1229	if( db_step(&q1)==SQLITE_ROW ){
1103		- append_all_ticket_fields(&raw, &q1);
	1230	+ if( iTitle<0 ){
	1231	+ int n = db_column_count(&q1);
	1232	+ for(iTitle=0; iTitle<n; iTitle++){
	1233	+ if( fossil_stricmp(db_column_name(&q1,iTitle),"title")==0 ) break;
	1234	+ }
	1235	+ }
	1236	+ append_all_ticket_fields(pOut, &q1, iTitle);
1104	1237	}
1105	1238	db_reset(&q1);
1106	1239	if( db_table_exists("repository","ticketchng") ){
1107	1240	static Stmt q2;
1108	1241	db_static_prepare(&q2, "SELECT * FROM ticketchng WHERE tkt_id=:rid"
1109	1242	" ORDER BY tkt_mtime");
1110	1243	db_bind_int(&q2, ":rid", rid);
1111	1244	while( db_step(&q2)==SQLITE_ROW ){
1112		- append_all_ticket_fields(&raw, &q2);
	1245	+ append_all_ticket_fields(pOut, &q2, -1);
1113	1246	}
1114	1247	db_reset(&q2);
1115	1248	}
1116		- html_to_plaintext(blob_str(&raw), pOut);
1117		- blob_reset(&raw);
1118	1249	break;
1119	1250	}
1120	1251	}
1121	1252	}
	1253	+
	1254	+/*
	1255	+** This routine is a wrapper around search_stext().
	1256	+**
	1257	+** This routine looks up the search text, stores it in an internal
	1258	+** buffer, and returns a pointer to the text. Subsequent requests
	1259	+** for the same document return the same pointer. The returned pointer
	1260	+** is valid until the next invocation of this routine. Call this routine
	1261	+** with an eType of 0 to clear the cache.
	1262	+*/
	1263	+char *search_stext_cached(
	1264	+ char cType, /* Type of document */
	1265	+ int rid, /* BLOB.RID or TAG.TAGID value for document */
	1266	+ const char zName, / Auxiliary information */
	1267	+ int pnTitle / OUT: length of title in bytes excluding \n */
	1268	+){
	1269	+ static struct {
	1270	+ Blob stext; /* Cached search text */
	1271	+ char cType; /* The type */
	1272	+ int rid; /* The RID */
	1273	+ int nTitle; /* Number of bytes in title */
	1274	+ } cache;
	1275	+ int i;
	1276	+ char *z;
	1277	+ if( cType!=cache.cType \|\| rid!=cache.rid ){
	1278	+ if( cache.rid>0 ){
	1279	+ blob_reset(&cache.stext);
	1280	+ }else{
	1281	+ blob_init(&cache.stext,0,0);
	1282	+ }
	1283	+ cache.cType = cType;
	1284	+ cache.rid = rid;
	1285	+ if( cType==0 ) return 0;
	1286	+ search_stext(cType, rid, zName, &cache.stext);
	1287	+ z = blob_str(&cache.stext);
	1288	+ for(i=0; z[i] && z[i]!='\n'; i++){}
	1289	+ cache.nTitle = i;
	1290	+ }
	1291	+ if( pnTitle ) *pnTitle = cache.nTitle;
	1292	+ return blob_str(&cache.stext);
	1293	+}
1122	1294
1123	1295	/*
1124	1296	** COMMAND: test-search-stext
1125	1297	**
1126	1298	** Usage: fossil test-search-stext TYPE ARG1 ARG2
		@@ -1131,10 +1303,30 @@
1131	1303	if( g.argc!=5 ) usage("TYPE RID NAME");
1132	1304	search_stext(g.argv[2][0], atoi(g.argv[3]), g.argv[4], &out);
1133	1305	fossil_print("%s\n",blob_str(&out));
1134	1306	blob_reset(&out);
1135	1307	}
	1308	+
	1309	+/*
	1310	+** COMMAND: test-convert-stext
	1311	+**
	1312	+** Usage: fossil test-convert-stext FILE MIMETYPE
	1313	+**
	1314	+** Read the content of FILE and convert it to stext according to MIMETYPE.
	1315	+** Send the result to standard output.
	1316	+*/
	1317	+void test_convert_stext(void){
	1318	+ Blob in, out;
	1319	+ db_find_and_open_repository(0,0);
	1320	+ if( g.argc!=4 ) usage("FILENAME MIMETYPE");
	1321	+ blob_read_from_file(&in, g.argv[2]);
	1322	+ blob_init(&out, 0, 0);
	1323	+ get_stext_by_mimetype(&in, g.argv[3], &out);
	1324	+ fossil_print("%s\n",blob_str(&out));
	1325	+ blob_reset(&in);
	1326	+ blob_reset(&out);
	1327	+}
1136	1328
1137	1329	/* The schema for the full-text index
1138	1330	*/
1139	1331	static const char zFtsSchema[] =
1140	1332	@ -- One entry for each possible search result
		@@ -1145,20 +1337,21 @@
1145	1337	@ name TEXT, -- Additional document description
1146	1338	@ idxed BOOLEAN, -- True if currently in the index
1147	1339	@ label TEXT, -- Label to print on search results
1148	1340	@ url TEXT, -- URL to access this document
1149	1341	@ mtime DATE, -- Date when document created
	1342	+@ bx TEXT, -- Temporary "body" content cache
1150	1343	@ UNIQUE(type,rid)
1151	1344	@ );
1152	1345	@ CREATE INDEX "%w".ftsdocIdxed ON ftsdocs(type,rid,name) WHERE idxed==0;
1153	1346	@ CREATE INDEX "%w".ftsdocName ON ftsdocs(name) WHERE type='w';
1154	1347	@ CREATE VIEW IF NOT EXISTS "%w".ftscontent AS
1155	1348	@ SELECT rowid, type, rid, name, idxed, label, url, mtime,
1156		-@ stext(type,rid,name) AS 'stext'
	1349	+@ title(type,rid,name) AS 'title', body(type,rid,name) AS 'body'
1157	1350	@ FROM ftsdocs;
1158	1351	@ CREATE VIRTUAL TABLE IF NOT EXISTS "%w".ftsidx
1159		-@ USING fts4(content="ftscontent", stext);
	1352	+@ USING fts4(content="ftscontent", title, body%s);
1160	1353	;
1161	1354	static const char zFtsDrop[] =
1162	1355	@ DROP TABLE IF EXISTS "%w".ftsidx;
1163	1356	@ DROP VIEW IF EXISTS "%w".ftscontent;
1164	1357	@ DROP TABLE IF EXISTS "%w".ftsdocs;
		@@ -1168,13 +1361,15 @@
1168	1361	** Create or drop the tables associated with a full-text index.
1169	1362	*/
1170	1363	static int searchIdxExists = -1;
1171	1364	void search_create_index(void){
1172	1365	const char *zDb = db_name("repository");
	1366	+ int useStemmer = db_get_boolean("search-stemmer",0);
	1367	+ const char *zExtra = useStemmer ? ",tokenize=porter" : "";
1173	1368	search_sql_setup(g.db);
1174		- db_multi_exec(zFtsSchema/works-like:"%w%w%w%w%w"/,
1175		- zDb, zDb, zDb, zDb, zDb);
	1369	+ db_multi_exec(zFtsSchema/works-like:"%w%w%w%w%w%s"/,
	1370	+ zDb, zDb, zDb, zDb, zDb, zExtra/safe-for-%s/);
1176	1371	searchIdxExists = 1;
1177	1372	}
1178	1373	void search_drop_index(void){
1179	1374	const char *zDb = db_name("repository");
1180	1375	db_multi_exec(zFtsDrop/works-like:"%w%w%w"/, zDb, zDb, zDb);
		@@ -1292,34 +1487,39 @@
1292	1487	db_multi_exec(
1293	1488	"DELETE FROM ftsdocs WHERE type='d'"
1294	1489	" AND rid NOT IN (SELECT rid FROM current_docs)"
1295	1490	);
1296	1491	db_multi_exec(
1297		- "INSERT OR IGNORE INTO ftsdocs(type,rid,name,idxed,label,url,mtime)"
	1492	+ "INSERT OR IGNORE INTO ftsdocs(type,rid,name,idxed,label,bx,url,mtime)"
1298	1493	" SELECT 'd', rid, name, 0,"
1299		- " printf('Document: %%s',name),"
	1494	+ " title('d',rid,name),"
	1495	+ " body('d',rid,name),"
1300	1496	" printf('/doc/%q/%%s',urlencode(name)),"
1301	1497	" %.17g"
1302	1498	" FROM current_docs",
1303	1499	zBrUuid, rTime
1304	1500	);
1305	1501	db_multi_exec(
1306		- "INSERT INTO ftsidx(docid,stext)"
1307		- " SELECT rowid, stext FROM ftscontent WHERE type='d' AND NOT idxed"
	1502	+ "INSERT INTO ftsidx(docid,title,body)"
	1503	+ " SELECT rowid, label, bx FROM ftsdocs WHERE type='d' AND NOT idxed"
1308	1504	);
1309	1505	db_multi_exec(
1310		- "UPDATE ftsdocs SET idxed=1 WHERE type='d' AND NOT idxed"
	1506	+ "UPDATE ftsdocs SET"
	1507	+ " idxed=1,"
	1508	+ " bx=NULL,"
	1509	+ " label='Document: '\|\|label"
	1510	+ " WHERE type='d' AND NOT idxed"
1311	1511	);
1312	1512	}
1313	1513
1314	1514	/*
1315	1515	** Deal with all of the unindexed 'c' terms in FTSDOCS
1316	1516	*/
1317	1517	static void search_update_checkin_index(void){
1318	1518	db_multi_exec(
1319		- "INSERT INTO ftsidx(docid,stext)"
1320		- " SELECT rowid, stext('c',rid,NULL) FROM ftsdocs"
	1519	+ "INSERT INTO ftsidx(docid,title,body)"
	1520	+ " SELECT rowid, '', body('c',rid,NULL) FROM ftsdocs"
1321	1521	" WHERE type='c' AND NOT idxed;"
1322	1522	);
1323	1523	db_multi_exec(
1324	1524	"REPLACE INTO ftsdocs(rowid,idxed,type,rid,name,label,url,mtime)"
1325	1525	" SELECT ftsdocs.rowid, 1, 'c', ftsdocs.rid, NULL,"
		@@ -1336,19 +1536,20 @@
1336	1536	/*
1337	1537	** Deal with all of the unindexed 't' terms in FTSDOCS
1338	1538	*/
1339	1539	static void search_update_ticket_index(void){
1340	1540	db_multi_exec(
1341		- "INSERT INTO ftsidx(docid,stext)"
1342		- " SELECT rowid, stext('t',rid,NULL) FROM ftsdocs"
	1541	+ "INSERT INTO ftsidx(docid,title,body)"
	1542	+ " SELECT rowid, title('t',rid,NULL), body('t',rid,NULL) FROM ftsdocs"
1343	1543	" WHERE type='t' AND NOT idxed;"
1344	1544	);
1345	1545	if( db_changes()==0 ) return;
1346	1546	db_multi_exec(
1347	1547	"REPLACE INTO ftsdocs(rowid,idxed,type,rid,name,label,url,mtime)"
1348	1548	" SELECT ftsdocs.rowid, 1, 't', ftsdocs.rid, NULL,"
1349		- " printf('Ticket [%%.16s] on %%s',tkt_uuid,datetime(tkt_mtime)),"
	1549	+ " printf('Ticket: %%s (%%s)',title('t',tkt_id,null),"
	1550	+ " datetime(tkt_mtime)),"
1350	1551	" printf('/tktview/%%.20s',tkt_uuid),"
1351	1552	" tkt_mtime"
1352	1553	" FROM ftsdocs, ticket"
1353	1554	" WHERE ftsdocs.type='t' AND NOT ftsdocs.idxed"
1354	1555	" AND ticket.tkt_id=ftsdocs.rid"
		@@ -1358,12 +1559,12 @@
1358	1559	/*
1359	1560	** Deal with all of the unindexed 'w' terms in FTSDOCS
1360	1561	*/
1361	1562	static void search_update_wiki_index(void){
1362	1563	db_multi_exec(
1363		- "INSERT INTO ftsidx(docid,stext)"
1364		- " SELECT rowid, stext('w',rid,NULL) FROM ftsdocs"
	1564	+ "INSERT INTO ftsidx(docid,title,body)"
	1565	+ " SELECT rowid, title('w',rid,NULL),body('w',rid,NULL) FROM ftsdocs"
1365	1566	" WHERE type='w' AND NOT idxed;"
1366	1567	);
1367	1568	if( db_changes()==0 ) return;
1368	1569	db_multi_exec(
1369	1570	"REPLACE INTO ftsdocs(rowid,idxed,type,rid,name,label,url,mtime)"
		@@ -1416,19 +1617,22 @@
1416	1617	** Usage: fossil fts-config ?SUBCOMMAND? ?ARGUMENT?
1417	1618	**
1418	1619	** The "fossil fts-config" command configures the full-text search capabilities
1419	1620	** of the repository. Subcommands:
1420	1621	**
1421		-** reindex Rebuild the search index. Create it if it does
1422		-** not already exist
	1622	+** reindex Rebuild the search index. This is a no-op if
	1623	+** index search is disabled
1423	1624	**
1424	1625	** index (on\|off) Turn the search index on or off
1425	1626	**
1426	1627	** enable cdtw Enable various kinds of search. c=Check-ins,
1427	1628	** d=Documents, t=Tickets, w=Wiki.
1428	1629	**
1429	1630	** disable cdtw Disable versious kinds of search
	1631	+**
	1632	+** stemmer (on\|off) Turn the Porter stemmer on or off for indexed
	1633	+** search. (Unindexed search is never stemmed.)
1430	1634	**
1431	1635	** The current search settings are displayed after any changes are applied.
1432	1636	** Run this command with no arguments to simply see the settings.
1433	1637	*/
1434	1638	void test_fts_cmd(void){
		@@ -1435,16 +1639,17 @@
1435	1639	static const struct { int iCmd; const char *z; } aCmd[] = {
1436	1640	{ 1, "reindex" },
1437	1641	{ 2, "index" },
1438	1642	{ 3, "disable" },
1439	1643	{ 4, "enable" },
	1644	+ { 5, "stemmer" },
1440	1645	};
1441	1646	static const struct { char zSetting; char zName; char *zSw; } aSetng[] = {
1442		- { "search-ckin", "check-in search:", "c" },
1443		- { "search-doc", "document search:", "d" },
1444		- { "search-tkt", "ticket search:", "t" },
1445		- { "search-wiki", "wiki search:", "w" },
	1647	+ { "search-ckin", "check-in search:", "c" },
	1648	+ { "search-doc", "document search:", "d" },
	1649	+ { "search-tkt", "ticket search:", "t" },
	1650	+ { "search-wiki", "wiki search:", "w" },
1446	1651	};
1447	1652	char *zSubCmd;
1448	1653	int i, j, n;
1449	1654	int iCmd = 0;
1450	1655	int iAction = 0;
		@@ -1464,11 +1669,11 @@
1464	1669	return;
1465	1670	}
1466	1671	iCmd = aCmd[i].iCmd;
1467	1672	}
1468	1673	if( iCmd==1 ){
1469		- iAction = 2;
	1674	+ if( search_index_exists() ) iAction = 2;
1470	1675	}
1471	1676	if( iCmd==2 ){
1472	1677	if( g.argc<3 ) usage("index (on\|off)");
1473	1678	iAction = 1 + is_truth(g.argv[3]);
1474	1679	}
		@@ -1475,18 +1680,23 @@
1475	1680	db_begin_transaction();
1476	1681
1477	1682	/* Adjust search settings */
1478	1683	if( iCmd==3 \|\| iCmd==4 ){
1479	1684	const char *zCtrl;
1480		- if( g.argc<4 ) usage("enable STRING");
	1685	+ if( g.argc<4 ) usage(mprintf("%s STRING",zSubCmd));
1481	1686	zCtrl = g.argv[3];
1482	1687	for(j=0; j<ArraySize(aSetng); j++){
1483	1688	if( strchr(zCtrl, aSetng[j].zSw[0])!=0 ){
1484	1689	db_set_int(aSetng[j].zSetting, iCmd-3, 0);
1485	1690	}
1486	1691	}
1487	1692	}
	1693	+ if( iCmd==5 ){
	1694	+ if( g.argc<4 ) usage("porter ON/OFF");
	1695	+ db_set_int("search-stemmer", is_truth(g.argv[3]), 0);
	1696	+ }
	1697	+
1488	1698
1489	1699	/* destroy or rebuild the index, if requested */
1490	1700	if( iAction>=1 ){
1491	1701	search_drop_index();
1492	1702	}
		@@ -1497,14 +1707,16 @@
1497	1707	/* Always show the status before ending */
1498	1708	for(i=0; i<ArraySize(aSetng); i++){
1499	1709	fossil_print("%-16s %s\n", aSetng[i].zName,
1500	1710	db_get_boolean(aSetng[i].zSetting,0) ? "on" : "off");
1501	1711	}
	1712	+ fossil_print("%-16s %s\n", "Porter stemmer:",
	1713	+ db_get_boolean("search-stemmer",0) ? "on" : "off");
1502	1714	if( search_index_exists() ){
1503	1715	fossil_print("%-16s enabled\n", "full-text index:");
1504	1716	fossil_print("%-16s %d\n", "documents:",
1505	1717	db_int(0, "SELECT count(*) FROM ftsdocs"));
1506	1718	}else{
1507	1719	fossil_print("%-16s disabled\n", "full-text index:");
1508	1720	}
1509	1721	db_end_transaction(0);
1510	1722	}
1511	1723

	--- src/search.c
	+++ src/search.c
	@@ -213,11 +213,11 @@
213	aiLastDoc[j] = iDoc;
214	aiLastOfst[j] = i;
215	for(k=1; j-k>=0 && anMatch[j-k] && aiWordIdx[j-k]==iWord-k; k++){}
216	for(ii=0; ii<k; ii++){
217	if( anMatch[j-ii]<k ){
218	anMatch[j-ii] = k;
219	aiBestDoc[j-ii] = aiLastDoc[j-ii];
220	aiBestOfst[j-ii] = aiLastOfst[j-ii];
221	}
222	}
223	break;
	@@ -396,14 +396,18 @@
396	static void search_match_sqlfunc(
397	sqlite3_context *context,
398	int argc,
399	sqlite3_value **argv
400	){
401	const char zSText = (const char)sqlite3_value_text(argv[0]);

402	int rc;
403	if( zSText==0 ) return;
404	rc = search_match(&gSearch, 1, &zSText);



405	sqlite3_result_int(context, rc);
406	}
407
408	/*
409	** These SQL functions return the results of the last
	@@ -435,16 +439,43 @@
435	static void search_stext_sqlfunc(
436	sqlite3_context *context,
437	int argc,
438	sqlite3_value **argv
439	){
440	Blob txt;


























441	const char zType = (const char)sqlite3_value_text(argv[0]);
442	int rid = sqlite3_value_int(argv[1]);
443	const char zName = (const char)sqlite3_value_text(argv[2]);
444	search_stext(zType[0], rid, zName, &txt);
445	sqlite3_result_text(context, blob_materialize(&txt), -1, fossil_free);

446	}
447
448	/*
449	** Encode a string for use as a query parameter in a URL
450	*/
	@@ -463,20 +494,24 @@
463	** do not delete the Search object.
464	*/
465	void search_sql_setup(sqlite3 *db){
466	static int once = 0;
467	if( once++ ) return;
468	sqlite3_create_function(db, "search_match", 1, SQLITE_UTF8, 0,
469	search_match_sqlfunc, 0, 0);
470	sqlite3_create_function(db, "search_score", 0, SQLITE_UTF8, 0,
471	search_score_sqlfunc, 0, 0);
472	sqlite3_create_function(db, "search_snippet", 0, SQLITE_UTF8, 0,
473	search_snippet_sqlfunc, 0, 0);
474	sqlite3_create_function(db, "search_init", -1, SQLITE_UTF8, 0,
475	search_init_sqlfunc, 0, 0);
476	sqlite3_create_function(db, "stext", 3, SQLITE_UTF8, 0,
477	search_stext_sqlfunc, 0, 0);




478	sqlite3_create_function(db, "urlencode", 1, SQLITE_UTF8, 0,
479	search_urlencode_sqlfunc, 0, 0);
480	}
481
482	/*
	@@ -616,21 +651,23 @@
616	if( zDocGlob && zDocGlob[0] && zDocBr && zDocBr[0] ){
617	db_multi_exec(
618	"CREATE VIRTUAL TABLE IF NOT EXISTS temp.foci USING files_of_checkin;"
619	);
620	db_multi_exec(
621	"INSERT INTO x(label,url,score,date,snip)"
622	" SELECT printf('Document: %%s',foci.filename),"
623	" printf('/doc/%T/%%s',foci.filename),"
624	" search_score(),"

625	" (SELECT datetime(event.mtime) FROM event"
626	" WHERE objid=symbolic_name_to_rid('trunk')),"
627	" search_snippet()"
628	" FROM foci CROSS JOIN blob"
629	" WHERE checkinID=symbolic_name_to_rid('trunk')"
630	" AND blob.uuid=foci.uuid"
631	" AND search_match(stext('d',blob.rid,foci.filename))"

632	" AND %z",
633	zDocBr, glob_expr("foci.filename", zDocGlob)
634	);
635	}
636	}
	@@ -641,18 +678,19 @@
641	" FROM tag, tagxref"
642	" WHERE tag.tagname GLOB 'wiki-*'"
643	" AND tagxref.tagid=tag.tagid"
644	" GROUP BY 1"
645	")"
646	"INSERT INTO x(label,url,score,date,snip)"
647	" SELECT printf('Wiki: %%s',name),"
648	" printf('/wiki?name=%%s',urlencode(name)),"
649	" search_score(),"

650	" datetime(mtime),"
651	" search_snippet()"
652	" FROM wiki"
653	" WHERE search_match(stext('w',rid,name));"
654	);
655	}
656	if( (srchFlags & SRCH_CKIN)!=0 ){
657	db_multi_exec(
658	"WITH ckin(uuid,rid,mtime) AS ("
	@@ -659,34 +697,45 @@
659	" SELECT blob.uuid, event.objid, event.mtime"
660	" FROM event, blob"
661	" WHERE event.type='ci'"
662	" AND blob.rid=event.objid"
663	")"
664	"INSERT INTO x(label,url,score,date,snip)"
665	" SELECT printf('Check-in [%%.10s] on %%s',uuid,datetime(mtime)),"
666	" printf('/timeline?c=%%s&n=8&y=ci',uuid),"
667	" search_score(),"

668	" datetime(mtime),"
669	" search_snippet()"
670	" FROM ckin"
671	" WHERE search_match(stext('c',rid,NULL));"
672	);
673	}
674	if( (srchFlags & SRCH_TKT)!=0 ){
675	db_multi_exec(
676	"INSERT INTO x(label,url,score, date,snip)"
677	" SELECT printf('Ticket [%%.17s] on %%s',"
678	"tkt_uuid,datetime(tkt_mtime)),"
679	" printf('/tktview/%%.20s',tkt_uuid),"
680	" search_score(),"

681	" datetime(tkt_mtime),"
682	" search_snippet()"
683	" FROM ticket"
684	" WHERE search_match(stext('t',tkt_id,NULL));"
685	);
686	}
687	}









688
689	/*
690	** Implemenation of the rank() function used with rank(matchinfo(*,'pcsx')).
691	*/
692	static void search_rank_sqlfunc(
	@@ -694,24 +743,45 @@
694	int argc,
695	sqlite3_value **argv
696	){
697	const unsigned aVal = (unsigned int)sqlite3_value_blob(argv[0]);
698	int nVal = sqlite3_value_bytes(argv[0])/4;

699	int nTerm; /* Number of search terms in the query */
700	int i; /* Loop counter */
701	double r = 1.0; /* Score */

702
703	if( nVal<6 ) return;
704	if( aVal[1]!=1 ) return;
705	nTerm = aVal[0];
706	r = 1<<((30(aVal[2]-1))/nTerm);
707	for(i=1; i<=nTerm; i++){
708	int hits_this_row = aVal[3*i];
709	int hits_all_rows = aVal[3*i+1];
710	int rows_with_hit = aVal[3*i+2];
711	double avg_hits_per_row = (double)hits_all_rows/(double)rows_with_hit;
712	r *= hits_this_row/avg_hits_per_row;




















713	}
714	#define SEARCH_DEBUG_RANK 0
715	#if SEARCH_DEBUG_RANK
716	{
717	Blob x;
	@@ -746,14 +816,15 @@
746	if( srchFlags==0 ) return;
747	sqlite3_create_function(g.db, "rank", 1, SQLITE_UTF8, 0,
748	search_rank_sqlfunc, 0, 0);
749	blob_init(&sql, 0, 0);
750	blob_appendf(&sql,
751	"INSERT INTO x(label,url,score,date,snip) "
752	" SELECT ftsdocs.label,"
753	" ftsdocs.url,"
754	" rank(matchinfo(ftsidx,'pcsx')),"

755	" datetime(ftsdocs.mtime),"
756	" snippet(ftsidx,'<mark>','</mark>',' ... ',-1,35)"
757	" FROM ftsidx CROSS JOIN ftsdocs"
758	" WHERE ftsidx MATCH %Q"
759	" AND ftsdocs.rowid=ftsidx.docid",
	@@ -838,29 +909,30 @@
838	**
839	** Return the number of rows.
840	*/
841	int search_run_and_output(
842	const char zPattern, / The query pattern */
843	unsigned int srchFlags /* What to search over */

844	){
845	Stmt q;
846	int nRow = 0;
847
848	srchFlags = search_restrict(srchFlags);
849	if( srchFlags==0 ) return 0;
850	search_sql_setup(g.db);
851	add_content_sql_commands(g.db);
852	db_multi_exec(
853	"CREATE TEMP TABLE x(label,url,score,date,snip);"
854	);
855	if( !search_index_exists() ){
856	search_fullscan(zPattern, srchFlags);
857	}else{
858	search_update_index(srchFlags);
859	search_indexed(zPattern, srchFlags);
860	}
861	db_prepare(&q, "SELECT url, snip, label"
862	" FROM x"
863	" ORDER BY score DESC, date DESC;");
864	while( db_step(&q)==SQLITE_ROW ){
865	const char *zUrl = db_column_text(&q, 0);
866	const char *zSnippet = db_column_text(&q, 1);
	@@ -867,12 +939,15 @@
867	const char *zLabel = db_column_text(&q, 2);
868	if( nRow==0 ){
869	@ <ol>
870	}
871	nRow++;
872	@ <li><p><a href='%R%s(zUrl)'>%h(zLabel)</a><br>
873	@ <span class='snippet'>%z(cleanSnippet(zSnippet))</span></li>



874	}
875	db_finalize(&q);
876	if( nRow ){
877	@ </ol>
878	}
	@@ -900,10 +975,11 @@
900	const char *zType = 0;
901	const char *zClass = 0;
902	const char *zDisable1;
903	const char *zDisable2;
904	const char *zPattern;

905	srchFlags = search_restrict(srchFlags);
906	switch( srchFlags ){
907	case SRCH_CKIN: zType = " Check-ins"; zClass = "Ckin"; break;
908	case SRCH_DOC: zType = " Docs"; zClass = "Doc"; break;
909	case SRCH_TKT: zType = " Tickets"; zClass = "Tkt"; break;
	@@ -947,10 +1023,13 @@
947	cgi_printf(">%s</option>\n", aY[i].zNm);
948	}
949	@ </select>
950	srchFlags = newFlags;
951	}



952	@ <input type="submit" value="Search%s(zType)"%s(zDisable2)>
953	if( srchFlags==0 ){
954	@ <p class="generalError">Search is disabled</p>
955	}
956	@ </div></form>
	@@ -959,11 +1038,11 @@
959	if( zClass ){
960	@ <div class='searchResult searchResult%s(zClass)'>
961	}else{
962	@ <div class='searchResult'>
963	}
964	if( search_run_and_output(zPattern, srchFlags)==0 ){
965	@ <p class='searchEmpty'>No matches for: <span>%h(zPattern)</span></p>
966	}
967	@ </div>
968	}
969	}
	@@ -983,10 +1062,14 @@
983
984
985	/*
986	** This is a helper function for search_stext(). Writing into pOut
987	** the search text obtained from pIn according to zMimetype.




988	*/
989	static void get_stext_by_mimetype(
990	Blob *pIn,
991	const char *zMimetype,
992	Blob *pOut
	@@ -994,41 +1077,74 @@
994	Blob html, title;
995	blob_init(&html, 0, 0);
996	blob_init(&title, 0, 0);
997	if( zMimetype==0 ) zMimetype = "text/plain";
998	if( fossil_strcmp(zMimetype,"text/x-fossil-wiki")==0 ){
999	wiki_convert(pIn, &html, 0);









1000	html_to_plaintext(blob_str(&html), pOut);
1001	}else if( fossil_strcmp(zMimetype,"text/x-markdown")==0 ){
1002	markdown_to_html(pIn, &title, &html);





1003	html_to_plaintext(blob_str(&html), pOut);
1004	}else if( fossil_strcmp(zMimetype,"text/html")==0 ){



1005	html_to_plaintext(blob_str(pIn), pOut);
1006	}else{
1007	pOut = pIn;
1008	blob_init(pIn, 0, 0);
1009	}
1010	blob_reset(&html);
1011	blob_reset(&title);
1012	}
1013
1014	/*
1015	** Query pQuery is pointing at a single row of output. Append a text
1016	** representation of every text-compatible column to pAccum.
1017	*/
1018	static void append_all_ticket_fields(Blob pAccum, Stmt pQuery){
1019	int n = db_column_count(pQuery);
1020	int i;







1021	for(i=0; i<n; i++){
1022	const char *zColName = db_column_name(pQuery,i);


1023	if( fossil_strnicmp(zColName,"tkt_",4)==0 ) continue;
1024	if( fossil_stricmp(zColName,"mimetype")==0 ) continue;
1025	switch( db_column_type(pQuery,i) ){
1026	case SQLITE_INTEGER:
1027	case SQLITE_FLOAT:
1028	case SQLITE_TEXT:
1029	blob_appendf(pAccum, "%s: %s \|\n", zColName, db_column_text(pQuery,i));








1030	}
1031	}
1032	}
1033
1034
	@@ -1054,11 +1170,11 @@
1054	){
1055	blob_init(pOut, 0, 0);
1056	switch( cType ){
1057	case 'd': { /* Documents */
1058	Blob doc;
1059	content_get(rid, &doc);
1060	blob_to_utf8_no_bom(&doc, 0);
1061	get_stext_by_mimetype(&doc, mimetype_from_name(zName), pOut);
1062	blob_reset(&doc);
1063	break;
1064	}
	@@ -1073,10 +1189,11 @@
1073	manifest_destroy(pWiki);
1074	break;
1075	}
1076	case 'c': { /* Check-in Comments */
1077	static Stmt q;

1078	db_static_prepare(&q,
1079	"SELECT coalesce(ecomment,comment)"
1080	" \|\|' (user: '\|\|coalesce(euser,user,'?')"
1081	" \|\|', tags: '\|\|"
1082	" (SELECT group_concat(substr(tag.tagname,5),',')"
	@@ -1083,44 +1200,99 @@
1083	" FROM tag, tagxref"
1084	" WHERE tagname GLOB 'sym-*' AND tag.tagid=tagxref.tagid"
1085	" AND tagxref.rid=event.objid AND tagxref.tagtype>0)"
1086	" \|\|')'"
1087	" FROM event WHERE objid=:x AND type='ci'");



1088	db_bind_int(&q, ":x", rid);
1089	if( db_step(&q)==SQLITE_ROW ){
1090	db_column_blob(&q, 0, pOut);
1091	blob_append(pOut, "\n", 1);









1092	}
1093	db_reset(&q);
1094	break;
1095	}
1096	case 't': { /* Tickets */
1097	static Stmt q1;
1098	Blob raw;
1099	db_static_prepare(&q1, "SELECT * FROM ticket WHERE tkt_id=:rid");
1100	blob_init(&raw,0,0);
1101	db_bind_int(&q1, ":rid", rid);
1102	if( db_step(&q1)==SQLITE_ROW ){
1103	append_all_ticket_fields(&raw, &q1);






1104	}
1105	db_reset(&q1);
1106	if( db_table_exists("repository","ticketchng") ){
1107	static Stmt q2;
1108	db_static_prepare(&q2, "SELECT * FROM ticketchng WHERE tkt_id=:rid"
1109	" ORDER BY tkt_mtime");
1110	db_bind_int(&q2, ":rid", rid);
1111	while( db_step(&q2)==SQLITE_ROW ){
1112	append_all_ticket_fields(&raw, &q2);
1113	}
1114	db_reset(&q2);
1115	}
1116	html_to_plaintext(blob_str(&raw), pOut);
1117	blob_reset(&raw);
1118	break;
1119	}
1120	}
1121	}









































1122
1123	/*
1124	** COMMAND: test-search-stext
1125	**
1126	** Usage: fossil test-search-stext TYPE ARG1 ARG2
	@@ -1131,10 +1303,30 @@
1131	if( g.argc!=5 ) usage("TYPE RID NAME");
1132	search_stext(g.argv[2][0], atoi(g.argv[3]), g.argv[4], &out);
1133	fossil_print("%s\n",blob_str(&out));
1134	blob_reset(&out);
1135	}




















1136
1137	/* The schema for the full-text index
1138	*/
1139	static const char zFtsSchema[] =
1140	@ -- One entry for each possible search result
	@@ -1145,20 +1337,21 @@
1145	@ name TEXT, -- Additional document description
1146	@ idxed BOOLEAN, -- True if currently in the index
1147	@ label TEXT, -- Label to print on search results
1148	@ url TEXT, -- URL to access this document
1149	@ mtime DATE, -- Date when document created

1150	@ UNIQUE(type,rid)
1151	@ );
1152	@ CREATE INDEX "%w".ftsdocIdxed ON ftsdocs(type,rid,name) WHERE idxed==0;
1153	@ CREATE INDEX "%w".ftsdocName ON ftsdocs(name) WHERE type='w';
1154	@ CREATE VIEW IF NOT EXISTS "%w".ftscontent AS
1155	@ SELECT rowid, type, rid, name, idxed, label, url, mtime,
1156	@ stext(type,rid,name) AS 'stext'
1157	@ FROM ftsdocs;
1158	@ CREATE VIRTUAL TABLE IF NOT EXISTS "%w".ftsidx
1159	@ USING fts4(content="ftscontent", stext);
1160	;
1161	static const char zFtsDrop[] =
1162	@ DROP TABLE IF EXISTS "%w".ftsidx;
1163	@ DROP VIEW IF EXISTS "%w".ftscontent;
1164	@ DROP TABLE IF EXISTS "%w".ftsdocs;
	@@ -1168,13 +1361,15 @@
1168	** Create or drop the tables associated with a full-text index.
1169	*/
1170	static int searchIdxExists = -1;
1171	void search_create_index(void){
1172	const char *zDb = db_name("repository");


1173	search_sql_setup(g.db);
1174	db_multi_exec(zFtsSchema/works-like:"%w%w%w%w%w"/,
1175	zDb, zDb, zDb, zDb, zDb);
1176	searchIdxExists = 1;
1177	}
1178	void search_drop_index(void){
1179	const char *zDb = db_name("repository");
1180	db_multi_exec(zFtsDrop/works-like:"%w%w%w"/, zDb, zDb, zDb);
	@@ -1292,34 +1487,39 @@
1292	db_multi_exec(
1293	"DELETE FROM ftsdocs WHERE type='d'"
1294	" AND rid NOT IN (SELECT rid FROM current_docs)"
1295	);
1296	db_multi_exec(
1297	"INSERT OR IGNORE INTO ftsdocs(type,rid,name,idxed,label,url,mtime)"
1298	" SELECT 'd', rid, name, 0,"
1299	" printf('Document: %%s',name),"

1300	" printf('/doc/%q/%%s',urlencode(name)),"
1301	" %.17g"
1302	" FROM current_docs",
1303	zBrUuid, rTime
1304	);
1305	db_multi_exec(
1306	"INSERT INTO ftsidx(docid,stext)"
1307	" SELECT rowid, stext FROM ftscontent WHERE type='d' AND NOT idxed"
1308	);
1309	db_multi_exec(
1310	"UPDATE ftsdocs SET idxed=1 WHERE type='d' AND NOT idxed"




1311	);
1312	}
1313
1314	/*
1315	** Deal with all of the unindexed 'c' terms in FTSDOCS
1316	*/
1317	static void search_update_checkin_index(void){
1318	db_multi_exec(
1319	"INSERT INTO ftsidx(docid,stext)"
1320	" SELECT rowid, stext('c',rid,NULL) FROM ftsdocs"
1321	" WHERE type='c' AND NOT idxed;"
1322	);
1323	db_multi_exec(
1324	"REPLACE INTO ftsdocs(rowid,idxed,type,rid,name,label,url,mtime)"
1325	" SELECT ftsdocs.rowid, 1, 'c', ftsdocs.rid, NULL,"
	@@ -1336,19 +1536,20 @@
1336	/*
1337	** Deal with all of the unindexed 't' terms in FTSDOCS
1338	*/
1339	static void search_update_ticket_index(void){
1340	db_multi_exec(
1341	"INSERT INTO ftsidx(docid,stext)"
1342	" SELECT rowid, stext('t',rid,NULL) FROM ftsdocs"
1343	" WHERE type='t' AND NOT idxed;"
1344	);
1345	if( db_changes()==0 ) return;
1346	db_multi_exec(
1347	"REPLACE INTO ftsdocs(rowid,idxed,type,rid,name,label,url,mtime)"
1348	" SELECT ftsdocs.rowid, 1, 't', ftsdocs.rid, NULL,"
1349	" printf('Ticket [%%.16s] on %%s',tkt_uuid,datetime(tkt_mtime)),"

1350	" printf('/tktview/%%.20s',tkt_uuid),"
1351	" tkt_mtime"
1352	" FROM ftsdocs, ticket"
1353	" WHERE ftsdocs.type='t' AND NOT ftsdocs.idxed"
1354	" AND ticket.tkt_id=ftsdocs.rid"
	@@ -1358,12 +1559,12 @@
1358	/*
1359	** Deal with all of the unindexed 'w' terms in FTSDOCS
1360	*/
1361	static void search_update_wiki_index(void){
1362	db_multi_exec(
1363	"INSERT INTO ftsidx(docid,stext)"
1364	" SELECT rowid, stext('w',rid,NULL) FROM ftsdocs"
1365	" WHERE type='w' AND NOT idxed;"
1366	);
1367	if( db_changes()==0 ) return;
1368	db_multi_exec(
1369	"REPLACE INTO ftsdocs(rowid,idxed,type,rid,name,label,url,mtime)"
	@@ -1416,19 +1617,22 @@
1416	** Usage: fossil fts-config ?SUBCOMMAND? ?ARGUMENT?
1417	**
1418	** The "fossil fts-config" command configures the full-text search capabilities
1419	** of the repository. Subcommands:
1420	**
1421	** reindex Rebuild the search index. Create it if it does
1422	** not already exist
1423	**
1424	** index (on\|off) Turn the search index on or off
1425	**
1426	** enable cdtw Enable various kinds of search. c=Check-ins,
1427	** d=Documents, t=Tickets, w=Wiki.
1428	**
1429	** disable cdtw Disable versious kinds of search



1430	**
1431	** The current search settings are displayed after any changes are applied.
1432	** Run this command with no arguments to simply see the settings.
1433	*/
1434	void test_fts_cmd(void){
	@@ -1435,16 +1639,17 @@
1435	static const struct { int iCmd; const char *z; } aCmd[] = {
1436	{ 1, "reindex" },
1437	{ 2, "index" },
1438	{ 3, "disable" },
1439	{ 4, "enable" },

1440	};
1441	static const struct { char zSetting; char zName; char *zSw; } aSetng[] = {
1442	{ "search-ckin", "check-in search:", "c" },
1443	{ "search-doc", "document search:", "d" },
1444	{ "search-tkt", "ticket search:", "t" },
1445	{ "search-wiki", "wiki search:", "w" },
1446	};
1447	char *zSubCmd;
1448	int i, j, n;
1449	int iCmd = 0;
1450	int iAction = 0;
	@@ -1464,11 +1669,11 @@
1464	return;
1465	}
1466	iCmd = aCmd[i].iCmd;
1467	}
1468	if( iCmd==1 ){
1469	iAction = 2;
1470	}
1471	if( iCmd==2 ){
1472	if( g.argc<3 ) usage("index (on\|off)");
1473	iAction = 1 + is_truth(g.argv[3]);
1474	}
	@@ -1475,18 +1680,23 @@
1475	db_begin_transaction();
1476
1477	/* Adjust search settings */
1478	if( iCmd==3 \|\| iCmd==4 ){
1479	const char *zCtrl;
1480	if( g.argc<4 ) usage("enable STRING");
1481	zCtrl = g.argv[3];
1482	for(j=0; j<ArraySize(aSetng); j++){
1483	if( strchr(zCtrl, aSetng[j].zSw[0])!=0 ){
1484	db_set_int(aSetng[j].zSetting, iCmd-3, 0);
1485	}
1486	}
1487	}





1488
1489	/* destroy or rebuild the index, if requested */
1490	if( iAction>=1 ){
1491	search_drop_index();
1492	}
	@@ -1497,14 +1707,16 @@
1497	/* Always show the status before ending */
1498	for(i=0; i<ArraySize(aSetng); i++){
1499	fossil_print("%-16s %s\n", aSetng[i].zName,
1500	db_get_boolean(aSetng[i].zSetting,0) ? "on" : "off");
1501	}


1502	if( search_index_exists() ){
1503	fossil_print("%-16s enabled\n", "full-text index:");
1504	fossil_print("%-16s %d\n", "documents:",
1505	db_int(0, "SELECT count(*) FROM ftsdocs"));
1506	}else{
1507	fossil_print("%-16s disabled\n", "full-text index:");
1508	}
1509	db_end_transaction(0);
1510	}
1511

	--- src/search.c
	+++ src/search.c
	@@ -213,11 +213,11 @@
213	aiLastDoc[j] = iDoc;
214	aiLastOfst[j] = i;
215	for(k=1; j-k>=0 && anMatch[j-k] && aiWordIdx[j-k]==iWord-k; k++){}
216	for(ii=0; ii<k; ii++){
217	if( anMatch[j-ii]<k ){
218	anMatch[j-ii] = k*(nDoc-iDoc);
219	aiBestDoc[j-ii] = aiLastDoc[j-ii];
220	aiBestOfst[j-ii] = aiLastOfst[j-ii];
221	}
222	}
223	break;
	@@ -396,14 +396,18 @@
396	static void search_match_sqlfunc(
397	sqlite3_context *context,
398	int argc,
399	sqlite3_value **argv
400	){
401	const char *azDoc[5];
402	int nDoc;
403	int rc;
404	for(nDoc=0; nDoc<ArraySize(azDoc) && nDoc<argc; nDoc++){
405	azDoc[nDoc] = (const char*)sqlite3_value_text(argv[nDoc]);
406	if( azDoc[nDoc]==0 ) azDoc[nDoc] = "";
407	}
408	rc = search_match(&gSearch, nDoc, azDoc);
409	sqlite3_result_int(context, rc);
410	}
411
412	/*
413	** These SQL functions return the results of the last
	@@ -435,16 +439,43 @@
439	static void search_stext_sqlfunc(
440	sqlite3_context *context,
441	int argc,
442	sqlite3_value **argv
443	){
444	const char zType = (const char)sqlite3_value_text(argv[0]);
445	int rid = sqlite3_value_int(argv[1]);
446	const char zName = (const char)sqlite3_value_text(argv[2]);
447	sqlite3_result_text(context, search_stext_cached(zType[0],rid,zName,0), -1,
448	SQLITE_TRANSIENT);
449	}
450	static void search_title_sqlfunc(
451	sqlite3_context *context,
452	int argc,
453	sqlite3_value **argv
454	){
455	const char zType = (const char)sqlite3_value_text(argv[0]);
456	int rid = sqlite3_value_int(argv[1]);
457	const char zName = (const char)sqlite3_value_text(argv[2]);
458	int nHdr;
459	char *z = search_stext_cached(zType[0], rid, zName, &nHdr);
460	if( nHdr \|\| zType[0]!='d' ){
461	sqlite3_result_text(context, z, nHdr, SQLITE_TRANSIENT);
462	}else{
463	sqlite3_result_value(context, argv[2]);
464	}
465	}
466	static void search_body_sqlfunc(
467	sqlite3_context *context,
468	int argc,
469	sqlite3_value **argv
470	){
471	const char zType = (const char)sqlite3_value_text(argv[0]);
472	int rid = sqlite3_value_int(argv[1]);
473	const char zName = (const char)sqlite3_value_text(argv[2]);
474	int nHdr;
475	char *z = search_stext_cached(zType[0], rid, zName, &nHdr);
476	sqlite3_result_text(context, z+nHdr+1, -1, SQLITE_TRANSIENT);
477	}
478
479	/*
480	** Encode a string for use as a query parameter in a URL
481	*/
	@@ -463,20 +494,24 @@
494	** do not delete the Search object.
495	*/
496	void search_sql_setup(sqlite3 *db){
497	static int once = 0;
498	if( once++ ) return;
499	sqlite3_create_function(db, "search_match", -1, SQLITE_UTF8, 0,
500	search_match_sqlfunc, 0, 0);
501	sqlite3_create_function(db, "search_score", 0, SQLITE_UTF8, 0,
502	search_score_sqlfunc, 0, 0);
503	sqlite3_create_function(db, "search_snippet", 0, SQLITE_UTF8, 0,
504	search_snippet_sqlfunc, 0, 0);
505	sqlite3_create_function(db, "search_init", -1, SQLITE_UTF8, 0,
506	search_init_sqlfunc, 0, 0);
507	sqlite3_create_function(db, "stext", 3, SQLITE_UTF8, 0,
508	search_stext_sqlfunc, 0, 0);
509	sqlite3_create_function(db, "title", 3, SQLITE_UTF8, 0,
510	search_title_sqlfunc, 0, 0);
511	sqlite3_create_function(db, "body", 3, SQLITE_UTF8, 0,
512	search_body_sqlfunc, 0, 0);
513	sqlite3_create_function(db, "urlencode", 1, SQLITE_UTF8, 0,
514	search_urlencode_sqlfunc, 0, 0);
515	}
516
517	/*
	@@ -616,21 +651,23 @@
651	if( zDocGlob && zDocGlob[0] && zDocBr && zDocBr[0] ){
652	db_multi_exec(
653	"CREATE VIRTUAL TABLE IF NOT EXISTS temp.foci USING files_of_checkin;"
654	);
655	db_multi_exec(
656	"INSERT INTO x(label,url,score,id,date,snip)"
657	" SELECT printf('Document: %%s',title('d',blob.rid,foci.filename)),"
658	" printf('/doc/%T/%%s',foci.filename),"
659	" search_score(),"
660	" 'd'\|\|blob.rid,"
661	" (SELECT datetime(event.mtime) FROM event"
662	" WHERE objid=symbolic_name_to_rid('trunk')),"
663	" search_snippet()"
664	" FROM foci CROSS JOIN blob"
665	" WHERE checkinID=symbolic_name_to_rid('trunk')"
666	" AND blob.uuid=foci.uuid"
667	" AND search_match(title('d',blob.rid,foci.filename),"
668	" body('d',blob.rid,foci.filename))"
669	" AND %z",
670	zDocBr, glob_expr("foci.filename", zDocGlob)
671	);
672	}
673	}
	@@ -641,18 +678,19 @@
678	" FROM tag, tagxref"
679	" WHERE tag.tagname GLOB 'wiki-*'"
680	" AND tagxref.tagid=tag.tagid"
681	" GROUP BY 1"
682	")"
683	"INSERT INTO x(label,url,score,id,date,snip)"
684	" SELECT printf('Wiki: %%s',name),"
685	" printf('/wiki?name=%%s',urlencode(name)),"
686	" search_score(),"
687	" 'w'\|\|rid,"
688	" datetime(mtime),"
689	" search_snippet()"
690	" FROM wiki"
691	" WHERE search_match(title('w',rid,name),body('w',rid,name));"
692	);
693	}
694	if( (srchFlags & SRCH_CKIN)!=0 ){
695	db_multi_exec(
696	"WITH ckin(uuid,rid,mtime) AS ("
	@@ -659,34 +697,45 @@
697	" SELECT blob.uuid, event.objid, event.mtime"
698	" FROM event, blob"
699	" WHERE event.type='ci'"
700	" AND blob.rid=event.objid"
701	")"
702	"INSERT INTO x(label,url,score,id,date,snip)"
703	" SELECT printf('Check-in [%%.10s] on %%s',uuid,datetime(mtime)),"
704	" printf('/timeline?c=%%s&n=8&y=ci',uuid),"
705	" search_score(),"
706	" 'c'\|\|rid,"
707	" datetime(mtime),"
708	" search_snippet()"
709	" FROM ckin"
710	" WHERE search_match('',body('c',rid,NULL));"
711	);
712	}
713	if( (srchFlags & SRCH_TKT)!=0 ){
714	db_multi_exec(
715	"INSERT INTO x(label,url,score,id,date,snip)"
716	" SELECT printf('Ticket: %%s (%%s)',title('t',tkt_id,NULL),"
717	"datetime(tkt_mtime)),"
718	" printf('/tktview/%%.20s',tkt_uuid),"
719	" search_score(),"
720	" 't'\|\|tkt_id,"
721	" datetime(tkt_mtime),"
722	" search_snippet()"
723	" FROM ticket"
724	" WHERE search_match(title('t',tkt_id,NULL),body('t',tkt_id,NULL));"
725	);
726	}
727	}
728
729	/*
730	** Number of significant bits in a u32
731	*/
732	static int nbits(u32 x){
733	int n = 0;
734	while( x ){ n++; x >>= 1; }
735	return n;
736	}
737
738	/*
739	** Implemenation of the rank() function used with rank(matchinfo(*,'pcsx')).
740	*/
741	static void search_rank_sqlfunc(
	@@ -694,24 +743,45 @@
743	int argc,
744	sqlite3_value **argv
745	){
746	const unsigned aVal = (unsigned int)sqlite3_value_blob(argv[0]);
747	int nVal = sqlite3_value_bytes(argv[0])/4;
748	int nCol; /* Number of columns in the index */
749	int nTerm; /* Number of search terms in the query */
750	int i, j; /* Loop counter */
751	double r = 0.0; /* Score */
752	const unsigned aX, aS;
753
754	if( nVal<2 ) return;

755	nTerm = aVal[0];
756	nCol = aVal[1];
757	if( nVal<2+3nColnTerm+nCol ) return;
758	aS = aVal+2;
759	aX = aS+nCol;
760	for(j=0; j<nCol; j++){
761	double x;
762	if( aS[j]>0 ){
763	x = 0.0;
764	for(i=0; i<nTerm; i++){
765	int hits_this_row;
766	int hits_all_rows;
767	int rows_with_hit;
768	double avg_hits_per_row;
769
770	hits_this_row = aX[j + inCol3];
771	if( hits_this_row==0 )continue;
772	hits_all_rows = aX[j + inCol3 + 1];
773	rows_with_hit = aX[j + inCol3 + 2];
774	if( rows_with_hit==0 ) continue;
775	avg_hits_per_row = hits_all_rows/(double)rows_with_hit;
776	x += hits_this_row/(avg_hits_per_row*nbits(rows_with_hit));
777	}
778	x = (1<<((30(aS[j]-1))/nTerm));
779	}else{
780	x = 0.0;
781	}
782	r = r*10.0 + x;
783	}
784	#define SEARCH_DEBUG_RANK 0
785	#if SEARCH_DEBUG_RANK
786	{
787	Blob x;
	@@ -746,14 +816,15 @@
816	if( srchFlags==0 ) return;
817	sqlite3_create_function(g.db, "rank", 1, SQLITE_UTF8, 0,
818	search_rank_sqlfunc, 0, 0);
819	blob_init(&sql, 0, 0);
820	blob_appendf(&sql,
821	"INSERT INTO x(label,url,score,id,date,snip) "
822	" SELECT ftsdocs.label,"
823	" ftsdocs.url,"
824	" rank(matchinfo(ftsidx,'pcsx')),"
825	" ftsdocs.type \|\| ftsdocs.rid,"
826	" datetime(ftsdocs.mtime),"
827	" snippet(ftsidx,'<mark>','</mark>',' ... ',-1,35)"
828	" FROM ftsidx CROSS JOIN ftsdocs"
829	" WHERE ftsidx MATCH %Q"
830	" AND ftsdocs.rowid=ftsidx.docid",
	@@ -838,29 +909,30 @@
909	**
910	** Return the number of rows.
911	*/
912	int search_run_and_output(
913	const char zPattern, / The query pattern */
914	unsigned int srchFlags, /* What to search over */
915	int fDebug /* Extra debugging output */
916	){
917	Stmt q;
918	int nRow = 0;
919
920	srchFlags = search_restrict(srchFlags);
921	if( srchFlags==0 ) return 0;
922	search_sql_setup(g.db);
923	add_content_sql_commands(g.db);
924	db_multi_exec(
925	"CREATE TEMP TABLE x(label,url,score,id,date,snip);"
926	);
927	if( !search_index_exists() ){
928	search_fullscan(zPattern, srchFlags);
929	}else{
930	search_update_index(srchFlags);
931	search_indexed(zPattern, srchFlags);
932	}
933	db_prepare(&q, "SELECT url, snip, label, score, id"
934	" FROM x"
935	" ORDER BY score DESC, date DESC;");
936	while( db_step(&q)==SQLITE_ROW ){
937	const char *zUrl = db_column_text(&q, 0);
938	const char *zSnippet = db_column_text(&q, 1);
	@@ -867,12 +939,15 @@
939	const char *zLabel = db_column_text(&q, 2);
940	if( nRow==0 ){
941	@ <ol>
942	}
943	nRow++;
944	@ <li><p><a href='%R%s(zUrl)'>%h(zLabel)</a>
945	if( fDebug ){
946	@ (%e(db_column_double(&q,3)), %s(db_column_text(&q,4)))
947	}
948	@ <br><span class='snippet'>%z(cleanSnippet(zSnippet))</span></li>
949	}
950	db_finalize(&q);
951	if( nRow ){
952	@ </ol>
953	}
	@@ -900,10 +975,11 @@
975	const char *zType = 0;
976	const char *zClass = 0;
977	const char *zDisable1;
978	const char *zDisable2;
979	const char *zPattern;
980	int fDebug = PB("debug");
981	srchFlags = search_restrict(srchFlags);
982	switch( srchFlags ){
983	case SRCH_CKIN: zType = " Check-ins"; zClass = "Ckin"; break;
984	case SRCH_DOC: zType = " Docs"; zClass = "Doc"; break;
985	case SRCH_TKT: zType = " Tickets"; zClass = "Tkt"; break;
	@@ -947,10 +1023,13 @@
1023	cgi_printf(">%s</option>\n", aY[i].zNm);
1024	}
1025	@ </select>
1026	srchFlags = newFlags;
1027	}
1028	if( fDebug ){
1029	@ <input type="hidden" name="debug" value="1">
1030	}
1031	@ <input type="submit" value="Search%s(zType)"%s(zDisable2)>
1032	if( srchFlags==0 ){
1033	@ <p class="generalError">Search is disabled</p>
1034	}
1035	@ </div></form>
	@@ -959,11 +1038,11 @@
1038	if( zClass ){
1039	@ <div class='searchResult searchResult%s(zClass)'>
1040	}else{
1041	@ <div class='searchResult'>
1042	}
1043	if( search_run_and_output(zPattern, srchFlags, fDebug)==0 ){
1044	@ <p class='searchEmpty'>No matches for: <span>%h(zPattern)</span></p>
1045	}
1046	@ </div>
1047	}
1048	}
	@@ -983,10 +1062,14 @@
1062
1063
1064	/*
1065	** This is a helper function for search_stext(). Writing into pOut
1066	** the search text obtained from pIn according to zMimetype.
1067	**
1068	** The title of the document is the first line of text. All subsequent
1069	** lines are the body. If the document has no title, the first line
1070	** is blank.
1071	*/
1072	static void get_stext_by_mimetype(
1073	Blob *pIn,
1074	const char *zMimetype,
1075	Blob *pOut
	@@ -994,41 +1077,74 @@
1077	Blob html, title;
1078	blob_init(&html, 0, 0);
1079	blob_init(&title, 0, 0);
1080	if( zMimetype==0 ) zMimetype = "text/plain";
1081	if( fossil_strcmp(zMimetype,"text/x-fossil-wiki")==0 ){
1082	Blob tail;
1083	blob_init(&tail, 0, 0);
1084	if( wiki_find_title(pIn, &title, &tail) ){
1085	blob_appendf(pOut, "%s\n", blob_str(&title));
1086	wiki_convert(&tail, &html, 0);
1087	blob_reset(&tail);
1088	}else{
1089	blob_append(pOut, "\n", 1);
1090	wiki_convert(pIn, &html, 0);
1091	}
1092	html_to_plaintext(blob_str(&html), pOut);
1093	}else if( fossil_strcmp(zMimetype,"text/x-markdown")==0 ){
1094	markdown_to_html(pIn, &title, &html);
1095	if( blob_size(&title) ){
1096	blob_appendf(pOut, "%s\n", blob_str(&title));
1097	}else{
1098	blob_append(pOut, "\n", 1);
1099	}
1100	html_to_plaintext(blob_str(&html), pOut);
1101	}else if( fossil_strcmp(zMimetype,"text/html")==0 ){
1102	if( doc_is_embedded_html(pIn, &title) ){
1103	blob_appendf(pOut, "%s\n", blob_str(&title));
1104	}
1105	html_to_plaintext(blob_str(pIn), pOut);
1106	}else{
1107	blob_append(pOut, blob_buffer(pIn), blob_size(pIn));

1108	}
1109	blob_reset(&html);
1110	blob_reset(&title);
1111	}
1112
1113	/*
1114	** Query pQuery is pointing at a single row of output. Append a text
1115	** representation of every text-compatible column to pAccum.
1116	*/
1117	static void append_all_ticket_fields(Blob pAccum, Stmt pQuery, int iTitle){
1118	int n = db_column_count(pQuery);
1119	int i;
1120	const char *zMime = 0;
1121	if( iTitle>=0 && iTitle<n ){
1122	if( db_column_type(pQuery,iTitle)==SQLITE_TEXT ){
1123	blob_append(pAccum, db_column_text(pQuery,iTitle), -1);
1124	}
1125	blob_append(pAccum, "\n", 1);
1126	}
1127	for(i=0; i<n; i++){
1128	const char *zColName = db_column_name(pQuery,i);
1129	int eType = db_column_type(pQuery,i);
1130	if( i==iTitle ) continue;
1131	if( fossil_strnicmp(zColName,"tkt_",4)==0 ) continue;
1132	if( fossil_strnicmp(zColName,"private_",8)==0 ) continue;
1133	if( eType==SQLITE_BLOB \|\| eType==SQLITE_NULL ) continue;
1134	if( fossil_stricmp(zColName,"mimetype")==0 ){
1135	zMime = db_column_text(pQuery,i);
1136	if( fossil_strcmp(zMime,"text/plain")==0 ) zMime = 0;
1137	}else if( zMime==0 \|\| eType!=SQLITE_TEXT ){
1138	blob_appendf(pAccum, "%s: %s \|\n", zColName, db_column_text(pQuery,i));
1139	}else{
1140	Blob txt;
1141	blob_init(&txt, db_column_text(pQuery,i), -1);
1142	blob_appendf(pAccum, "%s: ", zColName);
1143	get_stext_by_mimetype(&txt, zMime, pAccum);
1144	blob_append(pAccum, " \|", 2);
1145	blob_reset(&txt);
1146	}
1147	}
1148	}
1149
1150
	@@ -1054,11 +1170,11 @@
1170	){
1171	blob_init(pOut, 0, 0);
1172	switch( cType ){
1173	case 'd': { /* Documents */
1174	Blob doc;
1175	content_get(rid, &doc);
1176	blob_to_utf8_no_bom(&doc, 0);
1177	get_stext_by_mimetype(&doc, mimetype_from_name(zName), pOut);
1178	blob_reset(&doc);
1179	break;
1180	}
	@@ -1073,10 +1189,11 @@
1189	manifest_destroy(pWiki);
1190	break;
1191	}
1192	case 'c': { /* Check-in Comments */
1193	static Stmt q;
1194	static int isPlainText = -1;
1195	db_static_prepare(&q,
1196	"SELECT coalesce(ecomment,comment)"
1197	" \|\|' (user: '\|\|coalesce(euser,user,'?')"
1198	" \|\|', tags: '\|\|"
1199	" (SELECT group_concat(substr(tag.tagname,5),',')"
	@@ -1083,44 +1200,99 @@
1200	" FROM tag, tagxref"
1201	" WHERE tagname GLOB 'sym-*' AND tag.tagid=tagxref.tagid"
1202	" AND tagxref.rid=event.objid AND tagxref.tagtype>0)"
1203	" \|\|')'"
1204	" FROM event WHERE objid=:x AND type='ci'");
1205	if( isPlainText<0 ){
1206	isPlainText = db_get_boolean("timeline-plaintext",0);
1207	}
1208	db_bind_int(&q, ":x", rid);
1209	if( db_step(&q)==SQLITE_ROW ){

1210	blob_append(pOut, "\n", 1);
1211	if( isPlainText ){
1212	db_column_blob(&q, 0, pOut);
1213	}else{
1214	Blob x;
1215	blob_init(&x,0,0);
1216	db_column_blob(&q, 0, &x);
1217	get_stext_by_mimetype(&x, "text/x-fossil-wiki", pOut);
1218	blob_reset(&x);
1219	}
1220	}
1221	db_reset(&q);
1222	break;
1223	}
1224	case 't': { /* Tickets */
1225	static Stmt q1;
1226	static int iTitle = -1;
1227	db_static_prepare(&q1, "SELECT * FROM ticket WHERE tkt_id=:rid");

1228	db_bind_int(&q1, ":rid", rid);
1229	if( db_step(&q1)==SQLITE_ROW ){
1230	if( iTitle<0 ){
1231	int n = db_column_count(&q1);
1232	for(iTitle=0; iTitle<n; iTitle++){
1233	if( fossil_stricmp(db_column_name(&q1,iTitle),"title")==0 ) break;
1234	}
1235	}
1236	append_all_ticket_fields(pOut, &q1, iTitle);
1237	}
1238	db_reset(&q1);
1239	if( db_table_exists("repository","ticketchng") ){
1240	static Stmt q2;
1241	db_static_prepare(&q2, "SELECT * FROM ticketchng WHERE tkt_id=:rid"
1242	" ORDER BY tkt_mtime");
1243	db_bind_int(&q2, ":rid", rid);
1244	while( db_step(&q2)==SQLITE_ROW ){
1245	append_all_ticket_fields(pOut, &q2, -1);
1246	}
1247	db_reset(&q2);
1248	}


1249	break;
1250	}
1251	}
1252	}
1253
1254	/*
1255	** This routine is a wrapper around search_stext().
1256	**
1257	** This routine looks up the search text, stores it in an internal
1258	** buffer, and returns a pointer to the text. Subsequent requests
1259	** for the same document return the same pointer. The returned pointer
1260	** is valid until the next invocation of this routine. Call this routine
1261	** with an eType of 0 to clear the cache.
1262	*/
1263	char *search_stext_cached(
1264	char cType, /* Type of document */
1265	int rid, /* BLOB.RID or TAG.TAGID value for document */
1266	const char zName, / Auxiliary information */
1267	int pnTitle / OUT: length of title in bytes excluding \n */
1268	){
1269	static struct {
1270	Blob stext; /* Cached search text */
1271	char cType; /* The type */
1272	int rid; /* The RID */
1273	int nTitle; /* Number of bytes in title */
1274	} cache;
1275	int i;
1276	char *z;
1277	if( cType!=cache.cType \|\| rid!=cache.rid ){
1278	if( cache.rid>0 ){
1279	blob_reset(&cache.stext);
1280	}else{
1281	blob_init(&cache.stext,0,0);
1282	}
1283	cache.cType = cType;
1284	cache.rid = rid;
1285	if( cType==0 ) return 0;
1286	search_stext(cType, rid, zName, &cache.stext);
1287	z = blob_str(&cache.stext);
1288	for(i=0; z[i] && z[i]!='\n'; i++){}
1289	cache.nTitle = i;
1290	}
1291	if( pnTitle ) *pnTitle = cache.nTitle;
1292	return blob_str(&cache.stext);
1293	}
1294
1295	/*
1296	** COMMAND: test-search-stext
1297	**
1298	** Usage: fossil test-search-stext TYPE ARG1 ARG2
	@@ -1131,10 +1303,30 @@
1303	if( g.argc!=5 ) usage("TYPE RID NAME");
1304	search_stext(g.argv[2][0], atoi(g.argv[3]), g.argv[4], &out);
1305	fossil_print("%s\n",blob_str(&out));
1306	blob_reset(&out);
1307	}
1308
1309	/*
1310	** COMMAND: test-convert-stext
1311	**
1312	** Usage: fossil test-convert-stext FILE MIMETYPE
1313	**
1314	** Read the content of FILE and convert it to stext according to MIMETYPE.
1315	** Send the result to standard output.
1316	*/
1317	void test_convert_stext(void){
1318	Blob in, out;
1319	db_find_and_open_repository(0,0);
1320	if( g.argc!=4 ) usage("FILENAME MIMETYPE");
1321	blob_read_from_file(&in, g.argv[2]);
1322	blob_init(&out, 0, 0);
1323	get_stext_by_mimetype(&in, g.argv[3], &out);
1324	fossil_print("%s\n",blob_str(&out));
1325	blob_reset(&in);
1326	blob_reset(&out);
1327	}
1328
1329	/* The schema for the full-text index
1330	*/
1331	static const char zFtsSchema[] =
1332	@ -- One entry for each possible search result
	@@ -1145,20 +1337,21 @@
1337	@ name TEXT, -- Additional document description
1338	@ idxed BOOLEAN, -- True if currently in the index
1339	@ label TEXT, -- Label to print on search results
1340	@ url TEXT, -- URL to access this document
1341	@ mtime DATE, -- Date when document created
1342	@ bx TEXT, -- Temporary "body" content cache
1343	@ UNIQUE(type,rid)
1344	@ );
1345	@ CREATE INDEX "%w".ftsdocIdxed ON ftsdocs(type,rid,name) WHERE idxed==0;
1346	@ CREATE INDEX "%w".ftsdocName ON ftsdocs(name) WHERE type='w';
1347	@ CREATE VIEW IF NOT EXISTS "%w".ftscontent AS
1348	@ SELECT rowid, type, rid, name, idxed, label, url, mtime,
1349	@ title(type,rid,name) AS 'title', body(type,rid,name) AS 'body'
1350	@ FROM ftsdocs;
1351	@ CREATE VIRTUAL TABLE IF NOT EXISTS "%w".ftsidx
1352	@ USING fts4(content="ftscontent", title, body%s);
1353	;
1354	static const char zFtsDrop[] =
1355	@ DROP TABLE IF EXISTS "%w".ftsidx;
1356	@ DROP VIEW IF EXISTS "%w".ftscontent;
1357	@ DROP TABLE IF EXISTS "%w".ftsdocs;
	@@ -1168,13 +1361,15 @@
1361	** Create or drop the tables associated with a full-text index.
1362	*/
1363	static int searchIdxExists = -1;
1364	void search_create_index(void){
1365	const char *zDb = db_name("repository");
1366	int useStemmer = db_get_boolean("search-stemmer",0);
1367	const char *zExtra = useStemmer ? ",tokenize=porter" : "";
1368	search_sql_setup(g.db);
1369	db_multi_exec(zFtsSchema/works-like:"%w%w%w%w%w%s"/,
1370	zDb, zDb, zDb, zDb, zDb, zExtra/safe-for-%s/);
1371	searchIdxExists = 1;
1372	}
1373	void search_drop_index(void){
1374	const char *zDb = db_name("repository");
1375	db_multi_exec(zFtsDrop/works-like:"%w%w%w"/, zDb, zDb, zDb);
	@@ -1292,34 +1487,39 @@
1487	db_multi_exec(
1488	"DELETE FROM ftsdocs WHERE type='d'"
1489	" AND rid NOT IN (SELECT rid FROM current_docs)"
1490	);
1491	db_multi_exec(
1492	"INSERT OR IGNORE INTO ftsdocs(type,rid,name,idxed,label,bx,url,mtime)"
1493	" SELECT 'd', rid, name, 0,"
1494	" title('d',rid,name),"
1495	" body('d',rid,name),"
1496	" printf('/doc/%q/%%s',urlencode(name)),"
1497	" %.17g"
1498	" FROM current_docs",
1499	zBrUuid, rTime
1500	);
1501	db_multi_exec(
1502	"INSERT INTO ftsidx(docid,title,body)"
1503	" SELECT rowid, label, bx FROM ftsdocs WHERE type='d' AND NOT idxed"
1504	);
1505	db_multi_exec(
1506	"UPDATE ftsdocs SET"
1507	" idxed=1,"
1508	" bx=NULL,"
1509	" label='Document: '\|\|label"
1510	" WHERE type='d' AND NOT idxed"
1511	);
1512	}
1513
1514	/*
1515	** Deal with all of the unindexed 'c' terms in FTSDOCS
1516	*/
1517	static void search_update_checkin_index(void){
1518	db_multi_exec(
1519	"INSERT INTO ftsidx(docid,title,body)"
1520	" SELECT rowid, '', body('c',rid,NULL) FROM ftsdocs"
1521	" WHERE type='c' AND NOT idxed;"
1522	);
1523	db_multi_exec(
1524	"REPLACE INTO ftsdocs(rowid,idxed,type,rid,name,label,url,mtime)"
1525	" SELECT ftsdocs.rowid, 1, 'c', ftsdocs.rid, NULL,"
	@@ -1336,19 +1536,20 @@
1536	/*
1537	** Deal with all of the unindexed 't' terms in FTSDOCS
1538	*/
1539	static void search_update_ticket_index(void){
1540	db_multi_exec(
1541	"INSERT INTO ftsidx(docid,title,body)"
1542	" SELECT rowid, title('t',rid,NULL), body('t',rid,NULL) FROM ftsdocs"
1543	" WHERE type='t' AND NOT idxed;"
1544	);
1545	if( db_changes()==0 ) return;
1546	db_multi_exec(
1547	"REPLACE INTO ftsdocs(rowid,idxed,type,rid,name,label,url,mtime)"
1548	" SELECT ftsdocs.rowid, 1, 't', ftsdocs.rid, NULL,"
1549	" printf('Ticket: %%s (%%s)',title('t',tkt_id,null),"
1550	" datetime(tkt_mtime)),"
1551	" printf('/tktview/%%.20s',tkt_uuid),"
1552	" tkt_mtime"
1553	" FROM ftsdocs, ticket"
1554	" WHERE ftsdocs.type='t' AND NOT ftsdocs.idxed"
1555	" AND ticket.tkt_id=ftsdocs.rid"
	@@ -1358,12 +1559,12 @@
1559	/*
1560	** Deal with all of the unindexed 'w' terms in FTSDOCS
1561	*/
1562	static void search_update_wiki_index(void){
1563	db_multi_exec(
1564	"INSERT INTO ftsidx(docid,title,body)"
1565	" SELECT rowid, title('w',rid,NULL),body('w',rid,NULL) FROM ftsdocs"
1566	" WHERE type='w' AND NOT idxed;"
1567	);
1568	if( db_changes()==0 ) return;
1569	db_multi_exec(
1570	"REPLACE INTO ftsdocs(rowid,idxed,type,rid,name,label,url,mtime)"
	@@ -1416,19 +1617,22 @@
1617	** Usage: fossil fts-config ?SUBCOMMAND? ?ARGUMENT?
1618	**
1619	** The "fossil fts-config" command configures the full-text search capabilities
1620	** of the repository. Subcommands:
1621	**
1622	** reindex Rebuild the search index. This is a no-op if
1623	** index search is disabled
1624	**
1625	** index (on\|off) Turn the search index on or off
1626	**
1627	** enable cdtw Enable various kinds of search. c=Check-ins,
1628	** d=Documents, t=Tickets, w=Wiki.
1629	**
1630	** disable cdtw Disable versious kinds of search
1631	**
1632	** stemmer (on\|off) Turn the Porter stemmer on or off for indexed
1633	** search. (Unindexed search is never stemmed.)
1634	**
1635	** The current search settings are displayed after any changes are applied.
1636	** Run this command with no arguments to simply see the settings.
1637	*/
1638	void test_fts_cmd(void){
	@@ -1435,16 +1639,17 @@
1639	static const struct { int iCmd; const char *z; } aCmd[] = {
1640	{ 1, "reindex" },
1641	{ 2, "index" },
1642	{ 3, "disable" },
1643	{ 4, "enable" },
1644	{ 5, "stemmer" },
1645	};
1646	static const struct { char zSetting; char zName; char *zSw; } aSetng[] = {
1647	{ "search-ckin", "check-in search:", "c" },
1648	{ "search-doc", "document search:", "d" },
1649	{ "search-tkt", "ticket search:", "t" },
1650	{ "search-wiki", "wiki search:", "w" },
1651	};
1652	char *zSubCmd;
1653	int i, j, n;
1654	int iCmd = 0;
1655	int iAction = 0;
	@@ -1464,11 +1669,11 @@
1669	return;
1670	}
1671	iCmd = aCmd[i].iCmd;
1672	}
1673	if( iCmd==1 ){
1674	if( search_index_exists() ) iAction = 2;
1675	}
1676	if( iCmd==2 ){
1677	if( g.argc<3 ) usage("index (on\|off)");
1678	iAction = 1 + is_truth(g.argv[3]);
1679	}
	@@ -1475,18 +1680,23 @@
1680	db_begin_transaction();
1681
1682	/* Adjust search settings */
1683	if( iCmd==3 \|\| iCmd==4 ){
1684	const char *zCtrl;
1685	if( g.argc<4 ) usage(mprintf("%s STRING",zSubCmd));
1686	zCtrl = g.argv[3];
1687	for(j=0; j<ArraySize(aSetng); j++){
1688	if( strchr(zCtrl, aSetng[j].zSw[0])!=0 ){
1689	db_set_int(aSetng[j].zSetting, iCmd-3, 0);
1690	}
1691	}
1692	}
1693	if( iCmd==5 ){
1694	if( g.argc<4 ) usage("porter ON/OFF");
1695	db_set_int("search-stemmer", is_truth(g.argv[3]), 0);
1696	}
1697
1698
1699	/* destroy or rebuild the index, if requested */
1700	if( iAction>=1 ){
1701	search_drop_index();
1702	}
	@@ -1497,14 +1707,16 @@
1707	/* Always show the status before ending */
1708	for(i=0; i<ArraySize(aSetng); i++){
1709	fossil_print("%-16s %s\n", aSetng[i].zName,
1710	db_get_boolean(aSetng[i].zSetting,0) ? "on" : "off");
1711	}
1712	fossil_print("%-16s %s\n", "Porter stemmer:",
1713	db_get_boolean("search-stemmer",0) ? "on" : "off");
1714	if( search_index_exists() ){
1715	fossil_print("%-16s enabled\n", "full-text index:");
1716	fossil_print("%-16s %d\n", "documents:",
1717	db_int(0, "SELECT count(*) FROM ftsdocs"));
1718	}else{
1719	fossil_print("%-16s disabled\n", "full-text index:");
1720	}
1721	db_end_transaction(0);
1722	}
1723

M src/setup.c

		--- src/setup.c
		+++ src/setup.c
		@@ -2266,16 +2266,18 @@
2266	2266	search_update_index(search_restrict(SRCH_ALL));
2267	2267	}
2268	2268	if( search_index_exists() ){
2269	2269	@ <p>Currently using an SQLite FTS4 search index. This makes search
2270	2270	@ run faster, especially on large repositories, but takes up space.</p>
	2271	+ onoff_attribute("Use Porter Stemmer","search-stemmer","ss",0,0);
2271	2272	@ <p><input type="submit" name="fts0" value="Delete The Full-Text Index">
2272	2273	@ <input type="submit" name="fts1" value="Rebuild The Full-Text Index">
2273	2274	}else{
2274	2275	@ <p>The SQLite FTS4 search index is disabled. All searching will be
2275	2276	@ a full-text scan. This usually works fine, but can be slow for
2276	2277	@ larger repositories.</p>
	2278	+ onoff_attribute("Use Porter Stemmer","search-stemmer","ss",0,0);
2277	2279	@ <p><input type="submit" name="fts1" value="Create A Full-Text Index">
2278	2280	}
2279	2281	@ </div></form>
2280	2282	style_footer();
2281	2283	}
2282	2284

	--- src/setup.c
	+++ src/setup.c
	@@ -2266,16 +2266,18 @@
2266	search_update_index(search_restrict(SRCH_ALL));
2267	}
2268	if( search_index_exists() ){
2269	@ <p>Currently using an SQLite FTS4 search index. This makes search
2270	@ run faster, especially on large repositories, but takes up space.</p>

2271	@ <p><input type="submit" name="fts0" value="Delete The Full-Text Index">
2272	@ <input type="submit" name="fts1" value="Rebuild The Full-Text Index">
2273	}else{
2274	@ <p>The SQLite FTS4 search index is disabled. All searching will be
2275	@ a full-text scan. This usually works fine, but can be slow for
2276	@ larger repositories.</p>

2277	@ <p><input type="submit" name="fts1" value="Create A Full-Text Index">
2278	}
2279	@ </div></form>
2280	style_footer();
2281	}
2282

	--- src/setup.c
	+++ src/setup.c
	@@ -2266,16 +2266,18 @@
2266	search_update_index(search_restrict(SRCH_ALL));
2267	}
2268	if( search_index_exists() ){
2269	@ <p>Currently using an SQLite FTS4 search index. This makes search
2270	@ run faster, especially on large repositories, but takes up space.</p>
2271	onoff_attribute("Use Porter Stemmer","search-stemmer","ss",0,0);
2272	@ <p><input type="submit" name="fts0" value="Delete The Full-Text Index">
2273	@ <input type="submit" name="fts1" value="Rebuild The Full-Text Index">
2274	}else{
2275	@ <p>The SQLite FTS4 search index is disabled. All searching will be
2276	@ a full-text scan. This usually works fine, but can be slow for
2277	@ larger repositories.</p>
2278	onoff_attribute("Use Porter Stemmer","search-stemmer","ss",0,0);
2279	@ <p><input type="submit" name="fts1" value="Create A Full-Text Index">
2280	}
2281	@ </div></form>
2282	style_footer();
2283	}
2284

M src/setup.c

		--- src/setup.c
		+++ src/setup.c
		@@ -2266,16 +2266,18 @@
2266	2266	search_update_index(search_restrict(SRCH_ALL));
2267	2267	}
2268	2268	if( search_index_exists() ){
2269	2269	@ <p>Currently using an SQLite FTS4 search index. This makes search
2270	2270	@ run faster, especially on large repositories, but takes up space.</p>
	2271	+ onoff_attribute("Use Porter Stemmer","search-stemmer","ss",0,0);
2271	2272	@ <p><input type="submit" name="fts0" value="Delete The Full-Text Index">
2272	2273	@ <input type="submit" name="fts1" value="Rebuild The Full-Text Index">
2273	2274	}else{
2274	2275	@ <p>The SQLite FTS4 search index is disabled. All searching will be
2275	2276	@ a full-text scan. This usually works fine, but can be slow for
2276	2277	@ larger repositories.</p>
	2278	+ onoff_attribute("Use Porter Stemmer","search-stemmer","ss",0,0);
2277	2279	@ <p><input type="submit" name="fts1" value="Create A Full-Text Index">
2278	2280	}
2279	2281	@ </div></form>
2280	2282	style_footer();
2281	2283	}
2282	2284

	--- src/setup.c
	+++ src/setup.c
	@@ -2266,16 +2266,18 @@
2266	search_update_index(search_restrict(SRCH_ALL));
2267	}
2268	if( search_index_exists() ){
2269	@ <p>Currently using an SQLite FTS4 search index. This makes search
2270	@ run faster, especially on large repositories, but takes up space.</p>

2271	@ <p><input type="submit" name="fts0" value="Delete The Full-Text Index">
2272	@ <input type="submit" name="fts1" value="Rebuild The Full-Text Index">
2273	}else{
2274	@ <p>The SQLite FTS4 search index is disabled. All searching will be
2275	@ a full-text scan. This usually works fine, but can be slow for
2276	@ larger repositories.</p>

2277	@ <p><input type="submit" name="fts1" value="Create A Full-Text Index">
2278	}
2279	@ </div></form>
2280	style_footer();
2281	}
2282

	--- src/setup.c
	+++ src/setup.c
	@@ -2266,16 +2266,18 @@
2266	search_update_index(search_restrict(SRCH_ALL));
2267	}
2268	if( search_index_exists() ){
2269	@ <p>Currently using an SQLite FTS4 search index. This makes search
2270	@ run faster, especially on large repositories, but takes up space.</p>
2271	onoff_attribute("Use Porter Stemmer","search-stemmer","ss",0,0);
2272	@ <p><input type="submit" name="fts0" value="Delete The Full-Text Index">
2273	@ <input type="submit" name="fts1" value="Rebuild The Full-Text Index">
2274	}else{
2275	@ <p>The SQLite FTS4 search index is disabled. All searching will be
2276	@ a full-text scan. This usually works fine, but can be slow for
2277	@ larger repositories.</p>
2278	onoff_attribute("Use Porter Stemmer","search-stemmer","ss",0,0);
2279	@ <p><input type="submit" name="fts1" value="Create A Full-Text Index">
2280	}
2281	@ </div></form>
2282	style_footer();
2283	}
2284

M src/wikiformat.c

+63 -8

		--- src/wikiformat.c
		+++ src/wikiformat.c
		@@ -1965,17 +1965,26 @@
1965	1965	** z points to the start of a token. Return the number of
1966	1966	** characters in that token.
1967	1967	*/
1968	1968	static int nextHtmlToken(const char *z){
1969	1969	int n;
1970		- if( z[0]=='<' ){
	1970	+ char c;
	1971	+ if( (c=z[0])=='<' ){
1971	1972	n = markupLength(z);
1972	1973	if( n<=0 ) n = 1;
1973		- }else if( fossil_isspace(z[0]) ){
	1974	+ }else if( fossil_isspace(c) ){
1974	1975	for(n=1; z[n] && fossil_isspace(z[n]); n++){}
	1976	+ }else if( c=='&' ){
	1977	+ n = z[1]=='#' ? 2 : 1;
	1978	+ while( fossil_isalnum(z[n]) ) n++;
	1979	+ if( z[n]==';' ) n++;
1975	1980	}else{
1976		- for(n=1; z[n] && z[n]!='<' && !fossil_isspace(z[n]); n++){}
	1981	+ n = 1;
	1982	+ for(n=1; 1; n++){
	1983	+ if( (c = z[n]) > '<' ) continue;
	1984	+ if( c=='<' \|\| c=='&' \|\| fossil_isspace(c) \|\| c==0 ) break;
	1985	+ }
1977	1986	}
1978	1987	return n;
1979	1988	}
1980	1989
1981	1990	/*
		@@ -2100,16 +2109,22 @@
2100	2109	}
2101	2110
2102	2111	/*
2103	2112	** Remove all HTML markup from the input text. The output written into
2104	2113	** pOut is pure text.
	2114	+**
	2115	+** Put the title on the first line, if there is any <title> markup.
	2116	+** If there is no <title>, then create a blank first line.
2105	2117	*/
2106	2118	void html_to_plaintext(const char zIn, Blob pOut){
2107	2119	int n;
2108	2120	int i, j;
	2121	+ int inTitle = 0; /* True between <title>...</title> */
	2122	+ int seenText = 0; /* True after first non-whitespace seen */
2109	2123	int nNL = 0; /* Number of \n characters at the end of pOut */
2110	2124	int nWS = 0; /* True if pOut ends with whitespace */
	2125	+ while( fossil_isspace(zIn[0]) ) zIn++;
2111	2126	while( zIn[0] ){
2112	2127	n = nextHtmlToken(zIn);
2113	2128	if( zIn[0]=='<' && n>1 ){
2114	2129	int isCloseTag;
2115	2130	int eTag;
		@@ -2130,26 +2145,66 @@
2130	2145	zIn += n;
2131	2146	}
2132	2147	if( zIn[0]=='<' ) zIn += n;
2133	2148	continue;
2134	2149	}
2135		- if( !isCloseTag && (eType & (MUTYPE_BLOCK\|MUTYPE_TABLE))!=0 ){
	2150	+ if( eTag==MARKUP_TITLE ){
	2151	+ inTitle = !isCloseTag;
	2152	+ }
	2153	+ if( !isCloseTag && seenText && (eType & (MUTYPE_BLOCK\|MUTYPE_TABLE))!=0 ){
2136	2154	if( nNL==0 ){
2137	2155	blob_append(pOut, "\n", 1);
2138	2156	nNL++;
2139	2157	}
2140	2158	nWS = 1;
2141	2159	}
2142	2160	}else if( fossil_isspace(zIn[0]) ){
2143		- for(i=nNL=0; i<n; i++) if( zIn[i]=='\n' ) nNL++;
2144		- if( !nWS ){
2145		- blob_append(pOut, nNL ? "\n" : " ", 1);
	2161	+ if( seenText ){
	2162	+ nNL = 0;
	2163	+ if( !inTitle ){ /* '\n' -> ' ' within <title> */
	2164	+ for(i=0; i<n; i++) if( zIn[i]=='\n' ) nNL++;
	2165	+ }
	2166	+ if( !nWS ){
	2167	+ blob_append(pOut, nNL ? "\n" : " ", 1);
	2168	+ nWS = 1;
	2169	+ }
	2170	+ }
	2171	+ }else if( zIn[0]=='&' ){
	2172	+ char c = '?';
	2173	+ if( zIn[1]=='#' ){
	2174	+ int x = atoi(&zIn[1]);
	2175	+ if( x>0 && x<=127 ) c = x;
	2176	+ }else{
	2177	+ static const struct { int n; char c; char *z; } aEntity[] = {
	2178	+ { 5, '&', "&" },
	2179	+ { 4, '<', "<" },
	2180	+ { 4, '>', ">" },
	2181	+ { 6, ' ', " " },
	2182	+ };
	2183	+ int jj;
	2184	+ for(jj=0; jj<ArraySize(aEntity); jj++){
	2185	+ if( aEntity[jj].n==n && strncmp(aEntity[jj].z,zIn,n)==0 ){
	2186	+ c = aEntity[jj].c;
	2187	+ break;
	2188	+ }
	2189	+ }
	2190	+ }
	2191	+ if( fossil_isspace(c) ){
	2192	+ if( nWS==0 && seenText ) blob_append(pOut, &c, 1);
2146	2193	nWS = 1;
	2194	+ nNL = c=='\n';
	2195	+ }else{
	2196	+ if( !seenText && !inTitle ) blob_append(pOut, "\n", 1);
	2197	+ seenText = 1;
	2198	+ nNL = nWS = 0;
	2199	+ blob_append(pOut, &c, 1);
2147	2200	}
2148	2201	}else{
2149		- blob_append(pOut, zIn, n);
	2202	+ if( !seenText && !inTitle ) blob_append(pOut, "\n", 1);
	2203	+ seenText = 1;
2150	2204	nNL = nWS = 0;
	2205	+ blob_append(pOut, zIn, n);
2151	2206	}
2152	2207	zIn += n;
2153	2208	}
2154	2209	if( nNL==0 ) blob_append(pOut, "\n", 1);
2155	2210	}
2156	2211

	--- src/wikiformat.c
	+++ src/wikiformat.c
	@@ -1965,17 +1965,26 @@
1965	** z points to the start of a token. Return the number of
1966	** characters in that token.
1967	*/
1968	static int nextHtmlToken(const char *z){
1969	int n;
1970	if( z[0]=='<' ){

1971	n = markupLength(z);
1972	if( n<=0 ) n = 1;
1973	}else if( fossil_isspace(z[0]) ){
1974	for(n=1; z[n] && fossil_isspace(z[n]); n++){}




1975	}else{
1976	for(n=1; z[n] && z[n]!='<' && !fossil_isspace(z[n]); n++){}




1977	}
1978	return n;
1979	}
1980
1981	/*
	@@ -2100,16 +2109,22 @@
2100	}
2101
2102	/*
2103	** Remove all HTML markup from the input text. The output written into
2104	** pOut is pure text.



2105	*/
2106	void html_to_plaintext(const char zIn, Blob pOut){
2107	int n;
2108	int i, j;


2109	int nNL = 0; /* Number of \n characters at the end of pOut */
2110	int nWS = 0; /* True if pOut ends with whitespace */

2111	while( zIn[0] ){
2112	n = nextHtmlToken(zIn);
2113	if( zIn[0]=='<' && n>1 ){
2114	int isCloseTag;
2115	int eTag;
	@@ -2130,26 +2145,66 @@
2130	zIn += n;
2131	}
2132	if( zIn[0]=='<' ) zIn += n;
2133	continue;
2134	}
2135	if( !isCloseTag && (eType & (MUTYPE_BLOCK\|MUTYPE_TABLE))!=0 ){



2136	if( nNL==0 ){
2137	blob_append(pOut, "\n", 1);
2138	nNL++;
2139	}
2140	nWS = 1;
2141	}
2142	}else if( fossil_isspace(zIn[0]) ){
2143	for(i=nNL=0; i<n; i++) if( zIn[i]=='\n' ) nNL++;
2144	if( !nWS ){
2145	blob_append(pOut, nNL ? "\n" : " ", 1);





























2146	nWS = 1;






2147	}
2148	}else{
2149	blob_append(pOut, zIn, n);

2150	nNL = nWS = 0;

2151	}
2152	zIn += n;
2153	}
2154	if( nNL==0 ) blob_append(pOut, "\n", 1);
2155	}
2156

	--- src/wikiformat.c
	+++ src/wikiformat.c
	@@ -1965,17 +1965,26 @@
1965	** z points to the start of a token. Return the number of
1966	** characters in that token.
1967	*/
1968	static int nextHtmlToken(const char *z){
1969	int n;
1970	char c;
1971	if( (c=z[0])=='<' ){
1972	n = markupLength(z);
1973	if( n<=0 ) n = 1;
1974	}else if( fossil_isspace(c) ){
1975	for(n=1; z[n] && fossil_isspace(z[n]); n++){}
1976	}else if( c=='&' ){
1977	n = z[1]=='#' ? 2 : 1;
1978	while( fossil_isalnum(z[n]) ) n++;
1979	if( z[n]==';' ) n++;
1980	}else{
1981	n = 1;
1982	for(n=1; 1; n++){
1983	if( (c = z[n]) > '<' ) continue;
1984	if( c=='<' \|\| c=='&' \|\| fossil_isspace(c) \|\| c==0 ) break;
1985	}
1986	}
1987	return n;
1988	}
1989
1990	/*
	@@ -2100,16 +2109,22 @@
2109	}
2110
2111	/*
2112	** Remove all HTML markup from the input text. The output written into
2113	** pOut is pure text.
2114	**
2115	** Put the title on the first line, if there is any <title> markup.
2116	** If there is no <title>, then create a blank first line.
2117	*/
2118	void html_to_plaintext(const char zIn, Blob pOut){
2119	int n;
2120	int i, j;
2121	int inTitle = 0; /* True between <title>...</title> */
2122	int seenText = 0; /* True after first non-whitespace seen */
2123	int nNL = 0; /* Number of \n characters at the end of pOut */
2124	int nWS = 0; /* True if pOut ends with whitespace */
2125	while( fossil_isspace(zIn[0]) ) zIn++;
2126	while( zIn[0] ){
2127	n = nextHtmlToken(zIn);
2128	if( zIn[0]=='<' && n>1 ){
2129	int isCloseTag;
2130	int eTag;
	@@ -2130,26 +2145,66 @@
2145	zIn += n;
2146	}
2147	if( zIn[0]=='<' ) zIn += n;
2148	continue;
2149	}
2150	if( eTag==MARKUP_TITLE ){
2151	inTitle = !isCloseTag;
2152	}
2153	if( !isCloseTag && seenText && (eType & (MUTYPE_BLOCK\|MUTYPE_TABLE))!=0 ){
2154	if( nNL==0 ){
2155	blob_append(pOut, "\n", 1);
2156	nNL++;
2157	}
2158	nWS = 1;
2159	}
2160	}else if( fossil_isspace(zIn[0]) ){
2161	if( seenText ){
2162	nNL = 0;
2163	if( !inTitle ){ /* '\n' -> ' ' within <title> */
2164	for(i=0; i<n; i++) if( zIn[i]=='\n' ) nNL++;
2165	}
2166	if( !nWS ){
2167	blob_append(pOut, nNL ? "\n" : " ", 1);
2168	nWS = 1;
2169	}
2170	}
2171	}else if( zIn[0]=='&' ){
2172	char c = '?';
2173	if( zIn[1]=='#' ){
2174	int x = atoi(&zIn[1]);
2175	if( x>0 && x<=127 ) c = x;
2176	}else{
2177	static const struct { int n; char c; char *z; } aEntity[] = {
2178	{ 5, '&', "&" },
2179	{ 4, '<', "<" },
2180	{ 4, '>', ">" },
2181	{ 6, ' ', " " },
2182	};
2183	int jj;
2184	for(jj=0; jj<ArraySize(aEntity); jj++){
2185	if( aEntity[jj].n==n && strncmp(aEntity[jj].z,zIn,n)==0 ){
2186	c = aEntity[jj].c;
2187	break;
2188	}
2189	}
2190	}
2191	if( fossil_isspace(c) ){
2192	if( nWS==0 && seenText ) blob_append(pOut, &c, 1);
2193	nWS = 1;
2194	nNL = c=='\n';
2195	}else{
2196	if( !seenText && !inTitle ) blob_append(pOut, "\n", 1);
2197	seenText = 1;
2198	nNL = nWS = 0;
2199	blob_append(pOut, &c, 1);
2200	}
2201	}else{
2202	if( !seenText && !inTitle ) blob_append(pOut, "\n", 1);
2203	seenText = 1;
2204	nNL = nWS = 0;
2205	blob_append(pOut, zIn, n);
2206	}
2207	zIn += n;
2208	}
2209	if( nNL==0 ) blob_append(pOut, "\n", 1);
2210	}
2211

Fossil SCM

Keyboard Shortcuts