Fossil SCM

Teach /chat search to use the same tokenizer as the main search index, defaulting to porter if the main search index is off, and reindex chat if the tokenizer is changed. The search config should arguably be expanded to provide the option of disabling chat search altogether, but that is beyond today's ambitions. Minor search result layout tweaks but there are still some fixes to do there.

stephan 2024-07-01 17:25 fts5-chat-search

Commit 778efb30f75a1804005562e3815baa70a40deaacef6af739ac97a5bae9e3575d

Parent b5281f4e6127551…

3 files changed +36 -16 +22 -8 +2 -13

~ src/chat.c ~ src/search.c ~ src/style.chat.css

M src/chat.c

+36 -16

		--- src/chat.c
		+++ src/chat.c
		@@ -283,12 +283,42 @@
283	283	@ );
284	284	;
285	285
286	286
287	287	/*
288		-** Make sure the repository data tables used by chat exist. Create them
289		-** if they do not.
	288	+** Create or rebuild the /chat search index. Requires that the
	289	+** repository.chat table exists. If bForce is true, it will drop the
	290	+** chatfts1 table and recreate/reindex it. If bForce is 0, it will
	291	+** only index the chat content if the chatfts1 table does not already
	292	+** exist.
	293	+*/
	294	+void chat_rebuild_index(int bForce){
	295	+ if( bForce!=0 ){
	296	+ db_multi_exec("DROP TABLE IF EXISTS chatfts1");
	297	+ }
	298	+ if( bForce!=0 \|\| !db_table_exists("repository", "chatfts1") ){
	299	+ const int tokType = search_tokenizer_type(0);
	300	+ const char *zTokenizer = search_tokenize_arg_for_type(
	301	+ tokType==FTS5TOK_NONE ? FTS5TOK_PORTER : tokType
	302	+ /* Special case: if fts search is disabled for the main repo
	303	+ ** content, use a default tokenizer here. */
	304	+ );
	305	+ assert( zTokenizer && zTokenizer[0] );
	306	+ db_multi_exec(
	307	+ "CREATE VIRTUAL TABLE repository.chatfts1 USING fts5("
	308	+ " xmsg, content=chat, content_rowid=msgid%s"
	309	+ ");"
	310	+ "INSERT INTO repository.chatfts1(chatfts1) VALUES('rebuild');",
	311	+ zTokenizer/safe-for-%s/
	312	+ );
	313	+ }
	314	+}
	315	+
	316	+/*
	317	+** Make sure the repository data tables used by chat exist. Create
	318	+** them if they do not. Set up TEMP triggers (if needed) to update the
	319	+** chatfts1 table as the chat table is updated.
290	320	*/
291	321	static void chat_create_tables(void){
292	322	if( !db_table_exists("repository","chat") ){
293	323	db_multi_exec(zChatSchema1/works-like:""/);
294	324	}else if( !db_table_has_column("repository","chat","lmtime") ){
		@@ -295,26 +325,16 @@
295	325	if( !db_table_has_column("repository","chat","mdel") ){
296	326	db_multi_exec("ALTER TABLE chat ADD COLUMN mdel INT");
297	327	}
298	328	db_multi_exec("ALTER TABLE chat ADD COLUMN lmtime TEXT");
299	329	}
300		-
301		- if( !db_table_exists("repository", "chatfts1") ){
302		- db_multi_exec(
303		- "CREATE VIRTUAL TABLE repository.chatfts1 USING fts5("
304		- " xmsg, content=chat, content_rowid=msgid, tokenize=porter"
305		- ");"
306		- "INSERT INTO repository.chatfts1(chatfts1) VALUES('rebuild');"
307		- );
308		- }
309		- db_multi_exec(
310		- "DROP TRIGGER IF EXISTS chat_ai;"
311		- "DROP TRIGGER IF EXISTS chat_ad;"
312		- "CREATE TEMP TRIGGER chat_ai AFTER INSERT ON chat BEGIN "
	330	+ chat_rebuild_index(0);
	331	+ db_multi_exec(
	332	+ "CREATE TEMP TRIGGER IF NOT EXISTS chat_ai AFTER INSERT ON chat BEGIN "
313	333	" INSERT INTO chatfts1(rowid, xmsg) VALUES(new.msgid, new.xmsg);"
314	334	"END;"
315		- "CREATE TEMP TRIGGER chat_ad AFTER DELETE ON chat BEGIN "
	335	+ "CREATE TEMP TRIGGER IF NOT EXISTS chat_ad AFTER DELETE ON chat BEGIN "
316	336	" INSERT INTO chatfts1(chatfts1, rowid, xmsg) "
317	337	" VALUES('delete', old.msgid, old.xmsg);"
318	338	"END;"
319	339	);
320	340	}
321	341

	--- src/chat.c
	+++ src/chat.c
	@@ -283,12 +283,42 @@
283	@ );
284	;
285
286
287	/*
288	** Make sure the repository data tables used by chat exist. Create them
289	** if they do not.






























290	*/
291	static void chat_create_tables(void){
292	if( !db_table_exists("repository","chat") ){
293	db_multi_exec(zChatSchema1/works-like:""/);
294	}else if( !db_table_has_column("repository","chat","lmtime") ){
	@@ -295,26 +325,16 @@
295	if( !db_table_has_column("repository","chat","mdel") ){
296	db_multi_exec("ALTER TABLE chat ADD COLUMN mdel INT");
297	}
298	db_multi_exec("ALTER TABLE chat ADD COLUMN lmtime TEXT");
299	}
300
301	if( !db_table_exists("repository", "chatfts1") ){
302	db_multi_exec(
303	"CREATE VIRTUAL TABLE repository.chatfts1 USING fts5("
304	" xmsg, content=chat, content_rowid=msgid, tokenize=porter"
305	");"
306	"INSERT INTO repository.chatfts1(chatfts1) VALUES('rebuild');"
307	);
308	}
309	db_multi_exec(
310	"DROP TRIGGER IF EXISTS chat_ai;"
311	"DROP TRIGGER IF EXISTS chat_ad;"
312	"CREATE TEMP TRIGGER chat_ai AFTER INSERT ON chat BEGIN "
313	" INSERT INTO chatfts1(rowid, xmsg) VALUES(new.msgid, new.xmsg);"
314	"END;"
315	"CREATE TEMP TRIGGER chat_ad AFTER DELETE ON chat BEGIN "
316	" INSERT INTO chatfts1(chatfts1, rowid, xmsg) "
317	" VALUES('delete', old.msgid, old.xmsg);"
318	"END;"
319	);
320	}
321

	--- src/chat.c
	+++ src/chat.c
	@@ -283,12 +283,42 @@
283	@ );
284	;
285
286
287	/*
288	** Create or rebuild the /chat search index. Requires that the
289	** repository.chat table exists. If bForce is true, it will drop the
290	** chatfts1 table and recreate/reindex it. If bForce is 0, it will
291	** only index the chat content if the chatfts1 table does not already
292	** exist.
293	*/
294	void chat_rebuild_index(int bForce){
295	if( bForce!=0 ){
296	db_multi_exec("DROP TABLE IF EXISTS chatfts1");
297	}
298	if( bForce!=0 \|\| !db_table_exists("repository", "chatfts1") ){
299	const int tokType = search_tokenizer_type(0);
300	const char *zTokenizer = search_tokenize_arg_for_type(
301	tokType==FTS5TOK_NONE ? FTS5TOK_PORTER : tokType
302	/* Special case: if fts search is disabled for the main repo
303	** content, use a default tokenizer here. */
304	);
305	assert( zTokenizer && zTokenizer[0] );
306	db_multi_exec(
307	"CREATE VIRTUAL TABLE repository.chatfts1 USING fts5("
308	" xmsg, content=chat, content_rowid=msgid%s"
309	");"
310	"INSERT INTO repository.chatfts1(chatfts1) VALUES('rebuild');",
311	zTokenizer/safe-for-%s/
312	);
313	}
314	}
315
316	/*
317	** Make sure the repository data tables used by chat exist. Create
318	** them if they do not. Set up TEMP triggers (if needed) to update the
319	** chatfts1 table as the chat table is updated.
320	*/
321	static void chat_create_tables(void){
322	if( !db_table_exists("repository","chat") ){
323	db_multi_exec(zChatSchema1/works-like:""/);
324	}else if( !db_table_has_column("repository","chat","lmtime") ){
	@@ -295,26 +325,16 @@
325	if( !db_table_has_column("repository","chat","mdel") ){
326	db_multi_exec("ALTER TABLE chat ADD COLUMN mdel INT");
327	}
328	db_multi_exec("ALTER TABLE chat ADD COLUMN lmtime TEXT");
329	}
330	chat_rebuild_index(0);
331	db_multi_exec(
332	"CREATE TEMP TRIGGER IF NOT EXISTS chat_ai AFTER INSERT ON chat BEGIN "










333	" INSERT INTO chatfts1(rowid, xmsg) VALUES(new.msgid, new.xmsg);"
334	"END;"
335	"CREATE TEMP TRIGGER IF NOT EXISTS chat_ad AFTER DELETE ON chat BEGIN "
336	" INSERT INTO chatfts1(chatfts1, rowid, xmsg) "
337	" VALUES('delete', old.msgid, old.xmsg);"
338	"END;"
339	);
340	}
341

M src/search.c

+22 -8

		--- src/search.c
		+++ src/search.c
		@@ -1637,10 +1637,11 @@
1637	1637	;
1638	1638	static const char zFtsDrop[] =
1639	1639	@ DROP TABLE IF EXISTS repository.ftsidx;
1640	1640	@ DROP VIEW IF EXISTS repository.ftscontent;
1641	1641	@ DROP TABLE IF EXISTS repository.ftsdocs;
	1642	+@ DROP TABLE IF EXISTS repository.chatfts1;
1642	1643	;
1643	1644
1644	1645	#if INTERFACE
1645	1646	/*
1646	1647	** Values for the search-tokenizer config option.
		@@ -1681,10 +1682,25 @@
1681	1682	iFtsTokenizer = is_truth(z) ? FTS5TOK_PORTER : FTS5TOK_NONE;
1682	1683	}
1683	1684	fossil_free(z);
1684	1685	return iFtsTokenizer;
1685	1686	}
	1687	+
	1688	+/*
	1689	+** Returns a string in the form ",tokenize=X", where X is the string
	1690	+** counterpart of the given FTS5TOK_xyz value. Returns "" if tokType
	1691	+** does not correspond to a known FTS5 tokenizer.
	1692	+*/
	1693	+const char * search_tokenize_arg_for_type(int tokType){
	1694	+ switch( tokType ){
	1695	+ case FTS5TOK_PORTER: return ",tokenize=porter";
	1696	+ case FTS5TOK_UNICODE61: return ",tokenize=unicode61";
	1697	+ case FTS5TOK_TRIGRAM: return ",tokenize=trigram";
	1698	+ case FTS5TOK_NONE:
	1699	+ default: return "";
	1700	+ }
	1701	+}
1686	1702
1687	1703	/*
1688	1704	** Returns a string value suitable for use as the search-tokenizer
1689	1705	** setting's value, depending on the value of z. If z is 0 then the
1690	1706	** current search-tokenizer value is used as the basis for formulating
		@@ -1726,18 +1742,13 @@
1726	1742	/*
1727	1743	** Create or drop the tables associated with a full-text index.
1728	1744	*/
1729	1745	static int searchIdxExists = -1;
1730	1746	void search_create_index(void){
1731		- const int useTokenizer = search_tokenizer_type(0);
1732		- const char *zExtra;
1733		- switch(useTokenizer){
1734		- case FTS5TOK_PORTER: zExtra = ",tokenize=porter"; break;
1735		- case FTS5TOK_UNICODE61: zExtra = ",tokenize=unicode61"; break;
1736		- case FTS5TOK_TRIGRAM: zExtra = ",tokenize=trigram"; break;
1737		- default: zExtra = ""; break;
1738		- }
	1747	+ const char *zExtra =
	1748	+ search_tokenize_arg_for_type(search_tokenizer_type(0));
	1749	+ assert( zExtra );
1739	1750	search_sql_setup(g.db);
1740	1751	db_multi_exec(zFtsSchema/works-like:"%s"/, zExtra/safe-for-%s/);
1741	1752	searchIdxExists = 1;
1742	1753	}
1743	1754	void search_drop_index(void){
		@@ -2057,10 +2068,13 @@
2057	2068	fossil_print("rebuilding the search index...");
2058	2069	fflush(stdout);
2059	2070	search_create_index();
2060	2071	search_fill_index();
2061	2072	search_update_index(search_restrict(SRCH_ALL));
	2073	+ if( db_table_exists("repository","chat") ){
	2074	+ chat_rebuild_index(1);
	2075	+ }
2062	2076	fossil_print(" done\n");
2063	2077	}
2064	2078
2065	2079	/*
2066	2080	** COMMAND: fts-config*
2067	2081

	--- src/search.c
	+++ src/search.c
	@@ -1637,10 +1637,11 @@
1637	;
1638	static const char zFtsDrop[] =
1639	@ DROP TABLE IF EXISTS repository.ftsidx;
1640	@ DROP VIEW IF EXISTS repository.ftscontent;
1641	@ DROP TABLE IF EXISTS repository.ftsdocs;

1642	;
1643
1644	#if INTERFACE
1645	/*
1646	** Values for the search-tokenizer config option.
	@@ -1681,10 +1682,25 @@
1681	iFtsTokenizer = is_truth(z) ? FTS5TOK_PORTER : FTS5TOK_NONE;
1682	}
1683	fossil_free(z);
1684	return iFtsTokenizer;
1685	}















1686
1687	/*
1688	** Returns a string value suitable for use as the search-tokenizer
1689	** setting's value, depending on the value of z. If z is 0 then the
1690	** current search-tokenizer value is used as the basis for formulating
	@@ -1726,18 +1742,13 @@
1726	/*
1727	** Create or drop the tables associated with a full-text index.
1728	*/
1729	static int searchIdxExists = -1;
1730	void search_create_index(void){
1731	const int useTokenizer = search_tokenizer_type(0);
1732	const char *zExtra;
1733	switch(useTokenizer){
1734	case FTS5TOK_PORTER: zExtra = ",tokenize=porter"; break;
1735	case FTS5TOK_UNICODE61: zExtra = ",tokenize=unicode61"; break;
1736	case FTS5TOK_TRIGRAM: zExtra = ",tokenize=trigram"; break;
1737	default: zExtra = ""; break;
1738	}
1739	search_sql_setup(g.db);
1740	db_multi_exec(zFtsSchema/works-like:"%s"/, zExtra/safe-for-%s/);
1741	searchIdxExists = 1;
1742	}
1743	void search_drop_index(void){
	@@ -2057,10 +2068,13 @@
2057	fossil_print("rebuilding the search index...");
2058	fflush(stdout);
2059	search_create_index();
2060	search_fill_index();
2061	search_update_index(search_restrict(SRCH_ALL));



2062	fossil_print(" done\n");
2063	}
2064
2065	/*
2066	** COMMAND: fts-config*
2067

	--- src/search.c
	+++ src/search.c
	@@ -1637,10 +1637,11 @@
1637	;
1638	static const char zFtsDrop[] =
1639	@ DROP TABLE IF EXISTS repository.ftsidx;
1640	@ DROP VIEW IF EXISTS repository.ftscontent;
1641	@ DROP TABLE IF EXISTS repository.ftsdocs;
1642	@ DROP TABLE IF EXISTS repository.chatfts1;
1643	;
1644
1645	#if INTERFACE
1646	/*
1647	** Values for the search-tokenizer config option.
	@@ -1681,10 +1682,25 @@
1682	iFtsTokenizer = is_truth(z) ? FTS5TOK_PORTER : FTS5TOK_NONE;
1683	}
1684	fossil_free(z);
1685	return iFtsTokenizer;
1686	}
1687
1688	/*
1689	** Returns a string in the form ",tokenize=X", where X is the string
1690	** counterpart of the given FTS5TOK_xyz value. Returns "" if tokType
1691	** does not correspond to a known FTS5 tokenizer.
1692	*/
1693	const char * search_tokenize_arg_for_type(int tokType){
1694	switch( tokType ){
1695	case FTS5TOK_PORTER: return ",tokenize=porter";
1696	case FTS5TOK_UNICODE61: return ",tokenize=unicode61";
1697	case FTS5TOK_TRIGRAM: return ",tokenize=trigram";
1698	case FTS5TOK_NONE:
1699	default: return "";
1700	}
1701	}
1702
1703	/*
1704	** Returns a string value suitable for use as the search-tokenizer
1705	** setting's value, depending on the value of z. If z is 0 then the
1706	** current search-tokenizer value is used as the basis for formulating
	@@ -1726,18 +1742,13 @@
1742	/*
1743	** Create or drop the tables associated with a full-text index.
1744	*/
1745	static int searchIdxExists = -1;
1746	void search_create_index(void){
1747	const char *zExtra =
1748	search_tokenize_arg_for_type(search_tokenizer_type(0));
1749	assert( zExtra );





1750	search_sql_setup(g.db);
1751	db_multi_exec(zFtsSchema/works-like:"%s"/, zExtra/safe-for-%s/);
1752	searchIdxExists = 1;
1753	}
1754	void search_drop_index(void){
	@@ -2057,10 +2068,13 @@
2068	fossil_print("rebuilding the search index...");
2069	fflush(stdout);
2070	search_create_index();
2071	search_fill_index();
2072	search_update_index(search_restrict(SRCH_ALL));
2073	if( db_table_exists("repository","chat") ){
2074	chat_rebuild_index(1);
2075	}
2076	fossil_print(" done\n");
2077	}
2078
2079	/*
2080	** COMMAND: fts-config*
2081

M src/style.chat.css

+2 -13

		--- src/style.chat.css
		+++ src/style.chat.css
		@@ -613,31 +613,20 @@
613	613	body.chat #chat-user-list .chat-user.selected {
614	614	font-weight: bold;
615	615	text-decoration: underline;
616	616	}
617	617
618		-body.cpage-chat-search .searchForm {
	618	+body.chat .searchForm {
619	619	margin-top: 1em;
620	620	}
621		-body.cpage-chat-search .spacer-widget button {
	621	+body.chat .spacer-widget button {
622	622	margin-left: 1ex;
623	623	margin-right: 1ex;
624	624	display: block;
625		-}
626		-
627		-body.chat .spacer-widget-buttons .up {
628	625	margin-top: 0.5em;
629		- margin-bottom: 1em;
630		-}
631		-body.chat .spacer-widget-buttons .down {
632		- margin-top: 1em;
633	626	margin-bottom: 0.5em;
634	627	}
635		-body.chat .spacer-widget-buttons .all {
636		- margin-bottom: 0.75em;
637		-}
638		-
639	628
640	629	body.chat .anim-rotate-360 {
641	630	animation: rotate-360 750ms linear;
642	631	}
643	632	@keyframes rotate-360 {
644	633

	--- src/style.chat.css
	+++ src/style.chat.css
	@@ -613,31 +613,20 @@
613	body.chat #chat-user-list .chat-user.selected {
614	font-weight: bold;
615	text-decoration: underline;
616	}
617
618	body.cpage-chat-search .searchForm {
619	margin-top: 1em;
620	}
621	body.cpage-chat-search .spacer-widget button {
622	margin-left: 1ex;
623	margin-right: 1ex;
624	display: block;
625	}
626
627	body.chat .spacer-widget-buttons .up {
628	margin-top: 0.5em;
629	margin-bottom: 1em;
630	}
631	body.chat .spacer-widget-buttons .down {
632	margin-top: 1em;
633	margin-bottom: 0.5em;
634	}
635	body.chat .spacer-widget-buttons .all {
636	margin-bottom: 0.75em;
637	}
638
639
640	body.chat .anim-rotate-360 {
641	animation: rotate-360 750ms linear;
642	}
643	@keyframes rotate-360 {
644

	--- src/style.chat.css
	+++ src/style.chat.css
	@@ -613,31 +613,20 @@
613	body.chat #chat-user-list .chat-user.selected {
614	font-weight: bold;
615	text-decoration: underline;
616	}
617
618	body.chat .searchForm {
619	margin-top: 1em;
620	}
621	body.chat .spacer-widget button {
622	margin-left: 1ex;
623	margin-right: 1ex;
624	display: block;



625	margin-top: 0.5em;




626	margin-bottom: 0.5em;
627	}




628
629	body.chat .anim-rotate-360 {
630	animation: rotate-360 750ms linear;
631	}
632	@keyframes rotate-360 {
633

Fossil SCM

Keyboard Shortcuts