Fossil SCM

Update the SQLite used internally to the first 3.6.21 release candidate.

drh 2009-11-23 13:21 UTC trunk

Commit f3d4a2db535a12c11b4ab28eba94421ce1880d7c

Parent a7822bcc001e9fc…

2 files changed +6131 -7812 +3 -3

M src/sqlite3.c

+6131 -7812

		--- src/sqlite3.c
		+++ src/sqlite3.c
		@@ -1,8 +1,8 @@
1	1	/******************************************************************************
2	2	** This file is an amalgamation of many separate C source files from SQLite
3		-** version 3.6.20. By combining all the individual C code files into this
	3	+** version 3.6.21. By combining all the individual C code files into this
4	4	** single large file, the entire code can be compiled as a one translation
5	5	** unit. This allows many compilers to do optimizations that would not be
6	6	** possible if the files were compiled separately. Performance improvements
7	7	** of 5% are more are commonly seen when SQLite is compiled as a single
8	8	** translation unit.
		@@ -91,12 +91,10 @@
91	91	** May you share freely, never taking more than you give.
92	92	**
93	93	*************************************************************************
94	94	**
95	95	** This file defines various limits of what SQLite can process.
96		-**
97		-** @(#) $Id: sqliteLimit.h,v 1.10 2009/01/10 16:15:09 danielk1977 Exp $
98	96	*/
99	97
100	98	/*
101	99	** The maximum length of a TEXT or BLOB in bytes. This also
102	100	** limits the size of a row in a table or index.
		@@ -277,15 +275,11 @@
277	275	** A value of 1 means that a trigger program will not be able to itself
278	276	** fire any triggers. A value of 0 means that no trigger programs at all
279	277	** may be executed.
280	278	*/
281	279	#ifndef SQLITE_MAX_TRIGGER_DEPTH
282		-#if defined(SQLITE_SMALL_STACK)
283		-# define SQLITE_MAX_TRIGGER_DEPTH 10
284		-#else
285	280	# define SQLITE_MAX_TRIGGER_DEPTH 1000
286		-#endif
287	281	#endif
288	282
289	283	/************ End of sqliteLimit.h ***************************************/
290	284	/************ Continuing where we left off in sqliteInt.h ****************/
291	285
		@@ -647,13 +641,13 @@
647	641	** [sqlite3_libversion_number()], [sqlite3_sourceid()],
648	642	** [sqlite_version()] and [sqlite_source_id()].
649	643	**
650	644	** Requirements: [H10011] [H10014]
651	645	*/
652		-#define SQLITE_VERSION "3.6.20"
653		-#define SQLITE_VERSION_NUMBER 3006020
654		-#define SQLITE_SOURCE_ID "2009-11-04 23:02:52 f1c09acaca3e205acf5b077c9b2d0fe35f035c1e"
	646	+#define SQLITE_VERSION "3.6.21"
	647	+#define SQLITE_VERSION_NUMBER 3006021
	648	+#define SQLITE_SOURCE_ID "2009-11-23 13:17:27 39214aee6553db76309851e7aa74fcc02d4f59b7"
655	649
656	650	/*
657	651	** CAPI3REF: Run-Time Library Version Numbers {H10020} <S60100>
658	652	** KEYWORDS: sqlite3_version
659	653	**
		@@ -6316,12 +6310,10 @@
6316	6310	** May you share freely, never taking more than you give.
6317	6311	**
6318	6312	*************************************************************************
6319	6313	** This is the header file for the generic hash-table implemenation
6320	6314	** used in SQLite.
6321		-**
6322		-** $Id: hash.h,v 1.15 2009/05/02 13:29:38 drh Exp $
6323	6315	*/
6324	6316	#ifndef _SQLITE_HASH_H_
6325	6317	#define _SQLITE_HASH_H_
6326	6318
6327	6319	/* Forward declarations of structures. */
		@@ -6920,12 +6912,10 @@
6920	6912	**
6921	6913	*************************************************************************
6922	6914	** This header file defines the interface that the sqlite B-Tree file
6923	6915	** subsystem. See comments in the source code for a detailed description
6924	6916	** of what each interface routine does.
6925		-**
6926		-** @(#) $Id: btree.h,v 1.120 2009/07/22 00:35:24 drh Exp $
6927	6917	*/
6928	6918	#ifndef _BTREE_H_
6929	6919	#define _BTREE_H_
6930	6920
6931	6921	/* TODO: This definition is just included so other modules compile. It
		@@ -7058,10 +7048,11 @@
7058	7048	int wrFlag, /* 1 for writing. 0 for read-only */
7059	7049	struct KeyInfo, / First argument to compare function */
7060	7050	BtCursor pCursor / Space to write cursor structure */
7061	7051	);
7062	7052	SQLITE_PRIVATE int sqlite3BtreeCursorSize(void);
	7053	+SQLITE_PRIVATE void sqlite3BtreeCursorZero(BtCursor*);
7063	7054
7064	7055	SQLITE_PRIVATE int sqlite3BtreeCloseCursor(BtCursor*);
7065	7056	SQLITE_PRIVATE int sqlite3BtreeMovetoUnpacked(
7066	7057	BtCursor*,
7067	7058	UnpackedRecord *pUnKey,
		@@ -7169,12 +7160,10 @@
7169	7160	** Header file for the Virtual DataBase Engine (VDBE)
7170	7161	**
7171	7162	** This header defines the interface to the virtual database engine
7172	7163	** or VDBE. The VDBE implements an abstract machine that runs a
7173	7164	** simple program to access and modify the underlying database.
7174		-**
7175		-** $Id: vdbe.h,v 1.142 2009/07/24 17:58:53 danielk1977 Exp $
7176	7165	*/
7177	7166	#ifndef _SQLITE_VDBE_H_
7178	7167	#define _SQLITE_VDBE_H_
7179	7168
7180	7169	/*
		@@ -7198,11 +7187,11 @@
7198	7187	** as an instance of the following structure:
7199	7188	*/
7200	7189	struct VdbeOp {
7201	7190	u8 opcode; /* What operation to perform */
7202	7191	signed char p4type; /* One of the P4_xxx constants for p4 */
7203		- u8 opflags; /* Not currently used */
	7192	+ u8 opflags; /* Mask of the OPFLG_* flags in opcodes.h */
7204	7193	u8 p5; /* Fifth parameter is an unsigned character */
7205	7194	int p1; /* First operand */
7206	7195	int p2; /* Second parameter (often the jump destination) */
7207	7196	int p3; /* The third parameter */
7208	7197	union { /* fourth parameter */
		@@ -7476,26 +7465,27 @@
7476	7465	#define OPFLG_JUMP 0x0001 /* jump: P2 holds jmp target */
7477	7466	#define OPFLG_OUT2_PRERELEASE 0x0002 /* out2-prerelease: */
7478	7467	#define OPFLG_IN1 0x0004 /* in1: P1 is an input */
7479	7468	#define OPFLG_IN2 0x0008 /* in2: P2 is an input */
7480	7469	#define OPFLG_IN3 0x0010 /* in3: P3 is an input */
7481		-#define OPFLG_OUT3 0x0020 /* out3: P3 is an output */
	7470	+#define OPFLG_OUT2 0x0020 /* out2: P2 is an output */
	7471	+#define OPFLG_OUT3 0x0040 /* out3: P3 is an output */
7482	7472	#define OPFLG_INITIALIZER {\
7483		-/* 0 */ 0x00, 0x01, 0x01, 0x04, 0x04, 0x10, 0x00, 0x02,\
7484		-/* 8 */ 0x02, 0x02, 0x02, 0x02, 0x00, 0x00, 0x04, 0x04,\
7485		-/* 16 */ 0x00, 0x00, 0x00, 0x04, 0x04, 0x05, 0x04, 0x00,\
	7473	+/* 0 */ 0x00, 0x01, 0x05, 0x04, 0x04, 0x10, 0x00, 0x02,\
	7474	+/* 8 */ 0x02, 0x02, 0x02, 0x02, 0x00, 0x00, 0x24, 0x24,\
	7475	+/* 16 */ 0x00, 0x00, 0x00, 0x24, 0x04, 0x05, 0x04, 0x00,\
7486	7476	/* 24 */ 0x00, 0x01, 0x05, 0x05, 0x00, 0x00, 0x00, 0x02,\
7487	7477	/* 32 */ 0x00, 0x00, 0x00, 0x02, 0x10, 0x00, 0x00, 0x00,\
7488	7478	/* 40 */ 0x00, 0x00, 0x00, 0x11, 0x11, 0x11, 0x11, 0x08,\
7489	7479	/* 48 */ 0x11, 0x11, 0x11, 0x11, 0x02, 0x02, 0x00, 0x00,\
7490	7480	/* 56 */ 0x00, 0x00, 0x00, 0x00, 0x02, 0x00, 0x01, 0x01,\
7491		-/* 64 */ 0x01, 0x01, 0x01, 0x08, 0x2c, 0x2c, 0x00, 0x02,\
7492		-/* 72 */ 0x11, 0x05, 0x05, 0x15, 0x15, 0x15, 0x15, 0x15,\
7493		-/* 80 */ 0x15, 0x11, 0x2c, 0x2c, 0x2c, 0x2c, 0x2c, 0x2c,\
7494		-/* 88 */ 0x2c, 0x2c, 0x2c, 0x2c, 0x02, 0x04, 0x02, 0x00,\
	7481	+/* 64 */ 0x01, 0x01, 0x01, 0x08, 0x4c, 0x4c, 0x00, 0x02,\
	7482	+/* 72 */ 0x01, 0x05, 0x05, 0x15, 0x15, 0x15, 0x15, 0x15,\
	7483	+/* 80 */ 0x15, 0x01, 0x4c, 0x4c, 0x4c, 0x4c, 0x4c, 0x4c,\
	7484	+/* 88 */ 0x4c, 0x4c, 0x4c, 0x4c, 0x02, 0x24, 0x02, 0x00,\
7495	7485	/* 96 */ 0x02, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,\
7496		-/* 104 */ 0x08, 0x21, 0x15, 0x01, 0x02, 0x00, 0x01, 0x08,\
	7486	+/* 104 */ 0x0c, 0x45, 0x15, 0x01, 0x02, 0x00, 0x01, 0x08,\
7497	7487	/* 112 */ 0x05, 0x05, 0x05, 0x00, 0x00, 0x00, 0x01, 0x00,\
7498	7488	/* 120 */ 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x01,\
7499	7489	/* 128 */ 0x00, 0x00, 0x02, 0x02, 0x00, 0x00, 0x00, 0x00,\
7500	7490	/* 136 */ 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, 0x04, 0x04,\
7501	7491	/* 144 */ 0x04, 0x04,}
		@@ -7511,10 +7501,11 @@
7511	7501	SQLITE_PRIVATE int sqlite3VdbeAddOp0(Vdbe*,int);
7512	7502	SQLITE_PRIVATE int sqlite3VdbeAddOp1(Vdbe*,int,int);
7513	7503	SQLITE_PRIVATE int sqlite3VdbeAddOp2(Vdbe*,int,int,int);
7514	7504	SQLITE_PRIVATE int sqlite3VdbeAddOp3(Vdbe*,int,int,int,int);
7515	7505	SQLITE_PRIVATE int sqlite3VdbeAddOp4(Vdbe,int,int,int,int,const char zP4,int);
	7506	+SQLITE_PRIVATE int sqlite3VdbeAddOp4Int(Vdbe*,int,int,int,int,int);
7516	7507	SQLITE_PRIVATE int sqlite3VdbeAddOpList(Vdbe, int nOp, VdbeOpList const aOp);
7517	7508	SQLITE_PRIVATE void sqlite3VdbeChangeP1(Vdbe*, int addr, int P1);
7518	7509	SQLITE_PRIVATE void sqlite3VdbeChangeP2(Vdbe*, int addr, int P2);
7519	7510	SQLITE_PRIVATE void sqlite3VdbeChangeP3(Vdbe*, int addr, int P3);
7520	7511	SQLITE_PRIVATE void sqlite3VdbeChangeP5(Vdbe*, u8 P5);
		@@ -7579,12 +7570,10 @@
7579	7570	**
7580	7571	*************************************************************************
7581	7572	** This header file defines the interface that the sqlite page cache
7582	7573	** subsystem. The page cache subsystem reads and writes a file a page
7583	7574	** at a time and provides a journal for rollback.
7584		-**
7585		-** @(#) $Id: pager.h,v 1.104 2009/07/24 19:01:19 drh Exp $
7586	7575	*/
7587	7576
7588	7577	#ifndef _PAGER_H_
7589	7578	#define _PAGER_H_
7590	7579
		@@ -7750,12 +7739,10 @@
7750	7739	** May you share freely, never taking more than you give.
7751	7740	**
7752	7741	*************************************************************************
7753	7742	** This header file defines the interface that the sqlite page cache
7754	7743	** subsystem.
7755		-**
7756		-** @(#) $Id: pcache.h,v 1.20 2009/07/25 11:46:49 danielk1977 Exp $
7757	7744	*/
7758	7745
7759	7746	#ifndef _PCACHE_H_
7760	7747
7761	7748	typedef struct PgHdr PgHdr;
		@@ -7918,12 +7905,10 @@
7918	7905	** "os.c") attempt to abstract the underlying operating system so that
7919	7906	** the SQLite library will work on both POSIX and windows systems.
7920	7907	**
7921	7908	** This header file is #include-ed by sqliteInt.h and thus ends up
7922	7909	** being included by every source file.
7923		-**
7924		-** $Id: os.h,v 1.108 2009/02/05 16:31:46 drh Exp $
7925	7910	*/
7926	7911	#ifndef _SQLITE_OS_H_
7927	7912	#define _SQLITE_OS_H_
7928	7913
7929	7914	/*
		@@ -8198,12 +8183,10 @@
8198	8183	** better organized.
8199	8184	**
8200	8185	** NOTE: source files should not #include this header file directly.
8201	8186	** Source files should #include the sqliteInt.h file and let that file
8202	8187	** include this one indirectly.
8203		-**
8204		-** $Id: mutex.h,v 1.9 2008/10/07 15:25:48 drh Exp $
8205	8188	*/
8206	8189
8207	8190
8208	8191	/*
8209	8192	** Figure out what version of the code to use. The choices are
		@@ -8576,10 +8559,11 @@
8576	8559	#define SQLITE_FUNC_CASE 0x02 /* Case-sensitive LIKE-type function */
8577	8560	#define SQLITE_FUNC_EPHEM 0x04 /* Ephemeral. Delete with VDBE */
8578	8561	#define SQLITE_FUNC_NEEDCOLL 0x08 /* sqlite3GetFuncCollSeq() might be called */
8579	8562	#define SQLITE_FUNC_PRIVATE 0x10 /* Allowed for internal use only */
8580	8563	#define SQLITE_FUNC_COUNT 0x20 /* Built-in count() aggregate /
	8564	+#define SQLITE_FUNC_COALESCE 0x40 /* Built-in coalesce() or ifnull() function */
8581	8565
8582	8566	/*
8583	8567	** The following three macros, FUNCTION(), LIKEFUNC() and AGGREGATE() are
8584	8568	** used to create the initializers for the FuncDef structures.
8585	8569	**
		@@ -10290,10 +10274,13 @@
10290	10274	SQLITE_PRIVATE void sqlite3CloseSavepoints(sqlite3 *);
10291	10275	SQLITE_PRIVATE int sqlite3ExprIsConstant(Expr*);
10292	10276	SQLITE_PRIVATE int sqlite3ExprIsConstantNotJoin(Expr*);
10293	10277	SQLITE_PRIVATE int sqlite3ExprIsConstantOrFunction(Expr*);
10294	10278	SQLITE_PRIVATE int sqlite3ExprIsInteger(Expr, int);
	10279	+SQLITE_PRIVATE int sqlite3ExprCanBeNull(const Expr*);
	10280	+SQLITE_PRIVATE void sqlite3ExprCodeIsNullJump(Vdbe, const Expr, int, int);
	10281	+SQLITE_PRIVATE int sqlite3ExprNeedsNoAffinityChange(const Expr*, char);
10295	10282	SQLITE_PRIVATE int sqlite3IsRowid(const char*);
10296	10283	SQLITE_PRIVATE void sqlite3GenerateRowDelete(Parse, Table, int, int, int, Trigger *, int);
10297	10284	SQLITE_PRIVATE void sqlite3GenerateRowIndexDelete(Parse, Table, int, int*);
10298	10285	SQLITE_PRIVATE int sqlite3GenerateIndexKey(Parse, Index, int, int, int);
10299	10286	SQLITE_PRIVATE void sqlite3GenerateConstraintChecks(Parse,Table,int,int,
		@@ -10461,10 +10448,11 @@
10461	10448	SQLITE_PRIVATE char sqlite3Utf8to16(sqlite3 , u8, char , int, int );
10462	10449	#endif
10463	10450	SQLITE_PRIVATE int sqlite3ValueFromExpr(sqlite3 , Expr , u8, u8, sqlite3_value **);
10464	10451	SQLITE_PRIVATE void sqlite3ValueApplyAffinity(sqlite3_value *, u8, u8);
10465	10452	#ifndef SQLITE_AMALGAMATION
	10453	+SQLITE_PRIVATE const unsigned char sqlite3OpcodeProperty[];
10466	10454	SQLITE_PRIVATE const unsigned char sqlite3UpperToLower[];
10467	10455	SQLITE_PRIVATE const unsigned char sqlite3CtypeMap[];
10468	10456	SQLITE_PRIVATE SQLITE_WSD struct Sqlite3Config sqlite3Config;
10469	10457	SQLITE_PRIVATE SQLITE_WSD FuncDefHash sqlite3GlobalFunctions;
10470	10458	SQLITE_PRIVATE int sqlite3PendingByte;
		@@ -10702,11 +10690,10 @@
10702	10690	*************************************************************************
10703	10691	**
10704	10692	** This file contains definitions of global variables and contants.
10705	10693	*/
10706	10694
10707		-
10708	10695	/* An array to map all upper-case characters into their corresponding
10709	10696	** lower-case character.
10710	10697	**
10711	10698	** SQLite only considers US-ASCII (or EBCDIC) characters. We do not
10712	10699	** handle case conversions for the UTF character set since the tables
		@@ -10758,10 +10745,11 @@
10758	10745	** isalpha() 0x02
10759	10746	** isdigit() 0x04
10760	10747	** isalnum() 0x06
10761	10748	** isxdigit() 0x08
10762	10749	** toupper() 0x20
	10750	+** SQLite identifier character 0x40
10763	10751	**
10764	10752	** Bit 0x20 is set if the mapped character requires translation to upper
10765	10753	** case. i.e. if the character is a lower-case ASCII character.
10766	10754	** If x is a lower-case ASCII character, then its upper-case equivalent
10767	10755	** is (x - 0x20). Therefore toupper() can be implemented as:
		@@ -10768,10 +10756,15 @@
10768	10756	**
10769	10757	** (x & ~(map[x]&0x20))
10770	10758	**
10771	10759	** Standard function tolower() is implemented using the sqlite3UpperToLower[]
10772	10760	** array. tolower() is used more often than toupper() by SQLite.
	10761	+**
	10762	+** Bit 0x40 is set if the character non-alphanumeric and can be used in an
	10763	+** SQLite identifier. Identifiers are alphanumerics, "_", "$", and any
	10764	+** non-ASCII UTF character. Hence the test for whether or not a character is
	10765	+** part of an identifier is 0x46.
10773	10766	**
10774	10767	** SQLite's versions are identical to the standard versions assuming a
10775	10768	** locale of "C". They are implemented as macros in sqliteInt.h.
10776	10769	*/
10777	10770	#ifdef SQLITE_ASCII
		@@ -10778,41 +10771,41 @@
10778	10771	SQLITE_PRIVATE const unsigned char sqlite3CtypeMap[256] = {
10779	10772	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 00..07 ........ */
10780	10773	0x00, 0x01, 0x01, 0x01, 0x01, 0x01, 0x00, 0x00, /* 08..0f ........ */
10781	10774	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 10..17 ........ */
10782	10775	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 18..1f ........ */
10783		- 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 20..27 !"#$%&' */
	10776	+ 0x01, 0x00, 0x00, 0x00, 0x40, 0x00, 0x00, 0x00, /* 20..27 !"#$%&' */
10784	10777	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 28..2f ()+,-./ /
10785	10778	0x0c, 0x0c, 0x0c, 0x0c, 0x0c, 0x0c, 0x0c, 0x0c, /* 30..37 01234567 */
10786	10779	0x0c, 0x0c, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 38..3f 89:;<=>? */
10787	10780
10788	10781	0x00, 0x0a, 0x0a, 0x0a, 0x0a, 0x0a, 0x0a, 0x02, /* 40..47 @ABCDEFG */
10789	10782	0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, /* 48..4f HIJKLMNO */
10790	10783	0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, /* 50..57 PQRSTUVW */
10791		- 0x02, 0x02, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, /* 58..5f XYZ[\]^_ */
	10784	+ 0x02, 0x02, 0x02, 0x00, 0x00, 0x00, 0x00, 0x40, /* 58..5f XYZ[\]^_ */
10792	10785	0x00, 0x2a, 0x2a, 0x2a, 0x2a, 0x2a, 0x2a, 0x22, /* 60..67 `abcdefg */
10793	10786	0x22, 0x22, 0x22, 0x22, 0x22, 0x22, 0x22, 0x22, /* 68..6f hijklmno */
10794	10787	0x22, 0x22, 0x22, 0x22, 0x22, 0x22, 0x22, 0x22, /* 70..77 pqrstuvw */
10795	10788	0x22, 0x22, 0x22, 0x00, 0x00, 0x00, 0x00, 0x00, /* 78..7f xyz{\|}~. */
10796	10789
10797		- 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 80..87 ........ */
10798		- 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 88..8f ........ */
10799		- 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 90..97 ........ */
10800		- 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 98..9f ........ */
10801		- 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* a0..a7 ........ */
10802		- 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* a8..af ........ */
10803		- 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* b0..b7 ........ */
10804		- 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* b8..bf ........ */
10805		-
10806		- 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* c0..c7 ........ */
10807		- 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* c8..cf ........ */
10808		- 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* d0..d7 ........ */
10809		- 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* d8..df ........ */
10810		- 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* e0..e7 ........ */
10811		- 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* e8..ef ........ */
10812		- 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* f0..f7 ........ */
10813		- 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 /* f8..ff ........ */
	10790	+ 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, /* 80..87 ........ */
	10791	+ 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, /* 88..8f ........ */
	10792	+ 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, /* 90..97 ........ */
	10793	+ 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, /* 98..9f ........ */
	10794	+ 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, /* a0..a7 ........ */
	10795	+ 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, /* a8..af ........ */
	10796	+ 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, /* b0..b7 ........ */
	10797	+ 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, /* b8..bf ........ */
	10798	+
	10799	+ 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, /* c0..c7 ........ */
	10800	+ 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, /* c8..cf ........ */
	10801	+ 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, /* d0..d7 ........ */
	10802	+ 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, /* d8..df ........ */
	10803	+ 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, /* e0..e7 ........ */
	10804	+ 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, /* e8..ef ........ */
	10805	+ 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, /* f0..f7 ........ */
	10806	+ 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40 /* f8..ff ........ */
10814	10807	};
10815	10808	#endif
10816	10809
10817	10810
10818	10811
		@@ -10877,10 +10870,18 @@
10877	10870	** Changing the pending byte during operating results in undefined
10878	10871	** and dileterious behavior.
10879	10872	*/
10880	10873	SQLITE_PRIVATE int sqlite3PendingByte = 0x40000000;
10881	10874
	10875	+/*
	10876	+** Properties of opcodes. The OPFLG_INITIALIZER macro is
	10877	+** created by mkopcodeh.awk during compilation. Data is obtained
	10878	+** from the comments following the "case OP_xxxx:" statements in
	10879	+** the vdbe.c file.
	10880	+*/
	10881	+SQLITE_PRIVATE const unsigned char sqlite3OpcodeProperty[] = OPFLG_INITIALIZER;
	10882	+
10882	10883	/************ End of global.c ********************************************/
10883	10884	/************ Begin file status.c ****************************************/
10884	10885	/*
10885	10886	** 2008 June 18
10886	10887	**
		@@ -10893,12 +10894,10 @@
10893	10894	**
10894	10895	*************************************************************************
10895	10896	**
10896	10897	** This module implements the sqlite3_status() interface and related
10897	10898	** functionality.
10898		-**
10899		-** $Id: status.c,v 1.9 2008/09/02 00:52:52 drh Exp $
10900	10899	*/
10901	10900
10902	10901	/*
10903	10902	** Variables in which to record status information.
10904	10903	*/
		@@ -11021,12 +11020,10 @@
11021	11020	**
11022	11021	** There is only one exported symbol in this file - the function
11023	11022	** sqlite3RegisterDateTimeFunctions() found at the bottom of the file.
11024	11023	** All other code has file scope.
11025	11024	**
11026		-** $Id: date.c,v 1.107 2009/05/03 20:23:53 drh Exp $
11027		-**
11028	11025	** SQLite processes all times and dates as Julian Day numbers. The
11029	11026	** dates and times are stored as the number of days since noon
11030	11027	** in Greenwich on November 24, 4714 B.C. according to the Gregorian
11031	11028	** calendar system.
11032	11029	**
		@@ -12125,12 +12122,10 @@
12125	12122	**
12126	12123	******************************************************************************
12127	12124	**
12128	12125	** This file contains OS interface code that is common to all
12129	12126	** architectures.
12130		-**
12131		-** $Id: os.c,v 1.127 2009/07/27 11:41:21 danielk1977 Exp $
12132	12127	*/
12133	12128	#define _SQLITE_OS_C_ 1
12134	12129	#undef _SQLITE_OS_C_
12135	12130
12136	12131	/*
		@@ -12252,10 +12247,11 @@
12252	12247	sqlite3_vfs *pVfs,
12253	12248	const char *zPath,
12254	12249	int nPathOut,
12255	12250	char *zPathOut
12256	12251	){
	12252	+ zPathOut[0] = 0;
12257	12253	return pVfs->xFullPathname(pVfs, zPath, nPathOut, zPathOut);
12258	12254	}
12259	12255	#ifndef SQLITE_OMIT_LOAD_EXTENSION
12260	12256	SQLITE_PRIVATE void sqlite3OsDlOpen(sqlite3_vfs pVfs, const char *zPath){
12261	12257	return pVfs->xDlOpen(pVfs, zPath);
		@@ -12423,14 +12419,10 @@
12423	12419	** May you find forgiveness for yourself and forgive others.
12424	12420	** May you share freely, never taking more than you give.
12425	12421	**
12426	12422	*************************************************************************
12427	12423	**
12428		-** $Id: fault.c,v 1.11 2008/09/02 00:52:52 drh Exp $
12429		-*/
12430		-
12431		-/*
12432	12424	** This file contains code to support the concept of "benign"
12433	12425	** malloc failures (when the xMalloc() or xRealloc() method of the
12434	12426	** sqlite3_mem_methods structure fails to allocate a block of memory
12435	12427	** and returns 0).
12436	12428	**
		@@ -12521,12 +12513,10 @@
12521	12513	** This file contains a no-op memory allocation drivers for use when
12522	12514	** SQLITE_ZERO_MALLOC is defined. The allocation drivers implemented
12523	12515	** here always fail. SQLite will not operate with these drivers. These
12524	12516	** are merely placeholders. Real drivers must be substituted using
12525	12517	** sqlite3_config() before SQLite will operate.
12526		-**
12527		-** $Id: mem0.c,v 1.1 2008/10/28 18:58:20 drh Exp $
12528	12518	*/
12529	12519
12530	12520	/*
12531	12521	** This version of the memory allocator is the default. It is
12532	12522	** used when no other memory allocator is specified using compile-time
		@@ -12585,12 +12575,10 @@
12585	12575	** SQLite will use the standard C-library malloc/realloc/free interface
12586	12576	** to obtain the memory it needs.
12587	12577	**
12588	12578	** This file contains implementations of the low-level memory allocation
12589	12579	** routines specified in the sqlite3_mem_methods object.
12590		-**
12591		-** $Id: mem1.c,v 1.30 2009/03/23 04:33:33 danielk1977 Exp $
12592	12580	*/
12593	12581
12594	12582	/*
12595	12583	** This version of the memory allocator is the default. It is
12596	12584	** used when no other memory allocator is specified using compile-time
		@@ -12734,12 +12722,10 @@
12734	12722	** information to each allocation in order to help detect and fix memory
12735	12723	** leaks and memory usage errors.
12736	12724	**
12737	12725	** This file contains implementations of the low-level memory allocation
12738	12726	** routines specified in the sqlite3_mem_methods object.
12739		-**
12740		-** $Id: mem2.c,v 1.45 2009/03/23 04:33:33 danielk1977 Exp $
12741	12727	*/
12742	12728
12743	12729	/*
12744	12730	** This version of the memory allocator is used only if the
12745	12731	** SQLITE_MEMDEBUG macro is defined
		@@ -13183,12 +13169,10 @@
13183	13169	** the amount of memory available to SQLite is fixed and cannot
13184	13170	** be changed.
13185	13171	**
13186	13172	** This version of the memory allocation subsystem is included
13187	13173	** in the build only if SQLITE_ENABLE_MEMSYS3 is defined.
13188		-**
13189		-** $Id: mem3.c,v 1.25 2008/11/19 16:52:44 danielk1977 Exp $
13190	13174	*/
13191	13175
13192	13176	/*
13193	13177	** This version of the memory allocator is only built into the library
13194	13178	** SQLITE_ENABLE_MEMSYS3 is defined. Defining this symbol does not
		@@ -14442,13 +14426,10 @@
14442	14426	**
14443	14427	*************************************************************************
14444	14428	** This file contains the C functions that implement mutexes.
14445	14429	**
14446	14430	** This file contains code that is common across all mutex implementations.
14447		-
14448		-**
14449		-** $Id: mutex.c,v 1.31 2009/07/16 18:21:18 drh Exp $
14450	14431	*/
14451	14432
14452	14433	#if defined(SQLITE_DEBUG) && !defined(SQLITE_MUTEX_OMIT)
14453	14434	/*
14454	14435	** For debugging purposes, record when the mutex subsystem is initialized
		@@ -14609,12 +14590,10 @@
14609	14590	** interface.
14610	14591	**
14611	14592	** If compiled with SQLITE_DEBUG, then additional logic is inserted
14612	14593	** that does error checking on mutexes to make sure they are being
14613	14594	** called correctly.
14614		-**
14615		-** $Id: mutex_noop.c,v 1.3 2008/12/05 17:17:08 drh Exp $
14616	14595	*/
14617	14596
14618	14597
14619	14598	#if defined(SQLITE_MUTEX_NOOP) && !defined(SQLITE_DEBUG)
14620	14599	/*
		@@ -14783,12 +14762,10 @@
14783	14762	** May you find forgiveness for yourself and forgive others.
14784	14763	** May you share freely, never taking more than you give.
14785	14764	**
14786	14765	*************************************************************************
14787	14766	** This file contains the C functions that implement mutexes for OS/2
14788		-**
14789		-** $Id: mutex_os2.c,v 1.11 2008/11/22 19:50:54 pweilbacher Exp $
14790	14767	*/
14791	14768
14792	14769	/*
14793	14770	** The code in this file is only used if SQLITE_MUTEX_OS2 is defined.
14794	14771	** See the mutex.h file for details.
		@@ -15058,12 +15035,10 @@
15058	15035	** May you find forgiveness for yourself and forgive others.
15059	15036	** May you share freely, never taking more than you give.
15060	15037	**
15061	15038	*************************************************************************
15062	15039	** This file contains the C functions that implement mutexes for pthreads
15063		-**
15064		-** $Id: mutex_unix.c,v 1.16 2008/12/08 18:19:18 drh Exp $
15065	15040	*/
15066	15041
15067	15042	/*
15068	15043	** The code in this file is only used if we are compiling threadsafe
15069	15044	** under unix with pthreads.
		@@ -15389,12 +15364,10 @@
15389	15364	** May you find forgiveness for yourself and forgive others.
15390	15365	** May you share freely, never taking more than you give.
15391	15366	**
15392	15367	*************************************************************************
15393	15368	** This file contains the C functions that implement mutexes for win32
15394		-**
15395		-** $Id: mutex_w32.c,v 1.18 2009/08/10 03:23:21 shane Exp $
15396	15369	*/
15397	15370
15398	15371	/*
15399	15372	** The code in this file is only used if we are compiling multithreaded
15400	15373	** on a win32 system.
		@@ -15676,12 +15649,10 @@
15676	15649	** May you share freely, never taking more than you give.
15677	15650	**
15678	15651	*************************************************************************
15679	15652	**
15680	15653	** Memory allocation functions used throughout sqlite.
15681		-**
15682		-** $Id: malloc.c,v 1.66 2009/07/17 11:44:07 drh Exp $
15683	15654	*/
15684	15655
15685	15656	/*
15686	15657	** This routine runs when the memory allocator sees that the
15687	15658	** total memory allocation is about to exceed the soft heap
		@@ -16383,12 +16354,10 @@
16383	16354	** the public domain. The original comments are included here for
16384	16355	** completeness. They are very out-of-date but might be useful as
16385	16356	** an historical reference. Most of the "enhancements" have been backed
16386	16357	** out so that the functionality is now the same as standard printf().
16387	16358	**
16388		-** $Id: printf.c,v 1.104 2009/06/03 01:24:54 drh Exp $
16389		-**
16390	16359	**************************************************************************
16391	16360	**
16392	16361	** The following modules is an enhanced replacement for the "printf" subroutines
16393	16362	** found in the standard C library. The following enhancements are
16394	16363	** supported:
		@@ -17352,12 +17321,10 @@
17352	17321	** This file contains code to implement a pseudo-random number
17353	17322	** generator (PRNG) for SQLite.
17354	17323	**
17355	17324	** Random numbers are used by some of the database backends in order
17356	17325	** to generate random integer keys for tables or random filenames.
17357		-**
17358		-** $Id: random.c,v 1.29 2008/12/10 19:26:24 drh Exp $
17359	17326	*/
17360	17327
17361	17328
17362	17329	/* All threads share a single random number generator.
17363	17330	** This structure is the current state of the generator.
		@@ -17499,12 +17466,10 @@
17499	17466	**
17500	17467	*************************************************************************
17501	17468	** This file contains routines used to translate between UTF-8,
17502	17469	** UTF-16, UTF-16BE, and UTF-16LE.
17503	17470	**
17504		-** $Id: utf.c,v 1.73 2009/04/01 18:40:32 drh Exp $
17505		-**
17506	17471	** Notes on UTF-8:
17507	17472	**
17508	17473	** Byte-0 Byte-1 Byte-2 Byte-3 Value
17509	17474	** 0xxxxxxx 00000000 00000000 0xxxxxxx
17510	17475	** 110yyyyy 10xxxxxx 00000000 00000yyy yyxxxxxx
		@@ -17540,12 +17505,10 @@
17540	17505	** This is the header file for information that is private to the
17541	17506	** VDBE. This information used to all be at the top of the single
17542	17507	** source code file "vdbe.c". When that file became too big (over
17543	17508	** 6000 lines long) it was split up into several smaller files and
17544	17509	** this header information was factored out.
17545		-**
17546		-** $Id: vdbeInt.h,v 1.174 2009/06/23 14:15:04 drh Exp $
17547	17510	*/
17548	17511	#ifndef _VDBEINT_H_
17549	17512	#define _VDBEINT_H_
17550	17513
17551	17514	/*
		@@ -17907,11 +17870,10 @@
17907	17870	SQLITE_PRIVATE int sqlite3VdbeMemFromBtree(BtCursor,int,int,int,Mem);
17908	17871	SQLITE_PRIVATE void sqlite3VdbeMemRelease(Mem *p);
17909	17872	SQLITE_PRIVATE void sqlite3VdbeMemReleaseExternal(Mem *p);
17910	17873	SQLITE_PRIVATE int sqlite3VdbeMemFinalize(Mem, FuncDef);
17911	17874	SQLITE_PRIVATE const char *sqlite3OpcodeName(int);
17912		-SQLITE_PRIVATE int sqlite3VdbeOpcodeHasProperty(int, int);
17913	17875	SQLITE_PRIVATE int sqlite3VdbeMemGrow(Mem *pMem, int n, int preserve);
17914	17876	SQLITE_PRIVATE int sqlite3VdbeCloseStatement(Vdbe *, int);
17915	17877	SQLITE_PRIVATE void sqlite3VdbeFrameDelete(VdbeFrame*);
17916	17878	SQLITE_PRIVATE int sqlite3VdbeFrameRestore(VdbeFrame *);
17917	17879	SQLITE_PRIVATE void sqlite3VdbeMemStoreType(Mem *pMem);
		@@ -19574,12 +19536,10 @@
19574	19536	** May you share freely, never taking more than you give.
19575	19537	**
19576	19538	*************************************************************************
19577	19539	** This is the implementation of generic hash-tables
19578	19540	** used in SQLite.
19579		-**
19580		-** $Id: hash.c,v 1.38 2009/05/09 23:29:12 drh Exp $
19581	19541	*/
19582	19542
19583	19543	/* Turn bulk memory into a hash table object by initializing the
19584	19544	** fields of the Hash structure.
19585	19545	**
		@@ -20011,12 +19971,10 @@
20011	19971	** May you share freely, never taking more than you give.
20012	19972	**
20013	19973	******************************************************************************
20014	19974	**
20015	19975	** This file contains code that is specific to OS/2.
20016		-**
20017		-** $Id: os_os2.c,v 1.63 2008/12/10 19:26:24 drh Exp $
20018	19976	*/
20019	19977
20020	19978
20021	19979	#if SQLITE_OS_OS2
20022	19980
		@@ -20074,12 +20032,10 @@
20074	20032	** all of the platform-specific files (os_*.c) and is #included into those
20075	20033	** files.
20076	20034	**
20077	20035	** This file should be #included by the os_*.c files only. It is not a
20078	20036	** general purpose header file.
20079		-**
20080		-** $Id: os_common.h,v 1.38 2009/02/24 18:40:50 danielk1977 Exp $
20081	20037	*/
20082	20038	#ifndef _OS_COMMON_H_
20083	20039	#define _OS_COMMON_H_
20084	20040
20085	20041	/*
		@@ -20136,12 +20092,10 @@
20136	20092	**
20137	20093	******************************************************************************
20138	20094	**
20139	20095	** This file contains inline asm code for retrieving "high-performance"
20140	20096	** counters for x86 class CPUs.
20141		-**
20142		-** $Id: hwtime.h,v 1.3 2008/08/01 14:33:15 shane Exp $
20143	20097	*/
20144	20098	#ifndef _HWTIME_H_
20145	20099	#define _HWTIME_H_
20146	20100
20147	20101	/*
		@@ -21603,12 +21557,10 @@
21603	21557	** all of the platform-specific files (os_*.c) and is #included into those
21604	21558	** files.
21605	21559	**
21606	21560	** This file should be #included by the os_*.c files only. It is not a
21607	21561	** general purpose header file.
21608		-**
21609		-** $Id: os_common.h,v 1.38 2009/02/24 18:40:50 danielk1977 Exp $
21610	21562	*/
21611	21563	#ifndef _OS_COMMON_H_
21612	21564	#define _OS_COMMON_H_
21613	21565
21614	21566	/*
		@@ -21665,12 +21617,10 @@
21665	21617	**
21666	21618	******************************************************************************
21667	21619	**
21668	21620	** This file contains inline asm code for retrieving "high-performance"
21669	21621	** counters for x86 class CPUs.
21670		-**
21671		-** $Id: hwtime.h,v 1.3 2008/08/01 14:33:15 shane Exp $
21672	21622	*/
21673	21623	#ifndef _HWTIME_H_
21674	21624	#define _HWTIME_H_
21675	21625
21676	21626	/*
		@@ -24685,10 +24635,23 @@
24685	24635	rc = ftruncate(((unixFile*)id)->h, (off_t)nByte);
24686	24636	if( rc ){
24687	24637	((unixFile*)id)->lastErrno = errno;
24688	24638	return SQLITE_IOERR_TRUNCATE;
24689	24639	}else{
	24640	+#ifndef NDEBUG
	24641	+ /* If we are doing a normal write to a database file (as opposed to
	24642	+ ** doing a hot-journal rollback or a write to some file other than a
	24643	+ ** normal database file) and we truncate the file to zero length,
	24644	+ ** that effectively updates the change counter. This might happen
	24645	+ ** when restoring a database using the backup API from a zero-length
	24646	+ ** source.
	24647	+ */
	24648	+ if( ((unixFile*)id)->inNormalWrite && nByte==0 ){
	24649	+ ((unixFile*)id)->transCntrChng = 1;
	24650	+ }
	24651	+#endif
	24652	+
24690	24653	return SQLITE_OK;
24691	24654	}
24692	24655	}
24693	24656
24694	24657	/*
		@@ -27042,12 +27005,10 @@
27042	27005	** all of the platform-specific files (os_*.c) and is #included into those
27043	27006	** files.
27044	27007	**
27045	27008	** This file should be #included by the os_*.c files only. It is not a
27046	27009	** general purpose header file.
27047		-**
27048		-** $Id: os_common.h,v 1.38 2009/02/24 18:40:50 danielk1977 Exp $
27049	27010	*/
27050	27011	#ifndef _OS_COMMON_H_
27051	27012	#define _OS_COMMON_H_
27052	27013
27053	27014	/*
		@@ -27104,12 +27065,10 @@
27104	27065	**
27105	27066	******************************************************************************
27106	27067	**
27107	27068	** This file contains inline asm code for retrieving "high-performance"
27108	27069	** counters for x86 class CPUs.
27109		-**
27110		-** $Id: hwtime.h,v 1.3 2008/08/01 14:33:15 shane Exp $
27111	27070	*/
27112	27071	#ifndef _HWTIME_H_
27113	27072	#define _HWTIME_H_
27114	27073
27115	27074	/*
		@@ -29147,12 +29106,10 @@
29147	29106	** 5 and 500 set operations per Bitvec object, though the number of sets can
29148	29107	** sometimes grow into tens of thousands or larger. The size of the
29149	29108	** Bitvec object is the number of pages in the database file at the
29150	29109	** start of a transaction, and is thus usually less than a few thousand,
29151	29110	** but can be as large as 2 billion for a really big database.
29152		-**
29153		-** @(#) $Id: bitvec.c,v 1.17 2009/07/25 17:33:26 drh Exp $
29154	29111	*/
29155	29112
29156	29113	/* Size of the Bitvec structure in bytes. */
29157	29114	#define BITVEC_SZ (sizeof(void)128) /* 512 on 32bit. 1024 on 64bit */
29158	29115
		@@ -29536,12 +29493,10 @@
29536	29493	** May you find forgiveness for yourself and forgive others.
29537	29494	** May you share freely, never taking more than you give.
29538	29495	**
29539	29496	*************************************************************************
29540	29497	** This file implements that page cache.
29541		-**
29542		-** @(#) $Id: pcache.c,v 1.47 2009/07/25 11:46:49 danielk1977 Exp $
29543	29498	*/
29544	29499
29545	29500	/*
29546	29501	** A complete page cache is an instance of this structure.
29547	29502	*/
		@@ -30119,12 +30074,10 @@
30119	30074	** This file implements the default page cache implementation (the
30120	30075	** sqlite3_pcache interface). It also contains part of the implementation
30121	30076	** of the SQLITE_CONFIG_PAGECACHE and sqlite3_release_memory() features.
30122	30077	** If the default page cache implementation is overriden, then neither of
30123	30078	** these two features are available.
30124		-**
30125		-** @(#) $Id: pcache1.c,v 1.19 2009/07/17 11:44:07 drh Exp $
30126	30079	*/
30127	30080
30128	30081
30129	30082	typedef struct PCache1 PCache1;
30130	30083	typedef struct PgHdr1 PgHdr1;
		@@ -30770,19 +30723,11 @@
30770	30723
30771	30724	h = iNew%pCache->nHash;
30772	30725	pPage->iKey = iNew;
30773	30726	pPage->pNext = pCache->apHash[h];
30774	30727	pCache->apHash[h] = pPage;
30775		-
30776		- /* The xRekey() interface is only used to move pages earlier in the
30777		- ** database file (in order to move all free pages to the end of the
30778		- ** file where they can be truncated off.) Hence, it is not possible
30779		- ** for the new page number to be greater than the largest previously
30780		- ** fetched page. But we retain the following test in case xRekey()
30781		- ** begins to be used in different ways in the future.
30782		- */
30783		- if( NEVER(iNew>pCache->iMaxKey) ){
	30728	+ if( iNew>pCache->iMaxKey ){
30784	30729	pCache->iMaxKey = iNew;
30785	30730	}
30786	30731
30787	30732	pcache1LeaveMutex();
30788	30733	}
		@@ -30954,12 +30899,10 @@
30954	30899	** of the first SMALLEST is O(NlogN). Second and subsequent SMALLEST
30955	30900	** primitives are constant time. The cost of DESTROY is O(N).
30956	30901	**
30957	30902	** There is an added cost of O(N) when switching between TEST and
30958	30903	** SMALLEST primitives.
30959		-**
30960		-** $Id: rowset.c,v 1.7 2009/05/22 01:00:13 drh Exp $
30961	30904	*/
30962	30905
30963	30906
30964	30907	/*
30965	30908	** Target size for allocation chunks.
		@@ -31338,12 +31281,10 @@
31338	31281	** atomic commit and rollback through the use of a journal file that
31339	31282	** is separate from the database file. The pager also implements file
31340	31283	** locking to prevent two processes from writing the same database
31341	31284	** file simultaneously, or one process from reading the database while
31342	31285	** another is writing.
31343		-**
31344		-** @(#) $Id: pager.c,v 1.629 2009/08/10 17:48:57 drh Exp $
31345	31286	*/
31346	31287	#ifndef SQLITE_OMIT_DISKIO
31347	31288
31348	31289	/*
31349	31290	** Macros for troubleshooting. Normally turned off
		@@ -32233,14 +32174,14 @@
32233	32174	return rc;
32234	32175	}
32235	32176
32236	32177	/* Check that the values read from the page-size and sector-size fields
32237	32178	** are within range. To be 'in range', both values need to be a power
32238		- ** of two greater than or equal to 512, and not greater than their
	32179	+ ** of two greater than or equal to 512 or 32, and not greater than their
32239	32180	** respective compile time maximum limits.
32240	32181	*/
32241		- if( iPageSize<512 \|\| iSectorSize<512
	32182	+ if( iPageSize<512 \|\| iSectorSize<32
32242	32183	\|\| iPageSize>SQLITE_MAX_PAGE_SIZE \|\| iSectorSize>MAX_SECTOR_SIZE
32243	32184	\|\| ((iPageSize-1)&iPageSize)!=0 \|\| ((iSectorSize-1)&iSectorSize)!=0
32244	32185	){
32245	32186	/* If the either the page-size or sector-size in the journal-header is
32246	32187	** invalid, then the process that wrote the journal-header must have
		@@ -32743,28 +32684,28 @@
32743	32684	){
32744	32685	int rc;
32745	32686	PgHdr pPg; / An existing page in the cache */
32746	32687	Pgno pgno; /* The page number of a page in journal */
32747	32688	u32 cksum; /* Checksum used for sanity checking */
32748		- u8 aData; / Temporary storage for the page */
	32689	+ char aData; / Temporary storage for the page */
32749	32690	sqlite3_file jfd; / The file descriptor for the journal file */
32750	32691
32751	32692	assert( (isMainJrnl&~1)==0 ); /* isMainJrnl is 0 or 1 */
32752	32693	assert( (isSavepnt&~1)==0 ); /* isSavepnt is 0 or 1 */
32753	32694	assert( isMainJrnl \|\| pDone ); /* pDone always used on sub-journals */
32754	32695	assert( isSavepnt \|\| pDone==0 ); /* pDone never used on non-savepoint */
32755	32696
32756		- aData = (u8*)pPager->pTmpSpace;
	32697	+ aData = pPager->pTmpSpace;
32757	32698	assert( aData ); /* Temp storage must have already been allocated */
32758	32699
32759	32700	/* Read the page number and page data from the journal or sub-journal
32760	32701	** file. Return an error code to the caller if an IO error occurs.
32761	32702	*/
32762	32703	jfd = isMainJrnl ? pPager->jfd : pPager->sjfd;
32763	32704	rc = read32bits(jfd, *pOffset, &pgno);
32764	32705	if( rc!=SQLITE_OK ) return rc;
32765		- rc = sqlite3OsRead(jfd, aData, pPager->pageSize, (*pOffset)+4);
	32706	+ rc = sqlite3OsRead(jfd, (u8)aData, pPager->pageSize, (pOffset)+4);
32766	32707	if( rc!=SQLITE_OK ) return rc;
32767	32708	pOffset += pPager->pageSize + 4 + isMainJrnl4;
32768	32709
32769	32710	/* Sanity checking on the page. This is more important that I originally
32770	32711	** thought. If a power failure occurs while the journal is being written,
		@@ -32779,11 +32720,11 @@
32779	32720	return SQLITE_OK;
32780	32721	}
32781	32722	if( isMainJrnl ){
32782	32723	rc = read32bits(jfd, (*pOffset)-4, &cksum);
32783	32724	if( rc ) return rc;
32784		- if( !isSavepnt && pager_cksum(pPager, aData)!=cksum ){
	32725	+ if( !isSavepnt && pager_cksum(pPager, (u8*)aData)!=cksum ){
32785	32726	return SQLITE_DONE;
32786	32727	}
32787	32728	}
32788	32729
32789	32730	if( pDone && (rc = sqlite3BitvecSet(pDone, pgno))!=SQLITE_OK ){
		@@ -32825,27 +32766,27 @@
32825	32766	** Do not attempt to write if database file has never been opened.
32826	32767	*/
32827	32768	pPg = pager_lookup(pPager, pgno);
32828	32769	assert( pPg \|\| !MEMDB );
32829	32770	PAGERTRACE(("PLAYBACK %d page %d hash(%08x) %s\n",
32830		- PAGERID(pPager), pgno, pager_datahash(pPager->pageSize, aData),
32831		- (isMainJrnl?"main-journal":"sub-journal")
	32771	+ PAGERID(pPager), pgno, pager_datahash(pPager->pageSize, (u8*)aData),
	32772	+ (isMainJrnl?"main-journal":"sub-journal")
32832	32773	));
32833	32774	if( (pPager->state>=PAGER_EXCLUSIVE)
32834	32775	&& (pPg==0 \|\| 0==(pPg->flags&PGHDR_NEED_SYNC))
32835	32776	&& isOpen(pPager->fd)
32836	32777	&& !isUnsync
32837	32778	){
32838	32779	i64 ofst = (pgno-1)*(i64)pPager->pageSize;
32839		- rc = sqlite3OsWrite(pPager->fd, aData, pPager->pageSize, ofst);
	32780	+ rc = sqlite3OsWrite(pPager->fd, (u8*)aData, pPager->pageSize, ofst);
32840	32781	if( pgno>pPager->dbFileSize ){
32841	32782	pPager->dbFileSize = pgno;
32842	32783	}
32843	32784	if( pPager->pBackup ){
32844	32785	CODEC1(pPager, aData, pgno, 3, rc=SQLITE_NOMEM);
32845		- sqlite3BackupUpdate(pPager->pBackup, pgno, aData);
32846		- CODEC1(pPager, aData, pgno, 0, rc=SQLITE_NOMEM);
	32786	+ sqlite3BackupUpdate(pPager->pBackup, pgno, (u8*)aData);
	32787	+ CODEC2(pPager, aData, pgno, 7, rc=SQLITE_NOMEM, aData);
32847	32788	}
32848	32789	}else if( !isMainJrnl && pPg==0 ){
32849	32790	/* If this is a rollback of a savepoint and data was not written to
32850	32791	** the database and the page is not in-memory, there is a potential
32851	32792	** problem. When the page is next fetched by the b-tree layer, it
		@@ -32876,11 +32817,11 @@
32876	32817	** of an internal error resulting in an automatic call to
32877	32818	** sqlite3PagerRollback().
32878	32819	*/
32879	32820	void *pData;
32880	32821	pData = pPg->pData;
32881		- memcpy(pData, aData, pPager->pageSize);
	32822	+ memcpy(pData, (u8*)aData, pPager->pageSize);
32882	32823	pPager->xReiniter(pPg);
32883	32824	if( isMainJrnl && (!isSavepnt \|\| *pOffset<=pPager->journalHdr) ){
32884	32825	/* If the contents of this page were just restored from the main
32885	32826	** journal file, then its content must be as they were when the
32886	32827	** transaction was first opened. In this case we can mark the page
		@@ -33101,12 +33042,12 @@
33101	33042	** master journal pointers within created journal files.
33102	33043	**
33103	33044	** For temporary files the effective sector size is always 512 bytes.
33104	33045	**
33105	33046	** Otherwise, for non-temporary files, the effective sector size is
33106		-** the value returned by the xSectorSize() method rounded up to 512 if
33107		-** it is less than 512, or rounded down to MAX_SECTOR_SIZE if it
	33047	+** the value returned by the xSectorSize() method rounded up to 32 if
	33048	+** it is less than 32, or rounded down to MAX_SECTOR_SIZE if it
33108	33049	** is greater than MAX_SECTOR_SIZE.
33109	33050	*/
33110	33051	static void setSectorSize(Pager *pPager){
33111	33052	assert( isOpen(pPager->fd) \|\| pPager->tempFile );
33112	33053
		@@ -33115,11 +33056,11 @@
33115	33056	** may not have been opened yet, in which case the OsSectorSize()
33116	33057	** call will segfault.
33117	33058	*/
33118	33059	pPager->sectorSize = sqlite3OsSectorSize(pPager->fd);
33119	33060	}
33120		- if( pPager->sectorSize<512 ){
	33061	+ if( pPager->sectorSize<32 ){
33121	33062	pPager->sectorSize = 512;
33122	33063	}
33123	33064	if( pPager->sectorSize>MAX_SECTOR_SIZE ){
33124	33065	assert( MAX_SECTOR_SIZE>=512 );
33125	33066	pPager->sectorSize = MAX_SECTOR_SIZE;
		@@ -34193,11 +34134,13 @@
34193	34134	** than Pager.dbSize, this means sqlite3PagerTruncateImage() was called to
34194	34135	** make the file smaller (presumably by auto-vacuum code). Do not write
34195	34136	** any such pages to the file.
34196	34137	**
34197	34138	** Also, do not write out any page that has the PGHDR_DONT_WRITE flag
34198		- ** set (set by sqlite3PagerDontWrite()).
	34139	+ ** set (set by sqlite3PagerDontWrite()). Note that if compiled with
	34140	+ ** SQLITE_SECURE_DELETE the PGHDR_DONT_WRITE bit is never set and so
	34141	+ ** the second test is always true.
34199	34142	*/
34200	34143	if( pgno<=pPager->dbSize && 0==(pList->flags&PGHDR_DONT_WRITE) ){
34201	34144	i64 offset = (pgno-1)(i64)pPager->pageSize; / Offset to write */
34202	34145	char pData; / Data to write */
34203	34146
		@@ -35702,10 +35645,11 @@
35702	35645	SQLITE_PRIVATE int sqlite3PagerIswriteable(DbPage *pPg){
35703	35646	return pPg->flags&PGHDR_DIRTY;
35704	35647	}
35705	35648	#endif
35706	35649
	35650	+#ifndef SQLITE_SECURE_DELETE
35707	35651	/*
35708	35652	** A call to this routine tells the pager that it is not necessary to
35709	35653	** write the information on page pPg back to the disk, even though
35710	35654	** that page might be marked as dirty. This happens, for example, when
35711	35655	** the page has been added as a leaf of the freelist and so its
		@@ -35727,10 +35671,11 @@
35727	35671	#ifdef SQLITE_CHECK_PAGES
35728	35672	pPg->pageHash = pager_pagehash(pPg);
35729	35673	#endif
35730	35674	}
35731	35675	}
	35676	+#endif /* !defined(SQLITE_SECURE_DELETE) */
35732	35677
35733	35678	/*
35734	35679	** This routine is called to increment the value of the database file
35735	35680	** change-counter, stored as a 4-byte big-endian integer starting at
35736	35681	** byte offset 24 of the pager file.
		@@ -35766,11 +35711,11 @@
35766	35711	#else
35767	35712	# define DIRECT_MODE isDirectMode
35768	35713	#endif
35769	35714
35770	35715	assert( pPager->state>=PAGER_RESERVED );
35771		- if( !pPager->changeCountDone && ALWAYS(pPager->dbSize>0) ){
	35716	+ if( !pPager->changeCountDone && pPager->dbSize>0 ){
35772	35717	PgHdr pPgHdr; / Reference to page 1 */
35773	35718	u32 change_counter; /* Initial value of change-counter field */
35774	35719
35775	35720	assert( !pPager->tempFile && isOpen(pPager->fd) );
35776	35721
		@@ -36418,10 +36363,18 @@
36418	36363	Pgno needSyncPgno = 0; /* Old value of pPg->pgno, if sync is required */
36419	36364	int rc; /* Return code */
36420	36365	Pgno origPgno; /* The original page number */
36421	36366
36422	36367	assert( pPg->nRef>0 );
	36368	+
	36369	+ /* In order to be able to rollback, an in-memory database must journal
	36370	+ ** the page we are moving from.
	36371	+ */
	36372	+ if( MEMDB ){
	36373	+ rc = sqlite3PagerWrite(pPg);
	36374	+ if( rc ) return rc;
	36375	+ }
36423	36376
36424	36377	/* If the page being moved is dirty and has not been saved by the latest
36425	36378	** savepoint, then save the current contents of the page into the
36426	36379	** sub-journal now. This is required to handle the following scenario:
36427	36380	**
		@@ -36437,11 +36390,11 @@
36437	36390	**
36438	36391	** subjournalPage() may need to allocate space to store pPg->pgno into
36439	36392	** one or more savepoint bitvecs. This is the reason this function
36440	36393	** may return SQLITE_NOMEM.
36441	36394	*/
36442		- if( pPg->flags&PGHDR_DIRTY
	36395	+ if( pPg->flags&PGHDR_DIRTY
36443	36396	&& subjRequiresPage(pPg)
36444	36397	&& SQLITE_OK!=(rc = subjournalPage(pPg))
36445	36398	){
36446	36399	return rc;
36447	36400	}
		@@ -36472,11 +36425,18 @@
36472	36425	pPg->flags &= ~PGHDR_NEED_SYNC;
36473	36426	pPgOld = pager_lookup(pPager, pgno);
36474	36427	assert( !pPgOld \|\| pPgOld->nRef==1 );
36475	36428	if( pPgOld ){
36476	36429	pPg->flags \|= (pPgOld->flags&PGHDR_NEED_SYNC);
36477		- sqlite3PcacheDrop(pPgOld);
	36430	+ if( MEMDB ){
	36431	+ /* Do not discard pages from an in-memory database since we might
	36432	+ ** need to rollback later. Just move the page out of the way. */
	36433	+ assert( pPager->dbSizeValid );
	36434	+ sqlite3PcacheMove(pPgOld, pPager->dbSize+1);
	36435	+ }else{
	36436	+ sqlite3PcacheDrop(pPgOld);
	36437	+ }
36478	36438	}
36479	36439
36480	36440	origPgno = pPg->pgno;
36481	36441	sqlite3PcacheMove(pPg, pgno);
36482	36442	sqlite3PcacheMakeDirty(pPg);
		@@ -36517,22 +36477,16 @@
36517	36477	sqlite3PagerUnref(pPgHdr);
36518	36478	}
36519	36479
36520	36480	/*
36521	36481	** For an in-memory database, make sure the original page continues
36522		- ** to exist, in case the transaction needs to roll back. We allocate
36523		- ** the page now, instead of at rollback, because we can better deal
36524		- ** with an out-of-memory error now. Ticket #3761.
	36482	+ ** to exist, in case the transaction needs to roll back. Use pPgOld
	36483	+ ** as the original page since it has already been allocated.
36525	36484	*/
36526	36485	if( MEMDB ){
36527		- DbPage *pNew;
36528		- rc = sqlite3PagerAcquire(pPager, origPgno, &pNew, 1);
36529		- if( rc!=SQLITE_OK ){
36530		- sqlite3PcacheMove(pPg, origPgno);
36531		- return rc;
36532		- }
36533		- sqlite3PagerUnref(pNew);
	36486	+ sqlite3PcacheMove(pPgOld, origPgno);
	36487	+ sqlite3PagerUnref(pPgOld);
36534	36488	}
36535	36489
36536	36490	return SQLITE_OK;
36537	36491	}
36538	36492	#endif
		@@ -36655,12 +36609,10 @@
36655	36609	** May you find forgiveness for yourself and forgive others.
36656	36610	** May you share freely, never taking more than you give.
36657	36611	**
36658	36612	*************************************************************************
36659	36613	**
36660		-** $Id: btmutex.c,v 1.17 2009/07/20 12:33:33 drh Exp $
36661		-**
36662	36614	** This file contains code used to implement mutexes on Btree objects.
36663	36615	** This code really belongs in btree.c. But btree.c is getting too
36664	36616	** big and we want to break it down some. This packaged seemed like
36665	36617	** a good breakout.
36666	36618	*/
		@@ -36675,12 +36627,10 @@
36675	36627	** May you do good and not evil.
36676	36628	** May you find forgiveness for yourself and forgive others.
36677	36629	** May you share freely, never taking more than you give.
36678	36630	**
36679	36631	*************************************************************************
36680		-** $Id: btreeInt.h,v 1.52 2009/07/15 17:25:46 drh Exp $
36681		-**
36682	36632	** This file implements a external (disk-based) database using BTrees.
36683	36633	** For a detailed discussion of BTrees, refer to
36684	36634	**
36685	36635	** Donald E. Knuth, THE ART OF COMPUTER PROGRAMMING, Volume 3:
36686	36636	** "Sorting And Searching", pages 473-480. Addison-Wesley
		@@ -37660,12 +37610,10 @@
37660	37610	** May you do good and not evil.
37661	37611	** May you find forgiveness for yourself and forgive others.
37662	37612	** May you share freely, never taking more than you give.
37663	37613	**
37664	37614	*************************************************************************
37665		-** $Id: btree.c,v 1.705 2009/08/10 03:57:58 shane Exp $
37666		-**
37667	37615	** This file implements a external (disk-based) database using BTrees.
37668	37616	** See the header comment on "btreeInt.h" for additional information.
37669	37617	** Including a description of file format and an overview of operation.
37670	37618	*/
37671	37619
		@@ -40930,12 +40878,12 @@
40930	40878	**
40931	40879	** No checking is done to make sure that page iTable really is the
40932	40880	** root page of a b-tree. If it is not, then the cursor acquired
40933	40881	** will not work correctly.
40934	40882	**
40935		-** It is assumed that the sqlite3BtreeCursorSize() bytes of memory
40936		-** pointed to by pCur have been zeroed by the caller.
	40883	+** It is assumed that the sqlite3BtreeCursorZero() has been called
	40884	+** on pCur to initialize the memory space prior to invoking this routine.
40937	40885	*/
40938	40886	static int btreeCursor(
40939	40887	Btree p, / The btree */
40940	40888	int iTable, /* Root page of table to open */
40941	40889	int wrFlag, /* 1 to write. 0 read-only */
		@@ -41004,11 +40952,23 @@
41004	40952	** sufficient storage to hold a cursor. The BtCursor object is opaque
41005	40953	** to users so they cannot do the sizeof() themselves - they must call
41006	40954	** this routine.
41007	40955	*/
41008	40956	SQLITE_PRIVATE int sqlite3BtreeCursorSize(void){
41009		- return sizeof(BtCursor);
	40957	+ return ROUND8(sizeof(BtCursor));
	40958	+}
	40959	+
	40960	+/*
	40961	+** Initialize memory that will be converted into a BtCursor object.
	40962	+**
	40963	+** The simple approach here would be to memset() the entire object
	40964	+** to zero. But it turns out that the apPage[] and aiIdx[] arrays
	40965	+** do not need to be zeroed and they are large, so we can save a lot
	40966	+** of run-time by skipping the initialization of those elements.
	40967	+*/
	40968	+SQLITE_PRIVATE void sqlite3BtreeCursorZero(BtCursor *p){
	40969	+ memset(p, 0, offsetof(BtCursor, iPage));
41010	40970	}
41011	40971
41012	40972	/*
41013	40973	** Set the cached rowid value of every cursor in the same database file
41014	40974	** as pCur and having the same root page number as pCur. The value is
		@@ -45489,12 +45449,10 @@
45489	45449	** May you share freely, never taking more than you give.
45490	45450	**
45491	45451	*************************************************************************
45492	45452	** This file contains the implementation of the sqlite3_backup_XXX()
45493	45453	** API functions and the related features.
45494		-**
45495		-** $Id: backup.c,v 1.19 2009/07/06 19:03:13 drh Exp $
45496	45454	*/
45497	45455
45498	45456	/* Macro to find the minimum of two numeric values.
45499	45457	*/
45500	45458	#ifndef MIN
		@@ -46123,12 +46081,10 @@
46123	46081	**
46124	46082	** This file contains code use to manipulate "Mem" structure. A "Mem"
46125	46083	** stores a single value in the VDBE. Mem is an opaque structure visible
46126	46084	** only within the VDBE. Interface routines refer to a Mem using the
46127	46085	** name sqlite_value
46128		-**
46129		-** $Id: vdbemem.c,v 1.152 2009/07/22 18:07:41 drh Exp $
46130	46086	*/
46131	46087
46132	46088	/*
46133	46089	** Call sqlite3VdbeMemExpandBlob() on the supplied value (type Mem*)
46134	46090	** P if required.
		@@ -46863,13 +46819,10 @@
46863	46819	SQLITE_PRIVATE int sqlite3MemCompare(const Mem pMem1, const Mem pMem2, const CollSeq *pColl){
46864	46820	int rc;
46865	46821	int f1, f2;
46866	46822	int combined_flags;
46867	46823
46868		- /* Interchange pMem1 and pMem2 if the collating sequence specifies
46869		- ** DESC order.
46870		- */
46871	46824	f1 = pMem1->flags;
46872	46825	f2 = pMem2->flags;
46873	46826	combined_flags = f1\|f2;
46874	46827	assert( (combined_flags & MEM_RowSet)==0 );
46875	46828
		@@ -47116,11 +47069,11 @@
47116	47069	*ppVal = 0;
47117	47070	return SQLITE_OK;
47118	47071	}
47119	47072	op = pExpr->op;
47120	47073	if( op==TK_REGISTER ){
47121		- op = pExpr->op2;
	47074	+ op = pExpr->op2; /* This only happens with SQLITE_ENABLE_STAT2 */
47122	47075	}
47123	47076
47124	47077	if( op==TK_STRING \|\| op==TK_FLOAT \|\| op==TK_INTEGER ){
47125	47078	pVal = sqlite3ValueNew(db);
47126	47079	if( pVal==0 ) goto no_mem;
		@@ -47229,12 +47182,10 @@
47229	47182	*************************************************************************
47230	47183	** This file contains code used for creating, destroying, and populating
47231	47184	** a VDBE (or an "sqlite3_stmt" as it is known to the outside world.) Prior
47232	47185	** to version 2.8.7, all this code was combined into the vdbe.c source file.
47233	47186	** But that file was getting too big so this subroutines were split out.
47234		-**
47235		-** $Id: vdbeaux.c,v 1.480 2009/08/08 18:01:08 drh Exp $
47236	47187	*/
47237	47188
47238	47189
47239	47190
47240	47191	/*
		@@ -47410,10 +47361,26 @@
47410	47361	){
47411	47362	int addr = sqlite3VdbeAddOp3(p, op, p1, p2, p3);
47412	47363	sqlite3VdbeChangeP4(p, addr, zP4, p4type);
47413	47364	return addr;
47414	47365	}
	47366	+
	47367	+/*
	47368	+** Add an opcode that includes the p4 value as an integer.
	47369	+*/
	47370	+SQLITE_PRIVATE int sqlite3VdbeAddOp4Int(
	47371	+ Vdbe p, / Add the opcode to this VM */
	47372	+ int op, /* The new opcode */
	47373	+ int p1, /* The P1 operand */
	47374	+ int p2, /* The P2 operand */
	47375	+ int p3, /* The P3 operand */
	47376	+ int p4 /* The P4 operand as an integer */
	47377	+){
	47378	+ int addr = sqlite3VdbeAddOp3(p, op, p1, p2, p3);
	47379	+ sqlite3VdbeChangeP4(p, addr, SQLITE_INT_TO_PTR(p4), P4_INT32);
	47380	+ return addr;
	47381	+}
47415	47382
47416	47383	/*
47417	47384	** Create a new symbolic label for an instruction that has yet to be
47418	47385	** coded. The symbolic label is really just a negative number. The
47419	47386	** label can be used as the P2 value of an operation. Later, when
		@@ -47586,10 +47553,12 @@
47586	47553	** This routine is called once after all opcodes have been inserted.
47587	47554	**
47588	47555	** Variable *pMaxFuncArgs is set to the maximum value of any P2 argument
47589	47556	** to an OP_Function, OP_AggStep or OP_VFilter opcode. This is used by
47590	47557	** sqlite3VdbeMakeReady() to size the Vdbe.apArg[] array.
	47558	+**
	47559	+** The Op.opflags field is set on all opcodes.
47591	47560	*/
47592	47561	static void resolveP2Values(Vdbe p, int pMaxFuncArgs){
47593	47562	int i;
47594	47563	int nMaxArgs = *pMaxFuncArgs;
47595	47564	Op *pOp;
		@@ -47596,29 +47565,28 @@
47596	47565	int *aLabel = p->aLabel;
47597	47566	p->readOnly = 1;
47598	47567	for(pOp=p->aOp, i=p->nOp-1; i>=0; i--, pOp++){
47599	47568	u8 opcode = pOp->opcode;
47600	47569
	47570	+ pOp->opflags = sqlite3OpcodeProperty[opcode];
47601	47571	if( opcode==OP_Function \|\| opcode==OP_AggStep ){
47602	47572	if( pOp->p5>nMaxArgs ) nMaxArgs = pOp->p5;
	47573	+ }else if( opcode==OP_Transaction && pOp->p2!=0 ){
	47574	+ p->readOnly = 0;
47603	47575	#ifndef SQLITE_OMIT_VIRTUALTABLE
47604	47576	}else if( opcode==OP_VUpdate ){
47605	47577	if( pOp->p2>nMaxArgs ) nMaxArgs = pOp->p2;
47606		-#endif
47607		- }else if( opcode==OP_Transaction && pOp->p2!=0 ){
47608		- p->readOnly = 0;
47609		-#ifndef SQLITE_OMIT_VIRTUALTABLE
47610	47578	}else if( opcode==OP_VFilter ){
47611	47579	int n;
47612	47580	assert( p->nOp - i >= 3 );
47613	47581	assert( pOp[-1].opcode==OP_Integer );
47614	47582	n = pOp[-1].p1;
47615	47583	if( n>nMaxArgs ) nMaxArgs = n;
47616	47584	#endif
47617	47585	}
47618	47586
47619		- if( sqlite3VdbeOpcodeHasProperty(opcode, OPFLG_JUMP) && pOp->p2<0 ){
	47587	+ if( (pOp->opflags & OPFLG_JUMP)!=0 && pOp->p2<0 ){
47620	47588	assert( -1-pOp->p2<p->nLabel );
47621	47589	pOp->p2 = aLabel[-1-pOp->p2];
47622	47590	}
47623	47591	}
47624	47592	sqlite3DbFree(p->db, p->aLabel);
		@@ -47676,11 +47644,11 @@
47676	47644	for(i=0; i<nOp; i++, pIn++){
47677	47645	int p2 = pIn->p2;
47678	47646	VdbeOp *pOut = &p->aOp[i+addr];
47679	47647	pOut->opcode = pIn->opcode;
47680	47648	pOut->p1 = pIn->p1;
47681		- if( p2<0 && sqlite3VdbeOpcodeHasProperty(pOut->opcode, OPFLG_JUMP) ){
	47649	+ if( p2<0 && (sqlite3OpcodeProperty[pOut->opcode] & OPFLG_JUMP)!=0 ){
47682	47650	pOut->p2 = addr + ADDR(p2);
47683	47651	}else{
47684	47652	pOut->p2 = p2;
47685	47653	}
47686	47654	pOut->p3 = pIn->p3;
		@@ -49986,13 +49954,21 @@
49986	49954	Mem mem1;
49987	49955
49988	49956	pKeyInfo = pPKey2->pKeyInfo;
49989	49957	mem1.enc = pKeyInfo->enc;
49990	49958	mem1.db = pKeyInfo->db;
49991		- mem1.flags = 0;
49992		- mem1.u.i = 0; /* not needed, here to silence compiler warning */
49993		- mem1.zMalloc = 0;
	49959	+ /* mem1.flags = 0; // Will be initialized by sqlite3VdbeSerialGet() */
	49960	+ VVA_ONLY( mem1.zMalloc = 0; ) /* Only needed by assert() statements */
	49961	+
	49962	+ /* Compilers may complain that mem1.u.i is potentially uninitialized.
	49963	+ ** We could initialize it, as shown here, to silence those complaints.
	49964	+ ** But in fact, mem1.u.i will never actually be used initialized, and doing
	49965	+ ** the unnecessary initialization has a measurable negative performance
	49966	+ ** impact, since this routine is a very high runner. And so, we choose
	49967	+ ** to ignore the compiler warnings and leave this variable uninitialized.
	49968	+ */
	49969	+ /* mem1.u.i = 0; // not needed, here to silence compiler warning */
49994	49970
49995	49971	idx1 = getVarint32(aKey1, szHdr1);
49996	49972	d1 = szHdr1;
49997	49973	if( pPKey2->flags & UNPACKED_IGNORE_ROWID ){
49998	49974	szHdr1--;
		@@ -50012,51 +49988,56 @@
50012	49988	/* Do the comparison
50013	49989	*/
50014	49990	rc = sqlite3MemCompare(&mem1, &pPKey2->aMem[i],
50015	49991	i<nField ? pKeyInfo->aColl[i] : 0);
50016	49992	if( rc!=0 ){
50017		- break;
	49993	+ assert( mem1.zMalloc==0 ); /* See comment below */
	49994	+
	49995	+ /* Invert the result if we are using DESC sort order. */
	49996	+ if( pKeyInfo->aSortOrder && i<nField && pKeyInfo->aSortOrder[i] ){
	49997	+ rc = -rc;
	49998	+ }
	49999	+
	50000	+ /* If the PREFIX_SEARCH flag is set and all fields except the final
	50001	+ ** rowid field were equal, then clear the PREFIX_SEARCH flag and set
	50002	+ ** pPKey2->rowid to the value of the rowid field in (pKey1, nKey1).
	50003	+ ** This is used by the OP_IsUnique opcode.
	50004	+ */
	50005	+ if( (pPKey2->flags & UNPACKED_PREFIX_SEARCH) && i==(pPKey2->nField-1) ){
	50006	+ assert( idx1==szHdr1 && rc );
	50007	+ assert( mem1.flags & MEM_Int );
	50008	+ pPKey2->flags &= ~UNPACKED_PREFIX_SEARCH;
	50009	+ pPKey2->rowid = mem1.u.i;
	50010	+ }
	50011	+
	50012	+ return rc;
50018	50013	}
50019	50014	i++;
50020	50015	}
50021	50016
50022		- /* No memory allocation is ever used on mem1. */
50023		- if( NEVER(mem1.zMalloc) ) sqlite3VdbeMemRelease(&mem1);
50024		-
50025		- /* If the PREFIX_SEARCH flag is set and all fields except the final
50026		- ** rowid field were equal, then clear the PREFIX_SEARCH flag and set
50027		- ** pPKey2->rowid to the value of the rowid field in (pKey1, nKey1).
50028		- ** This is used by the OP_IsUnique opcode.
50029		- */
50030		- if( (pPKey2->flags & UNPACKED_PREFIX_SEARCH) && i==(pPKey2->nField-1) ){
50031		- assert( idx1==szHdr1 && rc );
50032		- assert( mem1.flags & MEM_Int );
50033		- pPKey2->flags &= ~UNPACKED_PREFIX_SEARCH;
50034		- pPKey2->rowid = mem1.u.i;
50035		- }
50036		-
50037		- if( rc==0 ){
50038		- /* rc==0 here means that one of the keys ran out of fields and
50039		- ** all the fields up to that point were equal. If the UNPACKED_INCRKEY
50040		- ** flag is set, then break the tie by treating key2 as larger.
50041		- ** If the UPACKED_PREFIX_MATCH flag is set, then keys with common prefixes
50042		- ** are considered to be equal. Otherwise, the longer key is the
50043		- ** larger. As it happens, the pPKey2 will always be the longer
50044		- ** if there is a difference.
50045		- */
50046		- if( pPKey2->flags & UNPACKED_INCRKEY ){
50047		- rc = -1;
50048		- }else if( pPKey2->flags & UNPACKED_PREFIX_MATCH ){
50049		- /* Leave rc==0 */
50050		- }else if( idx1<szHdr1 ){
50051		- rc = 1;
50052		- }
50053		- }else if( pKeyInfo->aSortOrder && i<pKeyInfo->nField
50054		- && pKeyInfo->aSortOrder[i] ){
50055		- rc = -rc;
50056		- }
50057		-
	50017	+ /* No memory allocation is ever used on mem1. Prove this using
	50018	+ ** the following assert(). If the assert() fails, it indicates a
	50019	+ ** memory leak and a need to call sqlite3VdbeMemRelease(&mem1).
	50020	+ */
	50021	+ assert( mem1.zMalloc==0 );
	50022	+
	50023	+ /* rc==0 here means that one of the keys ran out of fields and
	50024	+ ** all the fields up to that point were equal. If the UNPACKED_INCRKEY
	50025	+ ** flag is set, then break the tie by treating key2 as larger.
	50026	+ ** If the UPACKED_PREFIX_MATCH flag is set, then keys with common prefixes
	50027	+ ** are considered to be equal. Otherwise, the longer key is the
	50028	+ ** larger. As it happens, the pPKey2 will always be the longer
	50029	+ ** if there is a difference.
	50030	+ */
	50031	+ assert( rc==0 );
	50032	+ if( pPKey2->flags & UNPACKED_INCRKEY ){
	50033	+ rc = -1;
	50034	+ }else if( pPKey2->flags & UNPACKED_PREFIX_MATCH ){
	50035	+ /* Leave rc==0 */
	50036	+ }else if( idx1<szHdr1 ){
	50037	+ rc = 1;
	50038	+ }
50058	50039	return rc;
50059	50040	}
50060	50041
50061	50042
50062	50043	/*
		@@ -50272,12 +50253,10 @@
50272	50253	**
50273	50254	*************************************************************************
50274	50255	**
50275	50256	** This file contains code use to implement APIs that are part of the
50276	50257	** VDBE.
50277		-**
50278		-** $Id: vdbeapi.c,v 1.167 2009/06/25 01:47:12 drh Exp $
50279	50258	*/
50280	50259
50281	50260	#ifndef SQLITE_OMIT_DEPRECATED
50282	50261	/*
50283	50262	** Return TRUE (non-zero) of the statement supplied as an argument needs
		@@ -51528,12 +51507,10 @@
51528	51507	** Various scripts scan this source file in order to generate HTML
51529	51508	** documentation, headers files, or other derived files. The formatting
51530	51509	** of the code in this file is, therefore, important. See other comments
51531	51510	** in this file for details. If in doubt, do not deviate from existing
51532	51511	** commenting and indentation practices when changing or adding code.
51533		-**
51534		-** $Id: vdbe.c,v 1.874 2009/07/24 17:58:53 danielk1977 Exp $
51535	51512	*/
51536	51513
51537	51514	/*
51538	51515	** The following global variable is incremented every time a cursor
51539	51516	** moves, either by the OP_SeekXX, OP_Next, or OP_Prev opcodes. The test
		@@ -51656,27 +51633,10 @@
51656	51633	}else{
51657	51634	pMem->type = SQLITE_BLOB;
51658	51635	}
51659	51636	}
51660	51637
51661		-/*
51662		-** Properties of opcodes. The OPFLG_INITIALIZER macro is
51663		-** created by mkopcodeh.awk during compilation. Data is obtained
51664		-** from the comments following the "case OP_xxxx:" statements in
51665		-** this file.
51666		-*/
51667		-static const unsigned char opcodeProperty[] = OPFLG_INITIALIZER;
51668		-
51669		-/*
51670		-** Return true if an opcode has any of the OPFLG_xxx properties
51671		-** specified by mask.
51672		-*/
51673		-SQLITE_PRIVATE int sqlite3VdbeOpcodeHasProperty(int opcode, int mask){
51674		- assert( opcode>0 && opcode<(int)sizeof(opcodeProperty) );
51675		- return (opcodeProperty[opcode]&mask)!=0;
51676		-}
51677		-
51678	51638	/*
51679	51639	** Allocate VdbeCursor number iCur. Return a pointer to it. Return NULL
51680	51640	** if we run out of memory.
51681	51641	*/
51682	51642	static VdbeCursor *allocateCursor(
		@@ -51707,11 +51667,11 @@
51707	51667	Mem *pMem = &p->aMem[p->nMem-iCur];
51708	51668
51709	51669	int nByte;
51710	51670	VdbeCursor *pCx = 0;
51711	51671	nByte =
51712		- sizeof(VdbeCursor) +
	51672	+ ROUND8(sizeof(VdbeCursor)) +
51713	51673	(isBtreeCursor?sqlite3BtreeCursorSize():0) +
51714	51674	2nFieldsizeof(u32);
51715	51675
51716	51676	assert( iCur<p->nCursor );
51717	51677	if( p->apCsr[iCur] ){
		@@ -51718,19 +51678,20 @@
51718	51678	sqlite3VdbeFreeCursor(p, p->apCsr[iCur]);
51719	51679	p->apCsr[iCur] = 0;
51720	51680	}
51721	51681	if( SQLITE_OK==sqlite3VdbeMemGrow(pMem, nByte, 0) ){
51722	51682	p->apCsr[iCur] = pCx = (VdbeCursor*)pMem->z;
51723		- memset(pMem->z, 0, nByte);
	51683	+ memset(pCx, 0, sizeof(VdbeCursor));
51724	51684	pCx->iDb = iDb;
51725	51685	pCx->nField = nField;
51726	51686	if( nField ){
51727		- pCx->aType = (u32 *)&pMem->z[sizeof(VdbeCursor)];
	51687	+ pCx->aType = (u32 *)&pMem->z[ROUND8(sizeof(VdbeCursor))];
51728	51688	}
51729	51689	if( isBtreeCursor ){
51730	51690	pCx->pCursor = (BtCursor*)
51731		- &pMem->z[sizeof(VdbeCursor)+2nFieldsizeof(u32)];
	51691	+ &pMem->z[ROUND8(sizeof(VdbeCursor))+2nFieldsizeof(u32)];
	51692	+ sqlite3BtreeCursorZero(pCx->pCursor);
51732	51693	}
51733	51694	}
51734	51695	return pCx;
51735	51696	}
51736	51697
		@@ -51968,12 +51929,10 @@
51968	51929	**
51969	51930	******************************************************************************
51970	51931	**
51971	51932	** This file contains inline asm code for retrieving "high-performance"
51972	51933	** counters for x86 class CPUs.
51973		-**
51974		-** $Id: hwtime.h,v 1.3 2008/08/01 14:33:15 shane Exp $
51975	51934	*/
51976	51935	#ifndef _HWTIME_H_
51977	51936	#define _HWTIME_H_
51978	51937
51979	51938	/*
		@@ -52132,28 +52091,31 @@
52132	52091	*/
52133	52092	SQLITE_PRIVATE int sqlite3VdbeExec(
52134	52093	Vdbe p / The VDBE */
52135	52094	){
52136	52095	int pc; /* The program counter */
	52096	+ Op aOp = p->aOp; / Copy of p->aOp */
52137	52097	Op pOp; / Current operation */
52138	52098	int rc = SQLITE_OK; /* Value to return */
52139	52099	sqlite3 db = p->db; / The database */
	52100	+ u8 resetSchemaOnFault = 0; /* Reset schema after an error if true */
52140	52101	u8 encoding = ENC(db); /* The database encoding */
	52102	+#ifndef SQLITE_OMIT_PROGRESS_CALLBACK
	52103	+ u8 checkProgress; /* True if progress callbacks are enabled */
	52104	+ int nProgressOps = 0; /* Opcodes executed since progress callback. */
	52105	+#endif
	52106	+ Mem aMem = p->aMem; / Copy of p->aMem */
52141	52107	Mem pIn1 = 0; / 1st input operand */
52142	52108	Mem pIn2 = 0; / 2nd input operand */
52143	52109	Mem pIn3 = 0; / 3rd input operand */
52144	52110	Mem pOut = 0; / Output operand */
52145		- u8 opProperty;
52146	52111	int iCompare = 0; /* Result of last OP_Compare operation */
52147	52112	int aPermute = 0; / Permutation of columns for OP_Compare */
52148	52113	#ifdef VDBE_PROFILE
52149	52114	u64 start; /* CPU clock count at start of opcode */
52150	52115	int origPc; /* Program counter at start of opcode */
52151	52116	#endif
52152		-#ifndef SQLITE_OMIT_PROGRESS_CALLBACK
52153		- int nProgressOps = 0; /* Opcodes executed since progress callback. */
52154		-#endif
52155	52117	/********************************************************************
52156	52118	** Automatically generated code
52157	52119	**
52158	52120	** The following union is automatically generated by the
52159	52121	** vdbe-compress.tcl script. The purpose of this union is to
		@@ -52245,14 +52207,12 @@
52245	52207	int szHdr; /* Size of the header size field at start of record */
52246	52208	int avail; /* Number of bytes of available data */
52247	52209	Mem pReg; / PseudoTable input register */
52248	52210	} am;
52249	52211	struct OP_Affinity_stack_vars {
52250		- char zAffinity; / The affinity to be applied */
52251		- Mem pData0; / First register to which to apply affinity */
52252		- Mem pLast; / Last register to which to apply affinity */
52253		- Mem pRec; / Current register */
	52212	+ const char zAffinity; / The affinity to be applied */
	52213	+ char cAff; /* A single character of affinity */
52254	52214	} an;
52255	52215	struct OP_MakeRecord_stack_vars {
52256	52216	u8 zNewRecord; / A buffer to hold the data for the new record */
52257	52217	Mem pRec; / The new record */
52258	52218	u64 nData; /* Number of bytes of data space */
		@@ -52333,18 +52293,19 @@
52333	52293	struct OP_Found_stack_vars {
52334	52294	int alreadyExists;
52335	52295	VdbeCursor *pC;
52336	52296	int res;
52337	52297	UnpackedRecord *pIdxKey;
	52298	+ UnpackedRecord r;
52338	52299	char aTempRec[ROUND8(sizeof(UnpackedRecord)) + sizeof(Mem)*3 + 7];
52339	52300	} bb;
52340	52301	struct OP_IsUnique_stack_vars {
52341	52302	u16 ii;
52342	52303	VdbeCursor *pCx;
52343	52304	BtCursor *pCrsr;
52344	52305	u16 nField;
52345		- Mem *aMem;
	52306	+ Mem *aMx;
52346	52307	UnpackedRecord r; /* B-Tree index search key */
52347	52308	i64 R; /* Rowid stored in register P3 */
52348	52309	} bc;
52349	52310	struct OP_NotExists_stack_vars {
52350	52311	VdbeCursor *pC;
		@@ -52453,63 +52414,58 @@
52453	52414	int j; /* Loop counter */
52454	52415	int nErr; /* Number of errors reported */
52455	52416	char z; / Text of the error report */
52456	52417	Mem pnErr; / Register keeping track of errors remaining */
52457	52418	} bv;
52458		- struct OP_RowSetAdd_stack_vars {
52459		- Mem *pIdx;
52460		- Mem *pVal;
52461		- } bw;
52462	52419	struct OP_RowSetRead_stack_vars {
52463		- Mem *pIdx;
52464	52420	i64 val;
52465		- } bx;
	52421	+ } bw;
52466	52422	struct OP_RowSetTest_stack_vars {
52467	52423	int iSet;
52468	52424	int exists;
52469		- } by;
	52425	+ } bx;
52470	52426	struct OP_Program_stack_vars {
52471	52427	int nMem; /* Number of memory registers for sub-program */
52472	52428	int nByte; /* Bytes of runtime space required for sub-program */
52473	52429	Mem pRt; / Register to allocate runtime space */
52474	52430	Mem pMem; / Used to iterate through memory cells */
52475	52431	Mem pEnd; / Last memory cell in new array */
52476	52432	VdbeFrame pFrame; / New vdbe frame to execute in */
52477	52433	SubProgram pProgram; / Sub-program to execute */
52478	52434	void t; / Token identifying trigger */
52479		- } bz;
	52435	+ } by;
52480	52436	struct OP_Param_stack_vars {
52481	52437	VdbeFrame *pFrame;
52482	52438	Mem *pIn;
52483		- } ca;
	52439	+ } bz;
52484	52440	struct OP_MemMax_stack_vars {
52485	52441	Mem *pIn1;
52486	52442	VdbeFrame *pFrame;
52487		- } cb;
	52443	+ } ca;
52488	52444	struct OP_AggStep_stack_vars {
52489	52445	int n;
52490	52446	int i;
52491	52447	Mem *pMem;
52492	52448	Mem *pRec;
52493	52449	sqlite3_context ctx;
52494	52450	sqlite3_value **apVal;
52495		- } cc;
	52451	+ } cb;
52496	52452	struct OP_AggFinal_stack_vars {
52497	52453	Mem *pMem;
52498		- } cd;
	52454	+ } cc;
52499	52455	struct OP_IncrVacuum_stack_vars {
52500	52456	Btree *pBt;
52501		- } ce;
	52457	+ } cd;
52502	52458	struct OP_VBegin_stack_vars {
52503	52459	VTable *pVTab;
52504		- } cf;
	52460	+ } ce;
52505	52461	struct OP_VOpen_stack_vars {
52506	52462	VdbeCursor *pCur;
52507	52463	sqlite3_vtab_cursor *pVtabCursor;
52508	52464	sqlite3_vtab *pVtab;
52509	52465	sqlite3_module *pModule;
52510		- } cg;
	52466	+ } cf;
52511	52467	struct OP_VFilter_stack_vars {
52512	52468	int nArg;
52513	52469	int iQuery;
52514	52470	const sqlite3_module *pModule;
52515	52471	Mem *pQuery;
		@@ -52518,44 +52474,44 @@
52518	52474	sqlite3_vtab *pVtab;
52519	52475	VdbeCursor *pCur;
52520	52476	int res;
52521	52477	int i;
52522	52478	Mem **apArg;
52523		- } ch;
	52479	+ } cg;
52524	52480	struct OP_VColumn_stack_vars {
52525	52481	sqlite3_vtab *pVtab;
52526	52482	const sqlite3_module *pModule;
52527	52483	Mem *pDest;
52528	52484	sqlite3_context sContext;
52529		- } ci;
	52485	+ } ch;
52530	52486	struct OP_VNext_stack_vars {
52531	52487	sqlite3_vtab *pVtab;
52532	52488	const sqlite3_module *pModule;
52533	52489	int res;
52534	52490	VdbeCursor *pCur;
52535		- } cj;
	52491	+ } ci;
52536	52492	struct OP_VRename_stack_vars {
52537	52493	sqlite3_vtab *pVtab;
52538	52494	Mem *pName;
52539		- } ck;
	52495	+ } cj;
52540	52496	struct OP_VUpdate_stack_vars {
52541	52497	sqlite3_vtab *pVtab;
52542	52498	sqlite3_module *pModule;
52543	52499	int nArg;
52544	52500	int i;
52545	52501	sqlite_int64 rowid;
52546	52502	Mem **apArg;
52547	52503	Mem *pX;
52548		- } cl;
	52504	+ } ck;
52549	52505	struct OP_Pagecount_stack_vars {
52550	52506	int p1;
52551	52507	int nPage;
52552	52508	Pager *pPager;
52553		- } cm;
	52509	+ } cl;
52554	52510	struct OP_Trace_stack_vars {
52555	52511	char *zTrace;
52556		- } cn;
	52512	+ } cm;
52557	52513	} u;
52558	52514	/* End automatically generated code
52559	52515	********************************************************************/
52560	52516
52561	52517	assert( p->magic==VDBE_MAGIC_RUN ); /* sqlite3_step() verifies this */
		@@ -52571,20 +52527,23 @@
52571	52527	assert( p->explain==0 );
52572	52528	p->pResultSet = 0;
52573	52529	db->busyHandler.nBusy = 0;
52574	52530	CHECK_FOR_INTERRUPT;
52575	52531	sqlite3VdbeIOTraceSql(p);
	52532	+#ifndef SQLITE_OMIT_PROGRESS_CALLBACK
	52533	+ checkProgress = db->xProgress!=0;
	52534	+#endif
52576	52535	#ifdef SQLITE_DEBUG
52577	52536	sqlite3BeginBenignMalloc();
52578	52537	if( p->pc==0
52579	52538	&& ((p->db->flags & SQLITE_VdbeListing) \|\| fileExists(db, "vdbe_explain"))
52580	52539	){
52581	52540	int i;
52582	52541	printf("VDBE Program Listing:\n");
52583	52542	sqlite3VdbePrintSql(p);
52584	52543	for(i=0; i<p->nOp; i++){
52585		- sqlite3VdbePrintOp(stdout, i, &p->aOp[i]);
	52544	+ sqlite3VdbePrintOp(stdout, i, &aOp[i]);
52586	52545	}
52587	52546	}
52588	52547	if( fileExists(db, "vdbe_trace") ){
52589	52548	p->trace = stdout;
52590	52549	}
		@@ -52595,11 +52554,11 @@
52595	52554	if( db->mallocFailed ) goto no_mem;
52596	52555	#ifdef VDBE_PROFILE
52597	52556	origPc = pc;
52598	52557	start = sqlite3Hwtime();
52599	52558	#endif
52600		- pOp = &p->aOp[pc];
	52559	+ pOp = &aOp[pc];
52601	52560
52602	52561	/* Only allow tracing if SQLITE_DEBUG is defined.
52603	52562	*/
52604	52563	#ifdef SQLITE_DEBUG
52605	52564	if( p->trace ){
		@@ -52636,11 +52595,11 @@
52636	52595	** of VDBE ops have been executed (either since this invocation of
52637	52596	** sqlite3VdbeExec() or since last time the progress callback was called).
52638	52597	** If the progress callback returns non-zero, exit the virtual machine with
52639	52598	** a return code SQLITE_ABORT.
52640	52599	*/
52641		- if( db->xProgress ){
	52600	+ if( checkProgress ){
52642	52601	if( db->nProgressOps==nProgressOps ){
52643	52602	int prc;
52644	52603	if( sqlite3SafetyOff(db) ) goto abort_due_to_misuse;
52645	52604	prc =db->xProgress(db->pProgressArg);
52646	52605	if( sqlite3SafetyOn(db) ) goto abort_due_to_misuse;
		@@ -52652,70 +52611,51 @@
52652	52611	}
52653	52612	nProgressOps++;
52654	52613	}
52655	52614	#endif
52656	52615
52657		- /* Do common setup processing for any opcode that is marked
52658		- ** with the "out2-prerelease" tag. Such opcodes have a single
52659		- ** output which is specified by the P2 parameter. The P2 register
52660		- ** is initialized to a NULL.
	52616	+ /* On any opcode with the "out2-prerelase" tag, free any
	52617	+ ** external allocations out of mem[p2] and set mem[p2] to be
	52618	+ ** an undefined integer. Opcodes will either fill in the integer
	52619	+ ** value or convert mem[p2] to a different type.
52661	52620	*/
52662		- opProperty = opcodeProperty[pOp->opcode];
52663		- if( (opProperty & OPFLG_OUT2_PRERELEASE)!=0 ){
	52621	+ assert( pOp->opflags==sqlite3OpcodeProperty[pOp->opcode] );
	52622	+ if( pOp->opflags & OPFLG_OUT2_PRERELEASE ){
52664	52623	assert( pOp->p2>0 );
52665	52624	assert( pOp->p2<=p->nMem );
52666		- pOut = &p->aMem[pOp->p2];
	52625	+ pOut = &aMem[pOp->p2];
52667	52626	sqlite3VdbeMemReleaseExternal(pOut);
52668		- pOut->flags = MEM_Null;
52669		- pOut->n = 0;
52670		- }else
52671		-
52672		- /* Do common setup for opcodes marked with one of the following
52673		- ** combinations of properties.
52674		- **
52675		- ** in1
52676		- ** in1 in2
52677		- ** in1 in2 out3
52678		- ** in1 in3
52679		- **
52680		- ** Variables pIn1, pIn2, and pIn3 are made to point to appropriate
52681		- ** registers for inputs. Variable pOut points to the output register.
52682		- */
52683		- if( (opProperty & OPFLG_IN1)!=0 ){
	52627	+ pOut->flags = MEM_Int;
	52628	+ }
	52629	+
	52630	+ /* Sanity checking on other operands */
	52631	+#ifdef SQLITE_DEBUG
	52632	+ if( (pOp->opflags & OPFLG_IN1)!=0 ){
52684	52633	assert( pOp->p1>0 );
52685	52634	assert( pOp->p1<=p->nMem );
52686		- pIn1 = &p->aMem[pOp->p1];
52687		- REGISTER_TRACE(pOp->p1, pIn1);
52688		- if( (opProperty & OPFLG_IN2)!=0 ){
52689		- assert( pOp->p2>0 );
52690		- assert( pOp->p2<=p->nMem );
52691		- pIn2 = &p->aMem[pOp->p2];
52692		- REGISTER_TRACE(pOp->p2, pIn2);
52693		- /* As currently implemented, in2 implies out3. There is no reason
52694		- ** why this has to be, it just worked out that way. */
52695		- assert( (opProperty & OPFLG_OUT3)!=0 );
52696		- assert( pOp->p3>0 );
52697		- assert( pOp->p3<=p->nMem );
52698		- pOut = &p->aMem[pOp->p3];
52699		- }else if( (opProperty & OPFLG_IN3)!=0 ){
52700		- assert( pOp->p3>0 );
52701		- assert( pOp->p3<=p->nMem );
52702		- pIn3 = &p->aMem[pOp->p3];
52703		- REGISTER_TRACE(pOp->p3, pIn3);
52704		- }
52705		- }else if( (opProperty & OPFLG_IN2)!=0 ){
52706		- assert( pOp->p2>0 );
52707		- assert( pOp->p2<=p->nMem );
52708		- pIn2 = &p->aMem[pOp->p2];
52709		- REGISTER_TRACE(pOp->p2, pIn2);
52710		- }else if( (opProperty & OPFLG_IN3)!=0 ){
52711		- assert( pOp->p3>0 );
52712		- assert( pOp->p3<=p->nMem );
52713		- pIn3 = &p->aMem[pOp->p3];
52714		- REGISTER_TRACE(pOp->p3, pIn3);
52715		- }
52716		-
	52635	+ REGISTER_TRACE(pOp->p1, &aMem[pOp->p1]);
	52636	+ }
	52637	+ if( (pOp->opflags & OPFLG_IN2)!=0 ){
	52638	+ assert( pOp->p2>0 );
	52639	+ assert( pOp->p2<=p->nMem );
	52640	+ REGISTER_TRACE(pOp->p2, &aMem[pOp->p2]);
	52641	+ }
	52642	+ if( (pOp->opflags & OPFLG_IN3)!=0 ){
	52643	+ assert( pOp->p3>0 );
	52644	+ assert( pOp->p3<=p->nMem );
	52645	+ REGISTER_TRACE(pOp->p3, &aMem[pOp->p3]);
	52646	+ }
	52647	+ if( (pOp->opflags & OPFLG_OUT2)!=0 ){
	52648	+ assert( pOp->p2>0 );
	52649	+ assert( pOp->p2<=p->nMem );
	52650	+ }
	52651	+ if( (pOp->opflags & OPFLG_OUT3)!=0 ){
	52652	+ assert( pOp->p3>0 );
	52653	+ assert( pOp->p3<=p->nMem );
	52654	+ }
	52655	+#endif
	52656	+
52717	52657	switch( pOp->opcode ){
52718	52658
52719	52659	/*****************************************************************************
52720	52660	** What follows is a massive switch statement where each case implements a
52721	52661	** separate instruction in the virtual machine. If we follow the usual
		@@ -52767,14 +52707,12 @@
52767	52707	/* Opcode: Gosub P1 P2 * * *
52768	52708	**
52769	52709	** Write the current address onto register P1
52770	52710	** and then jump to address P2.
52771	52711	*/
52772		-case OP_Gosub: { /* jump */
52773		- assert( pOp->p1>0 );
52774		- assert( pOp->p1<=p->nMem );
52775		- pIn1 = &p->aMem[pOp->p1];
	52712	+case OP_Gosub: { /* jump, in1 */
	52713	+ pIn1 = &aMem[pOp->p1];
52776	52714	assert( (pIn1->flags & MEM_Dyn)==0 );
52777	52715	pIn1->flags = MEM_Int;
52778	52716	pIn1->u.i = pc;
52779	52717	REGISTER_TRACE(pOp->p1, pIn1);
52780	52718	pc = pOp->p2 - 1;
		@@ -52784,10 +52722,11 @@
52784	52722	/* Opcode: Return P1 * * * *
52785	52723	**
52786	52724	** Jump to the next instruction after the address in register P1.
52787	52725	*/
52788	52726	case OP_Return: { /* in1 */
	52727	+ pIn1 = &aMem[pOp->p1];
52789	52728	assert( pIn1->flags & MEM_Int );
52790	52729	pc = (int)pIn1->u.i;
52791	52730	break;
52792	52731	}
52793	52732
		@@ -52797,10 +52736,11 @@
52797	52736	*/
52798	52737	case OP_Yield: { /* in1 */
52799	52738	#if 0 /* local variables moved into u.aa */
52800	52739	int pcDest;
52801	52740	#endif /* local variables moved into u.aa */
	52741	+ pIn1 = &aMem[pOp->p1];
52802	52742	assert( (pIn1->flags & MEM_Dyn)==0 );
52803	52743	pIn1->flags = MEM_Int;
52804	52744	u.aa.pcDest = (int)pIn1->u.i;
52805	52745	pIn1->u.i = pc;
52806	52746	REGISTER_TRACE(pOp->p1, pIn1);
		@@ -52813,10 +52753,11 @@
52813	52753	** Check the value in register P3. If is is NULL then Halt using
52814	52754	** parameter P1, P2, and P4 as if this were a Halt instruction. If the
52815	52755	** value in register P3 is not NULL, then this routine is a no-op.
52816	52756	*/
52817	52757	case OP_HaltIfNull: { /* in3 */
	52758	+ pIn3 = &aMem[pOp->p3];
52818	52759	if( (pIn3->flags & MEM_Null)==0 ) break;
52819	52760	/* Fall through into OP_Halt */
52820	52761	}
52821	52762
52822	52763	/* Opcode: Halt P1 P2 * P4 *
		@@ -52852,10 +52793,12 @@
52852	52793	** instruction is set to OE_Ignore, then the sub-program is throwing
52853	52794	** an IGNORE exception. In this case jump to the address specified
52854	52795	** as the p2 of the calling OP_Program. */
52855	52796	pc = p->aOp[pc].p2-1;
52856	52797	}
	52798	+ aOp = p->aOp;
	52799	+ aMem = p->aMem;
52857	52800	break;
52858	52801	}
52859	52802
52860	52803	p->rc = pOp->p1;
52861	52804	p->errorAction = (u8)pOp->p2;
		@@ -52878,11 +52821,10 @@
52878	52821	/* Opcode: Integer P1 P2 * * *
52879	52822	**
52880	52823	** The 32-bit integer value P1 is written into register P2.
52881	52824	*/
52882	52825	case OP_Integer: { /* out2-prerelease */
52883		- pOut->flags = MEM_Int;
52884	52826	pOut->u.i = pOp->p1;
52885	52827	break;
52886	52828	}
52887	52829
52888	52830	/* Opcode: Int64 * P2 * P4 *
		@@ -52890,11 +52832,10 @@
52890	52832	** P4 is a pointer to a 64-bit integer value.
52891	52833	** Write that value into register P2.
52892	52834	*/
52893	52835	case OP_Int64: { /* out2-prerelease */
52894	52836	assert( pOp->p4.pI64!=0 );
52895		- pOut->flags = MEM_Int;
52896	52837	pOut->u.i = *pOp->p4.pI64;
52897	52838	break;
52898	52839	}
52899	52840
52900	52841	/* Opcode: Real * P2 * P4 *
		@@ -52960,10 +52901,11 @@
52960	52901	/* Opcode: Null * P2 * * *
52961	52902	**
52962	52903	** Write a NULL into register P2.
52963	52904	*/
52964	52905	case OP_Null: { /* out2-prerelease */
	52906	+ pOut->flags = MEM_Null;
52965	52907	break;
52966	52908	}
52967	52909
52968	52910
52969	52911	/* Opcode: Blob P1 P2 * P4
		@@ -53009,11 +52951,11 @@
53009	52951	while( u.ab.n-- > 0 ){
53010	52952	u.ab.pVar = &p->aVar[u.ab.p1++];
53011	52953	if( sqlite3VdbeMemTooBig(u.ab.pVar) ){
53012	52954	goto too_big;
53013	52955	}
53014		- pOut = &p->aMem[u.ab.p2++];
	52956	+ pOut = &aMem[u.ab.p2++];
53015	52957	sqlite3VdbeMemReleaseExternal(pOut);
53016	52958	pOut->flags = MEM_Null;
53017	52959	sqlite3VdbeMemShallowCopy(pOut, u.ab.pVar, MEM_Static);
53018	52960	UPDATE_MAX_BLOBSIZE(pOut);
53019	52961	}
		@@ -53039,15 +52981,15 @@
53039	52981	u.ac.p1 = pOp->p1;
53040	52982	u.ac.p2 = pOp->p2;
53041	52983	assert( u.ac.n>0 && u.ac.p1>0 && u.ac.p2>0 );
53042	52984	assert( u.ac.p1+u.ac.n<=u.ac.p2 \|\| u.ac.p2+u.ac.n<=u.ac.p1 );
53043	52985
53044		- pIn1 = &p->aMem[u.ac.p1];
53045		- pOut = &p->aMem[u.ac.p2];
	52986	+ pIn1 = &aMem[u.ac.p1];
	52987	+ pOut = &aMem[u.ac.p2];
53046	52988	while( u.ac.n-- ){
53047		- assert( pOut<=&p->aMem[p->nMem] );
53048		- assert( pIn1<=&p->aMem[p->nMem] );
	52989	+ assert( pOut<=&aMem[p->nMem] );
	52990	+ assert( pIn1<=&aMem[p->nMem] );
53049	52991	u.ac.zMalloc = pOut->zMalloc;
53050	52992	pOut->zMalloc = 0;
53051	52993	sqlite3VdbeMemMove(pOut, pIn1);
53052	52994	pIn1->zMalloc = u.ac.zMalloc;
53053	52995	REGISTER_TRACE(u.ac.p2++, pOut);
		@@ -53062,14 +53004,13 @@
53062	53004	** Make a copy of register P1 into register P2.
53063	53005	**
53064	53006	** This instruction makes a deep copy of the value. A duplicate
53065	53007	** is made of any string or blob constant. See also OP_SCopy.
53066	53008	*/
53067		-case OP_Copy: { /* in1 */
53068		- assert( pOp->p2>0 );
53069		- assert( pOp->p2<=p->nMem );
53070		- pOut = &p->aMem[pOp->p2];
	53009	+case OP_Copy: { /* in1, out2 */
	53010	+ pIn1 = &aMem[pOp->p1];
	53011	+ pOut = &aMem[pOp->p2];
53071	53012	assert( pOut!=pIn1 );
53072	53013	sqlite3VdbeMemShallowCopy(pOut, pIn1, MEM_Ephem);
53073	53014	Deephemeralize(pOut);
53074	53015	REGISTER_TRACE(pOp->p2, pOut);
53075	53016	break;
		@@ -53085,15 +53026,13 @@
53085	53026	** Worse, if the original is deallocated, the copy becomes invalid.
53086	53027	** Thus the program must guarantee that the original will not change
53087	53028	** during the lifetime of the copy. Use OP_Copy to make a complete
53088	53029	** copy.
53089	53030	*/
53090		-case OP_SCopy: { /* in1 */
53091		- REGISTER_TRACE(pOp->p1, pIn1);
53092		- assert( pOp->p2>0 );
53093		- assert( pOp->p2<=p->nMem );
53094		- pOut = &p->aMem[pOp->p2];
	53031	+case OP_SCopy: { /* in1, out2 */
	53032	+ pIn1 = &aMem[pOp->p1];
	53033	+ pOut = &aMem[pOp->p2];
53095	53034	assert( pOut!=pIn1 );
53096	53035	sqlite3VdbeMemShallowCopy(pOut, pIn1, MEM_Ephem);
53097	53036	REGISTER_TRACE(pOp->p2, pOut);
53098	53037	break;
53099	53038	}
		@@ -53150,11 +53089,11 @@
53150	53089
53151	53090	/* Make sure the results of the current row are \000 terminated
53152	53091	** and have an assigned type. The results are de-ephemeralized as
53153	53092	** as side effect.
53154	53093	*/
53155		- u.ad.pMem = p->pResultSet = &p->aMem[pOp->p1];
	53094	+ u.ad.pMem = p->pResultSet = &aMem[pOp->p1];
53156	53095	for(u.ad.i=0; u.ad.i<pOp->p2; u.ad.i++){
53157	53096	sqlite3VdbeMemNulTerminate(&u.ad.pMem[u.ad.i]);
53158	53097	sqlite3VdbeMemStoreType(&u.ad.pMem[u.ad.i]);
53159	53098	REGISTER_TRACE(pOp->p1+u.ad.i, &u.ad.pMem[u.ad.i]);
53160	53099	}
		@@ -53182,10 +53121,13 @@
53182	53121	case OP_Concat: { /* same as TK_CONCAT, in1, in2, out3 */
53183	53122	#if 0 /* local variables moved into u.ae */
53184	53123	i64 nByte;
53185	53124	#endif /* local variables moved into u.ae */
53186	53125
	53126	+ pIn1 = &aMem[pOp->p1];
	53127	+ pIn2 = &aMem[pOp->p2];
	53128	+ pOut = &aMem[pOp->p3];
53187	53129	assert( pIn1!=pOut );
53188	53130	if( (pIn1->flags \| pIn2->flags) & MEM_Null ){
53189	53131	sqlite3VdbeMemSetNull(pOut);
53190	53132	break;
53191	53133	}
		@@ -53257,12 +53199,15 @@
53257	53199	i64 iB; /* Integer value of right operand */
53258	53200	double rA; /* Real value of left operand */
53259	53201	double rB; /* Real value of right operand */
53260	53202	#endif /* local variables moved into u.af */
53261	53203
	53204	+ pIn1 = &aMem[pOp->p1];
53262	53205	applyNumericAffinity(pIn1);
	53206	+ pIn2 = &aMem[pOp->p2];
53263	53207	applyNumericAffinity(pIn2);
	53208	+ pOut = &aMem[pOp->p3];
53264	53209	u.af.flags = pIn1->flags \| pIn2->flags;
53265	53210	if( (u.af.flags & MEM_Null)!=0 ) goto arithmetic_result_is_null;
53266	53211	if( (pIn1->flags & pIn2->flags & MEM_Int)==MEM_Int ){
53267	53212	u.af.iA = pIn1->u.i;
53268	53213	u.af.iB = pIn2->u.i;
		@@ -53375,11 +53320,11 @@
53375	53320	u.ag.apVal = p->apArg;
53376	53321	assert( u.ag.apVal \|\| u.ag.n==0 );
53377	53322
53378	53323	assert( u.ag.n==0 \|\| (pOp->p2>0 && pOp->p2+u.ag.n<=p->nMem+1) );
53379	53324	assert( pOp->p3<pOp->p2 \|\| pOp->p3>=pOp->p2+u.ag.n );
53380		- u.ag.pArg = &p->aMem[pOp->p2];
	53325	+ u.ag.pArg = &aMem[pOp->p2];
53381	53326	for(u.ag.i=0; u.ag.i<u.ag.n; u.ag.i++, u.ag.pArg++){
53382	53327	u.ag.apVal[u.ag.i] = u.ag.pArg;
53383	53328	sqlite3VdbeMemStoreType(u.ag.pArg);
53384	53329	REGISTER_TRACE(pOp->p2, u.ag.pArg);
53385	53330	}
		@@ -53392,11 +53337,11 @@
53392	53337	u.ag.ctx.pVdbeFunc = (VdbeFunc*)pOp->p4.pVdbeFunc;
53393	53338	u.ag.ctx.pFunc = u.ag.ctx.pVdbeFunc->pFunc;
53394	53339	}
53395	53340
53396	53341	assert( pOp->p3>0 && pOp->p3<=p->nMem );
53397		- pOut = &p->aMem[pOp->p3];
	53342	+ pOut = &aMem[pOp->p3];
53398	53343	u.ag.ctx.s.flags = MEM_Null;
53399	53344	u.ag.ctx.s.db = db;
53400	53345	u.ag.ctx.s.xDel = 0;
53401	53346	u.ag.ctx.s.zMalloc = 0;
53402	53347
		@@ -53407,11 +53352,11 @@
53407	53352	sqlite3VdbeMemMove(&u.ag.ctx.s, pOut);
53408	53353	MemSetTypeFlag(&u.ag.ctx.s, MEM_Null);
53409	53354
53410	53355	u.ag.ctx.isError = 0;
53411	53356	if( u.ag.ctx.pFunc->flags & SQLITE_FUNC_NEEDCOLL ){
53412		- assert( pOp>p->aOp );
	53357	+ assert( pOp>aOp );
53413	53358	assert( pOp[-1].p4type==P4_COLLSEQ );
53414	53359	assert( pOp[-1].opcode==OP_CollSeq );
53415	53360	u.ag.ctx.pColl = pOp[-1].p4.pColl;
53416	53361	}
53417	53362	if( sqlite3SafetyOff(db) ) goto abort_due_to_misuse;
		@@ -53493,10 +53438,13 @@
53493	53438	#if 0 /* local variables moved into u.ah */
53494	53439	i64 a;
53495	53440	i64 b;
53496	53441	#endif /* local variables moved into u.ah */
53497	53442
	53443	+ pIn1 = &aMem[pOp->p1];
	53444	+ pIn2 = &aMem[pOp->p2];
	53445	+ pOut = &aMem[pOp->p3];
53498	53446	if( (pIn1->flags \| pIn2->flags) & MEM_Null ){
53499	53447	sqlite3VdbeMemSetNull(pOut);
53500	53448	break;
53501	53449	}
53502	53450	u.ah.a = sqlite3VdbeIntValue(pIn2);
		@@ -53519,10 +53467,11 @@
53519	53467	** The result is always an integer.
53520	53468	**
53521	53469	** To force any register to be an integer, just add 0.
53522	53470	*/
53523	53471	case OP_AddImm: { /* in1 */
	53472	+ pIn1 = &aMem[pOp->p1];
53524	53473	sqlite3VdbeMemIntegerify(pIn1);
53525	53474	pIn1->u.i += pOp->p2;
53526	53475	break;
53527	53476	}
53528	53477
		@@ -53532,10 +53481,11 @@
53532	53481	** in P1 is not an integer and cannot be converted into an integer
53533	53482	** without data loss, then jump immediately to P2, or if P2==0
53534	53483	** raise an SQLITE_MISMATCH exception.
53535	53484	*/
53536	53485	case OP_MustBeInt: { /* jump, in1 */
	53486	+ pIn1 = &aMem[pOp->p1];
53537	53487	applyAffinity(pIn1, SQLITE_AFF_NUMERIC, encoding);
53538	53488	if( (pIn1->flags & MEM_Int)==0 ){
53539	53489	if( pOp->p2==0 ){
53540	53490	rc = SQLITE_MISMATCH;
53541	53491	goto abort_due_to_error;
		@@ -53556,10 +53506,11 @@
53556	53506	** has REAL affinity. Such column values may still be stored as
53557	53507	** integers, for space efficiency, but after extraction we want them
53558	53508	** to have only a real value.
53559	53509	*/
53560	53510	case OP_RealAffinity: { /* in1 */
	53511	+ pIn1 = &aMem[pOp->p1];
53561	53512	if( pIn1->flags & MEM_Int ){
53562	53513	sqlite3VdbeMemRealify(pIn1);
53563	53514	}
53564	53515	break;
53565	53516	}
		@@ -53573,10 +53524,11 @@
53573	53524	** are afterwards simply interpreted as text.
53574	53525	**
53575	53526	** A NULL value is not changed by this routine. It remains NULL.
53576	53527	*/
53577	53528	case OP_ToText: { /* same as TK_TO_TEXT, in1 */
	53529	+ pIn1 = &aMem[pOp->p1];
53578	53530	if( pIn1->flags & MEM_Null ) break;
53579	53531	assert( MEM_Str==(MEM_Blob>>3) );
53580	53532	pIn1->flags \|= (pIn1->flags&MEM_Blob)>>3;
53581	53533	applyAffinity(pIn1, SQLITE_AFF_TEXT, encoding);
53582	53534	rc = ExpandBlob(pIn1);
		@@ -53594,10 +53546,11 @@
53594	53546	** to the underlying data.
53595	53547	**
53596	53548	** A NULL value is not changed by this routine. It remains NULL.
53597	53549	*/
53598	53550	case OP_ToBlob: { /* same as TK_TO_BLOB, in1 */
	53551	+ pIn1 = &aMem[pOp->p1];
53599	53552	if( pIn1->flags & MEM_Null ) break;
53600	53553	if( (pIn1->flags & MEM_Blob)==0 ){
53601	53554	applyAffinity(pIn1, SQLITE_AFF_TEXT, encoding);
53602	53555	assert( pIn1->flags & MEM_Str \|\| db->mallocFailed );
53603	53556	MemSetTypeFlag(pIn1, MEM_Blob);
		@@ -53617,10 +53570,11 @@
53617	53570	** is possible.
53618	53571	**
53619	53572	** A NULL value is not changed by this routine. It remains NULL.
53620	53573	*/
53621	53574	case OP_ToNumeric: { /* same as TK_TO_NUMERIC, in1 */
	53575	+ pIn1 = &aMem[pOp->p1];
53622	53576	if( (pIn1->flags & (MEM_Null\|MEM_Int\|MEM_Real))==0 ){
53623	53577	sqlite3VdbeMemNumerify(pIn1);
53624	53578	}
53625	53579	break;
53626	53580	}
		@@ -53634,10 +53588,11 @@
53634	53588	** equivalent of atoi() and store 0 if no such conversion is possible.
53635	53589	**
53636	53590	** A NULL value is not changed by this routine. It remains NULL.
53637	53591	*/
53638	53592	case OP_ToInt: { /* same as TK_TO_INT, in1 */
	53593	+ pIn1 = &aMem[pOp->p1];
53639	53594	if( (pIn1->flags & MEM_Null)==0 ){
53640	53595	sqlite3VdbeMemIntegerify(pIn1);
53641	53596	}
53642	53597	break;
53643	53598	}
		@@ -53651,10 +53606,11 @@
53651	53606	** equivalent of atoi() and store 0.0 if no such conversion is possible.
53652	53607	**
53653	53608	** A NULL value is not changed by this routine. It remains NULL.
53654	53609	*/
53655	53610	case OP_ToReal: { /* same as TK_TO_REAL, in1 */
	53611	+ pIn1 = &aMem[pOp->p1];
53656	53612	if( (pIn1->flags & MEM_Null)==0 ){
53657	53613	sqlite3VdbeMemRealify(pIn1);
53658	53614	}
53659	53615	break;
53660	53616	}
		@@ -53741,10 +53697,12 @@
53741	53697	#if 0 /* local variables moved into u.ai */
53742	53698	int res; /* Result of the comparison of pIn1 against pIn3 */
53743	53699	char affinity; /* Affinity to use for comparison */
53744	53700	#endif /* local variables moved into u.ai */
53745	53701
	53702	+ pIn1 = &aMem[pOp->p1];
	53703	+ pIn3 = &aMem[pOp->p3];
53746	53704	if( (pIn1->flags \| pIn3->flags)&MEM_Null ){
53747	53705	/* One or both operands are NULL */
53748	53706	if( pOp->p5 & SQLITE_NULLEQ ){
53749	53707	/* If SQLITE_NULLEQ is set (which will only happen if the operator is
53750	53708	** OP_Eq or OP_Ne) then take the jump or not depending on whether
		@@ -53756,11 +53714,11 @@
53756	53714	/* SQLITE_NULLEQ is clear and at least one operand is NULL,
53757	53715	** then the result is always NULL.
53758	53716	** The jump is taken if the SQLITE_JUMPIFNULL bit is set.
53759	53717	*/
53760	53718	if( pOp->p5 & SQLITE_STOREP2 ){
53761		- pOut = &p->aMem[pOp->p2];
	53719	+ pOut = &aMem[pOp->p2];
53762	53720	MemSetTypeFlag(pOut, MEM_Null);
53763	53721	REGISTER_TRACE(pOp->p2, pOut);
53764	53722	}else if( pOp->p5 & SQLITE_JUMPIFNULL ){
53765	53723	pc = pOp->p2-1;
53766	53724	}
		@@ -53788,11 +53746,11 @@
53788	53746	case OP_Gt: u.ai.res = u.ai.res>0; break;
53789	53747	default: u.ai.res = u.ai.res>=0; break;
53790	53748	}
53791	53749
53792	53750	if( pOp->p5 & SQLITE_STOREP2 ){
53793		- pOut = &p->aMem[pOp->p2];
	53751	+ pOut = &aMem[pOp->p2];
53794	53752	MemSetTypeFlag(pOut, MEM_Int);
53795	53753	pOut->u.i = u.ai.res;
53796	53754	REGISTER_TRACE(pOp->p2, pOut);
53797	53755	}else if( u.ai.res ){
53798	53756	pc = pOp->p2-1;
		@@ -53859,16 +53817,16 @@
53859	53817	assert( u.aj.p2>0 && u.aj.p2+u.aj.n<=p->nMem+1 );
53860	53818	}
53861	53819	#endif /* SQLITE_DEBUG */
53862	53820	for(u.aj.i=0; u.aj.i<u.aj.n; u.aj.i++){
53863	53821	u.aj.idx = aPermute ? aPermute[u.aj.i] : u.aj.i;
53864		- REGISTER_TRACE(u.aj.p1+u.aj.idx, &p->aMem[u.aj.p1+u.aj.idx]);
53865		- REGISTER_TRACE(u.aj.p2+u.aj.idx, &p->aMem[u.aj.p2+u.aj.idx]);
	53822	+ REGISTER_TRACE(u.aj.p1+u.aj.idx, &aMem[u.aj.p1+u.aj.idx]);
	53823	+ REGISTER_TRACE(u.aj.p2+u.aj.idx, &aMem[u.aj.p2+u.aj.idx]);
53866	53824	assert( u.aj.i<u.aj.pKeyInfo->nField );
53867	53825	u.aj.pColl = u.aj.pKeyInfo->aColl[u.aj.i];
53868	53826	u.aj.bRev = u.aj.pKeyInfo->aSortOrder[u.aj.i];
53869		- iCompare = sqlite3MemCompare(&p->aMem[u.aj.p1+u.aj.idx], &p->aMem[u.aj.p2+u.aj.idx], u.aj.pColl);
	53827	+ iCompare = sqlite3MemCompare(&aMem[u.aj.p1+u.aj.idx], &aMem[u.aj.p2+u.aj.idx], u.aj.pColl);
53870	53828	if( iCompare ){
53871	53829	if( u.aj.bRev ) iCompare = -iCompare;
53872	53830	break;
53873	53831	}
53874	53832	}
		@@ -53916,15 +53874,17 @@
53916	53874	#if 0 /* local variables moved into u.ak */
53917	53875	int v1; /* Left operand: 0==FALSE, 1==TRUE, 2==UNKNOWN or NULL */
53918	53876	int v2; /* Right operand: 0==FALSE, 1==TRUE, 2==UNKNOWN or NULL */
53919	53877	#endif /* local variables moved into u.ak */
53920	53878
	53879	+ pIn1 = &aMem[pOp->p1];
53921	53880	if( pIn1->flags & MEM_Null ){
53922	53881	u.ak.v1 = 2;
53923	53882	}else{
53924	53883	u.ak.v1 = sqlite3VdbeIntValue(pIn1)!=0;
53925	53884	}
	53885	+ pIn2 = &aMem[pOp->p2];
53926	53886	if( pIn2->flags & MEM_Null ){
53927	53887	u.ak.v2 = 2;
53928	53888	}else{
53929	53889	u.ak.v2 = sqlite3VdbeIntValue(pIn2)!=0;
53930	53890	}
		@@ -53933,10 +53893,11 @@
53933	53893	u.ak.v1 = and_logic[u.ak.v1*3+u.ak.v2];
53934	53894	}else{
53935	53895	static const unsigned char or_logic[] = { 0, 1, 2, 1, 1, 1, 2, 1, 2 };
53936	53896	u.ak.v1 = or_logic[u.ak.v1*3+u.ak.v2];
53937	53897	}
	53898	+ pOut = &aMem[pOp->p3];
53938	53899	if( u.ak.v1==2 ){
53939	53900	MemSetTypeFlag(pOut, MEM_Null);
53940	53901	}else{
53941	53902	pOut->u.i = u.ak.v1;
53942	53903	MemSetTypeFlag(pOut, MEM_Int);
		@@ -53948,12 +53909,13 @@
53948	53909	**
53949	53910	** Interpret the value in register P1 as a boolean value. Store the
53950	53911	** boolean complement in register P2. If the value in register P1 is
53951	53912	** NULL, then a NULL is stored in P2.
53952	53913	*/
53953		-case OP_Not: { /* same as TK_NOT, in1 */
53954		- pOut = &p->aMem[pOp->p2];
	53914	+case OP_Not: { /* same as TK_NOT, in1, out2 */
	53915	+ pIn1 = &aMem[pOp->p1];
	53916	+ pOut = &aMem[pOp->p2];
53955	53917	if( pIn1->flags & MEM_Null ){
53956	53918	sqlite3VdbeMemSetNull(pOut);
53957	53919	}else{
53958	53920	sqlite3VdbeMemSetInt64(pOut, !sqlite3VdbeIntValue(pIn1));
53959	53921	}
		@@ -53964,12 +53926,13 @@
53964	53926	**
53965	53927	** Interpret the content of register P1 as an integer. Store the
53966	53928	** ones-complement of the P1 value into register P2. If P1 holds
53967	53929	** a NULL then store a NULL in P2.
53968	53930	*/
53969		-case OP_BitNot: { /* same as TK_BITNOT, in1 */
53970		- pOut = &p->aMem[pOp->p2];
	53931	+case OP_BitNot: { /* same as TK_BITNOT, in1, out2 */
	53932	+ pIn1 = &aMem[pOp->p1];
	53933	+ pOut = &aMem[pOp->p2];
53971	53934	if( pIn1->flags & MEM_Null ){
53972	53935	sqlite3VdbeMemSetNull(pOut);
53973	53936	}else{
53974	53937	sqlite3VdbeMemSetInt64(pOut, ~sqlite3VdbeIntValue(pIn1));
53975	53938	}
		@@ -53991,10 +53954,11 @@
53991	53954	case OP_If: /* jump, in1 */
53992	53955	case OP_IfNot: { /* jump, in1 */
53993	53956	#if 0 /* local variables moved into u.al */
53994	53957	int c;
53995	53958	#endif /* local variables moved into u.al */
	53959	+ pIn1 = &aMem[pOp->p1];
53996	53960	if( pIn1->flags & MEM_Null ){
53997	53961	u.al.c = pOp->p3;
53998	53962	}else{
53999	53963	#ifdef SQLITE_OMIT_FLOATING_POINT
54000	53964	u.al.c = sqlite3VdbeIntValue(pIn1)!=0;
		@@ -54012,10 +53976,11 @@
54012	53976	/* Opcode: IsNull P1 P2 * * *
54013	53977	**
54014	53978	** Jump to P2 if the value in register P1 is NULL.
54015	53979	*/
54016	53980	case OP_IsNull: { /* same as TK_ISNULL, jump, in1 */
	53981	+ pIn1 = &aMem[pOp->p1];
54017	53982	if( (pIn1->flags & MEM_Null)!=0 ){
54018	53983	pc = pOp->p2 - 1;
54019	53984	}
54020	53985	break;
54021	53986	}
		@@ -54023,10 +53988,11 @@
54023	53988	/* Opcode: NotNull P1 P2 * * *
54024	53989	**
54025	53990	** Jump to P2 if the value in register P1 is not NULL.
54026	53991	*/
54027	53992	case OP_NotNull: { /* same as TK_NOTNULL, jump, in1 */
	53993	+ pIn1 = &aMem[pOp->p1];
54028	53994	if( (pIn1->flags & MEM_Null)==0 ){
54029	53995	pc = pOp->p2 - 1;
54030	53996	}
54031	53997	break;
54032	53998	}
		@@ -54081,11 +54047,11 @@
54081	54047	u.am.p2 = pOp->p2;
54082	54048	u.am.pC = 0;
54083	54049	memset(&u.am.sMem, 0, sizeof(u.am.sMem));
54084	54050	assert( u.am.p1<p->nCursor );
54085	54051	assert( pOp->p3>0 && pOp->p3<=p->nMem );
54086		- u.am.pDest = &p->aMem[pOp->p3];
	54052	+ u.am.pDest = &aMem[pOp->p3];
54087	54053	MemSetTypeFlag(u.am.pDest, MEM_Null);
54088	54054	u.am.zRec = 0;
54089	54055
54090	54056	/* This block sets the variable u.am.payloadSize to be the total number of
54091	54057	** bytes in the record.
		@@ -54127,11 +54093,11 @@
54127	54093	assert( sqlite3BtreeCursorIsValid(u.am.pCrsr) );
54128	54094	rc = sqlite3BtreeDataSize(u.am.pCrsr, &u.am.payloadSize);
54129	54095	assert( rc==SQLITE_OK ); /* DataSize() cannot fail */
54130	54096	}
54131	54097	}else if( u.am.pC->pseudoTableReg>0 ){
54132		- u.am.pReg = &p->aMem[u.am.pC->pseudoTableReg];
	54098	+ u.am.pReg = &aMem[u.am.pC->pseudoTableReg];
54133	54099	assert( u.am.pReg->flags & MEM_Blob );
54134	54100	u.am.payloadSize = u.am.pReg->n;
54135	54101	u.am.zRec = u.am.pReg->z;
54136	54102	u.am.pC->cacheStatus = (pOp->p5&OPFLAG_CLEARCACHE) ? CACHE_STALE : p->cacheCtr;
54137	54103	assert( u.am.payloadSize==0 \|\| u.am.zRec!=0 );
		@@ -54339,22 +54305,23 @@
54339	54305	** string indicates the column affinity that should be used for the nth
54340	54306	** memory cell in the range.
54341	54307	*/
54342	54308	case OP_Affinity: {
54343	54309	#if 0 /* local variables moved into u.an */
54344		- char zAffinity; / The affinity to be applied */
54345		- Mem pData0; / First register to which to apply affinity */
54346		- Mem pLast; / Last register to which to apply affinity */
54347		- Mem pRec; / Current register */
	54310	+ const char zAffinity; / The affinity to be applied */
	54311	+ char cAff; /* A single character of affinity */
54348	54312	#endif /* local variables moved into u.an */
54349	54313
54350	54314	u.an.zAffinity = pOp->p4.z;
54351		- u.an.pData0 = &p->aMem[pOp->p1];
54352		- u.an.pLast = &u.an.pData0[pOp->p2-1];
54353		- for(u.an.pRec=u.an.pData0; u.an.pRec<=u.an.pLast; u.an.pRec++){
54354		- ExpandBlob(u.an.pRec);
54355		- applyAffinity(u.an.pRec, u.an.zAffinity[u.an.pRec-u.an.pData0], encoding);
	54315	+ assert( u.an.zAffinity!=0 );
	54316	+ assert( u.an.zAffinity[pOp->p2]==0 );
	54317	+ pIn1 = &aMem[pOp->p1];
	54318	+ while( (u.an.cAff = *(u.an.zAffinity++))!=0 ){
	54319	+ assert( pIn1 <= &p->aMem[p->nMem] );
	54320	+ ExpandBlob(pIn1);
	54321	+ applyAffinity(pIn1, u.an.cAff, encoding);
	54322	+ pIn1++;
54356	54323	}
54357	54324	break;
54358	54325	}
54359	54326
54360	54327	/* Opcode: MakeRecord P1 P2 P3 P4 *
		@@ -54414,11 +54381,11 @@
54414	54381	u.ao.nByte = 0; /* Data space required for this record */
54415	54382	u.ao.nZero = 0; /* Number of zero bytes at the end of the record */
54416	54383	u.ao.nField = pOp->p1;
54417	54384	u.ao.zAffinity = pOp->p4.z;
54418	54385	assert( u.ao.nField>0 && pOp->p2>0 && pOp->p2+u.ao.nField<=p->nMem+1 );
54419		- u.ao.pData0 = &p->aMem[u.ao.nField];
	54386	+ u.ao.pData0 = &aMem[u.ao.nField];
54420	54387	u.ao.nField = pOp->p2;
54421	54388	u.ao.pLast = &u.ao.pData0[u.ao.nField-1];
54422	54389	u.ao.file_format = p->minWriteFileFormat;
54423	54390
54424	54391	/* Loop through the elements that will make up the record to figure
		@@ -54458,11 +54425,11 @@
54458	54425	** the new record. The output register (pOp->p3) is not allowed to
54459	54426	** be one of the input registers (because the following call to
54460	54427	** sqlite3VdbeMemGrow() could clobber the value before it is used).
54461	54428	*/
54462	54429	assert( pOp->p3<pOp->p1 \|\| pOp->p3>=pOp->p1+pOp->p2 );
54463		- pOut = &p->aMem[pOp->p3];
	54430	+ pOut = &aMem[pOp->p3];
54464	54431	if( sqlite3VdbeMemGrow(pOut, (int)u.ao.nByte, 0) ){
54465	54432	goto no_mem;
54466	54433	}
54467	54434	u.ao.zNewRecord = (u8 *)pOut->z;
54468	54435
		@@ -54507,11 +54474,10 @@
54507	54474	if( u.ap.pCrsr ){
54508	54475	rc = sqlite3BtreeCount(u.ap.pCrsr, &u.ap.nEntry);
54509	54476	}else{
54510	54477	u.ap.nEntry = 0;
54511	54478	}
54512		- pOut->flags = MEM_Int;
54513	54479	pOut->u.i = u.ap.nEntry;
54514	54480	break;
54515	54481	}
54516	54482	#endif
54517	54483
		@@ -54836,11 +54802,10 @@
54836	54802	assert( db->aDb[u.at.iDb].pBt!=0 );
54837	54803	assert( (p->btreeMask & (1<<u.at.iDb))!=0 );
54838	54804
54839	54805	sqlite3BtreeGetMeta(db->aDb[u.at.iDb].pBt, u.at.iCookie, (u32 *)&u.at.iMeta);
54840	54806	pOut->u.i = u.at.iMeta;
54841		- MemSetTypeFlag(pOut, MEM_Int);
54842	54807	break;
54843	54808	}
54844	54809
54845	54810	/* Opcode: SetCookie P1 P2 P3 * *
54846	54811	**
		@@ -54859,10 +54824,11 @@
54859	54824	assert( pOp->p2<SQLITE_N_BTREE_META );
54860	54825	assert( pOp->p1>=0 && pOp->p1<db->nDb );
54861	54826	assert( (p->btreeMask & (1<<pOp->p1))!=0 );
54862	54827	u.au.pDb = &db->aDb[pOp->p1];
54863	54828	assert( u.au.pDb->pBt!=0 );
	54829	+ pIn3 = &aMem[pOp->p3];
54864	54830	sqlite3VdbeMemIntegerify(pIn3);
54865	54831	/* See note about index shifting on OP_ReadCookie */
54866	54832	rc = sqlite3BtreeUpdateMeta(u.au.pDb->pBt, pOp->p2, (int)pIn3->u.i);
54867	54833	if( pOp->p2==BTREE_SCHEMA_VERSION ){
54868	54834	/* When the schema cookie changes, record the new cookie internally */
		@@ -55021,11 +54987,11 @@
55021	54987	u.aw.wrFlag = 0;
55022	54988	}
55023	54989	if( pOp->p5 ){
55024	54990	assert( u.aw.p2>0 );
55025	54991	assert( u.aw.p2<=p->nMem );
55026		- pIn2 = &p->aMem[u.aw.p2];
	54992	+ pIn2 = &aMem[u.aw.p2];
55027	54993	sqlite3VdbeMemIntegerify(pIn2);
55028	54994	u.aw.p2 = (int)pIn2->u.i;
55029	54995	/* The u.aw.p2 value always comes from a prior OP_CreateTable opcode and
55030	54996	** that opcode will always set the u.aw.p2 value to 2 or more or else fail.
55031	54997	** If there were a failure, the prepared statement would have halted
		@@ -55243,17 +55209,21 @@
55243	55209	assert( pOp->p1>=0 && pOp->p1<p->nCursor );
55244	55210	assert( pOp->p2!=0 );
55245	55211	u.az.pC = p->apCsr[pOp->p1];
55246	55212	assert( u.az.pC!=0 );
55247	55213	assert( u.az.pC->pseudoTableReg==0 );
	55214	+ assert( OP_SeekLe == OP_SeekLt+1 );
	55215	+ assert( OP_SeekGe == OP_SeekLt+2 );
	55216	+ assert( OP_SeekGt == OP_SeekLt+3 );
55248	55217	if( u.az.pC->pCursor!=0 ){
55249	55218	u.az.oc = pOp->opcode;
55250	55219	u.az.pC->nullRow = 0;
55251	55220	if( u.az.pC->isTable ){
55252	55221	/* The input value in P3 might be of any type: integer, real, string,
55253	55222	** blob, or NULL. But it needs to be an integer before we can do
55254	55223	** the seek, so covert it. */
	55224	+ pIn3 = &aMem[pOp->p3];
55255	55225	applyNumericAffinity(pIn3);
55256	55226	u.az.iKey = sqlite3VdbeIntValue(pIn3);
55257	55227	u.az.pC->rowidIsValid = 0;
55258	55228
55259	55229	/* If the P3 value could not be converted into an integer without
		@@ -55272,16 +55242,16 @@
55272	55242	if( u.az.iKey==SMALLEST_INT64 && (pIn3->r<(double)u.az.iKey \|\| pIn3->r>0) ){
55273	55243	/* The P3 value is too large in magnitude to be expressed as an
55274	55244	** integer. */
55275	55245	u.az.res = 1;
55276	55246	if( pIn3->r<0 ){
55277		- if( u.az.oc==OP_SeekGt \|\| u.az.oc==OP_SeekGe ){
	55247	+ if( u.az.oc>=OP_SeekGe ){ assert( u.az.oc==OP_SeekGe \|\| u.az.oc==OP_SeekGt );
55278	55248	rc = sqlite3BtreeFirst(u.az.pC->pCursor, &u.az.res);
55279	55249	if( rc!=SQLITE_OK ) goto abort_due_to_error;
55280	55250	}
55281	55251	}else{
55282		- if( u.az.oc==OP_SeekLt \|\| u.az.oc==OP_SeekLe ){
	55252	+ if( u.az.oc<=OP_SeekLe ){ assert( u.az.oc==OP_SeekLt \|\| u.az.oc==OP_SeekLe );
55283	55253	rc = sqlite3BtreeLast(u.az.pC->pCursor, &u.az.res);
55284	55254	if( rc!=SQLITE_OK ) goto abort_due_to_error;
55285	55255	}
55286	55256	}
55287	55257	if( u.az.res ){
		@@ -55309,16 +55279,26 @@
55309	55279	u.az.nField = pOp->p4.i;
55310	55280	assert( pOp->p4type==P4_INT32 );
55311	55281	assert( u.az.nField>0 );
55312	55282	u.az.r.pKeyInfo = u.az.pC->pKeyInfo;
55313	55283	u.az.r.nField = (u16)u.az.nField;
55314		- if( u.az.oc==OP_SeekGt \|\| u.az.oc==OP_SeekLe ){
55315		- u.az.r.flags = UNPACKED_INCRKEY;
55316		- }else{
55317		- u.az.r.flags = 0;
55318		- }
55319		- u.az.r.aMem = &p->aMem[pOp->p3];
	55284	+
	55285	+ /* The next line of code computes as follows, only faster:
	55286	+ ** if( u.az.oc==OP_SeekGt \|\| u.az.oc==OP_SeekLe ){
	55287	+ ** u.az.r.flags = UNPACKED_INCRKEY;
	55288	+ ** }else{
	55289	+ ** u.az.r.flags = 0;
	55290	+ ** }
	55291	+ */
	55292	+ u.az.r.flags = UNPACKED_INCRKEY * (1 & (u.az.oc - OP_SeekLt));
	55293	+ assert( u.az.oc!=OP_SeekGt \|\| u.az.r.flags==UNPACKED_INCRKEY );
	55294	+ assert( u.az.oc!=OP_SeekLe \|\| u.az.r.flags==UNPACKED_INCRKEY );
	55295	+ assert( u.az.oc!=OP_SeekGe \|\| u.az.r.flags==0 );
	55296	+ assert( u.az.oc!=OP_SeekLt \|\| u.az.r.flags==0 );
	55297	+
	55298	+ u.az.r.aMem = &aMem[pOp->p3];
	55299	+ ExpandBlob(u.az.r.aMem);
55320	55300	rc = sqlite3BtreeMovetoUnpacked(u.az.pC->pCursor, &u.az.r, 0, 0, &u.az.res);
55321	55301	if( rc!=SQLITE_OK ){
55322	55302	goto abort_due_to_error;
55323	55303	}
55324	55304	u.az.pC->rowidIsValid = 0;
		@@ -55326,11 +55306,11 @@
55326	55306	u.az.pC->deferredMoveto = 0;
55327	55307	u.az.pC->cacheStatus = CACHE_STALE;
55328	55308	#ifdef SQLITE_TEST
55329	55309	sqlite3_search_count++;
55330	55310	#endif
55331		- if( u.az.oc==OP_SeekGe \|\| u.az.oc==OP_SeekGt ){
	55311	+ if( u.az.oc>=OP_SeekGe ){ assert( u.az.oc==OP_SeekGe \|\| u.az.oc==OP_SeekGt );
55332	55312	if( u.az.res<0 \|\| (u.az.res==0 && u.az.oc==OP_SeekGt) ){
55333	55313	rc = sqlite3BtreeNext(u.az.pC->pCursor, &u.az.res);
55334	55314	if( rc!=SQLITE_OK ) goto abort_due_to_error;
55335	55315	u.az.pC->rowidIsValid = 0;
55336	55316	}else{
		@@ -55381,45 +55361,40 @@
55381	55361	u.ba.pC = p->apCsr[pOp->p1];
55382	55362	assert( u.ba.pC!=0 );
55383	55363	if( ALWAYS(u.ba.pC->pCursor!=0) ){
55384	55364	assert( u.ba.pC->isTable );
55385	55365	u.ba.pC->nullRow = 0;
	55366	+ pIn2 = &aMem[pOp->p2];
55386	55367	u.ba.pC->movetoTarget = sqlite3VdbeIntValue(pIn2);
55387	55368	u.ba.pC->rowidIsValid = 0;
55388	55369	u.ba.pC->deferredMoveto = 1;
55389	55370	}
55390	55371	break;
55391	55372	}
55392	55373
55393	55374
55394		-/* Opcode: Found P1 P2 P3 * *
55395		-**
55396		-** Register P3 holds a blob constructed by MakeRecord. P1 is an index.
55397		-** If an entry that matches the value in register p3 exists in P1 then
55398		-** jump to P2. If the P3 value does not match any entry in P1
55399		-** then fall thru. The P1 cursor is left pointing at the matching entry
55400		-** if it exists.
55401		-**
55402		-** This instruction is used to implement the IN operator where the
55403		-** left-hand side is a SELECT statement. P1 may be a true index, or it
55404		-** may be a temporary index that holds the results of the SELECT
55405		-** statement. This instruction is also used to implement the
55406		-** DISTINCT keyword in SELECT statements.
55407		-**
55408		-** This instruction checks if index P1 contains a record for which
55409		-** the first N serialized values exactly match the N serialized values
55410		-** in the record in register P3, where N is the total number of values in
55411		-** the P3 record (the P3 record is a prefix of the P1 record).
55412		-**
55413		-** See also: NotFound, IsUnique, NotExists
55414		-*/
55415		-/* Opcode: NotFound P1 P2 P3 * *
55416		-**
55417		-** Register P3 holds a blob constructed by MakeRecord. P1 is
55418		-** an index. If no entry exists in P1 that matches the blob then jump
55419		-** to P2. If an entry does existing, fall through. The cursor is left
55420		-** pointing to the entry that matches.
	55375	+/* Opcode: Found P1 P2 P3 P4 *
	55376	+**
	55377	+** If P4==0 then register P3 holds a blob constructed by MakeRecord. If
	55378	+** P4>0 then register P3 is the first of P4 registers that form an unpacked
	55379	+** record.
	55380	+**
	55381	+** Cursor P1 is on an index btree. If the record identified by P3 and P4
	55382	+** is a prefix of any entry in P1 then a jump is made to P2 and
	55383	+** P1 is left pointing at the matching entry.
	55384	+*/
	55385	+/* Opcode: NotFound P1 P2 P3 P4 *
	55386	+**
	55387	+** If P4==0 then register P3 holds a blob constructed by MakeRecord. If
	55388	+** P4>0 then register P3 is the first of P4 registers that form an unpacked
	55389	+** record.
	55390	+**
	55391	+** Cursor P1 is on an index btree. If the record identified by P3 and P4
	55392	+** is not the prefix of any entry in P1 then a jump is made to P2. If P1
	55393	+** does contain an entry whose prefix matches the P3/P4 record then control
	55394	+** falls through to the next instruction and P1 is left pointing at the
	55395	+** matching entry.
55421	55396	**
55422	55397	** See also: Found, NotExists, IsUnique
55423	55398	*/
55424	55399	case OP_NotFound: /* jump, in3 */
55425	55400	case OP_Found: { /* jump, in3 */
		@@ -55426,36 +55401,47 @@
55426	55401	#if 0 /* local variables moved into u.bb */
55427	55402	int alreadyExists;
55428	55403	VdbeCursor *pC;
55429	55404	int res;
55430	55405	UnpackedRecord *pIdxKey;
	55406	+ UnpackedRecord r;
55431	55407	char aTempRec[ROUND8(sizeof(UnpackedRecord)) + sizeof(Mem)*3 + 7];
55432	55408	#endif /* local variables moved into u.bb */
55433	55409
55434	55410	#ifdef SQLITE_TEST
55435	55411	sqlite3_found_count++;
55436	55412	#endif
55437	55413
55438	55414	u.bb.alreadyExists = 0;
55439	55415	assert( pOp->p1>=0 && pOp->p1<p->nCursor );
	55416	+ assert( pOp->p4type==P4_INT32 );
55440	55417	u.bb.pC = p->apCsr[pOp->p1];
55441	55418	assert( u.bb.pC!=0 );
	55419	+ pIn3 = &aMem[pOp->p3];
55442	55420	if( ALWAYS(u.bb.pC->pCursor!=0) ){
55443	55421
55444	55422	assert( u.bb.pC->isTable==0 );
55445		- assert( pIn3->flags & MEM_Blob );
55446		- ExpandBlob(pIn3);
55447		- u.bb.pIdxKey = sqlite3VdbeRecordUnpack(u.bb.pC->pKeyInfo, pIn3->n, pIn3->z,
55448		- u.bb.aTempRec, sizeof(u.bb.aTempRec));
55449		- if( u.bb.pIdxKey==0 ){
55450		- goto no_mem;
55451		- }
55452		- if( pOp->opcode==OP_Found ){
	55423	+ if( pOp->p4.i>0 ){
	55424	+ u.bb.r.pKeyInfo = u.bb.pC->pKeyInfo;
	55425	+ u.bb.r.nField = pOp->p4.i;
	55426	+ u.bb.r.aMem = pIn3;
	55427	+ u.bb.r.flags = UNPACKED_PREFIX_MATCH;
	55428	+ u.bb.pIdxKey = &u.bb.r;
	55429	+ }else{
	55430	+ assert( pIn3->flags & MEM_Blob );
	55431	+ ExpandBlob(pIn3);
	55432	+ u.bb.pIdxKey = sqlite3VdbeRecordUnpack(u.bb.pC->pKeyInfo, pIn3->n, pIn3->z,
	55433	+ u.bb.aTempRec, sizeof(u.bb.aTempRec));
	55434	+ if( u.bb.pIdxKey==0 ){
	55435	+ goto no_mem;
	55436	+ }
55453	55437	u.bb.pIdxKey->flags \|= UNPACKED_PREFIX_MATCH;
55454	55438	}
55455	55439	rc = sqlite3BtreeMovetoUnpacked(u.bb.pC->pCursor, u.bb.pIdxKey, 0, 0, &u.bb.res);
55456		- sqlite3VdbeDeleteUnpackedRecord(u.bb.pIdxKey);
	55440	+ if( pOp->p4.i==0 ){
	55441	+ sqlite3VdbeDeleteUnpackedRecord(u.bb.pIdxKey);
	55442	+ }
55457	55443	if( rc!=SQLITE_OK ){
55458	55444	break;
55459	55445	}
55460	55446	u.bb.alreadyExists = (u.bb.res==0);
55461	55447	u.bb.pC->deferredMoveto = 0;
		@@ -55469,13 +55455,14 @@
55469	55455	break;
55470	55456	}
55471	55457
55472	55458	/* Opcode: IsUnique P1 P2 P3 P4 *
55473	55459	**
55474		-** Cursor P1 is open on an index. So it has no data and its key consists
55475		-** of a record generated by OP_MakeRecord where the last field is the
55476		-** rowid of the entry that the index refers to.
	55460	+** Cursor P1 is open on an index b-tree - that is to say, a btree which
	55461	+** no data and where the key are records generated by OP_MakeRecord with
	55462	+** the list field being the integer ROWID of the entry that the index
	55463	+** entry refers to.
55477	55464	**
55478	55465	** The P3 register contains an integer record number. Call this record
55479	55466	** number R. Register P4 is the first in a set of N contiguous registers
55480	55467	** that make up an unpacked index key that can be used with cursor P1.
55481	55468	** The value of N can be inferred from the cursor. N includes the rowid
		@@ -55498,16 +55485,17 @@
55498	55485	#if 0 /* local variables moved into u.bc */
55499	55486	u16 ii;
55500	55487	VdbeCursor *pCx;
55501	55488	BtCursor *pCrsr;
55502	55489	u16 nField;
55503		- Mem *aMem;
	55490	+ Mem *aMx;
55504	55491	UnpackedRecord r; /* B-Tree index search key */
55505	55492	i64 R; /* Rowid stored in register P3 */
55506	55493	#endif /* local variables moved into u.bc */
55507	55494
55508		- u.bc.aMem = &p->aMem[pOp->p4.i];
	55495	+ pIn3 = &aMem[pOp->p3];
	55496	+ u.bc.aMx = &aMem[pOp->p4.i];
55509	55497	/* Assert that the values of parameters P1 and P4 are in range. */
55510	55498	assert( pOp->p4type==P4_INT32 );
55511	55499	assert( pOp->p4.i>0 && pOp->p4.i<=p->nMem );
55512	55500	assert( pOp->p1>=0 && pOp->p1<p->nCursor );
55513	55501
		@@ -55519,24 +55507,24 @@
55519	55507	u.bc.pCrsr = u.bc.pCx->pCursor;
55520	55508
55521	55509	/* If any of the values are NULL, take the jump. */
55522	55510	u.bc.nField = u.bc.pCx->pKeyInfo->nField;
55523	55511	for(u.bc.ii=0; u.bc.ii<u.bc.nField; u.bc.ii++){
55524		- if( u.bc.aMem[u.bc.ii].flags & MEM_Null ){
	55512	+ if( u.bc.aMx[u.bc.ii].flags & MEM_Null ){
55525	55513	pc = pOp->p2 - 1;
55526	55514	u.bc.pCrsr = 0;
55527	55515	break;
55528	55516	}
55529	55517	}
55530		- assert( (u.bc.aMem[u.bc.nField].flags & MEM_Null)==0 );
	55518	+ assert( (u.bc.aMx[u.bc.nField].flags & MEM_Null)==0 );
55531	55519
55532	55520	if( u.bc.pCrsr!=0 ){
55533	55521	/* Populate the index search key. */
55534	55522	u.bc.r.pKeyInfo = u.bc.pCx->pKeyInfo;
55535	55523	u.bc.r.nField = u.bc.nField + 1;
55536	55524	u.bc.r.flags = UNPACKED_PREFIX_SEARCH;
55537		- u.bc.r.aMem = u.bc.aMem;
	55525	+ u.bc.r.aMem = u.bc.aMx;
55538	55526
55539	55527	/* Extract the value of u.bc.R from register P3. */
55540	55528	sqlite3VdbeMemIntegerify(pIn3);
55541	55529	u.bc.R = pIn3->u.i;
55542	55530
		@@ -55573,10 +55561,11 @@
55573	55561	BtCursor *pCrsr;
55574	55562	int res;
55575	55563	u64 iKey;
55576	55564	#endif /* local variables moved into u.bd */
55577	55565
	55566	+ pIn3 = &aMem[pOp->p3];
55578	55567	assert( pIn3->flags & MEM_Int );
55579	55568	assert( pOp->p1>=0 && pOp->p1<p->nCursor );
55580	55569	u.bd.pC = p->apCsr[pOp->p1];
55581	55570	assert( u.bd.pC!=0 );
55582	55571	assert( u.bd.pC->isTable );
		@@ -55616,11 +55605,10 @@
55616	55605	*/
55617	55606	case OP_Sequence: { /* out2-prerelease */
55618	55607	assert( pOp->p1>=0 && pOp->p1<p->nCursor );
55619	55608	assert( p->apCsr[pOp->p1]!=0 );
55620	55609	pOut->u.i = p->apCsr[pOp->p1]->seqCount++;
55621		- MemSetTypeFlag(pOut, MEM_Int);
55622	55610	break;
55623	55611	}
55624	55612
55625	55613
55626	55614	/* Opcode: NewRowid P1 P2 P3 * *
		@@ -55712,11 +55700,11 @@
55712	55700	assert( pOp->p3<=u.be.pFrame->nMem );
55713	55701	u.be.pMem = &u.be.pFrame->aMem[pOp->p3];
55714	55702	}else{
55715	55703	/* Assert that P3 is a valid memory cell. */
55716	55704	assert( pOp->p3<=p->nMem );
55717		- u.be.pMem = &p->aMem[pOp->p3];
	55705	+ u.be.pMem = &aMem[pOp->p3];
55718	55706	}
55719	55707
55720	55708	REGISTER_TRACE(pOp->p3, u.be.pMem);
55721	55709	sqlite3VdbeMemIntegerify(u.be.pMem);
55722	55710	assert( (u.be.pMem->flags & MEM_Int)!=0 ); /* mem(P3) holds an integer */
		@@ -55755,11 +55743,10 @@
55755	55743	}
55756	55744	u.be.pC->rowidIsValid = 0;
55757	55745	u.be.pC->deferredMoveto = 0;
55758	55746	u.be.pC->cacheStatus = CACHE_STALE;
55759	55747	}
55760		- MemSetTypeFlag(pOut, MEM_Int);
55761	55748	pOut->u.i = u.be.v;
55762	55749	break;
55763	55750	}
55764	55751
55765	55752	/* Opcode: Insert P1 P2 P3 P4 P5
		@@ -55818,21 +55805,21 @@
55818	55805	const char zDb; / database name - used by the update hook */
55819	55806	const char zTbl; / Table name - used by the opdate hook */
55820	55807	int op; /* Opcode for update hook: SQLITE_UPDATE or SQLITE_INSERT */
55821	55808	#endif /* local variables moved into u.bf */
55822	55809
55823		- u.bf.pData = &p->aMem[pOp->p2];
	55810	+ u.bf.pData = &aMem[pOp->p2];
55824	55811	assert( pOp->p1>=0 && pOp->p1<p->nCursor );
55825	55812	u.bf.pC = p->apCsr[pOp->p1];
55826	55813	assert( u.bf.pC!=0 );
55827	55814	assert( u.bf.pC->pCursor!=0 );
55828	55815	assert( u.bf.pC->pseudoTableReg==0 );
55829	55816	assert( u.bf.pC->isTable );
55830	55817	REGISTER_TRACE(pOp->p2, u.bf.pData);
55831	55818
55832	55819	if( pOp->opcode==OP_Insert ){
55833		- u.bf.pKey = &p->aMem[pOp->p3];
	55820	+ u.bf.pKey = &aMem[pOp->p3];
55834	55821	assert( u.bf.pKey->flags & MEM_Int );
55835	55822	REGISTER_TRACE(pOp->p3, u.bf.pKey);
55836	55823	u.bf.iKey = u.bf.pKey->u.i;
55837	55824	}else{
55838	55825	assert( pOp->opcode==OP_InsertInt );
		@@ -55980,11 +55967,11 @@
55980	55967	BtCursor *pCrsr;
55981	55968	u32 n;
55982	55969	i64 n64;
55983	55970	#endif /* local variables moved into u.bh */
55984	55971
55985		- pOut = &p->aMem[pOp->p2];
	55972	+ pOut = &aMem[pOp->p2];
55986	55973
55987	55974	/* Note that RowKey and RowData are really exactly the same instruction */
55988	55975	assert( pOp->p1>=0 && pOp->p1<p->nCursor );
55989	55976	u.bh.pC = p->apCsr[pOp->p1];
55990	55977	assert( u.bh.pC->isTable \|\| pOp->opcode==OP_RowKey );
		@@ -56055,11 +56042,11 @@
56055	56042	assert( pOp->p1>=0 && pOp->p1<p->nCursor );
56056	56043	u.bi.pC = p->apCsr[pOp->p1];
56057	56044	assert( u.bi.pC!=0 );
56058	56045	assert( u.bi.pC->pseudoTableReg==0 );
56059	56046	if( u.bi.pC->nullRow ){
56060		- /* Do nothing so that reg[P2] remains NULL */
	56047	+ pOut->flags = MEM_Null;
56061	56048	break;
56062	56049	}else if( u.bi.pC->deferredMoveto ){
56063	56050	u.bi.v = u.bi.pC->movetoTarget;
56064	56051	#ifndef SQLITE_OMIT_VIRTUALTABLE
56065	56052	}else if( u.bi.pC->pVtabCursor ){
		@@ -56083,11 +56070,10 @@
56083	56070	rc = sqlite3BtreeKeySize(u.bi.pC->pCursor, &u.bi.v);
56084	56071	assert( rc==SQLITE_OK ); /* Always so because of CursorMoveto() above */
56085	56072	}
56086	56073	}
56087	56074	pOut->u.i = u.bi.v;
56088		- MemSetTypeFlag(pOut, MEM_Int);
56089	56075	break;
56090	56076	}
56091	56077
56092	56078	/* Opcode: NullRow P1 * * * *
56093	56079	**
		@@ -56278,10 +56264,11 @@
56278	56264	#endif /* local variables moved into u.bn */
56279	56265
56280	56266	assert( pOp->p1>=0 && pOp->p1<p->nCursor );
56281	56267	u.bn.pC = p->apCsr[pOp->p1];
56282	56268	assert( u.bn.pC!=0 );
	56269	+ pIn2 = &aMem[pOp->p2];
56283	56270	assert( pIn2->flags & MEM_Blob );
56284	56271	u.bn.pCrsr = u.bn.pC->pCursor;
56285	56272	if( ALWAYS(u.bn.pCrsr!=0) ){
56286	56273	assert( u.bn.pC->isTable==0 );
56287	56274	rc = ExpandBlob(pIn2);
		@@ -56320,11 +56307,11 @@
56320	56307	u.bo.pCrsr = u.bo.pC->pCursor;
56321	56308	if( ALWAYS(u.bo.pCrsr!=0) ){
56322	56309	u.bo.r.pKeyInfo = u.bo.pC->pKeyInfo;
56323	56310	u.bo.r.nField = (u16)pOp->p3;
56324	56311	u.bo.r.flags = 0;
56325		- u.bo.r.aMem = &p->aMem[pOp->p2];
	56312	+ u.bo.r.aMem = &aMem[pOp->p2];
56326	56313	rc = sqlite3BtreeMovetoUnpacked(u.bo.pCrsr, &u.bo.r, 0, 0, &u.bo.res);
56327	56314	if( rc==SQLITE_OK && u.bo.res==0 ){
56328	56315	rc = sqlite3BtreeDelete(u.bo.pCrsr);
56329	56316	}
56330	56317	assert( u.bo.pC->deferredMoveto==0 );
		@@ -56350,10 +56337,11 @@
56350	56337
56351	56338	assert( pOp->p1>=0 && pOp->p1<p->nCursor );
56352	56339	u.bp.pC = p->apCsr[pOp->p1];
56353	56340	assert( u.bp.pC!=0 );
56354	56341	u.bp.pCrsr = u.bp.pC->pCursor;
	56342	+ pOut->flags = MEM_Null;
56355	56343	if( ALWAYS(u.bp.pCrsr!=0) ){
56356	56344	rc = sqlite3VdbeCursorMoveto(u.bp.pC);
56357	56345	if( NEVER(rc) ) goto abort_due_to_error;
56358	56346	assert( u.bp.pC->deferredMoveto==0 );
56359	56347	assert( u.bp.pC->isTable==0 );
		@@ -56360,12 +56348,12 @@
56360	56348	if( !u.bp.pC->nullRow ){
56361	56349	rc = sqlite3VdbeIdxRowid(db, u.bp.pCrsr, &u.bp.rowid);
56362	56350	if( rc!=SQLITE_OK ){
56363	56351	goto abort_due_to_error;
56364	56352	}
56365		- MemSetTypeFlag(pOut, MEM_Int);
56366	56353	pOut->u.i = u.bp.rowid;
	56354	+ pOut->flags = MEM_Int;
56367	56355	}
56368	56356	}
56369	56357	break;
56370	56358	}
56371	56359
		@@ -56393,12 +56381,12 @@
56393	56381	** Otherwise fall through to the next instruction.
56394	56382	**
56395	56383	** If P5 is non-zero then the key value is increased by an epsilon prior
56396	56384	** to the comparison. This makes the opcode work like IdxLE.
56397	56385	*/
56398		-case OP_IdxLT: /* jump, in3 */
56399		-case OP_IdxGE: { /* jump, in3 */
	56386	+case OP_IdxLT: /* jump */
	56387	+case OP_IdxGE: { /* jump */
56400	56388	#if 0 /* local variables moved into u.bq */
56401	56389	VdbeCursor *pC;
56402	56390	int res;
56403	56391	UnpackedRecord r;
56404	56392	#endif /* local variables moved into u.bq */
		@@ -56415,11 +56403,11 @@
56415	56403	if( pOp->p5 ){
56416	56404	u.bq.r.flags = UNPACKED_INCRKEY \| UNPACKED_IGNORE_ROWID;
56417	56405	}else{
56418	56406	u.bq.r.flags = UNPACKED_IGNORE_ROWID;
56419	56407	}
56420		- u.bq.r.aMem = &p->aMem[pOp->p3];
	56408	+ u.bq.r.aMem = &aMem[pOp->p3];
56421	56409	rc = sqlite3VdbeIdxKeyCompare(u.bq.pC, &u.bq.r, &u.bq.res);
56422	56410	if( pOp->opcode==OP_IdxLT ){
56423	56411	u.bq.res = -u.bq.res;
56424	56412	}else{
56425	56413	assert( pOp->opcode==OP_IdxGE );
		@@ -56467,23 +56455,25 @@
56467	56455	}
56468	56456	}
56469	56457	#else
56470	56458	u.br.iCnt = db->activeVdbeCnt;
56471	56459	#endif
	56460	+ pOut->flags = MEM_Null;
56472	56461	if( u.br.iCnt>1 ){
56473	56462	rc = SQLITE_LOCKED;
56474	56463	p->errorAction = OE_Abort;
56475	56464	}else{
56476	56465	u.br.iDb = pOp->p3;
56477	56466	assert( u.br.iCnt==1 );
56478	56467	assert( (p->btreeMask & (1<<u.br.iDb))!=0 );
56479	56468	rc = sqlite3BtreeDropTable(db->aDb[u.br.iDb].pBt, pOp->p1, &u.br.iMoved);
56480		- MemSetTypeFlag(pOut, MEM_Int);
	56469	+ pOut->flags = MEM_Int;
56481	56470	pOut->u.i = u.br.iMoved;
56482	56471	#ifndef SQLITE_OMIT_AUTOVACUUM
56483	56472	if( rc==SQLITE_OK && u.br.iMoved!=0 ){
56484	56473	sqlite3RootPageMoved(&db->aDb[u.br.iDb], u.br.iMoved, pOp->p1);
	56474	+ resetSchemaOnFault = 1;
56485	56475	}
56486	56476	#endif
56487	56477	}
56488	56478	break;
56489	56479	}
		@@ -56517,11 +56507,11 @@
56517	56507	db->aDb[pOp->p2].pBt, pOp->p1, (pOp->p3 ? &u.bs.nChange : 0)
56518	56508	);
56519	56509	if( pOp->p3 ){
56520	56510	p->nChange += u.bs.nChange;
56521	56511	if( pOp->p3>0 ){
56522		- p->aMem[pOp->p3].u.i += u.bs.nChange;
	56512	+ aMem[pOp->p3].u.i += u.bs.nChange;
56523	56513	}
56524	56514	}
56525	56515	break;
56526	56516	}
56527	56517
		@@ -56566,11 +56556,10 @@
56566	56556	}else{
56567	56557	u.bt.flags = BTREE_ZERODATA;
56568	56558	}
56569	56559	rc = sqlite3BtreeCreateTable(u.bt.pDb->pBt, &u.bt.pgno, u.bt.flags);
56570	56560	pOut->u.i = u.bt.pgno;
56571		- MemSetTypeFlag(pOut, MEM_Int);
56572	56561	break;
56573	56562	}
56574	56563
56575	56564	/* Opcode: ParseSchema P1 P2 * P4 *
56576	56565	**
		@@ -56733,14 +56722,14 @@
56733	56722	u.bv.nRoot = pOp->p2;
56734	56723	assert( u.bv.nRoot>0 );
56735	56724	u.bv.aRoot = sqlite3DbMallocRaw(db, sizeof(int)*(u.bv.nRoot+1) );
56736	56725	if( u.bv.aRoot==0 ) goto no_mem;
56737	56726	assert( pOp->p3>0 && pOp->p3<=p->nMem );
56738		- u.bv.pnErr = &p->aMem[pOp->p3];
	56727	+ u.bv.pnErr = &aMem[pOp->p3];
56739	56728	assert( (u.bv.pnErr->flags & MEM_Int)!=0 );
56740	56729	assert( (u.bv.pnErr->flags & (MEM_Str\|MEM_Blob))==0 );
56741		- pIn1 = &p->aMem[pOp->p1];
	56730	+ pIn1 = &aMem[pOp->p1];
56742	56731	for(u.bv.j=0; u.bv.j<u.bv.nRoot; u.bv.j++){
56743	56732	u.bv.aRoot[u.bv.j] = (int)sqlite3VdbeIntValue(&pIn1[u.bv.j]);
56744	56733	}
56745	56734	u.bv.aRoot[u.bv.j] = 0;
56746	56735	assert( pOp->p5<db->nDb );
		@@ -56768,53 +56757,43 @@
56768	56757	** Insert the integer value held by register P2 into a boolean index
56769	56758	** held in register P1.
56770	56759	**
56771	56760	** An assertion fails if P2 is not an integer.
56772	56761	*/
56773		-case OP_RowSetAdd: { /* in2 */
56774		-#if 0 /* local variables moved into u.bw */
56775		- Mem *pIdx;
56776		- Mem *pVal;
56777		-#endif /* local variables moved into u.bw */
56778		- assert( pOp->p1>0 && pOp->p1<=p->nMem );
56779		- u.bw.pIdx = &p->aMem[pOp->p1];
56780		- assert( pOp->p2>0 && pOp->p2<=p->nMem );
56781		- u.bw.pVal = &p->aMem[pOp->p2];
56782		- assert( (u.bw.pVal->flags & MEM_Int)!=0 );
56783		- if( (u.bw.pIdx->flags & MEM_RowSet)==0 ){
56784		- sqlite3VdbeMemSetRowSet(u.bw.pIdx);
56785		- if( (u.bw.pIdx->flags & MEM_RowSet)==0 ) goto no_mem;
56786		- }
56787		- sqlite3RowSetInsert(u.bw.pIdx->u.pRowSet, u.bw.pVal->u.i);
	56762	+case OP_RowSetAdd: { /* in1, in2 */
	56763	+ pIn1 = &aMem[pOp->p1];
	56764	+ pIn2 = &aMem[pOp->p2];
	56765	+ assert( (pIn2->flags & MEM_Int)!=0 );
	56766	+ if( (pIn1->flags & MEM_RowSet)==0 ){
	56767	+ sqlite3VdbeMemSetRowSet(pIn1);
	56768	+ if( (pIn1->flags & MEM_RowSet)==0 ) goto no_mem;
	56769	+ }
	56770	+ sqlite3RowSetInsert(pIn1->u.pRowSet, pIn2->u.i);
56788	56771	break;
56789	56772	}
56790	56773
56791	56774	/* Opcode: RowSetRead P1 P2 P3 * *
56792	56775	**
56793	56776	** Extract the smallest value from boolean index P1 and put that value into
56794	56777	** register P3. Or, if boolean index P1 is initially empty, leave P3
56795	56778	** unchanged and jump to instruction P2.
56796	56779	*/
56797		-case OP_RowSetRead: { /* jump, out3 */
56798		-#if 0 /* local variables moved into u.bx */
56799		- Mem *pIdx;
	56780	+case OP_RowSetRead: { /* jump, in1, out3 */
	56781	+#if 0 /* local variables moved into u.bw */
56800	56782	i64 val;
56801		-#endif /* local variables moved into u.bx */
56802		- assert( pOp->p1>0 && pOp->p1<=p->nMem );
	56783	+#endif /* local variables moved into u.bw */
56803	56784	CHECK_FOR_INTERRUPT;
56804		- u.bx.pIdx = &p->aMem[pOp->p1];
56805		- pOut = &p->aMem[pOp->p3];
56806		- if( (u.bx.pIdx->flags & MEM_RowSet)==0
56807		- \|\| sqlite3RowSetNext(u.bx.pIdx->u.pRowSet, &u.bx.val)==0
	56785	+ pIn1 = &aMem[pOp->p1];
	56786	+ if( (pIn1->flags & MEM_RowSet)==0
	56787	+ \|\| sqlite3RowSetNext(pIn1->u.pRowSet, &u.bw.val)==0
56808	56788	){
56809	56789	/* The boolean index is empty */
56810		- sqlite3VdbeMemSetNull(u.bx.pIdx);
	56790	+ sqlite3VdbeMemSetNull(pIn1);
56811	56791	pc = pOp->p2 - 1;
56812	56792	}else{
56813	56793	/* A value was pulled from the index */
56814		- assert( pOp->p3>0 && pOp->p3<=p->nMem );
56815		- sqlite3VdbeMemSetInt64(pOut, u.bx.val);
	56794	+ sqlite3VdbeMemSetInt64(&aMem[pOp->p3], u.bw.val);
56816	56795	}
56817	56796	break;
56818	56797	}
56819	56798
56820	56799	/* Opcode: RowSetTest P1 P2 P3 P4
		@@ -56839,16 +56818,18 @@
56839	56818	** inserted, there is no need to search to see if the same value was
56840	56819	** previously inserted as part of set X (only if it was previously
56841	56820	** inserted as part of some other set).
56842	56821	*/
56843	56822	case OP_RowSetTest: { /* jump, in1, in3 */
56844		-#if 0 /* local variables moved into u.by */
	56823	+#if 0 /* local variables moved into u.bx */
56845	56824	int iSet;
56846	56825	int exists;
56847		-#endif /* local variables moved into u.by */
	56826	+#endif /* local variables moved into u.bx */
56848	56827
56849		- u.by.iSet = pOp->p4.i;
	56828	+ pIn1 = &aMem[pOp->p1];
	56829	+ pIn3 = &aMem[pOp->p3];
	56830	+ u.bx.iSet = pOp->p4.i;
56850	56831	assert( pIn3->flags&MEM_Int );
56851	56832
56852	56833	/* If there is anything other than a rowset object in memory cell P1,
56853	56834	** delete it now and initialize P1 with an empty rowset
56854	56835	*/
		@@ -56856,21 +56837,21 @@
56856	56837	sqlite3VdbeMemSetRowSet(pIn1);
56857	56838	if( (pIn1->flags & MEM_RowSet)==0 ) goto no_mem;
56858	56839	}
56859	56840
56860	56841	assert( pOp->p4type==P4_INT32 );
56861		- assert( u.by.iSet==-1 \|\| u.by.iSet>=0 );
56862		- if( u.by.iSet ){
56863		- u.by.exists = sqlite3RowSetTest(pIn1->u.pRowSet,
56864		- (u8)(u.by.iSet>=0 ? u.by.iSet & 0xf : 0xff),
	56842	+ assert( u.bx.iSet==-1 \|\| u.bx.iSet>=0 );
	56843	+ if( u.bx.iSet ){
	56844	+ u.bx.exists = sqlite3RowSetTest(pIn1->u.pRowSet,
	56845	+ (u8)(u.bx.iSet>=0 ? u.bx.iSet & 0xf : 0xff),
56865	56846	pIn3->u.i);
56866		- if( u.by.exists ){
	56847	+ if( u.bx.exists ){
56867	56848	pc = pOp->p2 - 1;
56868	56849	break;
56869	56850	}
56870	56851	}
56871		- if( u.by.iSet>=0 ){
	56852	+ if( u.bx.iSet>=0 ){
56872	56853	sqlite3RowSetInsert(pIn1->u.pRowSet, pIn3->u.i);
56873	56854	}
56874	56855	break;
56875	56856	}
56876	56857
		@@ -56889,24 +56870,24 @@
56889	56870	** memory required by the sub-vdbe at runtime.
56890	56871	**
56891	56872	** P4 is a pointer to the VM containing the trigger program.
56892	56873	*/
56893	56874	case OP_Program: { /* jump */
56894		-#if 0 /* local variables moved into u.bz */
	56875	+#if 0 /* local variables moved into u.by */
56895	56876	int nMem; /* Number of memory registers for sub-program */
56896	56877	int nByte; /* Bytes of runtime space required for sub-program */
56897	56878	Mem pRt; / Register to allocate runtime space */
56898	56879	Mem pMem; / Used to iterate through memory cells */
56899	56880	Mem pEnd; / Last memory cell in new array */
56900	56881	VdbeFrame pFrame; / New vdbe frame to execute in */
56901	56882	SubProgram pProgram; / Sub-program to execute */
56902	56883	void t; / Token identifying trigger */
56903		-#endif /* local variables moved into u.bz */
	56884	+#endif /* local variables moved into u.by */
56904	56885
56905		- u.bz.pProgram = pOp->p4.pProgram;
56906		- u.bz.pRt = &p->aMem[pOp->p3];
56907		- assert( u.bz.pProgram->nOp>0 );
	56886	+ u.by.pProgram = pOp->p4.pProgram;
	56887	+ u.by.pRt = &aMem[pOp->p3];
	56888	+ assert( u.by.pProgram->nOp>0 );
56908	56889
56909	56890	/* If the p5 flag is clear, then recursive invocation of triggers is
56910	56891	** disabled for backwards compatibility (p5 is set if this sub-program
56911	56892	** is really a trigger, not a foreign key action, and the flag set
56912	56893	** and cleared by the "PRAGMA recursive_triggers" command is clear).
		@@ -56916,79 +56897,79 @@
56916	56897	** SubProgram (if the trigger may be executed with more than one different
56917	56898	** ON CONFLICT algorithm). SubProgram structures associated with a
56918	56899	** single trigger all have the same value for the SubProgram.token
56919	56900	** variable. */
56920	56901	if( pOp->p5 ){
56921		- u.bz.t = u.bz.pProgram->token;
56922		- for(u.bz.pFrame=p->pFrame; u.bz.pFrame && u.bz.pFrame->token!=u.bz.t; u.bz.pFrame=u.bz.pFrame->pParent);
56923		- if( u.bz.pFrame ) break;
	56902	+ u.by.t = u.by.pProgram->token;
	56903	+ for(u.by.pFrame=p->pFrame; u.by.pFrame && u.by.pFrame->token!=u.by.t; u.by.pFrame=u.by.pFrame->pParent);
	56904	+ if( u.by.pFrame ) break;
56924	56905	}
56925	56906
56926	56907	if( p->nFrame>=db->aLimit[SQLITE_LIMIT_TRIGGER_DEPTH] ){
56927	56908	rc = SQLITE_ERROR;
56928	56909	sqlite3SetString(&p->zErrMsg, db, "too many levels of trigger recursion");
56929	56910	break;
56930	56911	}
56931	56912
56932		- /* Register u.bz.pRt is used to store the memory required to save the state
	56913	+ /* Register u.by.pRt is used to store the memory required to save the state
56933	56914	** of the current program, and the memory required at runtime to execute
56934		- ** the trigger program. If this trigger has been fired before, then u.bz.pRt
	56915	+ ** the trigger program. If this trigger has been fired before, then u.by.pRt
56935	56916	** is already allocated. Otherwise, it must be initialized. */
56936		- if( (u.bz.pRt->flags&MEM_Frame)==0 ){
	56917	+ if( (u.by.pRt->flags&MEM_Frame)==0 ){
56937	56918	/* SubProgram.nMem is set to the number of memory cells used by the
56938	56919	** program stored in SubProgram.aOp. As well as these, one memory
56939	56920	** cell is required for each cursor used by the program. Set local
56940		- ** variable u.bz.nMem (and later, VdbeFrame.nChildMem) to this value.
	56921	+ ** variable u.by.nMem (and later, VdbeFrame.nChildMem) to this value.
56941	56922	*/
56942		- u.bz.nMem = u.bz.pProgram->nMem + u.bz.pProgram->nCsr;
56943		- u.bz.nByte = ROUND8(sizeof(VdbeFrame))
56944		- + u.bz.nMem * sizeof(Mem)
56945		- + u.bz.pProgram->nCsr * sizeof(VdbeCursor *);
56946		- u.bz.pFrame = sqlite3DbMallocZero(db, u.bz.nByte);
56947		- if( !u.bz.pFrame ){
	56923	+ u.by.nMem = u.by.pProgram->nMem + u.by.pProgram->nCsr;
	56924	+ u.by.nByte = ROUND8(sizeof(VdbeFrame))
	56925	+ + u.by.nMem * sizeof(Mem)
	56926	+ + u.by.pProgram->nCsr * sizeof(VdbeCursor *);
	56927	+ u.by.pFrame = sqlite3DbMallocZero(db, u.by.nByte);
	56928	+ if( !u.by.pFrame ){
56948	56929	goto no_mem;
56949	56930	}
56950		- sqlite3VdbeMemRelease(u.bz.pRt);
56951		- u.bz.pRt->flags = MEM_Frame;
56952		- u.bz.pRt->u.pFrame = u.bz.pFrame;
56953		-
56954		- u.bz.pFrame->v = p;
56955		- u.bz.pFrame->nChildMem = u.bz.nMem;
56956		- u.bz.pFrame->nChildCsr = u.bz.pProgram->nCsr;
56957		- u.bz.pFrame->pc = pc;
56958		- u.bz.pFrame->aMem = p->aMem;
56959		- u.bz.pFrame->nMem = p->nMem;
56960		- u.bz.pFrame->apCsr = p->apCsr;
56961		- u.bz.pFrame->nCursor = p->nCursor;
56962		- u.bz.pFrame->aOp = p->aOp;
56963		- u.bz.pFrame->nOp = p->nOp;
56964		- u.bz.pFrame->token = u.bz.pProgram->token;
56965		-
56966		- u.bz.pEnd = &VdbeFrameMem(u.bz.pFrame)[u.bz.pFrame->nChildMem];
56967		- for(u.bz.pMem=VdbeFrameMem(u.bz.pFrame); u.bz.pMem!=u.bz.pEnd; u.bz.pMem++){
56968		- u.bz.pMem->flags = MEM_Null;
56969		- u.bz.pMem->db = db;
	56931	+ sqlite3VdbeMemRelease(u.by.pRt);
	56932	+ u.by.pRt->flags = MEM_Frame;
	56933	+ u.by.pRt->u.pFrame = u.by.pFrame;
	56934	+
	56935	+ u.by.pFrame->v = p;
	56936	+ u.by.pFrame->nChildMem = u.by.nMem;
	56937	+ u.by.pFrame->nChildCsr = u.by.pProgram->nCsr;
	56938	+ u.by.pFrame->pc = pc;
	56939	+ u.by.pFrame->aMem = p->aMem;
	56940	+ u.by.pFrame->nMem = p->nMem;
	56941	+ u.by.pFrame->apCsr = p->apCsr;
	56942	+ u.by.pFrame->nCursor = p->nCursor;
	56943	+ u.by.pFrame->aOp = p->aOp;
	56944	+ u.by.pFrame->nOp = p->nOp;
	56945	+ u.by.pFrame->token = u.by.pProgram->token;
	56946	+
	56947	+ u.by.pEnd = &VdbeFrameMem(u.by.pFrame)[u.by.pFrame->nChildMem];
	56948	+ for(u.by.pMem=VdbeFrameMem(u.by.pFrame); u.by.pMem!=u.by.pEnd; u.by.pMem++){
	56949	+ u.by.pMem->flags = MEM_Null;
	56950	+ u.by.pMem->db = db;
56970	56951	}
56971	56952	}else{
56972		- u.bz.pFrame = u.bz.pRt->u.pFrame;
56973		- assert( u.bz.pProgram->nMem+u.bz.pProgram->nCsr==u.bz.pFrame->nChildMem );
56974		- assert( u.bz.pProgram->nCsr==u.bz.pFrame->nChildCsr );
56975		- assert( pc==u.bz.pFrame->pc );
	56953	+ u.by.pFrame = u.by.pRt->u.pFrame;
	56954	+ assert( u.by.pProgram->nMem+u.by.pProgram->nCsr==u.by.pFrame->nChildMem );
	56955	+ assert( u.by.pProgram->nCsr==u.by.pFrame->nChildCsr );
	56956	+ assert( pc==u.by.pFrame->pc );
56976	56957	}
56977	56958
56978	56959	p->nFrame++;
56979		- u.bz.pFrame->pParent = p->pFrame;
56980		- u.bz.pFrame->lastRowid = db->lastRowid;
56981		- u.bz.pFrame->nChange = p->nChange;
	56960	+ u.by.pFrame->pParent = p->pFrame;
	56961	+ u.by.pFrame->lastRowid = db->lastRowid;
	56962	+ u.by.pFrame->nChange = p->nChange;
56982	56963	p->nChange = 0;
56983		- p->pFrame = u.bz.pFrame;
56984		- p->aMem = &VdbeFrameMem(u.bz.pFrame)[-1];
56985		- p->nMem = u.bz.pFrame->nChildMem;
56986		- p->nCursor = (u16)u.bz.pFrame->nChildCsr;
56987		- p->apCsr = (VdbeCursor **)&p->aMem[p->nMem+1];
56988		- p->aOp = u.bz.pProgram->aOp;
56989		- p->nOp = u.bz.pProgram->nOp;
	56964	+ p->pFrame = u.by.pFrame;
	56965	+ p->aMem = aMem = &VdbeFrameMem(u.by.pFrame)[-1];
	56966	+ p->nMem = u.by.pFrame->nChildMem;
	56967	+ p->nCursor = (u16)u.by.pFrame->nChildCsr;
	56968	+ p->apCsr = (VdbeCursor **)&aMem[p->nMem+1];
	56969	+ p->aOp = aOp = u.by.pProgram->aOp;
	56970	+ p->nOp = u.by.pProgram->nOp;
56990	56971	pc = -1;
56991	56972
56992	56973	break;
56993	56974	}
56994	56975
		@@ -57003,17 +56984,17 @@
57003	56984	** The address of the cell in the parent frame is determined by adding
57004	56985	** the value of the P1 argument to the value of the P1 argument to the
57005	56986	** calling OP_Program instruction.
57006	56987	*/
57007	56988	case OP_Param: { /* out2-prerelease */
57008		-#if 0 /* local variables moved into u.ca */
	56989	+#if 0 /* local variables moved into u.bz */
57009	56990	VdbeFrame *pFrame;
57010	56991	Mem *pIn;
57011		-#endif /* local variables moved into u.ca */
57012		- u.ca.pFrame = p->pFrame;
57013		- u.ca.pIn = &u.ca.pFrame->aMem[pOp->p1 + u.ca.pFrame->aOp[u.ca.pFrame->pc].p1];
57014		- sqlite3VdbeMemShallowCopy(pOut, u.ca.pIn, MEM_Ephem);
	56992	+#endif /* local variables moved into u.bz */
	56993	+ u.bz.pFrame = p->pFrame;
	56994	+ u.bz.pIn = &u.bz.pFrame->aMem[pOp->p1 + u.bz.pFrame->aOp[u.bz.pFrame->pc].p1];
	56995	+ sqlite3VdbeMemShallowCopy(pOut, u.bz.pIn, MEM_Ephem);
57015	56996	break;
57016	56997	}
57017	56998
57018	56999	#endif /* #ifndef SQLITE_OMIT_TRIGGER */
57019	57000
		@@ -57065,24 +57046,25 @@
57065	57046	**
57066	57047	** This instruction throws an error if the memory cell is not initially
57067	57048	** an integer.
57068	57049	*/
57069	57050	case OP_MemMax: { /* in2 */
57070		-#if 0 /* local variables moved into u.cb */
	57051	+#if 0 /* local variables moved into u.ca */
57071	57052	Mem *pIn1;
57072	57053	VdbeFrame *pFrame;
57073		-#endif /* local variables moved into u.cb */
	57054	+#endif /* local variables moved into u.ca */
57074	57055	if( p->pFrame ){
57075		- for(u.cb.pFrame=p->pFrame; u.cb.pFrame->pParent; u.cb.pFrame=u.cb.pFrame->pParent);
57076		- u.cb.pIn1 = &u.cb.pFrame->aMem[pOp->p1];
	57056	+ for(u.ca.pFrame=p->pFrame; u.ca.pFrame->pParent; u.ca.pFrame=u.ca.pFrame->pParent);
	57057	+ u.ca.pIn1 = &u.ca.pFrame->aMem[pOp->p1];
57077	57058	}else{
57078		- u.cb.pIn1 = &p->aMem[pOp->p1];
	57059	+ u.ca.pIn1 = &aMem[pOp->p1];
57079	57060	}
57080		- sqlite3VdbeMemIntegerify(u.cb.pIn1);
	57061	+ sqlite3VdbeMemIntegerify(u.ca.pIn1);
	57062	+ pIn2 = &aMem[pOp->p2];
57081	57063	sqlite3VdbeMemIntegerify(pIn2);
57082		- if( u.cb.pIn1->u.i<pIn2->u.i){
57083		- u.cb.pIn1->u.i = pIn2->u.i;
	57064	+ if( u.ca.pIn1->u.i<pIn2->u.i){
	57065	+ u.ca.pIn1->u.i = pIn2->u.i;
57084	57066	}
57085	57067	break;
57086	57068	}
57087	57069	#endif /* SQLITE_OMIT_AUTOINCREMENT */
57088	57070
		@@ -57092,10 +57074,11 @@
57092	57074	**
57093	57075	** It is illegal to use this instruction on a register that does
57094	57076	** not contain an integer. An assertion fault will result if you try.
57095	57077	*/
57096	57078	case OP_IfPos: { /* jump, in1 */
	57079	+ pIn1 = &aMem[pOp->p1];
57097	57080	assert( pIn1->flags&MEM_Int );
57098	57081	if( pIn1->u.i>0 ){
57099	57082	pc = pOp->p2 - 1;
57100	57083	}
57101	57084	break;
		@@ -57107,26 +57090,30 @@
57107	57090	**
57108	57091	** It is illegal to use this instruction on a register that does
57109	57092	** not contain an integer. An assertion fault will result if you try.
57110	57093	*/
57111	57094	case OP_IfNeg: { /* jump, in1 */
	57095	+ pIn1 = &aMem[pOp->p1];
57112	57096	assert( pIn1->flags&MEM_Int );
57113	57097	if( pIn1->u.i<0 ){
57114	57098	pc = pOp->p2 - 1;
57115	57099	}
57116	57100	break;
57117	57101	}
57118	57102
57119		-/* Opcode: IfZero P1 P2 * * *
	57103	+/* Opcode: IfZero P1 P2 P3 * *
57120	57104	**
57121		-** If the value of register P1 is exactly 0, jump to P2.
	57105	+** The register P1 must contain an integer. Add literal P3 to the
	57106	+** value in register P1. If the result is exactly 0, jump to P2.
57122	57107	**
57123	57108	** It is illegal to use this instruction on a register that does
57124	57109	** not contain an integer. An assertion fault will result if you try.
57125	57110	*/
57126	57111	case OP_IfZero: { /* jump, in1 */
	57112	+ pIn1 = &aMem[pOp->p1];
57127	57113	assert( pIn1->flags&MEM_Int );
	57114	+ pIn1->u.i += pOp->p3;
57128	57115	if( pIn1->u.i==0 ){
57129	57116	pc = pOp->p2 - 1;
57130	57117	}
57131	57118	break;
57132	57119	}
		@@ -57140,51 +57127,51 @@
57140	57127	**
57141	57128	** The P5 arguments are taken from register P2 and its
57142	57129	** successors.
57143	57130	*/
57144	57131	case OP_AggStep: {
57145		-#if 0 /* local variables moved into u.cc */
	57132	+#if 0 /* local variables moved into u.cb */
57146	57133	int n;
57147	57134	int i;
57148	57135	Mem *pMem;
57149	57136	Mem *pRec;
57150	57137	sqlite3_context ctx;
57151	57138	sqlite3_value **apVal;
57152		-#endif /* local variables moved into u.cc */
57153		-
57154		- u.cc.n = pOp->p5;
57155		- assert( u.cc.n>=0 );
57156		- u.cc.pRec = &p->aMem[pOp->p2];
57157		- u.cc.apVal = p->apArg;
57158		- assert( u.cc.apVal \|\| u.cc.n==0 );
57159		- for(u.cc.i=0; u.cc.i<u.cc.n; u.cc.i++, u.cc.pRec++){
57160		- u.cc.apVal[u.cc.i] = u.cc.pRec;
57161		- sqlite3VdbeMemStoreType(u.cc.pRec);
57162		- }
57163		- u.cc.ctx.pFunc = pOp->p4.pFunc;
	57139	+#endif /* local variables moved into u.cb */
	57140	+
	57141	+ u.cb.n = pOp->p5;
	57142	+ assert( u.cb.n>=0 );
	57143	+ u.cb.pRec = &aMem[pOp->p2];
	57144	+ u.cb.apVal = p->apArg;
	57145	+ assert( u.cb.apVal \|\| u.cb.n==0 );
	57146	+ for(u.cb.i=0; u.cb.i<u.cb.n; u.cb.i++, u.cb.pRec++){
	57147	+ u.cb.apVal[u.cb.i] = u.cb.pRec;
	57148	+ sqlite3VdbeMemStoreType(u.cb.pRec);
	57149	+ }
	57150	+ u.cb.ctx.pFunc = pOp->p4.pFunc;
57164	57151	assert( pOp->p3>0 && pOp->p3<=p->nMem );
57165		- u.cc.ctx.pMem = u.cc.pMem = &p->aMem[pOp->p3];
57166		- u.cc.pMem->n++;
57167		- u.cc.ctx.s.flags = MEM_Null;
57168		- u.cc.ctx.s.z = 0;
57169		- u.cc.ctx.s.zMalloc = 0;
57170		- u.cc.ctx.s.xDel = 0;
57171		- u.cc.ctx.s.db = db;
57172		- u.cc.ctx.isError = 0;
57173		- u.cc.ctx.pColl = 0;
57174		- if( u.cc.ctx.pFunc->flags & SQLITE_FUNC_NEEDCOLL ){
	57152	+ u.cb.ctx.pMem = u.cb.pMem = &aMem[pOp->p3];
	57153	+ u.cb.pMem->n++;
	57154	+ u.cb.ctx.s.flags = MEM_Null;
	57155	+ u.cb.ctx.s.z = 0;
	57156	+ u.cb.ctx.s.zMalloc = 0;
	57157	+ u.cb.ctx.s.xDel = 0;
	57158	+ u.cb.ctx.s.db = db;
	57159	+ u.cb.ctx.isError = 0;
	57160	+ u.cb.ctx.pColl = 0;
	57161	+ if( u.cb.ctx.pFunc->flags & SQLITE_FUNC_NEEDCOLL ){
57175	57162	assert( pOp>p->aOp );
57176	57163	assert( pOp[-1].p4type==P4_COLLSEQ );
57177	57164	assert( pOp[-1].opcode==OP_CollSeq );
57178		- u.cc.ctx.pColl = pOp[-1].p4.pColl;
57179		- }
57180		- (u.cc.ctx.pFunc->xStep)(&u.cc.ctx, u.cc.n, u.cc.apVal);
57181		- if( u.cc.ctx.isError ){
57182		- sqlite3SetString(&p->zErrMsg, db, "%s", sqlite3_value_text(&u.cc.ctx.s));
57183		- rc = u.cc.ctx.isError;
57184		- }
57185		- sqlite3VdbeMemRelease(&u.cc.ctx.s);
	57165	+ u.cb.ctx.pColl = pOp[-1].p4.pColl;
	57166	+ }
	57167	+ (u.cb.ctx.pFunc->xStep)(&u.cb.ctx, u.cb.n, u.cb.apVal);
	57168	+ if( u.cb.ctx.isError ){
	57169	+ sqlite3SetString(&p->zErrMsg, db, "%s", sqlite3_value_text(&u.cb.ctx.s));
	57170	+ rc = u.cb.ctx.isError;
	57171	+ }
	57172	+ sqlite3VdbeMemRelease(&u.cb.ctx.s);
57186	57173	break;
57187	57174	}
57188	57175
57189	57176	/* Opcode: AggFinal P1 P2 * P4 *
57190	57177	**
		@@ -57197,23 +57184,23 @@
57197	57184	** functions that can take varying numbers of arguments. The
57198	57185	** P4 argument is only needed for the degenerate case where
57199	57186	** the step function was not previously called.
57200	57187	*/
57201	57188	case OP_AggFinal: {
57202		-#if 0 /* local variables moved into u.cd */
	57189	+#if 0 /* local variables moved into u.cc */
57203	57190	Mem *pMem;
57204		-#endif /* local variables moved into u.cd */
	57191	+#endif /* local variables moved into u.cc */
57205	57192	assert( pOp->p1>0 && pOp->p1<=p->nMem );
57206		- u.cd.pMem = &p->aMem[pOp->p1];
57207		- assert( (u.cd.pMem->flags & ~(MEM_Null\|MEM_Agg))==0 );
57208		- rc = sqlite3VdbeMemFinalize(u.cd.pMem, pOp->p4.pFunc);
	57193	+ u.cc.pMem = &aMem[pOp->p1];
	57194	+ assert( (u.cc.pMem->flags & ~(MEM_Null\|MEM_Agg))==0 );
	57195	+ rc = sqlite3VdbeMemFinalize(u.cc.pMem, pOp->p4.pFunc);
57209	57196	if( rc ){
57210		- sqlite3SetString(&p->zErrMsg, db, "%s", sqlite3_value_text(u.cd.pMem));
	57197	+ sqlite3SetString(&p->zErrMsg, db, "%s", sqlite3_value_text(u.cc.pMem));
57211	57198	}
57212		- sqlite3VdbeChangeEncoding(u.cd.pMem, encoding);
57213		- UPDATE_MAX_BLOBSIZE(u.cd.pMem);
57214		- if( sqlite3VdbeMemTooBig(u.cd.pMem) ){
	57199	+ sqlite3VdbeChangeEncoding(u.cc.pMem, encoding);
	57200	+ UPDATE_MAX_BLOBSIZE(u.cc.pMem);
	57201	+ if( sqlite3VdbeMemTooBig(u.cc.pMem) ){
57215	57202	goto too_big;
57216	57203	}
57217	57204	break;
57218	57205	}
57219	57206
		@@ -57239,18 +57226,18 @@
57239	57226	** Perform a single step of the incremental vacuum procedure on
57240	57227	** the P1 database. If the vacuum has finished, jump to instruction
57241	57228	** P2. Otherwise, fall through to the next instruction.
57242	57229	*/
57243	57230	case OP_IncrVacuum: { /* jump */
57244		-#if 0 /* local variables moved into u.ce */
	57231	+#if 0 /* local variables moved into u.cd */
57245	57232	Btree *pBt;
57246		-#endif /* local variables moved into u.ce */
	57233	+#endif /* local variables moved into u.cd */
57247	57234
57248	57235	assert( pOp->p1>=0 && pOp->p1<db->nDb );
57249	57236	assert( (p->btreeMask & (1<<pOp->p1))!=0 );
57250		- u.ce.pBt = db->aDb[pOp->p1].pBt;
57251		- rc = sqlite3BtreeIncrVacuum(u.ce.pBt);
	57237	+ u.cd.pBt = db->aDb[pOp->p1].pBt;
	57238	+ rc = sqlite3BtreeIncrVacuum(u.cd.pBt);
57252	57239	if( rc==SQLITE_DONE ){
57253	57240	pc = pOp->p2 - 1;
57254	57241	rc = SQLITE_OK;
57255	57242	}
57256	57243	break;
		@@ -57316,19 +57303,19 @@
57316	57303	** Also, whether or not P4 is set, check that this is not being called from
57317	57304	** within a callback to a virtual table xSync() method. If it is, the error
57318	57305	** code will be set to SQLITE_LOCKED.
57319	57306	*/
57320	57307	case OP_VBegin: {
57321		-#if 0 /* local variables moved into u.cf */
	57308	+#if 0 /* local variables moved into u.ce */
57322	57309	VTable *pVTab;
57323		-#endif /* local variables moved into u.cf */
57324		- u.cf.pVTab = pOp->p4.pVtab;
57325		- rc = sqlite3VtabBegin(db, u.cf.pVTab);
57326		- if( u.cf.pVTab ){
	57310	+#endif /* local variables moved into u.ce */
	57311	+ u.ce.pVTab = pOp->p4.pVtab;
	57312	+ rc = sqlite3VtabBegin(db, u.ce.pVTab);
	57313	+ if( u.ce.pVTab ){
57327	57314	sqlite3DbFree(db, p->zErrMsg);
57328		- p->zErrMsg = u.cf.pVTab->pVtab->zErrMsg;
57329		- u.cf.pVTab->pVtab->zErrMsg = 0;
	57315	+ p->zErrMsg = u.ce.pVTab->pVtab->zErrMsg;
	57316	+ u.ce.pVTab->pVtab->zErrMsg = 0;
57330	57317	}
57331	57318	break;
57332	57319	}
57333	57320	#endif /* SQLITE_OMIT_VIRTUALTABLE */
57334	57321
		@@ -57364,40 +57351,40 @@
57364	57351	** P4 is a pointer to a virtual table object, an sqlite3_vtab structure.
57365	57352	** P1 is a cursor number. This opcode opens a cursor to the virtual
57366	57353	** table and stores that cursor in P1.
57367	57354	*/
57368	57355	case OP_VOpen: {
57369		-#if 0 /* local variables moved into u.cg */
	57356	+#if 0 /* local variables moved into u.cf */
57370	57357	VdbeCursor *pCur;
57371	57358	sqlite3_vtab_cursor *pVtabCursor;
57372	57359	sqlite3_vtab *pVtab;
57373	57360	sqlite3_module *pModule;
57374		-#endif /* local variables moved into u.cg */
	57361	+#endif /* local variables moved into u.cf */
57375	57362
57376		- u.cg.pCur = 0;
57377		- u.cg.pVtabCursor = 0;
57378		- u.cg.pVtab = pOp->p4.pVtab->pVtab;
57379		- u.cg.pModule = (sqlite3_module *)u.cg.pVtab->pModule;
57380		- assert(u.cg.pVtab && u.cg.pModule);
	57363	+ u.cf.pCur = 0;
	57364	+ u.cf.pVtabCursor = 0;
	57365	+ u.cf.pVtab = pOp->p4.pVtab->pVtab;
	57366	+ u.cf.pModule = (sqlite3_module *)u.cf.pVtab->pModule;
	57367	+ assert(u.cf.pVtab && u.cf.pModule);
57381	57368	if( sqlite3SafetyOff(db) ) goto abort_due_to_misuse;
57382		- rc = u.cg.pModule->xOpen(u.cg.pVtab, &u.cg.pVtabCursor);
	57369	+ rc = u.cf.pModule->xOpen(u.cf.pVtab, &u.cf.pVtabCursor);
57383	57370	sqlite3DbFree(db, p->zErrMsg);
57384		- p->zErrMsg = u.cg.pVtab->zErrMsg;
57385		- u.cg.pVtab->zErrMsg = 0;
	57371	+ p->zErrMsg = u.cf.pVtab->zErrMsg;
	57372	+ u.cf.pVtab->zErrMsg = 0;
57386	57373	if( sqlite3SafetyOn(db) ) goto abort_due_to_misuse;
57387	57374	if( SQLITE_OK==rc ){
57388	57375	/* Initialize sqlite3_vtab_cursor base class */
57389		- u.cg.pVtabCursor->pVtab = u.cg.pVtab;
	57376	+ u.cf.pVtabCursor->pVtab = u.cf.pVtab;
57390	57377
57391	57378	/* Initialise vdbe cursor object */
57392		- u.cg.pCur = allocateCursor(p, pOp->p1, 0, -1, 0);
57393		- if( u.cg.pCur ){
57394		- u.cg.pCur->pVtabCursor = u.cg.pVtabCursor;
57395		- u.cg.pCur->pModule = u.cg.pVtabCursor->pVtab->pModule;
	57379	+ u.cf.pCur = allocateCursor(p, pOp->p1, 0, -1, 0);
	57380	+ if( u.cf.pCur ){
	57381	+ u.cf.pCur->pVtabCursor = u.cf.pVtabCursor;
	57382	+ u.cf.pCur->pModule = u.cf.pVtabCursor->pVtab->pModule;
57396	57383	}else{
57397	57384	db->mallocFailed = 1;
57398		- u.cg.pModule->xClose(u.cg.pVtabCursor);
	57385	+ u.cf.pModule->xClose(u.cf.pVtabCursor);
57399	57386	}
57400	57387	}
57401	57388	break;
57402	57389	}
57403	57390	#endif /* SQLITE_OMIT_VIRTUALTABLE */
		@@ -57420,11 +57407,11 @@
57420	57407	** xFilter as argv. Register P3+2 becomes argv[0] when passed to xFilter.
57421	57408	**
57422	57409	** A jump is made to P2 if the result set after filtering would be empty.
57423	57410	*/
57424	57411	case OP_VFilter: { /* jump */
57425		-#if 0 /* local variables moved into u.ch */
	57412	+#if 0 /* local variables moved into u.cg */
57426	57413	int nArg;
57427	57414	int iQuery;
57428	57415	const sqlite3_module *pModule;
57429	57416	Mem *pQuery;
57430	57417	Mem *pArgc;
		@@ -57432,52 +57419,52 @@
57432	57419	sqlite3_vtab *pVtab;
57433	57420	VdbeCursor *pCur;
57434	57421	int res;
57435	57422	int i;
57436	57423	Mem **apArg;
57437		-#endif /* local variables moved into u.ch */
57438		-
57439		- u.ch.pQuery = &p->aMem[pOp->p3];
57440		- u.ch.pArgc = &u.ch.pQuery[1];
57441		- u.ch.pCur = p->apCsr[pOp->p1];
57442		- REGISTER_TRACE(pOp->p3, u.ch.pQuery);
57443		- assert( u.ch.pCur->pVtabCursor );
57444		- u.ch.pVtabCursor = u.ch.pCur->pVtabCursor;
57445		- u.ch.pVtab = u.ch.pVtabCursor->pVtab;
57446		- u.ch.pModule = u.ch.pVtab->pModule;
	57424	+#endif /* local variables moved into u.cg */
	57425	+
	57426	+ u.cg.pQuery = &aMem[pOp->p3];
	57427	+ u.cg.pArgc = &u.cg.pQuery[1];
	57428	+ u.cg.pCur = p->apCsr[pOp->p1];
	57429	+ REGISTER_TRACE(pOp->p3, u.cg.pQuery);
	57430	+ assert( u.cg.pCur->pVtabCursor );
	57431	+ u.cg.pVtabCursor = u.cg.pCur->pVtabCursor;
	57432	+ u.cg.pVtab = u.cg.pVtabCursor->pVtab;
	57433	+ u.cg.pModule = u.cg.pVtab->pModule;
57447	57434
57448	57435	/* Grab the index number and argc parameters */
57449		- assert( (u.ch.pQuery->flags&MEM_Int)!=0 && u.ch.pArgc->flags==MEM_Int );
57450		- u.ch.nArg = (int)u.ch.pArgc->u.i;
57451		- u.ch.iQuery = (int)u.ch.pQuery->u.i;
	57436	+ assert( (u.cg.pQuery->flags&MEM_Int)!=0 && u.cg.pArgc->flags==MEM_Int );
	57437	+ u.cg.nArg = (int)u.cg.pArgc->u.i;
	57438	+ u.cg.iQuery = (int)u.cg.pQuery->u.i;
57452	57439
57453	57440	/* Invoke the xFilter method */
57454	57441	{
57455		- u.ch.res = 0;
57456		- u.ch.apArg = p->apArg;
57457		- for(u.ch.i = 0; u.ch.i<u.ch.nArg; u.ch.i++){
57458		- u.ch.apArg[u.ch.i] = &u.ch.pArgc[u.ch.i+1];
57459		- sqlite3VdbeMemStoreType(u.ch.apArg[u.ch.i]);
	57442	+ u.cg.res = 0;
	57443	+ u.cg.apArg = p->apArg;
	57444	+ for(u.cg.i = 0; u.cg.i<u.cg.nArg; u.cg.i++){
	57445	+ u.cg.apArg[u.cg.i] = &u.cg.pArgc[u.cg.i+1];
	57446	+ sqlite3VdbeMemStoreType(u.cg.apArg[u.cg.i]);
57460	57447	}
57461	57448
57462	57449	if( sqlite3SafetyOff(db) ) goto abort_due_to_misuse;
57463	57450	p->inVtabMethod = 1;
57464		- rc = u.ch.pModule->xFilter(u.ch.pVtabCursor, u.ch.iQuery, pOp->p4.z, u.ch.nArg, u.ch.apArg);
	57451	+ rc = u.cg.pModule->xFilter(u.cg.pVtabCursor, u.cg.iQuery, pOp->p4.z, u.cg.nArg, u.cg.apArg);
57465	57452	p->inVtabMethod = 0;
57466	57453	sqlite3DbFree(db, p->zErrMsg);
57467		- p->zErrMsg = u.ch.pVtab->zErrMsg;
57468		- u.ch.pVtab->zErrMsg = 0;
	57454	+ p->zErrMsg = u.cg.pVtab->zErrMsg;
	57455	+ u.cg.pVtab->zErrMsg = 0;
57469	57456	if( rc==SQLITE_OK ){
57470		- u.ch.res = u.ch.pModule->xEof(u.ch.pVtabCursor);
	57457	+ u.cg.res = u.cg.pModule->xEof(u.cg.pVtabCursor);
57471	57458	}
57472	57459	if( sqlite3SafetyOn(db) ) goto abort_due_to_misuse;
57473	57460
57474		- if( u.ch.res ){
	57461	+ if( u.cg.res ){
57475	57462	pc = pOp->p2 - 1;
57476	57463	}
57477	57464	}
57478		- u.ch.pCur->nullRow = 0;
	57465	+ u.cg.pCur->nullRow = 0;
57479	57466
57480	57467	break;
57481	57468	}
57482	57469	#endif /* SQLITE_OMIT_VIRTUALTABLE */
57483	57470
		@@ -57487,60 +57474,60 @@
57487	57474	** Store the value of the P2-th column of
57488	57475	** the row of the virtual-table that the
57489	57476	** P1 cursor is pointing to into register P3.
57490	57477	*/
57491	57478	case OP_VColumn: {
57492		-#if 0 /* local variables moved into u.ci */
	57479	+#if 0 /* local variables moved into u.ch */
57493	57480	sqlite3_vtab *pVtab;
57494	57481	const sqlite3_module *pModule;
57495	57482	Mem *pDest;
57496	57483	sqlite3_context sContext;
57497		-#endif /* local variables moved into u.ci */
	57484	+#endif /* local variables moved into u.ch */
57498	57485
57499	57486	VdbeCursor *pCur = p->apCsr[pOp->p1];
57500	57487	assert( pCur->pVtabCursor );
57501	57488	assert( pOp->p3>0 && pOp->p3<=p->nMem );
57502		- u.ci.pDest = &p->aMem[pOp->p3];
	57489	+ u.ch.pDest = &aMem[pOp->p3];
57503	57490	if( pCur->nullRow ){
57504		- sqlite3VdbeMemSetNull(u.ci.pDest);
	57491	+ sqlite3VdbeMemSetNull(u.ch.pDest);
57505	57492	break;
57506	57493	}
57507		- u.ci.pVtab = pCur->pVtabCursor->pVtab;
57508		- u.ci.pModule = u.ci.pVtab->pModule;
57509		- assert( u.ci.pModule->xColumn );
57510		- memset(&u.ci.sContext, 0, sizeof(u.ci.sContext));
	57494	+ u.ch.pVtab = pCur->pVtabCursor->pVtab;
	57495	+ u.ch.pModule = u.ch.pVtab->pModule;
	57496	+ assert( u.ch.pModule->xColumn );
	57497	+ memset(&u.ch.sContext, 0, sizeof(u.ch.sContext));
57511	57498
57512	57499	/* The output cell may already have a buffer allocated. Move
57513		- ** the current contents to u.ci.sContext.s so in case the user-function
	57500	+ ** the current contents to u.ch.sContext.s so in case the user-function
57514	57501	** can use the already allocated buffer instead of allocating a
57515	57502	** new one.
57516	57503	*/
57517		- sqlite3VdbeMemMove(&u.ci.sContext.s, u.ci.pDest);
57518		- MemSetTypeFlag(&u.ci.sContext.s, MEM_Null);
	57504	+ sqlite3VdbeMemMove(&u.ch.sContext.s, u.ch.pDest);
	57505	+ MemSetTypeFlag(&u.ch.sContext.s, MEM_Null);
57519	57506
57520	57507	if( sqlite3SafetyOff(db) ) goto abort_due_to_misuse;
57521		- rc = u.ci.pModule->xColumn(pCur->pVtabCursor, &u.ci.sContext, pOp->p2);
	57508	+ rc = u.ch.pModule->xColumn(pCur->pVtabCursor, &u.ch.sContext, pOp->p2);
57522	57509	sqlite3DbFree(db, p->zErrMsg);
57523		- p->zErrMsg = u.ci.pVtab->zErrMsg;
57524		- u.ci.pVtab->zErrMsg = 0;
57525		- if( u.ci.sContext.isError ){
57526		- rc = u.ci.sContext.isError;
	57510	+ p->zErrMsg = u.ch.pVtab->zErrMsg;
	57511	+ u.ch.pVtab->zErrMsg = 0;
	57512	+ if( u.ch.sContext.isError ){
	57513	+ rc = u.ch.sContext.isError;
57527	57514	}
57528	57515
57529	57516	/* Copy the result of the function to the P3 register. We
57530	57517	** do this regardless of whether or not an error occurred to ensure any
57531		- ** dynamic allocation in u.ci.sContext.s (a Mem struct) is released.
	57518	+ ** dynamic allocation in u.ch.sContext.s (a Mem struct) is released.
57532	57519	*/
57533		- sqlite3VdbeChangeEncoding(&u.ci.sContext.s, encoding);
57534		- REGISTER_TRACE(pOp->p3, u.ci.pDest);
57535		- sqlite3VdbeMemMove(u.ci.pDest, &u.ci.sContext.s);
57536		- UPDATE_MAX_BLOBSIZE(u.ci.pDest);
	57520	+ sqlite3VdbeChangeEncoding(&u.ch.sContext.s, encoding);
	57521	+ REGISTER_TRACE(pOp->p3, u.ch.pDest);
	57522	+ sqlite3VdbeMemMove(u.ch.pDest, &u.ch.sContext.s);
	57523	+ UPDATE_MAX_BLOBSIZE(u.ch.pDest);
57537	57524
57538	57525	if( sqlite3SafetyOn(db) ){
57539	57526	goto abort_due_to_misuse;
57540	57527	}
57541		- if( sqlite3VdbeMemTooBig(u.ci.pDest) ){
	57528	+ if( sqlite3VdbeMemTooBig(u.ch.pDest) ){
57542	57529	goto too_big;
57543	57530	}
57544	57531	break;
57545	57532	}
57546	57533	#endif /* SQLITE_OMIT_VIRTUALTABLE */
		@@ -57551,46 +57538,46 @@
57551	57538	** Advance virtual table P1 to the next row in its result set and
57552	57539	** jump to instruction P2. Or, if the virtual table has reached
57553	57540	** the end of its result set, then fall through to the next instruction.
57554	57541	*/
57555	57542	case OP_VNext: { /* jump */
57556		-#if 0 /* local variables moved into u.cj */
	57543	+#if 0 /* local variables moved into u.ci */
57557	57544	sqlite3_vtab *pVtab;
57558	57545	const sqlite3_module *pModule;
57559	57546	int res;
57560	57547	VdbeCursor *pCur;
57561		-#endif /* local variables moved into u.cj */
	57548	+#endif /* local variables moved into u.ci */
57562	57549
57563		- u.cj.res = 0;
57564		- u.cj.pCur = p->apCsr[pOp->p1];
57565		- assert( u.cj.pCur->pVtabCursor );
57566		- if( u.cj.pCur->nullRow ){
	57550	+ u.ci.res = 0;
	57551	+ u.ci.pCur = p->apCsr[pOp->p1];
	57552	+ assert( u.ci.pCur->pVtabCursor );
	57553	+ if( u.ci.pCur->nullRow ){
57567	57554	break;
57568	57555	}
57569		- u.cj.pVtab = u.cj.pCur->pVtabCursor->pVtab;
57570		- u.cj.pModule = u.cj.pVtab->pModule;
57571		- assert( u.cj.pModule->xNext );
	57556	+ u.ci.pVtab = u.ci.pCur->pVtabCursor->pVtab;
	57557	+ u.ci.pModule = u.ci.pVtab->pModule;
	57558	+ assert( u.ci.pModule->xNext );
57572	57559
57573	57560	/* Invoke the xNext() method of the module. There is no way for the
57574	57561	** underlying implementation to return an error if one occurs during
57575	57562	** xNext(). Instead, if an error occurs, true is returned (indicating that
57576	57563	** data is available) and the error code returned when xColumn or
57577	57564	** some other method is next invoked on the save virtual table cursor.
57578	57565	*/
57579	57566	if( sqlite3SafetyOff(db) ) goto abort_due_to_misuse;
57580	57567	p->inVtabMethod = 1;
57581		- rc = u.cj.pModule->xNext(u.cj.pCur->pVtabCursor);
	57568	+ rc = u.ci.pModule->xNext(u.ci.pCur->pVtabCursor);
57582	57569	p->inVtabMethod = 0;
57583	57570	sqlite3DbFree(db, p->zErrMsg);
57584		- p->zErrMsg = u.cj.pVtab->zErrMsg;
57585		- u.cj.pVtab->zErrMsg = 0;
	57571	+ p->zErrMsg = u.ci.pVtab->zErrMsg;
	57572	+ u.ci.pVtab->zErrMsg = 0;
57586	57573	if( rc==SQLITE_OK ){
57587		- u.cj.res = u.cj.pModule->xEof(u.cj.pCur->pVtabCursor);
	57574	+ u.ci.res = u.ci.pModule->xEof(u.ci.pCur->pVtabCursor);
57588	57575	}
57589	57576	if( sqlite3SafetyOn(db) ) goto abort_due_to_misuse;
57590	57577
57591		- if( !u.cj.res ){
	57578	+ if( !u.ci.res ){
57592	57579	/* If there is data, jump to P2 */
57593	57580	pc = pOp->p2 - 1;
57594	57581	}
57595	57582	break;
57596	57583	}
		@@ -57602,25 +57589,25 @@
57602	57589	** P4 is a pointer to a virtual table object, an sqlite3_vtab structure.
57603	57590	** This opcode invokes the corresponding xRename method. The value
57604	57591	** in register P1 is passed as the zName argument to the xRename method.
57605	57592	*/
57606	57593	case OP_VRename: {
57607		-#if 0 /* local variables moved into u.ck */
	57594	+#if 0 /* local variables moved into u.cj */
57608	57595	sqlite3_vtab *pVtab;
57609	57596	Mem *pName;
57610		-#endif /* local variables moved into u.ck */
	57597	+#endif /* local variables moved into u.cj */
57611	57598
57612		- u.ck.pVtab = pOp->p4.pVtab->pVtab;
57613		- u.ck.pName = &p->aMem[pOp->p1];
57614		- assert( u.ck.pVtab->pModule->xRename );
57615		- REGISTER_TRACE(pOp->p1, u.ck.pName);
57616		- assert( u.ck.pName->flags & MEM_Str );
	57599	+ u.cj.pVtab = pOp->p4.pVtab->pVtab;
	57600	+ u.cj.pName = &aMem[pOp->p1];
	57601	+ assert( u.cj.pVtab->pModule->xRename );
	57602	+ REGISTER_TRACE(pOp->p1, u.cj.pName);
	57603	+ assert( u.cj.pName->flags & MEM_Str );
57617	57604	if( sqlite3SafetyOff(db) ) goto abort_due_to_misuse;
57618		- rc = u.ck.pVtab->pModule->xRename(u.ck.pVtab, u.ck.pName->z);
	57605	+ rc = u.cj.pVtab->pModule->xRename(u.cj.pVtab, u.cj.pName->z);
57619	57606	sqlite3DbFree(db, p->zErrMsg);
57620		- p->zErrMsg = u.ck.pVtab->zErrMsg;
57621		- u.ck.pVtab->zErrMsg = 0;
	57607	+ p->zErrMsg = u.cj.pVtab->zErrMsg;
	57608	+ u.cj.pVtab->zErrMsg = 0;
57622	57609	if( sqlite3SafetyOn(db) ) goto abort_due_to_misuse;
57623	57610
57624	57611	break;
57625	57612	}
57626	57613	#endif
		@@ -57648,41 +57635,41 @@
57648	57635	** P1 is a boolean flag. If it is set to true and the xUpdate call
57649	57636	** is successful, then the value returned by sqlite3_last_insert_rowid()
57650	57637	** is set to the value of the rowid for the row just inserted.
57651	57638	*/
57652	57639	case OP_VUpdate: {
57653		-#if 0 /* local variables moved into u.cl */
	57640	+#if 0 /* local variables moved into u.ck */
57654	57641	sqlite3_vtab *pVtab;
57655	57642	sqlite3_module *pModule;
57656	57643	int nArg;
57657	57644	int i;
57658	57645	sqlite_int64 rowid;
57659	57646	Mem **apArg;
57660	57647	Mem *pX;
57661		-#endif /* local variables moved into u.cl */
	57648	+#endif /* local variables moved into u.ck */
57662	57649
57663		- u.cl.pVtab = pOp->p4.pVtab->pVtab;
57664		- u.cl.pModule = (sqlite3_module *)u.cl.pVtab->pModule;
57665		- u.cl.nArg = pOp->p2;
	57650	+ u.ck.pVtab = pOp->p4.pVtab->pVtab;
	57651	+ u.ck.pModule = (sqlite3_module *)u.ck.pVtab->pModule;
	57652	+ u.ck.nArg = pOp->p2;
57666	57653	assert( pOp->p4type==P4_VTAB );
57667		- if( ALWAYS(u.cl.pModule->xUpdate) ){
57668		- u.cl.apArg = p->apArg;
57669		- u.cl.pX = &p->aMem[pOp->p3];
57670		- for(u.cl.i=0; u.cl.i<u.cl.nArg; u.cl.i++){
57671		- sqlite3VdbeMemStoreType(u.cl.pX);
57672		- u.cl.apArg[u.cl.i] = u.cl.pX;
57673		- u.cl.pX++;
	57654	+ if( ALWAYS(u.ck.pModule->xUpdate) ){
	57655	+ u.ck.apArg = p->apArg;
	57656	+ u.ck.pX = &aMem[pOp->p3];
	57657	+ for(u.ck.i=0; u.ck.i<u.ck.nArg; u.ck.i++){
	57658	+ sqlite3VdbeMemStoreType(u.ck.pX);
	57659	+ u.ck.apArg[u.ck.i] = u.ck.pX;
	57660	+ u.ck.pX++;
57674	57661	}
57675	57662	if( sqlite3SafetyOff(db) ) goto abort_due_to_misuse;
57676		- rc = u.cl.pModule->xUpdate(u.cl.pVtab, u.cl.nArg, u.cl.apArg, &u.cl.rowid);
	57663	+ rc = u.ck.pModule->xUpdate(u.ck.pVtab, u.ck.nArg, u.ck.apArg, &u.ck.rowid);
57677	57664	sqlite3DbFree(db, p->zErrMsg);
57678		- p->zErrMsg = u.cl.pVtab->zErrMsg;
57679		- u.cl.pVtab->zErrMsg = 0;
	57665	+ p->zErrMsg = u.ck.pVtab->zErrMsg;
	57666	+ u.ck.pVtab->zErrMsg = 0;
57680	57667	if( sqlite3SafetyOn(db) ) goto abort_due_to_misuse;
57681	57668	if( rc==SQLITE_OK && pOp->p1 ){
57682		- assert( u.cl.nArg>1 && u.cl.apArg[0] && (u.cl.apArg[0]->flags&MEM_Null) );
57683		- db->lastRowid = u.cl.rowid;
	57669	+ assert( u.ck.nArg>1 && u.ck.apArg[0] && (u.ck.apArg[0]->flags&MEM_Null) );
	57670	+ db->lastRowid = u.ck.rowid;
57684	57671	}
57685	57672	p->nChange++;
57686	57673	}
57687	57674	break;
57688	57675	}
		@@ -57692,25 +57679,24 @@
57692	57679	/* Opcode: Pagecount P1 P2 * * *
57693	57680	**
57694	57681	** Write the current number of pages in database P1 to memory cell P2.
57695	57682	*/
57696	57683	case OP_Pagecount: { /* out2-prerelease */
57697		-#if 0 /* local variables moved into u.cm */
	57684	+#if 0 /* local variables moved into u.cl */
57698	57685	int p1;
57699	57686	int nPage;
57700	57687	Pager *pPager;
57701		-#endif /* local variables moved into u.cm */
	57688	+#endif /* local variables moved into u.cl */
57702	57689
57703		- u.cm.p1 = pOp->p1;
57704		- u.cm.pPager = sqlite3BtreePager(db->aDb[u.cm.p1].pBt);
57705		- rc = sqlite3PagerPagecount(u.cm.pPager, &u.cm.nPage);
	57690	+ u.cl.p1 = pOp->p1;
	57691	+ u.cl.pPager = sqlite3BtreePager(db->aDb[u.cl.p1].pBt);
	57692	+ rc = sqlite3PagerPagecount(u.cl.pPager, &u.cl.nPage);
57706	57693	/* OP_Pagecount is always called from within a read transaction. The
57707	57694	** page count has already been successfully read and cached. So the
57708	57695	** sqlite3PagerPagecount() call above cannot fail. */
57709	57696	if( ALWAYS(rc==SQLITE_OK) ){
57710		- pOut->flags = MEM_Int;
57711		- pOut->u.i = u.cm.nPage;
	57697	+ pOut->u.i = u.cl.nPage;
57712	57698	}
57713	57699	break;
57714	57700	}
57715	57701	#endif
57716	57702
		@@ -57719,22 +57705,22 @@
57719	57705	**
57720	57706	** If tracing is enabled (by the sqlite3_trace()) interface, then
57721	57707	** the UTF-8 string contained in P4 is emitted on the trace callback.
57722	57708	*/
57723	57709	case OP_Trace: {
57724		-#if 0 /* local variables moved into u.cn */
	57710	+#if 0 /* local variables moved into u.cm */
57725	57711	char *zTrace;
57726		-#endif /* local variables moved into u.cn */
	57712	+#endif /* local variables moved into u.cm */
57727	57713
57728		- u.cn.zTrace = (pOp->p4.z ? pOp->p4.z : p->zSql);
57729		- if( u.cn.zTrace ){
	57714	+ u.cm.zTrace = (pOp->p4.z ? pOp->p4.z : p->zSql);
	57715	+ if( u.cm.zTrace ){
57730	57716	if( db->xTrace ){
57731		- db->xTrace(db->pTraceArg, u.cn.zTrace);
	57717	+ db->xTrace(db->pTraceArg, u.cm.zTrace);
57732	57718	}
57733	57719	#ifdef SQLITE_DEBUG
57734	57720	if( (db->flags & SQLITE_SqlTrace)!=0 ){
57735		- sqlite3DebugPrintf("SQL-trace: %s\n", u.cn.zTrace);
	57721	+ sqlite3DebugPrintf("SQL-trace: %s\n", u.cm.zTrace);
57736	57722	}
57737	57723	#endif /* SQLITE_DEBUG */
57738	57724	}
57739	57725	break;
57740	57726	}
		@@ -57769,11 +57755,11 @@
57769	57755	u64 elapsed = sqlite3Hwtime() - start;
57770	57756	pOp->cycles += elapsed;
57771	57757	pOp->cnt++;
57772	57758	#if 0
57773	57759	fprintf(stdout, "%10llu ", elapsed);
57774		- sqlite3VdbePrintOp(stdout, origPc, &p->aOp[origPc]);
	57760	+ sqlite3VdbePrintOp(stdout, origPc, &aOp[origPc]);
57775	57761	#endif
57776	57762	}
57777	57763	#endif
57778	57764
57779	57765	/* The following code adds nothing to the actual functionality
		@@ -57785,15 +57771,15 @@
57785	57771	assert( pc>=-1 && pc<p->nOp );
57786	57772
57787	57773	#ifdef SQLITE_DEBUG
57788	57774	if( p->trace ){
57789	57775	if( rc!=0 ) fprintf(p->trace,"rc=%d\n",rc);
57790		- if( opProperty & OPFLG_OUT2_PRERELEASE ){
57791		- registerTrace(p->trace, pOp->p2, pOut);
	57776	+ if( pOp->opflags & (OPFLG_OUT2_PRERELEASE\|OPFLG_OUT2) ){
	57777	+ registerTrace(p->trace, pOp->p2, &aMem[pOp->p2]);
57792	57778	}
57793		- if( opProperty & OPFLG_OUT3 ){
57794		- registerTrace(p->trace, pOp->p3, pOut);
	57779	+ if( pOp->opflags & OPFLG_OUT3 ){
	57780	+ registerTrace(p->trace, pOp->p3, &aMem[pOp->p3]);
57795	57781	}
57796	57782	}
57797	57783	#endif /* SQLITE_DEBUG */
57798	57784	#endif /* NDEBUG */
57799	57785	} /* The end of the for(;;) loop the loops through opcodes */
		@@ -57805,10 +57791,11 @@
57805	57791	assert( rc );
57806	57792	p->rc = rc;
57807	57793	sqlite3VdbeHalt(p);
57808	57794	if( rc==SQLITE_IOERR_NOMEM ) db->mallocFailed = 1;
57809	57795	rc = SQLITE_ERROR;
	57796	+ if( resetSchemaOnFault ) sqlite3ResetInternalSchema(db, 0);
57810	57797
57811	57798	/* This is the only way out of this procedure. We have to
57812	57799	** release the mutexes on btrees that were acquired at the
57813	57800	** top. */
57814	57801	vdbe_return:
		@@ -57872,12 +57859,10 @@
57872	57859	** May you share freely, never taking more than you give.
57873	57860	**
57874	57861	*************************************************************************
57875	57862	**
57876	57863	** This file contains code used to implement incremental BLOB I/O.
57877		-**
57878		-** $Id: vdbeblob.c,v 1.35 2009/07/02 07:47:33 danielk1977 Exp $
57879	57864	*/
57880	57865
57881	57866
57882	57867	#ifndef SQLITE_OMIT_INCRBLOB
57883	57868
		@@ -58266,16 +58251,10 @@
58266	58251	** May you find forgiveness for yourself and forgive others.
58267	58252	** May you share freely, never taking more than you give.
58268	58253	**
58269	58254	*************************************************************************
58270	58255	**
58271		-** @(#) $Id: journal.c,v 1.9 2009/01/20 17:06:27 danielk1977 Exp $
58272		-*/
58273		-
58274		-#ifdef SQLITE_ENABLE_ATOMIC_WRITE
58275		-
58276		-/*
58277	58256	** This file implements a special kind of sqlite3_file object used
58278	58257	** by SQLite to create journal files if the atomic-write optimization
58279	58258	** is enabled.
58280	58259	**
58281	58260	** The distinctive characteristic of this sqlite3_file is that the
		@@ -58286,11 +58265,11 @@
58286	58265	**
58287	58266	** 1) The in-memory representation grows too large for the allocated
58288	58267	** buffer, or
58289	58268	** 2) The sqlite3JournalCreate() function is called.
58290	58269	*/
58291		-
	58270	+#ifdef SQLITE_ENABLE_ATOMIC_WRITE
58292	58271
58293	58272
58294	58273	/*
58295	58274	** A JournalFile object is a subclass of sqlite3_file used by
58296	58275	** as an open file handle for journal files.
		@@ -58511,12 +58490,10 @@
58511	58490	*************************************************************************
58512	58491	**
58513	58492	** This file contains code use to implement an in-memory rollback journal.
58514	58493	** The in-memory rollback journal is used to journal transactions for
58515	58494	** ":memory:" databases and when the journal_mode=MEMORY pragma is used.
58516		-**
58517		-** @(#) $Id: memjournal.c,v 1.12 2009/05/04 11:42:30 danielk1977 Exp $
58518	58495	*/
58519	58496
58520	58497	/* Forward references to internal structures */
58521	58498	typedef struct MemJournal MemJournal;
58522	58499	typedef struct FilePoint FilePoint;
		@@ -58770,12 +58747,10 @@
58770	58747	** May you share freely, never taking more than you give.
58771	58748	**
58772	58749	*************************************************************************
58773	58750	** This file contains routines used for walking the parser tree for
58774	58751	** an SQL statement.
58775		-**
58776		-** $Id: walker.c,v 1.7 2009/06/15 23:15:59 drh Exp $
58777	58752	*/
58778	58753
58779	58754
58780	58755	/*
58781	58756	** Walk an expression tree. Invoke the callback once for each node
		@@ -58910,12 +58885,10 @@
58910	58885	*************************************************************************
58911	58886	**
58912	58887	** This file contains routines used for walking the parser tree and
58913	58888	** resolve all identifiers by associating them with a particular
58914	58889	** table and column.
58915		-**
58916		-** $Id: resolve.c,v 1.30 2009/06/15 23:15:59 drh Exp $
58917	58890	*/
58918	58891
58919	58892	/*
58920	58893	** Turn the pExpr expression into an alias for the iCol-th column of the
58921	58894	** result set in pEList.
		@@ -61342,10 +61315,98 @@
61342	61315	p->flags \|= EP_IntValue;
61343	61316	p->u.iValue = *pValue;
61344	61317	}
61345	61318	return rc;
61346	61319	}
	61320	+
	61321	+/*
	61322	+** Return FALSE if there is no chance that the expression can be NULL.
	61323	+**
	61324	+** If the expression might be NULL or if the expression is too complex
	61325	+** to tell return TRUE.
	61326	+**
	61327	+** This routine is used as an optimization, to skip OP_IsNull opcodes
	61328	+** when we know that a value cannot be NULL. Hence, a false positive
	61329	+** (returning TRUE when in fact the expression can never be NULL) might
	61330	+** be a small performance hit but is otherwise harmless. On the other
	61331	+** hand, a false negative (returning FALSE when the result could be NULL)
	61332	+** will likely result in an incorrect answer. So when in doubt, return
	61333	+** TRUE.
	61334	+*/
	61335	+SQLITE_PRIVATE int sqlite3ExprCanBeNull(const Expr *p){
	61336	+ u8 op;
	61337	+ while( p->op==TK_UPLUS \|\| p->op==TK_UMINUS ){ p = p->pLeft; }
	61338	+ op = p->op;
	61339	+ if( op==TK_REGISTER ) op = p->op2;
	61340	+ switch( op ){
	61341	+ case TK_INTEGER:
	61342	+ case TK_STRING:
	61343	+ case TK_FLOAT:
	61344	+ case TK_BLOB:
	61345	+ return 0;
	61346	+ default:
	61347	+ return 1;
	61348	+ }
	61349	+}
	61350	+
	61351	+/*
	61352	+** Generate an OP_IsNull instruction that tests register iReg and jumps
	61353	+** to location iDest if the value in iReg is NULL. The value in iReg
	61354	+** was computed by pExpr. If we can look at pExpr at compile-time and
	61355	+** determine that it can never generate a NULL, then the OP_IsNull operation
	61356	+** can be omitted.
	61357	+*/
	61358	+SQLITE_PRIVATE void sqlite3ExprCodeIsNullJump(
	61359	+ Vdbe v, / The VDBE under construction */
	61360	+ const Expr pExpr, / Only generate OP_IsNull if this expr can be NULL */
	61361	+ int iReg, /* Test the value in this register for NULL */
	61362	+ int iDest /* Jump here if the value is null */
	61363	+){
	61364	+ if( sqlite3ExprCanBeNull(pExpr) ){
	61365	+ sqlite3VdbeAddOp2(v, OP_IsNull, iReg, iDest);
	61366	+ }
	61367	+}
	61368	+
	61369	+/*
	61370	+** Return TRUE if the given expression is a constant which would be
	61371	+** unchanged by OP_Affinity with the affinity given in the second
	61372	+** argument.
	61373	+**
	61374	+** This routine is used to determine if the OP_Affinity operation
	61375	+** can be omitted. When in doubt return FALSE. A false negative
	61376	+** is harmless. A false positive, however, can result in the wrong
	61377	+** answer.
	61378	+*/
	61379	+SQLITE_PRIVATE int sqlite3ExprNeedsNoAffinityChange(const Expr *p, char aff){
	61380	+ u8 op;
	61381	+ if( aff==SQLITE_AFF_NONE ) return 1;
	61382	+ while( p->op==TK_UPLUS \|\| p->op==TK_UMINUS ){ p = p->pLeft; }
	61383	+ op = p->op;
	61384	+ if( op==TK_REGISTER ) op = p->op2;
	61385	+ switch( op ){
	61386	+ case TK_INTEGER: {
	61387	+ return aff==SQLITE_AFF_INTEGER \|\| aff==SQLITE_AFF_NUMERIC;
	61388	+ }
	61389	+ case TK_FLOAT: {
	61390	+ return aff==SQLITE_AFF_REAL \|\| aff==SQLITE_AFF_NUMERIC;
	61391	+ }
	61392	+ case TK_STRING: {
	61393	+ return aff==SQLITE_AFF_TEXT;
	61394	+ }
	61395	+ case TK_BLOB: {
	61396	+ return 1;
	61397	+ }
	61398	+ case TK_COLUMN: {
	61399	+ assert( p->iTable>=0 ); /* p cannot be part of a CHECK constraint */
	61400	+ return p->iColumn<0
	61401	+ && (aff==SQLITE_AFF_INTEGER \|\| aff==SQLITE_AFF_NUMERIC);
	61402	+ }
	61403	+ default: {
	61404	+ return 0;
	61405	+ }
	61406	+ }
	61407	+}
61347	61408
61348	61409	/*
61349	61410	** Return TRUE if the given string is a row-id column name.
61350	61411	*/
61351	61412	SQLITE_PRIVATE int sqlite3IsRowid(const char *z){
		@@ -61430,20 +61491,20 @@
61430	61491	** be found with <column> as its left-most column.
61431	61492	**
61432	61493	** When the b-tree is being used for membership tests, the calling function
61433	61494	** needs to know whether or not the structure contains an SQL NULL
61434	61495	** value in order to correctly evaluate expressions like "X IN (Y, Z)".
61435		-** If there is a chance that the b-tree might contain a NULL value at
	61496	+** If there is any chance that the (...) might contain a NULL value at
61436	61497	** runtime, then a register is allocated and the register number written
61437		-** to *prNotFound. If there is no chance that the b-tree contains a
	61498	+** to *prNotFound. If there is no chance that the (...) contains a
61438	61499	** NULL value, then *prNotFound is left unchanged.
61439	61500	**
61440	61501	** If a register is allocated and its location stored in *prNotFound, then
61441		-** its initial value is NULL. If the b-tree does not remain constant
61442		-** for the duration of the query (i.e. the SELECT that generates the b-tree
	61502	+** its initial value is NULL. If the (...) does not remain constant
	61503	+** for the duration of the query (i.e. the SELECT within the (...)
61443	61504	** is a correlated subquery) then the value of the allocated register is
61444		-** reset to NULL each time the b-tree is repopulated. This allows the
	61505	+** reset to NULL each time the subquery is rerun. This allows the
61445	61506	** caller to use vdbe code equivalent to the following:
61446	61507	**
61447	61508	** if( register==NULL ){
61448	61509	** has_null = <test if data structure contains null>
61449	61510	** register = 1
		@@ -61631,11 +61692,11 @@
61631	61692	}
61632	61693
61633	61694	affinity = sqlite3ExprAffinity(pLeft);
61634	61695
61635	61696	/* Whether this is an 'x IN(SELECT...)' or an 'x IN(<exprlist>)'
61636		- ** expression it is handled the same way. A virtual table is
	61697	+ ** expression it is handled the same way. An ephemeral table is
61637	61698	** filled with single-field index keys representing the results
61638	61699	** from the SELECT or the <exprlist>.
61639	61700	**
61640	61701	** If the 'x' expression is a column value, or the SELECT...
61641	61702	** statement returns a column value, then the affinity of that
		@@ -61779,10 +61840,132 @@
61779	61840
61780	61841	return rReg;
61781	61842	}
61782	61843	#endif /* SQLITE_OMIT_SUBQUERY */
61783	61844
	61845	+#ifndef SQLITE_OMIT_SUBQUERY
	61846	+/*
	61847	+** Generate code for an IN expression.
	61848	+**
	61849	+** x IN (SELECT ...)
	61850	+** x IN (value, value, ...)
	61851	+**
	61852	+** The left-hand side (LHS) is a scalar expression. The right-hand side (RHS)
	61853	+** is an array of zero or more values. The expression is true if the LHS is
	61854	+** contained within the RHS. The value of the expression is unknown (NULL)
	61855	+** if the LHS is NULL or if the LHS is not contained within the RHS and the
	61856	+** RHS contains one or more NULL values.
	61857	+**
	61858	+** This routine generates code will jump to destIfFalse if the LHS is not
	61859	+** contained within the RHS. If due to NULLs we cannot determine if the LHS
	61860	+** is contained in the RHS then jump to destIfNull. If the LHS is contained
	61861	+** within the RHS then fall through.
	61862	+*/
	61863	+static void sqlite3ExprCodeIN(
	61864	+ Parse pParse, / Parsing and code generating context */
	61865	+ Expr pExpr, / The IN expression */
	61866	+ int destIfFalse, /* Jump here if LHS is not contained in the RHS */
	61867	+ int destIfNull /* Jump here if the results are unknown due to NULLs */
	61868	+){
	61869	+ int rRhsHasNull = 0; /* Register that is true if RHS contains NULL values */
	61870	+ char affinity; /* Comparison affinity to use */
	61871	+ int eType; /* Type of the RHS */
	61872	+ int r1; /* Temporary use register */
	61873	+ Vdbe v; / Statement under construction */
	61874	+
	61875	+ /* Compute the RHS. After this step, the table with cursor
	61876	+ ** pExpr->iTable will contains the values that make up the RHS.
	61877	+ */
	61878	+ v = pParse->pVdbe;
	61879	+ assert( v!=0 ); /* OOM detected prior to this routine */
	61880	+ VdbeNoopComment((v, "begin IN expr"));
	61881	+ eType = sqlite3FindInIndex(pParse, pExpr, &rRhsHasNull);
	61882	+
	61883	+ /* Figure out the affinity to use to create a key from the results
	61884	+ ** of the expression. affinityStr stores a static string suitable for
	61885	+ ** P4 of OP_MakeRecord.
	61886	+ */
	61887	+ affinity = comparisonAffinity(pExpr);
	61888	+
	61889	+ /* Code the LHS, the <expr> from "<expr> IN (...)".
	61890	+ */
	61891	+ sqlite3ExprCachePush(pParse);
	61892	+ r1 = sqlite3GetTempReg(pParse);
	61893	+ sqlite3ExprCode(pParse, pExpr->pLeft, r1);
	61894	+ sqlite3VdbeAddOp2(v, OP_IsNull, r1, destIfNull);
	61895	+
	61896	+
	61897	+ if( eType==IN_INDEX_ROWID ){
	61898	+ /* In this case, the RHS is the ROWID of table b-tree
	61899	+ */
	61900	+ sqlite3VdbeAddOp2(v, OP_MustBeInt, r1, destIfFalse);
	61901	+ sqlite3VdbeAddOp3(v, OP_NotExists, pExpr->iTable, destIfFalse, r1);
	61902	+ }else{
	61903	+ /* In this case, the RHS is an index b-tree.
	61904	+ */
	61905	+ sqlite3VdbeAddOp4(v, OP_Affinity, r1, 1, 0, &affinity, 1);
	61906	+
	61907	+ /* If the set membership test fails, then the result of the
	61908	+ ** "x IN (...)" expression must be either 0 or NULL. If the set
	61909	+ ** contains no NULL values, then the result is 0. If the set
	61910	+ ** contains one or more NULL values, then the result of the
	61911	+ ** expression is also NULL.
	61912	+ */
	61913	+ if( rRhsHasNull==0 \|\| destIfFalse==destIfNull ){
	61914	+ /* This branch runs if it is known at compile time that the RHS
	61915	+ ** cannot contain NULL values. This happens as the result
	61916	+ ** of a "NOT NULL" constraint in the database schema.
	61917	+ **
	61918	+ ** Also run this branch if NULL is equivalent to FALSE
	61919	+ ** for this particular IN operator.
	61920	+ */
	61921	+ sqlite3VdbeAddOp4Int(v, OP_NotFound, pExpr->iTable, destIfFalse, r1, 1);
	61922	+
	61923	+ }else{
	61924	+ /* In this branch, the RHS of the IN might contain a NULL and
	61925	+ ** the presence of a NULL on the RHS makes a difference in the
	61926	+ ** outcome.
	61927	+ */
	61928	+ int j1, j2, j3;
	61929	+
	61930	+ /* First check to see if the LHS is contained in the RHS. If so,
	61931	+ ** then the presence of NULLs in the RHS does not matter, so jump
	61932	+ ** over all of the code that follows.
	61933	+ */
	61934	+ j1 = sqlite3VdbeAddOp4Int(v, OP_Found, pExpr->iTable, 0, r1, 1);
	61935	+
	61936	+ /* Here we begin generating code that runs if the LHS is not
	61937	+ ** contained within the RHS. Generate additional code that
	61938	+ ** tests the RHS for NULLs. If the RHS contains a NULL then
	61939	+ ** jump to destIfNull. If there are no NULLs in the RHS then
	61940	+ ** jump to destIfFalse.
	61941	+ */
	61942	+ j2 = sqlite3VdbeAddOp1(v, OP_NotNull, rRhsHasNull);
	61943	+ j3 = sqlite3VdbeAddOp4Int(v, OP_Found, pExpr->iTable, 0, rRhsHasNull, 1);
	61944	+ sqlite3VdbeAddOp2(v, OP_Integer, -1, rRhsHasNull);
	61945	+ sqlite3VdbeJumpHere(v, j3);
	61946	+ sqlite3VdbeAddOp2(v, OP_AddImm, rRhsHasNull, 1);
	61947	+ sqlite3VdbeJumpHere(v, j2);
	61948	+
	61949	+ /* Jump to the appropriate target depending on whether or not
	61950	+ ** the RHS contains a NULL
	61951	+ */
	61952	+ sqlite3VdbeAddOp2(v, OP_If, rRhsHasNull, destIfNull);
	61953	+ sqlite3VdbeAddOp2(v, OP_Goto, 0, destIfFalse);
	61954	+
	61955	+ /* The OP_Found at the top of this branch jumps here when true,
	61956	+ ** causing the overall IN expression evaluation to fall through.
	61957	+ */
	61958	+ sqlite3VdbeJumpHere(v, j1);
	61959	+ }
	61960	+ }
	61961	+ sqlite3ReleaseTempReg(pParse, r1);
	61962	+ sqlite3ExprCachePop(pParse, 1);
	61963	+ VdbeComment((v, "end IN expr"));
	61964	+}
	61965	+#endif /* SQLITE_OMIT_SUBQUERY */
	61966	+
61784	61967	/*
61785	61968	** Duplicate an 8-byte value
61786	61969	*/
61787	61970	static char dup8bytes(Vdbe v, const char *in){
61788	61971	char *out = sqlite3DbMallocRaw(sqlite3VdbeDb(v), 8);
		@@ -62479,10 +62662,31 @@
62479	62662	pDef = sqlite3FindFunction(db, zId, nId, nFarg, enc, 0);
62480	62663	if( pDef==0 ){
62481	62664	sqlite3ErrorMsg(pParse, "unknown function: %.*s()", nId, zId);
62482	62665	break;
62483	62666	}
	62667	+
	62668	+ /* Attempt a direct implementation of the built-in COALESCE() and
	62669	+ ** IFNULL() functions. This avoids unnecessary evalation of
	62670	+ ** arguments past the first non-NULL argument.
	62671	+ */
	62672	+ if( pDef->flags & SQLITE_FUNC_COALESCE ){
	62673	+ int endCoalesce = sqlite3VdbeMakeLabel(v);
	62674	+ assert( nFarg>=2 );
	62675	+ sqlite3ExprCode(pParse, pFarg->a[0].pExpr, target);
	62676	+ for(i=1; i<nFarg; i++){
	62677	+ sqlite3VdbeAddOp2(v, OP_NotNull, target, endCoalesce);
	62678	+ sqlite3ExprCacheRemove(pParse, target);
	62679	+ sqlite3ExprCachePush(pParse);
	62680	+ sqlite3ExprCode(pParse, pFarg->a[i].pExpr, target);
	62681	+ sqlite3ExprCachePop(pParse, 1);
	62682	+ }
	62683	+ sqlite3VdbeResolveLabel(v, endCoalesce);
	62684	+ break;
	62685	+ }
	62686	+
	62687	+
62484	62688	if( pFarg ){
62485	62689	r1 = sqlite3GetTempRange(pParse, nFarg);
62486	62690	sqlite3ExprCachePush(pParse); /* Ticket 2ea2425d34be */
62487	62691	sqlite3ExprCodeExprList(pParse, pFarg, r1, 1);
62488	62692	sqlite3ExprCachePop(pParse, 1); /* Ticket 2ea2425d34be */
		@@ -62536,99 +62740,23 @@
62536	62740	testcase( op==TK_SELECT );
62537	62741	inReg = sqlite3CodeSubselect(pParse, pExpr, 0, 0);
62538	62742	break;
62539	62743	}
62540	62744	case TK_IN: {
62541		- int rNotFound = 0;
62542		- int rMayHaveNull = 0;
62543		- int j2, j3, j4, j5;
62544		- char affinity;
62545		- int eType;
62546		-
62547		- VdbeNoopComment((v, "begin IN expr r%d", target));
62548		- eType = sqlite3FindInIndex(pParse, pExpr, &rMayHaveNull);
62549		- if( rMayHaveNull ){
62550		- rNotFound = ++pParse->nMem;
62551		- }
62552		-
62553		- /* Figure out the affinity to use to create a key from the results
62554		- ** of the expression. affinityStr stores a static string suitable for
62555		- ** P4 of OP_MakeRecord.
62556		- */
62557		- affinity = comparisonAffinity(pExpr);
62558		-
62559		-
62560		- /* Code the <expr> from "<expr> IN (...)". The temporary table
62561		- ** pExpr->iTable contains the values that make up the (...) set.
62562		- */
62563		- sqlite3ExprCachePush(pParse);
62564		- sqlite3ExprCode(pParse, pExpr->pLeft, target);
62565		- j2 = sqlite3VdbeAddOp1(v, OP_IsNull, target);
62566		- if( eType==IN_INDEX_ROWID ){
62567		- j3 = sqlite3VdbeAddOp1(v, OP_MustBeInt, target);
62568		- j4 = sqlite3VdbeAddOp3(v, OP_NotExists, pExpr->iTable, 0, target);
62569		- sqlite3VdbeAddOp2(v, OP_Integer, 1, target);
62570		- j5 = sqlite3VdbeAddOp0(v, OP_Goto);
62571		- sqlite3VdbeJumpHere(v, j3);
62572		- sqlite3VdbeJumpHere(v, j4);
62573		- sqlite3VdbeAddOp2(v, OP_Integer, 0, target);
62574		- }else{
62575		- r2 = regFree2 = sqlite3GetTempReg(pParse);
62576		-
62577		- /* Create a record and test for set membership. If the set contains
62578		- ** the value, then jump to the end of the test code. The target
62579		- ** register still contains the true (1) value written to it earlier.
62580		- */
62581		- sqlite3VdbeAddOp4(v, OP_MakeRecord, target, 1, r2, &affinity, 1);
62582		- sqlite3VdbeAddOp2(v, OP_Integer, 1, target);
62583		- j5 = sqlite3VdbeAddOp3(v, OP_Found, pExpr->iTable, 0, r2);
62584		-
62585		- /* If the set membership test fails, then the result of the
62586		- ** "x IN (...)" expression must be either 0 or NULL. If the set
62587		- ** contains no NULL values, then the result is 0. If the set
62588		- ** contains one or more NULL values, then the result of the
62589		- ** expression is also NULL.
62590		- */
62591		- if( rNotFound==0 ){
62592		- /* This branch runs if it is known at compile time (now) that
62593		- ** the set contains no NULL values. This happens as the result
62594		- ** of a "NOT NULL" constraint in the database schema. No need
62595		- ** to test the data structure at runtime in this case.
62596		- */
62597		- sqlite3VdbeAddOp2(v, OP_Integer, 0, target);
62598		- }else{
62599		- /* This block populates the rNotFound register with either NULL
62600		- ** or 0 (an integer value). If the data structure contains one
62601		- ** or more NULLs, then set rNotFound to NULL. Otherwise, set it
62602		- ** to 0. If register rMayHaveNull is already set to some value
62603		- ** other than NULL, then the test has already been run and
62604		- ** rNotFound is already populated.
62605		- */
62606		- static const char nullRecord[] = { 0x02, 0x00 };
62607		- j3 = sqlite3VdbeAddOp1(v, OP_NotNull, rMayHaveNull);
62608		- sqlite3VdbeAddOp2(v, OP_Null, 0, rNotFound);
62609		- sqlite3VdbeAddOp4(v, OP_Blob, 2, rMayHaveNull, 0,
62610		- nullRecord, P4_STATIC);
62611		- j4 = sqlite3VdbeAddOp3(v, OP_Found, pExpr->iTable, 0, rMayHaveNull);
62612		- sqlite3VdbeAddOp2(v, OP_Integer, 0, rNotFound);
62613		- sqlite3VdbeJumpHere(v, j4);
62614		- sqlite3VdbeJumpHere(v, j3);
62615		-
62616		- /* Copy the value of register rNotFound (which is either NULL or 0)
62617		- ** into the target register. This will be the result of the
62618		- ** expression.
62619		- */
62620		- sqlite3VdbeAddOp2(v, OP_Copy, rNotFound, target);
62621		- }
62622		- }
62623		- sqlite3VdbeJumpHere(v, j2);
62624		- sqlite3VdbeJumpHere(v, j5);
62625		- sqlite3ExprCachePop(pParse, 1);
62626		- VdbeComment((v, "end IN expr r%d", target));
62627		- break;
62628		- }
62629		-#endif
	62745	+ int destIfFalse = sqlite3VdbeMakeLabel(v);
	62746	+ int destIfNull = sqlite3VdbeMakeLabel(v);
	62747	+ sqlite3VdbeAddOp2(v, OP_Null, 0, target);
	62748	+ sqlite3ExprCodeIN(pParse, pExpr, destIfFalse, destIfNull);
	62749	+ sqlite3VdbeAddOp2(v, OP_Integer, 1, target);
	62750	+ sqlite3VdbeResolveLabel(v, destIfFalse);
	62751	+ sqlite3VdbeAddOp2(v, OP_AddImm, target, 0);
	62752	+ sqlite3VdbeResolveLabel(v, destIfNull);
	62753	+ break;
	62754	+ }
	62755	+#endif /* SQLITE_OMIT_SUBQUERY */
	62756	+
	62757	+
62630	62758	/*
62631	62759	** x BETWEEN y AND z
62632	62760	**
62633	62761	** This is equivalent to
62634	62762	**
		@@ -63054,10 +63182,66 @@
63054	63182	sqlite3ExprHardCopy(pParse, target, n);
63055	63183	}
63056	63184	}
63057	63185	return n;
63058	63186	}
	63187	+
	63188	+/*
	63189	+** Generate code for a BETWEEN operator.
	63190	+**
	63191	+** x BETWEEN y AND z
	63192	+**
	63193	+** The above is equivalent to
	63194	+**
	63195	+** x>=y AND x<=z
	63196	+**
	63197	+** Code it as such, taking care to do the common subexpression
	63198	+** elementation of x.
	63199	+*/
	63200	+static void exprCodeBetween(
	63201	+ Parse pParse, / Parsing and code generating context */
	63202	+ Expr pExpr, / The BETWEEN expression */
	63203	+ int dest, /* Jump here if the jump is taken */
	63204	+ int jumpIfTrue, /* Take the jump if the BETWEEN is true */
	63205	+ int jumpIfNull /* Take the jump if the BETWEEN is NULL */
	63206	+){
	63207	+ Expr exprAnd; /* The AND operator in x>=y AND x<=z */
	63208	+ Expr compLeft; /* The x>=y term */
	63209	+ Expr compRight; /* The x<=z term */
	63210	+ Expr exprX; /* The x subexpression */
	63211	+ int regFree1 = 0; /* Temporary use register */
	63212	+
	63213	+ assert( !ExprHasProperty(pExpr, EP_xIsSelect) );
	63214	+ exprX = *pExpr->pLeft;
	63215	+ exprAnd.op = TK_AND;
	63216	+ exprAnd.pLeft = &compLeft;
	63217	+ exprAnd.pRight = &compRight;
	63218	+ compLeft.op = TK_GE;
	63219	+ compLeft.pLeft = &exprX;
	63220	+ compLeft.pRight = pExpr->x.pList->a[0].pExpr;
	63221	+ compRight.op = TK_LE;
	63222	+ compRight.pLeft = &exprX;
	63223	+ compRight.pRight = pExpr->x.pList->a[1].pExpr;
	63224	+ exprX.iTable = sqlite3ExprCodeTemp(pParse, &exprX, &regFree1);
	63225	+ exprX.op = TK_REGISTER;
	63226	+ if( jumpIfTrue ){
	63227	+ sqlite3ExprIfTrue(pParse, &exprAnd, dest, jumpIfNull);
	63228	+ }else{
	63229	+ sqlite3ExprIfFalse(pParse, &exprAnd, dest, jumpIfNull);
	63230	+ }
	63231	+ sqlite3ReleaseTempReg(pParse, regFree1);
	63232	+
	63233	+ /* Ensure adequate test coverage */
	63234	+ testcase( jumpIfTrue==0 && jumpIfNull==0 && regFree1==0 );
	63235	+ testcase( jumpIfTrue==0 && jumpIfNull==0 && regFree1!=0 );
	63236	+ testcase( jumpIfTrue==0 && jumpIfNull!=0 && regFree1==0 );
	63237	+ testcase( jumpIfTrue==0 && jumpIfNull!=0 && regFree1!=0 );
	63238	+ testcase( jumpIfTrue!=0 && jumpIfNull==0 && regFree1==0 );
	63239	+ testcase( jumpIfTrue!=0 && jumpIfNull==0 && regFree1!=0 );
	63240	+ testcase( jumpIfTrue!=0 && jumpIfNull!=0 && regFree1==0 );
	63241	+ testcase( jumpIfTrue!=0 && jumpIfNull!=0 && regFree1!=0 );
	63242	+}
63059	63243
63060	63244	/*
63061	63245	** Generate code for a boolean expression such that a jump is made
63062	63246	** to the label "dest" if the expression is true but execution
63063	63247	** continues straight thru if the expression is false.
		@@ -63154,40 +63338,20 @@
63154	63338	sqlite3VdbeAddOp2(v, op, r1, dest);
63155	63339	testcase( regFree1==0 );
63156	63340	break;
63157	63341	}
63158	63342	case TK_BETWEEN: {
63159		- /* x BETWEEN y AND z
63160		- **
63161		- ** Is equivalent to
63162		- **
63163		- ** x>=y AND x<=z
63164		- **
63165		- ** Code it as such, taking care to do the common subexpression
63166		- ** elementation of x.
63167		- */
63168		- Expr exprAnd;
63169		- Expr compLeft;
63170		- Expr compRight;
63171		- Expr exprX;
63172		-
63173		- assert( !ExprHasProperty(pExpr, EP_xIsSelect) );
63174		- exprX = *pExpr->pLeft;
63175		- exprAnd.op = TK_AND;
63176		- exprAnd.pLeft = &compLeft;
63177		- exprAnd.pRight = &compRight;
63178		- compLeft.op = TK_GE;
63179		- compLeft.pLeft = &exprX;
63180		- compLeft.pRight = pExpr->x.pList->a[0].pExpr;
63181		- compRight.op = TK_LE;
63182		- compRight.pLeft = &exprX;
63183		- compRight.pRight = pExpr->x.pList->a[1].pExpr;
63184		- exprX.iTable = sqlite3ExprCodeTemp(pParse, &exprX, &regFree1);
63185		- testcase( regFree1==0 );
63186		- exprX.op = TK_REGISTER;
63187	63343	testcase( jumpIfNull==0 );
63188		- sqlite3ExprIfTrue(pParse, &exprAnd, dest, jumpIfNull);
	63344	+ exprCodeBetween(pParse, pExpr, dest, 1, jumpIfNull);
	63345	+ break;
	63346	+ }
	63347	+ case TK_IN: {
	63348	+ int destIfFalse = sqlite3VdbeMakeLabel(v);
	63349	+ int destIfNull = jumpIfNull ? dest : destIfFalse;
	63350	+ sqlite3ExprCodeIN(pParse, pExpr, destIfFalse, destIfNull);
	63351	+ sqlite3VdbeAddOp2(v, OP_Goto, 0, dest);
	63352	+ sqlite3VdbeResolveLabel(v, destIfFalse);
63189	63353	break;
63190	63354	}
63191	63355	default: {
63192	63356	r1 = sqlite3ExprCodeTemp(pParse, pExpr, &regFree1);
63193	63357	sqlite3VdbeAddOp3(v, OP_If, r1, dest, jumpIfNull!=0);
		@@ -63267,10 +63431,11 @@
63267	63431	sqlite3VdbeResolveLabel(v, d2);
63268	63432	sqlite3ExprCachePop(pParse, 1);
63269	63433	break;
63270	63434	}
63271	63435	case TK_NOT: {
	63436	+ testcase( jumpIfNull==0 );
63272	63437	sqlite3ExprIfTrue(pParse, pExpr->pLeft, dest, jumpIfNull);
63273	63438	break;
63274	63439	}
63275	63440	case TK_LT:
63276	63441	case TK_LE:
		@@ -63314,40 +63479,22 @@
63314	63479	sqlite3VdbeAddOp2(v, op, r1, dest);
63315	63480	testcase( regFree1==0 );
63316	63481	break;
63317	63482	}
63318	63483	case TK_BETWEEN: {
63319		- /* x BETWEEN y AND z
63320		- **
63321		- ** Is equivalent to
63322		- **
63323		- ** x>=y AND x<=z
63324		- **
63325		- ** Code it as such, taking care to do the common subexpression
63326		- ** elementation of x.
63327		- */
63328		- Expr exprAnd;
63329		- Expr compLeft;
63330		- Expr compRight;
63331		- Expr exprX;
63332		-
63333		- assert( !ExprHasProperty(pExpr, EP_xIsSelect) );
63334		- exprX = *pExpr->pLeft;
63335		- exprAnd.op = TK_AND;
63336		- exprAnd.pLeft = &compLeft;
63337		- exprAnd.pRight = &compRight;
63338		- compLeft.op = TK_GE;
63339		- compLeft.pLeft = &exprX;
63340		- compLeft.pRight = pExpr->x.pList->a[0].pExpr;
63341		- compRight.op = TK_LE;
63342		- compRight.pLeft = &exprX;
63343		- compRight.pRight = pExpr->x.pList->a[1].pExpr;
63344		- exprX.iTable = sqlite3ExprCodeTemp(pParse, &exprX, &regFree1);
63345		- testcase( regFree1==0 );
63346		- exprX.op = TK_REGISTER;
63347	63484	testcase( jumpIfNull==0 );
63348		- sqlite3ExprIfFalse(pParse, &exprAnd, dest, jumpIfNull);
	63485	+ exprCodeBetween(pParse, pExpr, dest, 0, jumpIfNull);
	63486	+ break;
	63487	+ }
	63488	+ case TK_IN: {
	63489	+ if( jumpIfNull ){
	63490	+ sqlite3ExprCodeIN(pParse, pExpr, dest, dest);
	63491	+ }else{
	63492	+ int destIfNull = sqlite3VdbeMakeLabel(v);
	63493	+ sqlite3ExprCodeIN(pParse, pExpr, dest, destIfNull);
	63494	+ sqlite3VdbeResolveLabel(v, destIfNull);
	63495	+ }
63349	63496	break;
63350	63497	}
63351	63498	default: {
63352	63499	r1 = sqlite3ExprCodeTemp(pParse, pExpr, &regFree1);
63353	63500	sqlite3VdbeAddOp3(v, OP_IfNot, r1, dest, jumpIfNull!=0);
		@@ -63692,12 +63839,10 @@
63692	63839	** May you share freely, never taking more than you give.
63693	63840	**
63694	63841	*************************************************************************
63695	63842	** This file contains C code routines that used to generate VDBE code
63696	63843	** that implements the ALTER TABLE command.
63697		-**
63698		-** $Id: alter.c,v 1.62 2009/07/24 17:58:53 danielk1977 Exp $
63699	63844	*/
63700	63845
63701	63846	/*
63702	63847	** The code in this file only exists if we are not omitting the
63703	63848	** ALTER TABLE logic from the build.
		@@ -64488,12 +64633,10 @@
64488	64633	** May you find forgiveness for yourself and forgive others.
64489	64634	** May you share freely, never taking more than you give.
64490	64635	**
64491	64636	*************************************************************************
64492	64637	** This file contains code associated with the ANALYZE command.
64493		-**
64494		-** @(#) $Id: analyze.c,v 1.52 2009/04/16 17:45:48 drh Exp $
64495	64638	*/
64496	64639	#ifndef SQLITE_OMIT_ANALYZE
64497	64640
64498	64641	/*
64499	64642	** This routine generates code that opens the sqlite_stat1 table for
		@@ -65143,12 +65286,10 @@
65143	65286	** May you find forgiveness for yourself and forgive others.
65144	65287	** May you share freely, never taking more than you give.
65145	65288	**
65146	65289	*************************************************************************
65147	65290	** This file contains code used to implement the ATTACH and DETACH commands.
65148		-**
65149		-** $Id: attach.c,v 1.93 2009/05/31 21:21:41 drh Exp $
65150	65291	*/
65151	65292
65152	65293	#ifndef SQLITE_OMIT_ATTACH
65153	65294	/*
65154	65295	** Resolve an expression that was part of an ATTACH or DETACH statement. This
		@@ -65685,12 +65826,10 @@
65685	65826	*************************************************************************
65686	65827	** This file contains code used to implement the sqlite3_set_authorizer()
65687	65828	** API. This facility is an optional feature of the library. Embedded
65688	65829	** systems that do not need this facility may omit it by recompiling
65689	65830	** the library with -DSQLITE_OMIT_AUTHORIZATION=1
65690		-**
65691		-** $Id: auth.c,v 1.32 2009/07/02 18:40:35 danielk1977 Exp $
65692	65831	*/
65693	65832
65694	65833	/*
65695	65834	** All of the code in this file may be omitted by defining a single
65696	65835	** macro.
		@@ -65946,12 +66085,10 @@
65946	66085	** DROP INDEX
65947	66086	** creating ID lists
65948	66087	** BEGIN TRANSACTION
65949	66088	** COMMIT
65950	66089	** ROLLBACK
65951		-**
65952		-** $Id: build.c,v 1.557 2009/07/24 17:58:53 danielk1977 Exp $
65953	66090	*/
65954	66091
65955	66092	/*
65956	66093	** This routine is called when a new SQL statement is beginning to
65957	66094	** be parsed. Initialize the pParse structure as needed.
		@@ -69645,12 +69782,10 @@
69645	69782	**
69646	69783	*************************************************************************
69647	69784	**
69648	69785	** This file contains functions used to access the internal hash tables
69649	69786	** of user defined functions and collation sequences.
69650		-**
69651		-** $Id: callback.c,v 1.42 2009/06/17 00:35:31 drh Exp $
69652	69787	*/
69653	69788
69654	69789
69655	69790	/*
69656	69791	** Invoke the 'collation needed' callback to request a collation sequence
		@@ -70098,12 +70233,10 @@
70098	70233	** May you share freely, never taking more than you give.
70099	70234	**
70100	70235	*************************************************************************
70101	70236	** This file contains C code routines that are called by the parser
70102	70237	** in order to generate code for DELETE FROM statements.
70103		-**
70104		-** $Id: delete.c,v 1.207 2009/08/08 18:01:08 drh Exp $
70105	70238	*/
70106	70239
70107	70240	/*
70108	70241	** Look up every table that is named in pSrc. If any table is not found,
70109	70242	** add an error message to pParse->zErrMsg and return NULL. If all tables
		@@ -70722,11 +70855,10 @@
70722	70855	}
70723	70856	sqlite3ReleaseTempRange(pParse, regBase, nCol+1);
70724	70857	return regBase;
70725	70858	}
70726	70859
70727		-
70728	70860	/************ End of delete.c ********************************************/
70729	70861	/************ Begin file func.c ******************************************/
70730	70862	/*
70731	70863	** 2002 February 23
70732	70864	**
		@@ -70880,10 +71012,12 @@
70880	71012	** p1 is 1-indexed. So substr(x,1,1) returns the first character
70881	71013	** of x. If x is text, then we actually count UTF-8 characters.
70882	71014	** If x is a blob, then we count bytes.
70883	71015	**
70884	71016	** If p1 is negative, then we begin abs(p1) from the end of x[].
	71017	+**
	71018	+** If p2 is negative, return the p2 characters preceeding p1.
70885	71019	*/
70886	71020	static void substrFunc(
70887	71021	sqlite3_context *context,
70888	71022	int argc,
70889	71023	sqlite3_value **argv
		@@ -70900,24 +71034,26 @@
70900	71034	\|\| (argc==3 && sqlite3_value_type(argv[2])==SQLITE_NULL)
70901	71035	){
70902	71036	return;
70903	71037	}
70904	71038	p0type = sqlite3_value_type(argv[0]);
	71039	+ p1 = sqlite3_value_int(argv[1]);
70905	71040	if( p0type==SQLITE_BLOB ){
70906	71041	len = sqlite3_value_bytes(argv[0]);
70907	71042	z = sqlite3_value_blob(argv[0]);
70908	71043	if( z==0 ) return;
70909	71044	assert( len==sqlite3_value_bytes(argv[0]) );
70910	71045	}else{
70911	71046	z = sqlite3_value_text(argv[0]);
70912	71047	if( z==0 ) return;
70913	71048	len = 0;
70914		- for(z2=z; *z2; len++){
70915		- SQLITE_SKIP_UTF8(z2);
	71049	+ if( p1<0 ){
	71050	+ for(z2=z; *z2; len++){
	71051	+ SQLITE_SKIP_UTF8(z2);
	71052	+ }
70916	71053	}
70917	71054	}
70918		- p1 = sqlite3_value_int(argv[1]);
70919	71055	if( argc==3 ){
70920	71056	p2 = sqlite3_value_int(argv[2]);
70921	71057	if( p2<0 ){
70922	71058	p2 = -p2;
70923	71059	negP2 = 1;
		@@ -70943,14 +71079,10 @@
70943	71079	p2 += p1;
70944	71080	p1 = 0;
70945	71081	}
70946	71082	}
70947	71083	assert( p1>=0 && p2>=0 );
70948		- if( p1+p2>len ){
70949		- p2 = len-p1;
70950		- if( p2<0 ) p2 = 0;
70951		- }
70952	71084	if( p0type!=SQLITE_BLOB ){
70953	71085	while( *z && p1 ){
70954	71086	SQLITE_SKIP_UTF8(z);
70955	71087	p1--;
70956	71088	}
		@@ -70957,10 +71089,14 @@
70957	71089	for(z2=z; *z2 && p2; p2--){
70958	71090	SQLITE_SKIP_UTF8(z2);
70959	71091	}
70960	71092	sqlite3_result_text(context, (char*)z, (int)(z2-z), SQLITE_TRANSIENT);
70961	71093	}else{
	71094	+ if( p1+p2>len ){
	71095	+ p2 = len-p1;
	71096	+ if( p2<0 ) p2 = 0;
	71097	+ }
70962	71098	sqlite3_result_blob(context, (char*)&z[p1], (int)p2, SQLITE_TRANSIENT);
70963	71099	}
70964	71100	}
70965	71101
70966	71102	/*
		@@ -71058,10 +71194,18 @@
71058	71194	sqlite3_result_text(context, (char *)z1, -1, sqlite3_free);
71059	71195	}
71060	71196	}
71061	71197	}
71062	71198
	71199	+
	71200	+#if 0 /* This function is never used. */
	71201	+/*
	71202	+** The COALESCE() and IFNULL() functions used to be implemented as shown
	71203	+** here. But now they are implemented as VDBE code so that unused arguments
	71204	+** do not have to be computed. This legacy implementation is retained as
	71205	+** comment.
	71206	+*/
71063	71207	/*
71064	71208	** Implementation of the IFNULL(), NVL(), and COALESCE() functions.
71065	71209	** All three do the same thing. They return the first non-NULL
71066	71210	** argument.
71067	71211	*/
		@@ -71076,10 +71220,12 @@
71076	71220	sqlite3_result_value(context, argv[i]);
71077	71221	break;
71078	71222	}
71079	71223	}
71080	71224	}
	71225	+#endif /* NOT USED */
	71226	+#define ifnullFunc versionFunc /* Substitute function - never called */
71081	71227
71082	71228	/*
71083	71229	** Implementation of random(). Return a random integer.
71084	71230	*/
71085	71231	static void randomFunc(
		@@ -72160,14 +72306,16 @@
72160	72306	FUNCTION(round, 2, 0, 0, roundFunc ),
72161	72307	#endif
72162	72308	FUNCTION(upper, 1, 0, 0, upperFunc ),
72163	72309	FUNCTION(lower, 1, 0, 0, lowerFunc ),
72164	72310	FUNCTION(coalesce, 1, 0, 0, 0 ),
72165		- FUNCTION(coalesce, -1, 0, 0, ifnullFunc ),
72166	72311	FUNCTION(coalesce, 0, 0, 0, 0 ),
	72312	+/* FUNCTION(coalesce, -1, 0, 0, ifnullFunc ), */
	72313	+ {-1,SQLITE_UTF8,SQLITE_FUNC_COALESCE,0,0,ifnullFunc,0,0,"coalesce",0},
72167	72314	FUNCTION(hex, 1, 0, 0, hexFunc ),
72168		- FUNCTION(ifnull, 2, 0, 1, ifnullFunc ),
	72315	+/* FUNCTION(ifnull, 2, 0, 0, ifnullFunc ), */
	72316	+ {2,SQLITE_UTF8,SQLITE_FUNC_COALESCE,0,0,ifnullFunc,0,0,"ifnull",0},
72169	72317	FUNCTION(random, 0, 0, 0, randomFunc ),
72170	72318	FUNCTION(randomblob, 1, 0, 0, randomBlob ),
72171	72319	FUNCTION(nullif, 2, 0, 1, nullifFunc ),
72172	72320	FUNCTION(sqlite_version, 0, 0, 0, versionFunc ),
72173	72321	FUNCTION(sqlite_source_id, 0, 0, 0, sourceidFunc ),
		@@ -72613,11 +72761,11 @@
72613	72761	sqlite3VdbeAddOp2(v, OP_Goto, 0, iOk);
72614	72762	}
72615	72763
72616	72764	sqlite3VdbeAddOp3(v, OP_MakeRecord, regTemp, nCol, regRec);
72617	72765	sqlite3VdbeChangeP4(v, -1, sqlite3IndexAffinityStr(v, pIdx), 0);
72618		- sqlite3VdbeAddOp3(v, OP_Found, iCur, iOk, regRec);
	72766	+ sqlite3VdbeAddOp4Int(v, OP_Found, iCur, iOk, regRec, 0);
72619	72767
72620	72768	sqlite3ReleaseTempReg(pParse, regRec);
72621	72769	sqlite3ReleaseTempRange(pParse, regTemp, nCol);
72622	72770	}
72623	72771	}
		@@ -73419,12 +73567,10 @@
73419	73567	** May you share freely, never taking more than you give.
73420	73568	**
73421	73569	*************************************************************************
73422	73570	** This file contains C code routines that are called by the parser
73423	73571	** to handle INSERT statements in SQLite.
73424		-**
73425		-** $Id: insert.c,v 1.270 2009/07/24 17:58:53 danielk1977 Exp $
73426	73572	*/
73427	73573
73428	73574	/*
73429	73575	** Generate code that will open a table for reading.
73430	73576	*/
		@@ -75240,12 +75386,10 @@
75240	75386	*************************************************************************
75241	75387	** Main file for the SQLite library. The routines in this file
75242	75388	** implement the programmer interface to the library. Routines in
75243	75389	** other files are for internal use by SQLite and should not be
75244	75390	** accessed by users of the library.
75245		-**
75246		-** $Id: legacy.c,v 1.35 2009/08/07 16:56:00 danielk1977 Exp $
75247	75391	*/
75248	75392
75249	75393
75250	75394	/*
75251	75395	** Execute SQL code. Return one of the SQLITE_ success/failure
		@@ -75386,12 +75530,10 @@
75386	75530	** May you share freely, never taking more than you give.
75387	75531	**
75388	75532	*************************************************************************
75389	75533	** This file contains code used to dynamically load extensions into
75390	75534	** the SQLite library.
75391		-**
75392		-** $Id: loadext.c,v 1.60 2009/06/03 01:24:54 drh Exp $
75393	75535	*/
75394	75536
75395	75537	#ifndef SQLITE_CORE
75396	75538	#define SQLITE_CORE 1 /* Disable the API redefinition in sqlite3ext.h */
75397	75539	#endif
		@@ -75411,12 +75553,10 @@
75411	75553	** This header file defines the SQLite interface for use by
75412	75554	** shared libraries that want to be imported as extensions into
75413	75555	** an SQLite instance. Shared libraries that intend to be loaded
75414	75556	** as extensions by SQLite should #include this file instead of
75415	75557	** sqlite3.h.
75416		-**
75417		-** @(#) $Id: sqlite3ext.h,v 1.25 2008/10/12 00:27:54 shane Exp $
75418	75558	*/
75419	75559	#ifndef _SQLITE3EXT_H_
75420	75560	#define _SQLITE3EXT_H_
75421	75561
75422	75562	typedef struct sqlite3_api_routines sqlite3_api_routines;
		@@ -76380,12 +76520,10 @@
76380	76520	** May you find forgiveness for yourself and forgive others.
76381	76521	** May you share freely, never taking more than you give.
76382	76522	**
76383	76523	*************************************************************************
76384	76524	** This file contains code used to implement the PRAGMA command.
76385		-**
76386		-** $Id: pragma.c,v 1.214 2009/07/02 07:47:33 danielk1977 Exp $
76387	76525	*/
76388	76526
76389	76527	/* Ignore this whole file if pragmas are disabled
76390	76528	*/
76391	76529	#if !defined(SQLITE_OMIT_PRAGMA)
		@@ -77499,10 +77637,11 @@
77499	77637	sqlite3VdbeAddOp2(v, OP_Integer, 0, 2); /* reg(2) will count entries */
77500	77638	loopTop = sqlite3VdbeAddOp2(v, OP_Rewind, 1, 0);
77501	77639	sqlite3VdbeAddOp2(v, OP_AddImm, 2, 1); /* increment entry count */
77502	77640	for(j=0, pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext, j++){
77503	77641	int jmp2;
	77642	+ int r1;
77504	77643	static const VdbeOpList idxErr[] = {
77505	77644	{ OP_AddImm, 1, -1, 0},
77506	77645	{ OP_String8, 0, 3, 0}, /* 1 */
77507	77646	{ OP_Rowid, 1, 4, 0},
77508	77647	{ OP_String8, 0, 5, 0}, /* 3 */
		@@ -77512,12 +77651,12 @@
77512	77651	{ OP_Concat, 6, 3, 3},
77513	77652	{ OP_ResultRow, 3, 1, 0},
77514	77653	{ OP_IfPos, 1, 0, 0}, /* 9 */
77515	77654	{ OP_Halt, 0, 0, 0},
77516	77655	};
77517		- sqlite3GenerateIndexKey(pParse, pIdx, 1, 3, 1);
77518		- jmp2 = sqlite3VdbeAddOp3(v, OP_Found, j+2, 0, 3);
	77656	+ r1 = sqlite3GenerateIndexKey(pParse, pIdx, 1, 3, 0);
	77657	+ jmp2 = sqlite3VdbeAddOp4Int(v, OP_Found, j+2, 0, r1, pIdx->nColumn+1);
77519	77658	addr = sqlite3VdbeAddOpList(v, ArraySize(idxErr), idxErr);
77520	77659	sqlite3VdbeChangeP4(v, addr+1, "rowid ", P4_STATIC);
77521	77660	sqlite3VdbeChangeP4(v, addr+3, " missing from index ", P4_STATIC);
77522	77661	sqlite3VdbeChangeP4(v, addr+4, pIdx->zName, P4_STATIC);
77523	77662	sqlite3VdbeJumpHere(v, addr+9);
		@@ -77820,12 +77959,10 @@
77820	77959	**
77821	77960	*************************************************************************
77822	77961	** This file contains the implementation of the sqlite3_prepare()
77823	77962	** interface, and routines that contribute to loading the database schema
77824	77963	** from disk.
77825		-**
77826		-** $Id: prepare.c,v 1.131 2009/08/06 17:43:31 drh Exp $
77827	77964	*/
77828	77965
77829	77966	/*
77830	77967	** Fill the InitData structure with an error message that indicates
77831	77968	** that the database is corrupt.
		@@ -78694,12 +78831,10 @@
78694	78831	** May you share freely, never taking more than you give.
78695	78832	**
78696	78833	*************************************************************************
78697	78834	** This file contains C code routines that are called by the parser
78698	78835	** to handle SELECT statements in SQLite.
78699		-**
78700		-** $Id: select.c,v 1.526 2009/08/01 15:09:58 drh Exp $
78701	78836	*/
78702	78837
78703	78838
78704	78839	/*
78705	78840	** Delete all the content of a Select structure but do not deallocate
		@@ -79123,12 +79258,12 @@
79123	79258	Vdbe *v;
79124	79259	int r1;
79125	79260
79126	79261	v = pParse->pVdbe;
79127	79262	r1 = sqlite3GetTempReg(pParse);
	79263	+ sqlite3VdbeAddOp4Int(v, OP_Found, iTab, addrRepeat, iMem, N);
79128	79264	sqlite3VdbeAddOp3(v, OP_MakeRecord, iMem, N, r1);
79129		- sqlite3VdbeAddOp3(v, OP_Found, iTab, addrRepeat, r1);
79130	79265	sqlite3VdbeAddOp2(v, OP_IdxInsert, iTab, r1);
79131	79266	sqlite3ReleaseTempReg(pParse, r1);
79132	79267	}
79133	79268
79134	79269	/*
		@@ -79365,12 +79500,11 @@
79365	79500	/* Jump to the end of the loop if the LIMIT is reached.
79366	79501	*/
79367	79502	if( p->iLimit ){
79368	79503	assert( pOrderBy==0 ); /* If there is an ORDER BY, the call to
79369	79504	** pushOntoSorter() would have cleared p->iLimit */
79370		- sqlite3VdbeAddOp2(v, OP_AddImm, p->iLimit, -1);
79371		- sqlite3VdbeAddOp2(v, OP_IfZero, p->iLimit, iBreak);
	79505	+ sqlite3VdbeAddOp3(v, OP_IfZero, p->iLimit, iBreak, -1);
79372	79506	}
79373	79507	}
79374	79508
79375	79509	/*
79376	79510	** Given an expression list, generate a KeyInfo structure that records
		@@ -79992,11 +80126,11 @@
79992	80126	*/
79993	80127	static void computeLimitRegisters(Parse pParse, Select p, int iBreak){
79994	80128	Vdbe *v = 0;
79995	80129	int iLimit = 0;
79996	80130	int iOffset;
79997		- int addr1;
	80131	+ int addr1, n;
79998	80132	if( p->iLimit ) return;
79999	80133
80000	80134	/*
80001	80135	** "LIMIT -1" always shows all rows. There is some
80002	80136	** contraversy about what the correct behavior should be.
		@@ -80007,14 +80141,22 @@
80007	80141	assert( p->pOffset==0 \|\| p->pLimit!=0 );
80008	80142	if( p->pLimit ){
80009	80143	p->iLimit = iLimit = ++pParse->nMem;
80010	80144	v = sqlite3GetVdbe(pParse);
80011	80145	if( NEVER(v==0) ) return; /* VDBE should have already been allocated */
80012		- sqlite3ExprCode(pParse, p->pLimit, iLimit);
80013		- sqlite3VdbeAddOp1(v, OP_MustBeInt, iLimit);
80014		- VdbeComment((v, "LIMIT counter"));
80015		- sqlite3VdbeAddOp2(v, OP_IfZero, iLimit, iBreak);
	80146	+ if( sqlite3ExprIsInteger(p->pLimit, &n) ){
	80147	+ sqlite3VdbeAddOp2(v, OP_Integer, n, iLimit);
	80148	+ VdbeComment((v, "LIMIT counter"));
	80149	+ if( n==0 ){
	80150	+ sqlite3VdbeAddOp2(v, OP_Goto, 0, iBreak);
	80151	+ }
	80152	+ }else{
	80153	+ sqlite3ExprCode(pParse, p->pLimit, iLimit);
	80154	+ sqlite3VdbeAddOp1(v, OP_MustBeInt, iLimit);
	80155	+ VdbeComment((v, "LIMIT counter"));
	80156	+ sqlite3VdbeAddOp2(v, OP_IfZero, iLimit, iBreak);
	80157	+ }
80016	80158	if( p->pOffset ){
80017	80159	p->iOffset = iOffset = ++pParse->nMem;
80018	80160	pParse->nMem++; /* Allocate an extra register for limit+offset */
80019	80161	sqlite3ExprCode(pParse, p->pOffset, iOffset);
80020	80162	sqlite3VdbeAddOp1(v, OP_MustBeInt, iOffset);
		@@ -80346,11 +80488,11 @@
80346	80488	iCont = sqlite3VdbeMakeLabel(v);
80347	80489	computeLimitRegisters(pParse, p, iBreak);
80348	80490	sqlite3VdbeAddOp2(v, OP_Rewind, tab1, iBreak);
80349	80491	r1 = sqlite3GetTempReg(pParse);
80350	80492	iStart = sqlite3VdbeAddOp2(v, OP_RowKey, tab1, r1);
80351		- sqlite3VdbeAddOp3(v, OP_NotFound, tab2, iCont, r1);
	80493	+ sqlite3VdbeAddOp4Int(v, OP_NotFound, tab2, iCont, r1, 0);
80352	80494	sqlite3ReleaseTempReg(pParse, r1);
80353	80495	selectInnerLoop(pParse, p, p->pEList, tab1, p->pEList->nExpr,
80354	80496	0, -1, &dest, iCont, iBreak);
80355	80497	sqlite3VdbeResolveLabel(v, iCont);
80356	80498	sqlite3VdbeAddOp2(v, OP_Next, tab1, iStart);
		@@ -80565,12 +80707,11 @@
80565	80707	}
80566	80708
80567	80709	/* Jump to the end of the loop if the LIMIT is reached.
80568	80710	*/
80569	80711	if( p->iLimit ){
80570		- sqlite3VdbeAddOp2(v, OP_AddImm, p->iLimit, -1);
80571		- sqlite3VdbeAddOp2(v, OP_IfZero, p->iLimit, iBreak);
	80712	+ sqlite3VdbeAddOp3(v, OP_IfZero, p->iLimit, iBreak, -1);
80572	80713	}
80573	80714
80574	80715	/* Generate the subroutine return
80575	80716	*/
80576	80717	sqlite3VdbeResolveLabel(v, iContinue);
		@@ -82934,12 +83075,10 @@
82934	83075	** interface routines. These are just wrappers around the main
82935	83076	** interface routine of sqlite3_exec().
82936	83077	**
82937	83078	** These routines are in a separate files so that they will not be linked
82938	83079	** if they are not used.
82939		-**
82940		-** $Id: table.c,v 1.40 2009/04/10 14:28:00 drh Exp $
82941	83080	*/
82942	83081
82943	83082	#ifndef SQLITE_OMIT_GET_TABLE
82944	83083
82945	83084	/*
		@@ -83126,13 +83265,11 @@
83126	83265	** May you do good and not evil.
83127	83266	** May you find forgiveness for yourself and forgive others.
83128	83267	** May you share freely, never taking more than you give.
83129	83268	**
83130	83269	*************************************************************************
83131		-**
83132		-**
83133		-** $Id: trigger.c,v 1.143 2009/08/10 03:57:58 shane Exp $
	83270	+** This file contains the implementation for TRIGGERs
83134	83271	*/
83135	83272
83136	83273	#ifndef SQLITE_OMIT_TRIGGER
83137	83274	/*
83138	83275	** Delete a linked list of TriggerStep structures.
		@@ -84210,12 +84347,10 @@
84210	84347	** May you share freely, never taking more than you give.
84211	84348	**
84212	84349	*************************************************************************
84213	84350	** This file contains C code routines that are called by the parser
84214	84351	** to handle UPDATE statements.
84215		-**
84216		-** $Id: update.c,v 1.207 2009/08/08 18:01:08 drh Exp $
84217	84352	*/
84218	84353
84219	84354	#ifndef SQLITE_OMIT_VIRTUALTABLE
84220	84355	/* Forward declaration */
84221	84356	static void updateVirtualTable(
		@@ -84843,12 +84978,10 @@
84843	84978	*************************************************************************
84844	84979	** This file contains code used to implement the VACUUM command.
84845	84980	**
84846	84981	** Most of the code in this file may be omitted by defining the
84847	84982	** SQLITE_OMIT_VACUUM macro.
84848		-**
84849		-** $Id: vacuum.c,v 1.91 2009/07/02 07:47:33 danielk1977 Exp $
84850	84983	*/
84851	84984
84852	84985	#if !defined(SQLITE_OMIT_VACUUM) && !defined(SQLITE_OMIT_ATTACH)
84853	84986	/*
84854	84987	** Execute zSql on database db. Return an error code.
		@@ -85149,12 +85282,10 @@
85149	85282	** May you find forgiveness for yourself and forgive others.
85150	85283	** May you share freely, never taking more than you give.
85151	85284	**
85152	85285	*************************************************************************
85153	85286	** This file contains code used to help implement virtual tables.
85154		-**
85155		-** $Id: vtab.c,v 1.94 2009/08/08 18:01:08 drh Exp $
85156	85287	*/
85157	85288	#ifndef SQLITE_OMIT_VIRTUALTABLE
85158	85289
85159	85290	/*
85160	85291	** The actual function that does the work of creating a new module.
		@@ -86118,12 +86249,10 @@
86118	86249	** the WHERE clause of SQL statements. This module is responsible for
86119	86250	** generating the code that loops through a table looking for applicable
86120	86251	** rows. Indices are selected and used to speed the search when doing
86121	86252	** so is applicable. Because this module is responsible for selecting
86122	86253	** indices, you might also think of this module as the "query optimizer".
86123		-**
86124		-** $Id: where.c,v 1.411 2009/07/31 06:14:52 danielk1977 Exp $
86125	86254	*/
86126	86255
86127	86256	/*
86128	86257	** Trace output macros
86129	86258	*/
		@@ -88698,20 +88827,43 @@
88698	88827
88699	88828	/*
88700	88829	** Code an OP_Affinity opcode to apply the column affinity string zAff
88701	88830	** to the n registers starting at base.
88702	88831	**
88703		-** Buffer zAff was allocated using sqlite3DbMalloc(). It is the
88704		-** responsibility of this function to arrange for it to be eventually
88705		-** freed using sqlite3DbFree().
	88832	+** As an optimization, SQLITE_AFF_NONE entries (which are no-ops) at the
	88833	+** beginning and end of zAff are ignored. If all entries in zAff are
	88834	+** SQLITE_AFF_NONE, then no code gets generated.
	88835	+**
	88836	+** This routine makes its own copy of zAff so that the caller is free
	88837	+** to modify zAff after this routine returns.
88706	88838	*/
88707	88839	static void codeApplyAffinity(Parse pParse, int base, int n, char zAff){
88708	88840	Vdbe *v = pParse->pVdbe;
	88841	+ if( zAff==0 ){
	88842	+ assert( pParse->db->mallocFailed );
	88843	+ return;
	88844	+ }
88709	88845	assert( v!=0 );
88710		- sqlite3VdbeAddOp2(v, OP_Affinity, base, n);
88711		- sqlite3VdbeChangeP4(v, -1, zAff, P4_DYNAMIC);
88712		- sqlite3ExprCacheAffinityChange(pParse, base, n);
	88846	+
	88847	+ /* Adjust base and n to skip over SQLITE_AFF_NONE entries at the beginning
	88848	+ ** and end of the affinity string.
	88849	+ */
	88850	+ while( n>0 && zAff[0]==SQLITE_AFF_NONE ){
	88851	+ n--;
	88852	+ base++;
	88853	+ zAff++;
	88854	+ }
	88855	+ while( n>1 && zAff[n-1]==SQLITE_AFF_NONE ){
	88856	+ n--;
	88857	+ }
	88858	+
	88859	+ /* Code the OP_Affinity opcode if there is anything left to do. */
	88860	+ if( n>0 ){
	88861	+ sqlite3VdbeAddOp2(v, OP_Affinity, base, n);
	88862	+ sqlite3VdbeChangeP4(v, -1, zAff, n);
	88863	+ sqlite3ExprCacheAffinityChange(pParse, base, n);
	88864	+ }
88713	88865	}
88714	88866
88715	88867
88716	88868	/*
88717	88869	** Generate code for a single equality term of the WHERE clause. An equality
		@@ -88778,11 +88930,11 @@
88778	88930	return iReg;
88779	88931	}
88780	88932
88781	88933	/*
88782	88934	** Generate code that will evaluate all == and IN constraints for an
88783		-** index. The values for all constraints are left on the stack.
	88935	+** index.
88784	88936	**
88785	88937	** For example, consider table t1(a,b,c,d,e,f) with index i1(a,b,c).
88786	88938	** Suppose the WHERE clause is this: a==5 AND b IN (1,2,3) AND c>5 AND c<10
88787	88939	** The index has as many as three equality constraints, but in this
88788	88940	** example, the third "c" value is an inequality. So only two
		@@ -88790,11 +88942,12 @@
88790	88942	** a==5 and b IN (1,2,3). The current values for a and b will be stored
88791	88943	** in consecutive registers and the index of the first register is returned.
88792	88944	**
88793	88945	** In the example above nEq==2. But this subroutine works for any value
88794	88946	** of nEq including 0. If nEq==0, this routine is nearly a no-op.
88795		-** The only thing it does is allocate the pLevel->iMem memory cell.
	88947	+** The only thing it does is allocate the pLevel->iMem memory cell and
	88948	+** compute the affinity string.
88796	88949	**
88797	88950	** This routine always allocates at least one memory cell and returns
88798	88951	** the index of that memory cell. The code that
88799	88952	** calls this routine will use that memory cell to store the termination
88800	88953	** key value of the loop. If one or more IN operators appear, then
		@@ -88868,15 +89021,19 @@
88868	89021	}
88869	89022	}
88870	89023	testcase( pTerm->eOperator & WO_ISNULL );
88871	89024	testcase( pTerm->eOperator & WO_IN );
88872	89025	if( (pTerm->eOperator & (WO_ISNULL\|WO_IN))==0 ){
88873		- sqlite3VdbeAddOp2(v, OP_IsNull, regBase+j, pLevel->addrBrk);
88874		- if( zAff
88875		- && sqlite3CompareAffinity(pTerm->pExpr->pRight, zAff[j])==SQLITE_AFF_NONE
88876		- ){
88877		- zAff[j] = SQLITE_AFF_NONE;
	89026	+ Expr *pRight = pTerm->pExpr->pRight;
	89027	+ sqlite3ExprCodeIsNullJump(v, pRight, regBase+j, pLevel->addrBrk);
	89028	+ if( zAff ){
	89029	+ if( sqlite3CompareAffinity(pRight, zAff[j])==SQLITE_AFF_NONE ){
	89030	+ zAff[j] = SQLITE_AFF_NONE;
	89031	+ }
	89032	+ if( sqlite3ExprNeedsNoAffinityChange(pRight, zAff[j]) ){
	89033	+ zAff[j] = SQLITE_AFF_NONE;
	89034	+ }
88878	89035	}
88879	89036	}
88880	89037	}
88881	89038	*pzAff = zAff;
88882	89039	return regBase;
		@@ -89198,19 +89355,22 @@
89198	89355	/* Seek the index cursor to the start of the range. */
89199	89356	nConstraint = nEq;
89200	89357	if( pRangeStart ){
89201	89358	Expr *pRight = pRangeStart->pExpr->pRight;
89202	89359	sqlite3ExprCode(pParse, pRight, regBase+nEq);
89203		- sqlite3VdbeAddOp2(v, OP_IsNull, regBase+nEq, addrNxt);
89204		- if( zAff
89205		- && sqlite3CompareAffinity(pRight, zAff[nConstraint])==SQLITE_AFF_NONE
89206		- ){
89207		- /* Since the comparison is to be performed with no conversions applied
89208		- ** to the operands, set the affinity to apply to pRight to
89209		- ** SQLITE_AFF_NONE. */
89210		- zAff[nConstraint] = SQLITE_AFF_NONE;
89211		- }
	89360	+ sqlite3ExprCodeIsNullJump(v, pRight, regBase+nEq, addrNxt);
	89361	+ if( zAff ){
	89362	+ if( sqlite3CompareAffinity(pRight, zAff[nConstraint])==SQLITE_AFF_NONE){
	89363	+ /* Since the comparison is to be performed with no conversions
	89364	+ ** applied to the operands, set the affinity to apply to pRight to
	89365	+ ** SQLITE_AFF_NONE. */
	89366	+ zAff[nConstraint] = SQLITE_AFF_NONE;
	89367	+ }
	89368	+ if( sqlite3ExprNeedsNoAffinityChange(pRight, zAff[nConstraint]) ){
	89369	+ zAff[nConstraint] = SQLITE_AFF_NONE;
	89370	+ }
	89371	+ }
89212	89372	nConstraint++;
89213	89373	}else if( isMinQuery ){
89214	89374	sqlite3VdbeAddOp2(v, OP_Null, 0, regBase+nEq);
89215	89375	nConstraint++;
89216	89376	startEq = 0;
		@@ -89223,34 +89383,36 @@
89223	89383	testcase( op==OP_Last );
89224	89384	testcase( op==OP_SeekGt );
89225	89385	testcase( op==OP_SeekGe );
89226	89386	testcase( op==OP_SeekLe );
89227	89387	testcase( op==OP_SeekLt );
89228		- sqlite3VdbeAddOp4(v, op, iIdxCur, addrNxt, regBase,
89229		- SQLITE_INT_TO_PTR(nConstraint), P4_INT32);
	89388	+ sqlite3VdbeAddOp4Int(v, op, iIdxCur, addrNxt, regBase, nConstraint);
89230	89389
89231	89390	/* Load the value for the inequality constraint at the end of the
89232	89391	** range (if any).
89233	89392	*/
89234	89393	nConstraint = nEq;
89235	89394	if( pRangeEnd ){
89236	89395	Expr *pRight = pRangeEnd->pExpr->pRight;
89237	89396	sqlite3ExprCacheRemove(pParse, regBase+nEq);
89238	89397	sqlite3ExprCode(pParse, pRight, regBase+nEq);
89239		- sqlite3VdbeAddOp2(v, OP_IsNull, regBase+nEq, addrNxt);
89240		- zAff = sqlite3DbStrDup(pParse->db, zAff);
89241		- if( zAff
89242		- && sqlite3CompareAffinity(pRight, zAff[nConstraint])==SQLITE_AFF_NONE
89243		- ){
89244		- /* Since the comparison is to be performed with no conversions applied
89245		- ** to the operands, set the affinity to apply to pRight to
89246		- ** SQLITE_AFF_NONE. */
89247		- zAff[nConstraint] = SQLITE_AFF_NONE;
89248		- }
	89398	+ sqlite3ExprCodeIsNullJump(v, pRight, regBase+nEq, addrNxt);
	89399	+ if( zAff ){
	89400	+ if( sqlite3CompareAffinity(pRight, zAff[nConstraint])==SQLITE_AFF_NONE){
	89401	+ /* Since the comparison is to be performed with no conversions
	89402	+ ** applied to the operands, set the affinity to apply to pRight to
	89403	+ ** SQLITE_AFF_NONE. */
	89404	+ zAff[nConstraint] = SQLITE_AFF_NONE;
	89405	+ }
	89406	+ if( sqlite3ExprNeedsNoAffinityChange(pRight, zAff[nConstraint]) ){
	89407	+ zAff[nConstraint] = SQLITE_AFF_NONE;
	89408	+ }
	89409	+ }
89249	89410	codeApplyAffinity(pParse, regBase, nEq+1, zAff);
89250	89411	nConstraint++;
89251	89412	}
	89413	+ sqlite3DbFree(pParse->db, zAff);
89252	89414
89253	89415	/* Top of the loop body */
89254	89416	pLevel->p2 = sqlite3VdbeCurrentAddr(v);
89255	89417
89256	89418	/* Check if the index cursor is past the end of the range. */
		@@ -89257,12 +89419,11 @@
89257	89419	op = aEndOp[(pRangeEnd \|\| nEq) * (1 + bRev)];
89258	89420	testcase( op==OP_Noop );
89259	89421	testcase( op==OP_IdxGE );
89260	89422	testcase( op==OP_IdxLT );
89261	89423	if( op!=OP_Noop ){
89262		- sqlite3VdbeAddOp4(v, op, iIdxCur, addrNxt, regBase,
89263		- SQLITE_INT_TO_PTR(nConstraint), P4_INT32);
	89424	+ sqlite3VdbeAddOp4Int(v, op, iIdxCur, addrNxt, regBase, nConstraint);
89264	89425	sqlite3VdbeChangeP5(v, endEq!=bRev ?1:0);
89265	89426	}
89266	89427
89267	89428	/* If there are inequality constraints, check that the value
89268	89429	** of the table column that the inequality contrains is not NULL.
		@@ -89387,13 +89548,12 @@
89387	89548	if( (wctrlFlags & WHERE_DUPLICATES_OK)==0 ){
89388	89549	int iSet = ((ii==pOrWc->nTerm-1)?-1:ii);
89389	89550	int r;
89390	89551	r = sqlite3ExprCodeGetColumn(pParse, pTabItem->pTab, -1, iCur,
89391	89552	regRowid, 0);
89392		- sqlite3VdbeAddOp4(v, OP_RowSetTest, regRowset,
89393		- sqlite3VdbeCurrentAddr(v)+2,
89394		- r, SQLITE_INT_TO_PTR(iSet), P4_INT32);
	89553	+ sqlite3VdbeAddOp4Int(v, OP_RowSetTest, regRowset,
	89554	+ sqlite3VdbeCurrentAddr(v)+2, r, iSet);
89395	89555	}
89396	89556	sqlite3VdbeAddOp2(v, OP_Gosub, regReturn, iLoopBody);
89397	89557
89398	89558	/* Finish the loop through table entries that match term pOrTerm. */
89399	89559	sqlite3WhereEnd(pSubWInfo);
		@@ -89920,11 +90080,12 @@
89920	90080	sqlite3OpenTable(pParse, pTabItem->iCursor, iDb, pTab, op);
89921	90081	if( !pWInfo->okOnePass && pTab->nCol<BMS ){
89922	90082	Bitmask b = pTabItem->colUsed;
89923	90083	int n = 0;
89924	90084	for(; b; b=b>>1, n++){}
89925		- sqlite3VdbeChangeP4(v, sqlite3VdbeCurrentAddr(v)-1, SQLITE_INT_TO_PTR(n), P4_INT32);
	90085	+ sqlite3VdbeChangeP4(v, sqlite3VdbeCurrentAddr(v)-1,
	90086	+ SQLITE_INT_TO_PTR(n), P4_INT32);
89926	90087	assert( n<=pTab->nCol );
89927	90088	}
89928	90089	}else{
89929	90090	sqlite3TableLock(pParse, iDb, pTab->tnum, 0, pTab->zName);
89930	90091	}
		@@ -90048,11 +90209,15 @@
90048	90209	}
90049	90210	sqlite3VdbeResolveLabel(v, pLevel->addrBrk);
90050	90211	if( pLevel->iLeftJoin ){
90051	90212	int addr;
90052	90213	addr = sqlite3VdbeAddOp1(v, OP_IfPos, pLevel->iLeftJoin);
90053		- sqlite3VdbeAddOp1(v, OP_NullRow, pTabList->a[i].iCursor);
	90214	+ assert( (pLevel->plan.wsFlags & WHERE_IDX_ONLY)==0
	90215	+ \|\| (pLevel->plan.wsFlags & WHERE_INDEXED)!=0 );
	90216	+ if( (pLevel->plan.wsFlags & WHERE_IDX_ONLY)==0 ){
	90217	+ sqlite3VdbeAddOp1(v, OP_NullRow, pTabList->a[i].iCursor);
	90218	+ }
90054	90219	if( pLevel->iIdxCur>=0 ){
90055	90220	sqlite3VdbeAddOp1(v, OP_NullRow, pLevel->iIdxCur);
90056	90221	}
90057	90222	if( pLevel->op==OP_Return ){
90058	90223	sqlite3VdbeAddOp2(v, OP_Gosub, pLevel->p1, pLevel->addrFirst);
		@@ -90099,11 +90264,10 @@
90099	90264	*/
90100	90265	if( (pLevel->plan.wsFlags & WHERE_INDEXED)!=0 && !db->mallocFailed){
90101	90266	int k, j, last;
90102	90267	VdbeOp *pOp;
90103	90268	Index *pIdx = pLevel->plan.u.pIdx;
90104		- int useIndexOnly = pLevel->plan.wsFlags & WHERE_IDX_ONLY;
90105	90269
90106	90270	assert( pIdx!=0 );
90107	90271	pOp = sqlite3VdbeGetOp(v, pWInfo->iTop);
90108	90272	last = sqlite3VdbeCurrentAddr(v);
90109	90273	for(k=pWInfo->iTop; k<last; k++, pOp++){
		@@ -90114,16 +90278,15 @@
90114	90278	pOp->p2 = j;
90115	90279	pOp->p1 = pLevel->iIdxCur;
90116	90280	break;
90117	90281	}
90118	90282	}
90119		- assert(!useIndexOnly \|\| j<pIdx->nColumn);
	90283	+ assert( (pLevel->plan.wsFlags & WHERE_IDX_ONLY)==0
	90284	+ \|\| j<pIdx->nColumn );
90120	90285	}else if( pOp->opcode==OP_Rowid ){
90121	90286	pOp->p1 = pLevel->iIdxCur;
90122	90287	pOp->opcode = OP_IdxRowid;
90123		- }else if( pOp->opcode==OP_NullRow && useIndexOnly ){
90124		- pOp->opcode = OP_Noop;
90125	90288	}
90126	90289	}
90127	90290	}
90128	90291	}
90129	90292
		@@ -90239,10 +90402,21 @@
90239	90402	){
90240	90403	pOut->pExpr = sqlite3PExpr(pParse, op, pOperand->pExpr, 0, 0);
90241	90404	pOut->zStart = pOperand->zStart;
90242	90405	pOut->zEnd = &pPostOp->z[pPostOp->n];
90243	90406	}
	90407	+
	90408	+ /* A routine to convert a binary TK_IS or TK_ISNOT expression into a
	90409	+ ** unary TK_ISNULL or TK_NOTNULL expression. */
	90410	+ static void binaryToUnaryIfNull(Parse pParse, Expr pY, Expr *pA, int op){
	90411	+ sqlite3 *db = pParse->db;
	90412	+ if( db->mallocFailed==0 && pY->op==TK_NULL ){
	90413	+ pA->op = op;
	90414	+ sqlite3ExprDelete(db, pA->pRight);
	90415	+ pA->pRight = 0;
	90416	+ }
	90417	+ }
90244	90418
90245	90419	/* Construct an expression node for a unary prefix operator
90246	90420	*/
90247	90421	static void spanUnaryPrefix(
90248	90422	ExprSpan pOut, / Write the new expression node here */
		@@ -92857,21 +93031,17 @@
92857	93031	{spanUnaryPostfix(&yygotominor.yy346,pParse,TK_NOTNULL,&yymsp[-2].minor.yy346,&yymsp[0].minor.yy0);}
92858	93032	break;
92859	93033	case 216: /* expr ::= expr IS expr */
92860	93034	{
92861	93035	spanBinaryExpr(&yygotominor.yy346,pParse,TK_IS,&yymsp[-2].minor.yy346,&yymsp[0].minor.yy346);
92862		- if( pParse->db->mallocFailed==0 && yymsp[0].minor.yy346.pExpr->op==TK_NULL ){
92863		- yygotominor.yy346.pExpr->op = TK_ISNULL;
92864		- }
	93036	+ binaryToUnaryIfNull(pParse, yymsp[0].minor.yy346.pExpr, yygotominor.yy346.pExpr, TK_ISNULL);
92865	93037	}
92866	93038	break;
92867	93039	case 217: /* expr ::= expr IS NOT expr */
92868	93040	{
92869	93041	spanBinaryExpr(&yygotominor.yy346,pParse,TK_ISNOT,&yymsp[-3].minor.yy346,&yymsp[0].minor.yy346);
92870		- if( pParse->db->mallocFailed==0 && yymsp[0].minor.yy346.pExpr->op==TK_NULL ){
92871		- yygotominor.yy346.pExpr->op = TK_NOTNULL;
92872		- }
	93042	+ binaryToUnaryIfNull(pParse, yymsp[0].minor.yy346.pExpr, yygotominor.yy346.pExpr, TK_NOTNULL);
92873	93043	}
92874	93044	break;
92875	93045	case 218: /* expr ::= NOT expr */
92876	93046	case 219: /* expr ::= BITNOT expr */ yytestcase(yyruleno==219);
92877	93047	{spanUnaryPrefix(&yygotominor.yy346,pParse,yymsp[-1].major,&yymsp[0].minor.yy346,&yymsp[-1].minor.yy0);}
		@@ -93551,12 +93721,10 @@
93551	93721	** An tokenizer for SQL
93552	93722	**
93553	93723	** This file contains C code that splits an SQL input string up into
93554	93724	** individual tokens and sends those tokens one-by-one over to the
93555	93725	** parser for analysis.
93556		-**
93557		-** $Id: tokenize.c,v 1.163 2009/07/03 22:54:37 drh Exp $
93558	93726	*/
93559	93727
93560	93728	/*
93561	93729	** The charMap() macro maps alphabetic characters into their
93562	93730	** lower-case ASCII equivalent. On ASCII machines, this is just
		@@ -93892,20 +94060,11 @@
93892	94060	** middle of identfiers. But many SQL implementations do.
93893	94061	** SQLite will allow '$' in identifiers for compatibility.
93894	94062	** But the feature is undocumented.
93895	94063	*/
93896	94064	#ifdef SQLITE_ASCII
93897		-SQLITE_PRIVATE const char sqlite3IsAsciiIdChar[] = {
93898		-/* x0 x1 x2 x3 x4 x5 x6 x7 x8 x9 xA xB xC xD xE xF */
93899		- 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 2x */
93900		- 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, /* 3x */
93901		- 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* 4x */
93902		- 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 1, /* 5x */
93903		- 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* 6x */
93904		- 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, /* 7x */
93905		-};
93906		-#define IdChar(C) (((c=C)&0x80)!=0 \|\| (c>0x1f && sqlite3IsAsciiIdChar[c-0x20]))
	94065	+#define IdChar(C) ((sqlite3CtypeMap[(unsigned char)C]&0x46)!=0)
93907	94066	#endif
93908	94067	#ifdef SQLITE_EBCDIC
93909	94068	SQLITE_PRIVATE const char sqlite3IsEbcdicIdChar[] = {
93910	94069	/* x0 x1 x2 x3 x4 x5 x6 x7 x8 x9 xA xB xC xD xE xF */
93911	94070	0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, /* 4x */
		@@ -94361,22 +94520,19 @@
94361	94520	**
94362	94521	** This file contains C code that implements the sqlite3_complete() API.
94363	94522	** This code used to be part of the tokenizer.c source file. But by
94364	94523	** separating it out, the code will be automatically omitted from
94365	94524	** static links that do not use it.
94366		-**
94367		-** $Id: complete.c,v 1.8 2009/04/28 04:46:42 drh Exp $
94368	94525	*/
94369	94526	#ifndef SQLITE_OMIT_COMPLETE
94370	94527
94371	94528	/*
94372	94529	** This is defined in tokenize.c. We just have to import the definition.
94373	94530	*/
94374	94531	#ifndef SQLITE_AMALGAMATION
94375	94532	#ifdef SQLITE_ASCII
94376		-SQLITE_PRIVATE const char sqlite3IsAsciiIdChar[];
94377		-#define IdChar(C) (((c=C)&0x80)!=0 \|\| (c>0x1f && sqlite3IsAsciiIdChar[c-0x20]))
	94533	+#define IdChar(C) ((sqlite3CtypeMap[(unsigned char)C]&0x46)!=0)
94378	94534	#endif
94379	94535	#ifdef SQLITE_EBCDIC
94380	94536	SQLITE_PRIVATE const char sqlite3IsEbcdicIdChar[];
94381	94537	#define IdChar(C) (((c=C)>=0x42 && sqlite3IsEbcdicIdChar[c-0x40]))
94382	94538	#endif
		@@ -94529,11 +94685,13 @@
94529	94685	if( *zSql==0 ) return 0;
94530	94686	token = tkOTHER;
94531	94687	break;
94532	94688	}
94533	94689	default: {
94534		- int c;
	94690	+#ifdef SQLITE_EBCDIC
	94691	+ unsigned char c;
	94692	+#endif
94535	94693	if( IdChar((u8)*zSql) ){
94536	94694	/* Keywords and unquoted identifiers */
94537	94695	int nId;
94538	94696	for(nId=1; IdChar(zSql[nId]); nId++){}
94539	94697	#ifdef SQLITE_OMIT_TRIGGER
		@@ -97015,12 +97173,10 @@
97015	97173	**
97016	97174	*************************************************************************
97017	97175	**
97018	97176	** This file contains the implementation of the sqlite3_unlock_notify()
97019	97177	** API method and its associated functionality.
97020		-**
97021		-** $Id: notify.c,v 1.4 2009/04/07 22:06:57 drh Exp $
97022	97178	*/
97023	97179
97024	97180	/* Omit this entire file if SQLITE_ENABLE_UNLOCK_NOTIFY is not defined. */
97025	97181	#ifdef SQLITE_ENABLE_UNLOCK_NOTIFY
97026	97182
		@@ -97615,380 +97771,17 @@
97615	97771
97616	97772	#if defined(SQLITE_ENABLE_FTS3) && !defined(SQLITE_CORE)
97617	97773	# define SQLITE_CORE 1
97618	97774	#endif
97619	97775
97620		-
97621		-/************ Include fts3_expr.h in the middle of fts3.c ****************/
97622		-/************ Begin file fts3_expr.h *************************************/
97623		-/*
97624		-** 2008 Nov 28
97625		-**
97626		-** The author disclaims copyright to this source code. In place of
97627		-** a legal notice, here is a blessing:
97628		-**
97629		-** May you do good and not evil.
97630		-** May you find forgiveness for yourself and forgive others.
97631		-** May you share freely, never taking more than you give.
97632		-**
97633		-******************************************************************************
97634		-**
97635		-*/
97636		-
97637		-/************ Include fts3_tokenizer.h in the middle of fts3_expr.h ******/
97638		-/************ Begin file fts3_tokenizer.h ********************************/
97639		-/*
97640		-** 2006 July 10
97641		-**
97642		-** The author disclaims copyright to this source code.
97643		-**
97644		-*************************************************************************
97645		-** Defines the interface to tokenizers used by fulltext-search. There
97646		-** are three basic components:
97647		-**
97648		-** sqlite3_tokenizer_module is a singleton defining the tokenizer
97649		-** interface functions. This is essentially the class structure for
97650		-** tokenizers.
97651		-**
97652		-** sqlite3_tokenizer is used to define a particular tokenizer, perhaps
97653		-** including customization information defined at creation time.
97654		-**
97655		-** sqlite3_tokenizer_cursor is generated by a tokenizer to generate
97656		-** tokens from a particular input.
97657		-*/
97658		-#ifndef _FTS3_TOKENIZER_H_
97659		-#define _FTS3_TOKENIZER_H_
97660		-
97661		-/* TODO(shess) Only used for SQLITE_OK and SQLITE_DONE at this time.
97662		-** If tokenizers are to be allowed to call sqlite3_*() functions, then
97663		-** we will need a way to register the API consistently.
97664		-*/
97665		-
97666		-/*
97667		-** Structures used by the tokenizer interface. When a new tokenizer
97668		-** implementation is registered, the caller provides a pointer to
97669		-** an sqlite3_tokenizer_module containing pointers to the callback
97670		-** functions that make up an implementation.
97671		-**
97672		-** When an fts3 table is created, it passes any arguments passed to
97673		-** the tokenizer clause of the CREATE VIRTUAL TABLE statement to the
97674		-** sqlite3_tokenizer_module.xCreate() function of the requested tokenizer
97675		-** implementation. The xCreate() function in turn returns an
97676		-** sqlite3_tokenizer structure representing the specific tokenizer to
97677		-** be used for the fts3 table (customized by the tokenizer clause arguments).
97678		-**
97679		-** To tokenize an input buffer, the sqlite3_tokenizer_module.xOpen()
97680		-** method is called. It returns an sqlite3_tokenizer_cursor object
97681		-** that may be used to tokenize a specific input buffer based on
97682		-** the tokenization rules supplied by a specific sqlite3_tokenizer
97683		-** object.
97684		-*/
97685		-typedef struct sqlite3_tokenizer_module sqlite3_tokenizer_module;
97686		-typedef struct sqlite3_tokenizer sqlite3_tokenizer;
97687		-typedef struct sqlite3_tokenizer_cursor sqlite3_tokenizer_cursor;
97688		-
97689		-struct sqlite3_tokenizer_module {
97690		-
97691		- /*
97692		- ** Structure version. Should always be set to 0.
97693		- */
97694		- int iVersion;
97695		-
97696		- /*
97697		- ** Create a new tokenizer. The values in the argv[] array are the
97698		- ** arguments passed to the "tokenizer" clause of the CREATE VIRTUAL
97699		- ** TABLE statement that created the fts3 table. For example, if
97700		- ** the following SQL is executed:
97701		- **
97702		- ** CREATE .. USING fts3( ... , tokenizer <tokenizer-name> arg1 arg2)
97703		- **
97704		- ** then argc is set to 2, and the argv[] array contains pointers
97705		- ** to the strings "arg1" and "arg2".
97706		- **
97707		- ** This method should return either SQLITE_OK (0), or an SQLite error
97708		- ** code. If SQLITE_OK is returned, then *ppTokenizer should be set
97709		- ** to point at the newly created tokenizer structure. The generic
97710		- ** sqlite3_tokenizer.pModule variable should not be initialised by
97711		- ** this callback. The caller will do so.
97712		- */
97713		- int (*xCreate)(
97714		- int argc, /* Size of argv array */
97715		- const char constargv, /* Tokenizer argument strings */
97716		- sqlite3_tokenizer *ppTokenizer / OUT: Created tokenizer */
97717		- );
97718		-
97719		- /*
97720		- ** Destroy an existing tokenizer. The fts3 module calls this method
97721		- ** exactly once for each successful call to xCreate().
97722		- */
97723		- int (xDestroy)(sqlite3_tokenizer pTokenizer);
97724		-
97725		- /*
97726		- ** Create a tokenizer cursor to tokenize an input buffer. The caller
97727		- ** is responsible for ensuring that the input buffer remains valid
97728		- ** until the cursor is closed (using the xClose() method).
97729		- */
97730		- int (*xOpen)(
97731		- sqlite3_tokenizer pTokenizer, / Tokenizer object */
97732		- const char pInput, int nBytes, / Input buffer */
97733		- sqlite3_tokenizer_cursor *ppCursor / OUT: Created tokenizer cursor */
97734		- );
97735		-
97736		- /*
97737		- ** Destroy an existing tokenizer cursor. The fts3 module calls this
97738		- ** method exactly once for each successful call to xOpen().
97739		- */
97740		- int (xClose)(sqlite3_tokenizer_cursor pCursor);
97741		-
97742		- /*
97743		- ** Retrieve the next token from the tokenizer cursor pCursor. This
97744		- ** method should either return SQLITE_OK and set the values of the
97745		- ** "OUT" variables identified below, or SQLITE_DONE to indicate that
97746		- ** the end of the buffer has been reached, or an SQLite error code.
97747		- **
97748		- ** *ppToken should be set to point at a buffer containing the
97749		- ** normalized version of the token (i.e. after any case-folding and/or
97750		- ** stemming has been performed). *pnBytes should be set to the length
97751		- ** of this buffer in bytes. The input text that generated the token is
97752		- ** identified by the byte offsets returned in *piStartOffset and
97753		- ** piEndOffset. piStartOffset should be set to the index of the first
97754		- ** byte of the token in the input buffer. *piEndOffset should be set
97755		- ** to the index of the first byte just past the end of the token in
97756		- ** the input buffer.
97757		- **
97758		- ** The buffer *ppToken is set to point at is managed by the tokenizer
97759		- ** implementation. It is only required to be valid until the next call
97760		- ** to xNext() or xClose().
97761		- */
97762		- /* TODO(shess) current implementation requires pInput to be
97763		- ** nul-terminated. This should either be fixed, or pInput/nBytes
97764		- ** should be converted to zInput.
97765		- */
97766		- int (*xNext)(
97767		- sqlite3_tokenizer_cursor pCursor, / Tokenizer cursor */
97768		- const char *ppToken, int pnBytes, /* OUT: Normalized text for token */
97769		- int piStartOffset, / OUT: Byte offset of token in input buffer */
97770		- int piEndOffset, / OUT: Byte offset of end of token in input buffer */
97771		- int piPosition / OUT: Number of tokens returned before this one */
97772		- );
97773		-};
97774		-
97775		-struct sqlite3_tokenizer {
97776		- const sqlite3_tokenizer_module pModule; / The module for this tokenizer */
97777		- /* Tokenizer implementations will typically add additional fields */
97778		-};
97779		-
97780		-struct sqlite3_tokenizer_cursor {
97781		- sqlite3_tokenizer pTokenizer; / Tokenizer for this cursor. */
97782		- /* Tokenizer implementations will typically add additional fields */
97783		-};
97784		-
97785		-#endif /* _FTS3_TOKENIZER_H_ */
97786		-
97787		-/************ End of fts3_tokenizer.h ************************************/
97788		-/************ Continuing where we left off in fts3_expr.h ****************/
97789		-
97790		-/*
97791		-** The following describes the syntax supported by the fts3 MATCH
97792		-** operator in a similar format to that used by the lemon parser
97793		-** generator. This module does not use actually lemon, it uses a
97794		-** custom parser.
97795		-**
97796		-** query ::= andexpr (OR andexpr)*.
97797		-**
97798		-** andexpr ::= notexpr (AND? notexpr)*.
97799		-**
97800		-** notexpr ::= nearexpr (NOT nearexpr\|-TOKEN)*.
97801		-** notexpr ::= LP query RP.
97802		-**
97803		-** nearexpr ::= phrase (NEAR distance_opt nearexpr)*.
97804		-**
97805		-** distance_opt ::= .
97806		-** distance_opt ::= / INTEGER.
97807		-**
97808		-** phrase ::= TOKEN.
97809		-** phrase ::= COLUMN:TOKEN.
97810		-** phrase ::= "TOKEN TOKEN TOKEN...".
97811		-*/
97812		-
97813		-typedef struct Fts3Expr Fts3Expr;
97814		-typedef struct Fts3Phrase Fts3Phrase;
97815		-
97816		-/*
97817		-** A "phrase" is a sequence of one or more tokens that must match in
97818		-** sequence. A single token is the base case and the most common case.
97819		-** For a sequence of tokens contained in "...", nToken will be the number
97820		-** of tokens in the string.
97821		-*/
97822		-struct Fts3Phrase {
97823		- int nToken; /* Number of tokens in the phrase */
97824		- int iColumn; /* Index of column this phrase must match */
97825		- int isNot; /* Phrase prefixed by unary not (-) operator */
97826		- struct PhraseToken {
97827		- char z; / Text of the token */
97828		- int n; /* Number of bytes in buffer pointed to by z */
97829		- int isPrefix; /* True if token ends in with a "" character /
97830		- } aToken[1]; /* One entry for each token in the phrase */
97831		-};
97832		-
97833		-/*
97834		-** A tree of these objects forms the RHS of a MATCH operator.
97835		-*/
97836		-struct Fts3Expr {
97837		- int eType; /* One of the FTSQUERY_XXX values defined below */
97838		- int nNear; /* Valid if eType==FTSQUERY_NEAR */
97839		- Fts3Expr pParent; / pParent->pLeft==this or pParent->pRight==this */
97840		- Fts3Expr pLeft; / Left operand */
97841		- Fts3Expr pRight; / Right operand */
97842		- Fts3Phrase pPhrase; / Valid if eType==FTSQUERY_PHRASE */
97843		-};
97844		-
97845		-SQLITE_PRIVATE int sqlite3Fts3ExprParse(sqlite3_tokenizer , char *, int, int,
97846		- const char , int, Fts3Expr *);
97847		-SQLITE_PRIVATE void sqlite3Fts3ExprFree(Fts3Expr *);
97848		-
97849		-/*
97850		-** Candidate values for Fts3Query.eType. Note that the order of the first
97851		-** four values is in order of precedence when parsing expressions. For
97852		-** example, the following:
97853		-**
97854		-** "a OR b AND c NOT d NEAR e"
97855		-**
97856		-** is equivalent to:
97857		-**
97858		-** "a OR (b AND (c NOT (d NEAR e)))"
97859		-*/
97860		-#define FTSQUERY_NEAR 1
97861		-#define FTSQUERY_NOT 2
97862		-#define FTSQUERY_AND 3
97863		-#define FTSQUERY_OR 4
97864		-#define FTSQUERY_PHRASE 5
97865		-
97866		-#ifdef SQLITE_TEST
97867		-SQLITE_PRIVATE void sqlite3Fts3ExprInitTestInterface(sqlite3 *db);
97868		-#endif
97869		-
97870		-/************ End of fts3_expr.h *****************************************/
97871		-/************ Continuing where we left off in fts3.c *********************/
97872		-/************ Include fts3_hash.h in the middle of fts3.c ****************/
97873		-/************ Begin file fts3_hash.h *************************************/
97874		-/*
97875		-** 2001 September 22
97876		-**
97877		-** The author disclaims copyright to this source code. In place of
97878		-** a legal notice, here is a blessing:
97879		-**
97880		-** May you do good and not evil.
97881		-** May you find forgiveness for yourself and forgive others.
97882		-** May you share freely, never taking more than you give.
97883		-**
97884		-*************************************************************************
97885		-** This is the header file for the generic hash-table implemenation
97886		-** used in SQLite. We've modified it slightly to serve as a standalone
97887		-** hash table implementation for the full-text indexing module.
97888		-**
97889		-*/
97890		-#ifndef _FTS3_HASH_H_
97891		-#define _FTS3_HASH_H_
97892		-
97893		-/* Forward declarations of structures. */
97894		-typedef struct fts3Hash fts3Hash;
97895		-typedef struct fts3HashElem fts3HashElem;
97896		-
97897		-/* A complete hash table is an instance of the following structure.
97898		-** The internals of this structure are intended to be opaque -- client
97899		-** code should not attempt to access or modify the fields of this structure
97900		-** directly. Change this structure only by using the routines below.
97901		-** However, many of the "procedures" and "functions" for modifying and
97902		-** accessing this structure are really macros, so we can't really make
97903		-** this structure opaque.
97904		-*/
97905		-struct fts3Hash {
97906		- char keyClass; /* HASH_INT, _POINTER, _STRING, _BINARY */
97907		- char copyKey; /* True if copy of key made on insert */
97908		- int count; /* Number of entries in this table */
97909		- fts3HashElem first; / The first element of the array */
97910		- int htsize; /* Number of buckets in the hash table */
97911		- struct _fts3ht { /* the hash table */
97912		- int count; /* Number of entries with this hash */
97913		- fts3HashElem chain; / Pointer to first entry with this hash */
97914		- } *ht;
97915		-};
97916		-
97917		-/* Each element in the hash table is an instance of the following
97918		-** structure. All elements are stored on a single doubly-linked list.
97919		-**
97920		-** Again, this structure is intended to be opaque, but it can't really
97921		-** be opaque because it is used by macros.
97922		-*/
97923		-struct fts3HashElem {
97924		- fts3HashElem next, prev; /* Next and previous elements in the table */
97925		- void data; / Data associated with this element */
97926		- void pKey; int nKey; / Key associated with this element */
97927		-};
97928		-
97929		-/*
97930		-** There are 2 different modes of operation for a hash table:
97931		-**
97932		-** FTS3_HASH_STRING pKey points to a string that is nKey bytes long
97933		-** (including the null-terminator, if any). Case
97934		-** is respected in comparisons.
97935		-**
97936		-** FTS3_HASH_BINARY pKey points to binary data nKey bytes long.
97937		-** memcmp() is used to compare keys.
97938		-**
97939		-** A copy of the key is made if the copyKey parameter to fts3HashInit is 1.
97940		-*/
97941		-#define FTS3_HASH_STRING 1
97942		-#define FTS3_HASH_BINARY 2
97943		-
97944		-/*
97945		-** Access routines. To delete, insert a NULL pointer.
97946		-*/
97947		-SQLITE_PRIVATE void sqlite3Fts3HashInit(fts3Hash*, int keytype, int copyKey);
97948		-SQLITE_PRIVATE void sqlite3Fts3HashInsert(fts3Hash, const void pKey, int nKey, void pData);
97949		-SQLITE_PRIVATE void sqlite3Fts3HashFind(const fts3Hash, const void *pKey, int nKey);
97950		-SQLITE_PRIVATE void sqlite3Fts3HashClear(fts3Hash*);
97951		-
97952		-/*
97953		-** Shorthand for the functions above
97954		-*/
97955		-#define fts3HashInit sqlite3Fts3HashInit
97956		-#define fts3HashInsert sqlite3Fts3HashInsert
97957		-#define fts3HashFind sqlite3Fts3HashFind
97958		-#define fts3HashClear sqlite3Fts3HashClear
97959		-
97960		-/*
97961		-** Macros for looping over all elements of a hash table. The idiom is
97962		-** like this:
97963		-**
97964		-** fts3Hash h;
97965		-** fts3HashElem *p;
97966		-** ...
97967		-** for(p=fts3HashFirst(&h); p; p=fts3HashNext(p)){
97968		-** SomeStructure *pData = fts3HashData(p);
97969		-** // do something with pData
97970		-** }
97971		-*/
97972		-#define fts3HashFirst(H) ((H)->first)
97973		-#define fts3HashNext(E) ((E)->next)
97974		-#define fts3HashData(E) ((E)->data)
97975		-#define fts3HashKey(E) ((E)->pKey)
97976		-#define fts3HashKeysize(E) ((E)->nKey)
97977		-
97978		-/*
97979		-** Number of entries in a hash table
97980		-*/
97981		-#define fts3HashCount(H) ((H)->count)
97982		-
97983		-#endif /* _FTS3_HASH_H_ */
97984		-
97985		-/************ End of fts3_hash.h *****************************************/
97986		-/************ Continuing where we left off in fts3.c *********************/
	97776	+#include "fts3Int.h"
	97777	+
	97778	+
97987	97779	#ifndef SQLITE_CORE
97988	97780	SQLITE_EXTENSION_INIT1
97989	97781	#endif
	97782	+
97990	97783
97991	97784
97992	97785	/* TODO(shess) MAN, this thing needs some refactoring. At minimum, it
97993	97786	** would be nice to order the file better, perhaps something along the
97994	97787	** lines of:
		@@ -98006,29 +97799,10 @@
98006	97799	# define FTSTRACE(A) printf A; fflush(stdout)
98007	97800	#else
98008	97801	# define FTSTRACE(A)
98009	97802	#endif
98010	97803
98011		-/* It is not safe to call isspace(), tolower(), or isalnum() on
98012		-** hi-bit-set characters. This is the same solution used in the
98013		-** tokenizer.
98014		-*/
98015		-/* TODO(shess) The snippet-generation code should be using the
98016		-** tokenizer-generated tokens rather than doing its own local
98017		-** tokenization.
98018		-*/
98019		-/* TODO(shess) Is __isascii() a portable version of (c&0x80)==0? */
98020		-static int safe_isspace(char c){
98021		- return (c&0x80)==0 ? isspace(c) : 0;
98022		-}
98023		-static int safe_tolower(char c){
98024		- return (c&0x80)==0 ? tolower(c) : c;
98025		-}
98026		-static int safe_isalnum(char c){
98027		- return (c&0x80)==0 ? isalnum(c) : 0;
98028		-}
98029		-
98030	97804	typedef enum DocListType {
98031	97805	DL_DOCIDS, /* docids only */
98032	97806	DL_POSITIONS, /* docids + positions */
98033	97807	DL_POSITIONS_OFFSETS /* docids + positions + offsets */
98034	97808	} DocListType;
		@@ -98050,15 +97824,10 @@
98050	97824	POS_END = 0, /* end of this position list */
98051	97825	POS_COLUMN, /* followed by new column number */
98052	97826	POS_BASE
98053	97827	};
98054	97828
98055		-/* MERGE_COUNT controls how often we merge segments (see comment at
98056		-** top of file).
98057		-*/
98058		-#define MERGE_COUNT 16
98059		-
98060	97829	/* utility functions */
98061	97830
98062	97831	/* CLEAR() and SCRAMBLE() abstract memset() on a pointer to a single
98063	97832	** record to prevent errors of the form:
98064	97833	**
		@@ -98073,2225 +97842,71 @@
98073	97842	# define SCRAMBLE(b) memset(b, 0x55, sizeof(*(b)))
98074	97843	#else
98075	97844	# define SCRAMBLE(b)
98076	97845	#endif
98077	97846
98078		-/* We may need up to VARINT_MAX bytes to store an encoded 64-bit integer. */
98079		-#define VARINT_MAX 10
98080		-
98081		-/* Write a 64-bit variable-length integer to memory starting at p[0].
98082		- * The length of data written will be between 1 and VARINT_MAX bytes.
98083		- * The number of bytes written is returned. */
98084		-static int fts3PutVarint(char *p, sqlite_int64 v){
	97847	+/*
	97848	+** Write a 64-bit variable-length integer to memory starting at p[0].
	97849	+** The length of data written will be between 1 and FTS3_VARINT_MAX bytes.
	97850	+** The number of bytes written is returned.
	97851	+*/
	97852	+SQLITE_PRIVATE int sqlite3Fts3PutVarint(char *p, sqlite_int64 v){
98085	97853	unsigned char q = (unsigned char ) p;
98086	97854	sqlite_uint64 vu = v;
98087	97855	do{
98088	97856	*q++ = (unsigned char) ((vu & 0x7f) \| 0x80);
98089	97857	vu >>= 7;
98090	97858	}while( vu!=0 );
98091	97859	q[-1] &= 0x7f; /* turn off high bit in final byte */
98092		- assert( q - (unsigned char *)p <= VARINT_MAX );
	97860	+ assert( q - (unsigned char *)p <= FTS3_VARINT_MAX );
98093	97861	return (int) (q - (unsigned char *)p);
98094	97862	}
98095	97863
98096		-/* Read a 64-bit variable-length integer from memory starting at p[0].
98097		- * Return the number of bytes read, or 0 on error.
98098		- * The value is stored in v. /
98099		-static int fts3GetVarint(const char p, sqlite_int64 v){
	97864	+/*
	97865	+** Read a 64-bit variable-length integer from memory starting at p[0].
	97866	+** Return the number of bytes read, or 0 on error.
	97867	+** The value is stored in *v.
	97868	+*/
	97869	+SQLITE_PRIVATE int sqlite3Fts3GetVarint(const char p, sqlite_int64 v){
98100	97870	const unsigned char q = (const unsigned char ) p;
98101	97871	sqlite_uint64 x = 0, y = 1;
98102	97872	while( (*q & 0x80) == 0x80 ){
98103	97873	x += y * (*q++ & 0x7f);
98104	97874	y <<= 7;
98105		- if( q - (unsigned char )p >= VARINT_MAX ){ / bad data */
	97875	+ if( q - (unsigned char )p >= FTS3_VARINT_MAX ){ / bad data */
98106	97876	assert( 0 );
98107	97877	return 0;
98108	97878	}
98109	97879	}
98110	97880	x += y * (*q++);
98111	97881	*v = (sqlite_int64) x;
98112	97882	return (int) (q - (unsigned char *)p);
98113	97883	}
98114	97884
98115		-static int fts3GetVarint32(const char p, int pi){
	97885	+/*
	97886	+** Similar to sqlite3Fts3GetVarint(), except that the output is truncated to a
	97887	+** 32-bit integer before it is returned.
	97888	+*/
	97889	+SQLITE_PRIVATE int sqlite3Fts3GetVarint32(const char p, int pi){
98116	97890	sqlite_int64 i;
98117		- int ret = fts3GetVarint(p, &i);
	97891	+ int ret = sqlite3Fts3GetVarint(p, &i);
98118	97892	*pi = (int) i;
98119	97893	assert( *pi==i );
98120	97894	return ret;
98121	97895	}
98122	97896
98123		-/*******************************************************************/
98124		-/* DataBuffer is used to collect data into a buffer in piecemeal
98125		-** fashion. It implements the usual distinction between amount of
98126		-** data currently stored (nData) and buffer capacity (nCapacity).
98127		-**
98128		-** dataBufferInit - create a buffer with given initial capacity.
98129		-** dataBufferReset - forget buffer's data, retaining capacity.
98130		-** dataBufferDestroy - free buffer's data.
98131		-** dataBufferSwap - swap contents of two buffers.
98132		-** dataBufferExpand - expand capacity without adding data.
98133		-** dataBufferAppend - append data.
98134		-** dataBufferAppend2 - append two pieces of data at once.
98135		-** dataBufferReplace - replace buffer's data.
98136		-*/
98137		-typedef struct DataBuffer {
98138		- char pData; / Pointer to malloc'ed buffer. */
98139		- int nCapacity; /* Size of pData buffer. */
98140		- int nData; /* End of data loaded into pData. */
98141		-} DataBuffer;
98142		-
98143		-static void dataBufferInit(DataBuffer *pBuffer, int nCapacity){
98144		- assert( nCapacity>=0 );
98145		- pBuffer->nData = 0;
98146		- pBuffer->nCapacity = nCapacity;
98147		- pBuffer->pData = nCapacity==0 ? NULL : sqlite3_malloc(nCapacity);
98148		-}
98149		-static void dataBufferReset(DataBuffer *pBuffer){
98150		- pBuffer->nData = 0;
98151		-}
98152		-static void dataBufferDestroy(DataBuffer *pBuffer){
98153		- if( pBuffer->pData!=NULL ) sqlite3_free(pBuffer->pData);
98154		- SCRAMBLE(pBuffer);
98155		-}
98156		-static void dataBufferSwap(DataBuffer pBuffer1, DataBuffer pBuffer2){
98157		- DataBuffer tmp = *pBuffer1;
98158		- pBuffer1 = pBuffer2;
98159		- *pBuffer2 = tmp;
98160		-}
98161		-static void dataBufferExpand(DataBuffer *pBuffer, int nAddCapacity){
98162		- assert( nAddCapacity>0 );
98163		- /* TODO(shess) Consider expanding more aggressively. Note that the
98164		- ** underlying malloc implementation may take care of such things for
98165		- ** us already.
98166		- */
98167		- if( pBuffer->nData+nAddCapacity>pBuffer->nCapacity ){
98168		- pBuffer->nCapacity = pBuffer->nData+nAddCapacity;
98169		- pBuffer->pData = sqlite3_realloc(pBuffer->pData, pBuffer->nCapacity);
98170		- }
98171		-}
98172		-static void dataBufferAppend(DataBuffer *pBuffer,
98173		- const char *pSource, int nSource){
98174		- assert( nSource>0 && pSource!=NULL );
98175		- dataBufferExpand(pBuffer, nSource);
98176		- memcpy(pBuffer->pData+pBuffer->nData, pSource, nSource);
98177		- pBuffer->nData += nSource;
98178		-}
98179		-static void dataBufferAppend2(DataBuffer *pBuffer,
98180		- const char *pSource1, int nSource1,
98181		- const char *pSource2, int nSource2){
98182		- assert( nSource1>0 && pSource1!=NULL );
98183		- assert( nSource2>0 && pSource2!=NULL );
98184		- dataBufferExpand(pBuffer, nSource1+nSource2);
98185		- memcpy(pBuffer->pData+pBuffer->nData, pSource1, nSource1);
98186		- memcpy(pBuffer->pData+pBuffer->nData+nSource1, pSource2, nSource2);
98187		- pBuffer->nData += nSource1+nSource2;
98188		-}
98189		-static void dataBufferReplace(DataBuffer *pBuffer,
98190		- const char *pSource, int nSource){
98191		- dataBufferReset(pBuffer);
98192		- dataBufferAppend(pBuffer, pSource, nSource);
98193		-}
98194		-
98195		-/* StringBuffer is a null-terminated version of DataBuffer. */
98196		-typedef struct StringBuffer {
98197		- DataBuffer b; /* Includes null terminator. */
98198		-} StringBuffer;
98199		-
98200		-static void initStringBuffer(StringBuffer *sb){
98201		- dataBufferInit(&sb->b, 100);
98202		- dataBufferReplace(&sb->b, "", 1);
98203		-}
98204		-static int stringBufferLength(StringBuffer *sb){
98205		- return sb->b.nData-1;
98206		-}
98207		-static char stringBufferData(StringBuffer sb){
98208		- return sb->b.pData;
98209		-}
98210		-static void stringBufferDestroy(StringBuffer *sb){
98211		- dataBufferDestroy(&sb->b);
98212		-}
98213		-
98214		-static void nappend(StringBuffer sb, const char zFrom, int nFrom){
98215		- assert( sb->b.nData>0 );
98216		- if( nFrom>0 ){
98217		- sb->b.nData--;
98218		- dataBufferAppend2(&sb->b, zFrom, nFrom, "", 1);
98219		- }
98220		-}
98221		-static void append(StringBuffer sb, const char zFrom){
98222		- nappend(sb, zFrom, strlen(zFrom));
98223		-}
98224		-
98225		-/* Append a list of strings separated by commas. */
98226		-static void appendList(StringBuffer sb, int nString, char *azString){
98227		- int i;
98228		- for(i=0; i<nString; ++i){
98229		- if( i>0 ) append(sb, ", ");
98230		- append(sb, azString[i]);
98231		- }
98232		-}
98233		-
98234		-static int endsInWhiteSpace(StringBuffer *p){
98235		- return stringBufferLength(p)>0 &&
98236		- safe_isspace(stringBufferData(p)[stringBufferLength(p)-1]);
98237		-}
98238		-
98239		-/* If the StringBuffer ends in something other than white space, add a
98240		-** single space character to the end.
98241		-*/
98242		-static void appendWhiteSpace(StringBuffer *p){
98243		- if( stringBufferLength(p)==0 ) return;
98244		- if( !endsInWhiteSpace(p) ) append(p, " ");
98245		-}
98246		-
98247		-/* Remove white space from the end of the StringBuffer */
98248		-static void trimWhiteSpace(StringBuffer *p){
98249		- while( endsInWhiteSpace(p) ){
98250		- p->b.pData[--p->b.nData-1] = '\0';
98251		- }
98252		-}
98253		-
98254		-/*******************************************************************/
98255		-/* DLReader is used to read document elements from a doclist. The
98256		-** current docid is cached, so dlrDocid() is fast. DLReader does not
98257		-** own the doclist buffer.
98258		-**
98259		-** dlrAtEnd - true if there's no more data to read.
98260		-** dlrDocid - docid of current document.
98261		-** dlrDocData - doclist data for current document (including docid).
98262		-** dlrDocDataBytes - length of same.
98263		-** dlrAllDataBytes - length of all remaining data.
98264		-** dlrPosData - position data for current document.
98265		-** dlrPosDataLen - length of pos data for current document (incl POS_END).
98266		-** dlrStep - step to current document.
98267		-** dlrInit - initial for doclist of given type against given data.
98268		-** dlrDestroy - clean up.
98269		-**
98270		-** Expected usage is something like:
98271		-**
98272		-** DLReader reader;
98273		-** dlrInit(&reader, pData, nData);
98274		-** while( !dlrAtEnd(&reader) ){
98275		-** // calls to dlrDocid() and kin.
98276		-** dlrStep(&reader);
98277		-** }
98278		-** dlrDestroy(&reader);
98279		-*/
98280		-typedef struct DLReader {
98281		- DocListType iType;
98282		- const char *pData;
98283		- int nData;
98284		-
98285		- sqlite_int64 iDocid;
98286		- int nElement;
98287		-} DLReader;
98288		-
98289		-static int dlrAtEnd(DLReader *pReader){
98290		- assert( pReader->nData>=0 );
98291		- return pReader->nData==0;
98292		-}
98293		-static sqlite_int64 dlrDocid(DLReader *pReader){
98294		- assert( !dlrAtEnd(pReader) );
98295		- return pReader->iDocid;
98296		-}
98297		-static const char dlrDocData(DLReader pReader){
98298		- assert( !dlrAtEnd(pReader) );
98299		- return pReader->pData;
98300		-}
98301		-static int dlrDocDataBytes(DLReader *pReader){
98302		- assert( !dlrAtEnd(pReader) );
98303		- return pReader->nElement;
98304		-}
98305		-static int dlrAllDataBytes(DLReader *pReader){
98306		- assert( !dlrAtEnd(pReader) );
98307		- return pReader->nData;
98308		-}
98309		-/* TODO(shess) Consider adding a field to track iDocid varint length
98310		-** to make these two functions faster. This might matter (a tiny bit)
98311		-** for queries.
98312		-*/
98313		-static const char dlrPosData(DLReader pReader){
98314		- sqlite_int64 iDummy;
98315		- int n = fts3GetVarint(pReader->pData, &iDummy);
98316		- assert( !dlrAtEnd(pReader) );
98317		- return pReader->pData+n;
98318		-}
98319		-static int dlrPosDataLen(DLReader *pReader){
98320		- sqlite_int64 iDummy;
98321		- int n = fts3GetVarint(pReader->pData, &iDummy);
98322		- assert( !dlrAtEnd(pReader) );
98323		- return pReader->nElement-n;
98324		-}
98325		-static void dlrStep(DLReader *pReader){
98326		- assert( !dlrAtEnd(pReader) );
98327		-
98328		- /* Skip past current doclist element. */
98329		- assert( pReader->nElement<=pReader->nData );
98330		- pReader->pData += pReader->nElement;
98331		- pReader->nData -= pReader->nElement;
98332		-
98333		- /* If there is more data, read the next doclist element. */
98334		- if( pReader->nData!=0 ){
98335		- sqlite_int64 iDocidDelta;
98336		- int iDummy, n = fts3GetVarint(pReader->pData, &iDocidDelta);
98337		- pReader->iDocid += iDocidDelta;
98338		- if( pReader->iType>=DL_POSITIONS ){
98339		- assert( n<pReader->nData );
98340		- while( 1 ){
98341		- n += fts3GetVarint32(pReader->pData+n, &iDummy);
98342		- assert( n<=pReader->nData );
98343		- if( iDummy==POS_END ) break;
98344		- if( iDummy==POS_COLUMN ){
98345		- n += fts3GetVarint32(pReader->pData+n, &iDummy);
98346		- assert( n<pReader->nData );
98347		- }else if( pReader->iType==DL_POSITIONS_OFFSETS ){
98348		- n += fts3GetVarint32(pReader->pData+n, &iDummy);
98349		- n += fts3GetVarint32(pReader->pData+n, &iDummy);
98350		- assert( n<pReader->nData );
98351		- }
98352		- }
98353		- }
98354		- pReader->nElement = n;
98355		- assert( pReader->nElement<=pReader->nData );
98356		- }
98357		-}
98358		-static void dlrInit(DLReader *pReader, DocListType iType,
98359		- const char *pData, int nData){
98360		- assert( pData!=NULL && nData!=0 );
98361		- pReader->iType = iType;
98362		- pReader->pData = pData;
98363		- pReader->nData = nData;
98364		- pReader->nElement = 0;
98365		- pReader->iDocid = 0;
98366		-
98367		- /* Load the first element's data. There must be a first element. */
98368		- dlrStep(pReader);
98369		-}
98370		-static void dlrDestroy(DLReader *pReader){
98371		- SCRAMBLE(pReader);
98372		-}
98373		-
98374		-#ifndef NDEBUG
98375		-/* Verify that the doclist can be validly decoded. Also returns the
98376		-** last docid found because it is convenient in other assertions for
98377		-** DLWriter.
98378		-*/
98379		-static void docListValidate(DocListType iType, const char *pData, int nData,
98380		- sqlite_int64 *pLastDocid){
98381		- sqlite_int64 iPrevDocid = 0;
98382		- assert( nData>0 );
98383		- assert( pData!=0 );
98384		- assert( pData+nData>pData );
98385		- while( nData!=0 ){
98386		- sqlite_int64 iDocidDelta;
98387		- int n = fts3GetVarint(pData, &iDocidDelta);
98388		- iPrevDocid += iDocidDelta;
98389		- if( iType>DL_DOCIDS ){
98390		- int iDummy;
98391		- while( 1 ){
98392		- n += fts3GetVarint32(pData+n, &iDummy);
98393		- if( iDummy==POS_END ) break;
98394		- if( iDummy==POS_COLUMN ){
98395		- n += fts3GetVarint32(pData+n, &iDummy);
98396		- }else if( iType>DL_POSITIONS ){
98397		- n += fts3GetVarint32(pData+n, &iDummy);
98398		- n += fts3GetVarint32(pData+n, &iDummy);
98399		- }
98400		- assert( n<=nData );
98401		- }
98402		- }
98403		- assert( n<=nData );
98404		- pData += n;
98405		- nData -= n;
98406		- }
98407		- if( pLastDocid ) *pLastDocid = iPrevDocid;
98408		-}
98409		-#define ASSERT_VALID_DOCLIST(i, p, n, o) docListValidate(i, p, n, o)
98410		-#else
98411		-#define ASSERT_VALID_DOCLIST(i, p, n, o) assert( 1 )
98412		-#endif
98413		-
98414		-/*******************************************************************/
98415		-/* DLWriter is used to write doclist data to a DataBuffer. DLWriter
98416		-** always appends to the buffer and does not own it.
98417		-**
98418		-** dlwInit - initialize to write a given type doclistto a buffer.
98419		-** dlwDestroy - clear the writer's memory. Does not free buffer.
98420		-** dlwAppend - append raw doclist data to buffer.
98421		-** dlwCopy - copy next doclist from reader to writer.
98422		-** dlwAdd - construct doclist element and append to buffer.
98423		-** Only apply dlwAdd() to DL_DOCIDS doclists (else use PLWriter).
98424		-*/
98425		-typedef struct DLWriter {
98426		- DocListType iType;
98427		- DataBuffer *b;
98428		- sqlite_int64 iPrevDocid;
98429		-#ifndef NDEBUG
98430		- int has_iPrevDocid;
98431		-#endif
98432		-} DLWriter;
98433		-
98434		-static void dlwInit(DLWriter pWriter, DocListType iType, DataBuffer b){
98435		- pWriter->b = b;
98436		- pWriter->iType = iType;
98437		- pWriter->iPrevDocid = 0;
98438		-#ifndef NDEBUG
98439		- pWriter->has_iPrevDocid = 0;
98440		-#endif
98441		-}
98442		-static void dlwDestroy(DLWriter *pWriter){
98443		- SCRAMBLE(pWriter);
98444		-}
98445		-/* iFirstDocid is the first docid in the doclist in pData. It is
98446		-** needed because pData may point within a larger doclist, in which
98447		-** case the first item would be delta-encoded.
98448		-**
98449		-** iLastDocid is the final docid in the doclist in pData. It is
98450		-** needed to create the new iPrevDocid for future delta-encoding. The
98451		-** code could decode the passed doclist to recreate iLastDocid, but
98452		-** the only current user (docListMerge) already has decoded this
98453		-** information.
98454		-*/
98455		-/* TODO(shess) This has become just a helper for docListMerge.
98456		-** Consider a refactor to make this cleaner.
98457		-*/
98458		-static void dlwAppend(DLWriter *pWriter,
98459		- const char *pData, int nData,
98460		- sqlite_int64 iFirstDocid, sqlite_int64 iLastDocid){
98461		- sqlite_int64 iDocid = 0;
98462		- char c[VARINT_MAX];
98463		- int nFirstOld, nFirstNew; /* Old and new varint len of first docid. */
98464		-#ifndef NDEBUG
98465		- sqlite_int64 iLastDocidDelta;
98466		-#endif
98467		-
98468		- /* Recode the initial docid as delta from iPrevDocid. */
98469		- nFirstOld = fts3GetVarint(pData, &iDocid);
98470		- assert( nFirstOld<nData \|\| (nFirstOld==nData && pWriter->iType==DL_DOCIDS) );
98471		- nFirstNew = fts3PutVarint(c, iFirstDocid-pWriter->iPrevDocid);
98472		-
98473		- /* Verify that the incoming doclist is valid AND that it ends with
98474		- ** the expected docid. This is essential because we'll trust this
98475		- ** docid in future delta-encoding.
98476		- */
98477		- ASSERT_VALID_DOCLIST(pWriter->iType, pData, nData, &iLastDocidDelta);
98478		- assert( iLastDocid==iFirstDocid-iDocid+iLastDocidDelta );
98479		-
98480		- /* Append recoded initial docid and everything else. Rest of docids
98481		- ** should have been delta-encoded from previous initial docid.
98482		- */
98483		- if( nFirstOld<nData ){
98484		- dataBufferAppend2(pWriter->b, c, nFirstNew,
98485		- pData+nFirstOld, nData-nFirstOld);
98486		- }else{
98487		- dataBufferAppend(pWriter->b, c, nFirstNew);
98488		- }
98489		- pWriter->iPrevDocid = iLastDocid;
98490		-}
98491		-static void dlwCopy(DLWriter pWriter, DLReader pReader){
98492		- dlwAppend(pWriter, dlrDocData(pReader), dlrDocDataBytes(pReader),
98493		- dlrDocid(pReader), dlrDocid(pReader));
98494		-}
98495		-static void dlwAdd(DLWriter *pWriter, sqlite_int64 iDocid){
98496		- char c[VARINT_MAX];
98497		- int n = fts3PutVarint(c, iDocid-pWriter->iPrevDocid);
98498		-
98499		- /* Docids must ascend. */
98500		- assert( !pWriter->has_iPrevDocid \|\| iDocid>pWriter->iPrevDocid );
98501		- assert( pWriter->iType==DL_DOCIDS );
98502		-
98503		- dataBufferAppend(pWriter->b, c, n);
98504		- pWriter->iPrevDocid = iDocid;
98505		-#ifndef NDEBUG
98506		- pWriter->has_iPrevDocid = 1;
98507		-#endif
98508		-}
98509		-
98510		-/*******************************************************************/
98511		-/* PLReader is used to read data from a document's position list. As
98512		-** the caller steps through the list, data is cached so that varints
98513		-** only need to be decoded once.
98514		-**
98515		-** plrInit, plrDestroy - create/destroy a reader.
98516		-** plrColumn, plrPosition, plrStartOffset, plrEndOffset - accessors
98517		-** plrAtEnd - at end of stream, only call plrDestroy once true.
98518		-** plrStep - step to the next element.
98519		-*/
98520		-typedef struct PLReader {
98521		- /* These refer to the next position's data. nData will reach 0 when
98522		- ** reading the last position, so plrStep() signals EOF by setting
98523		- ** pData to NULL.
98524		- */
98525		- const char *pData;
98526		- int nData;
98527		-
98528		- DocListType iType;
98529		- int iColumn; /* the last column read */
98530		- int iPosition; /* the last position read */
98531		- int iStartOffset; /* the last start offset read */
98532		- int iEndOffset; /* the last end offset read */
98533		-} PLReader;
98534		-
98535		-static int plrAtEnd(PLReader *pReader){
98536		- return pReader->pData==NULL;
98537		-}
98538		-static int plrColumn(PLReader *pReader){
98539		- assert( !plrAtEnd(pReader) );
98540		- return pReader->iColumn;
98541		-}
98542		-static int plrPosition(PLReader *pReader){
98543		- assert( !plrAtEnd(pReader) );
98544		- return pReader->iPosition;
98545		-}
98546		-static int plrStartOffset(PLReader *pReader){
98547		- assert( !plrAtEnd(pReader) );
98548		- return pReader->iStartOffset;
98549		-}
98550		-static int plrEndOffset(PLReader *pReader){
98551		- assert( !plrAtEnd(pReader) );
98552		- return pReader->iEndOffset;
98553		-}
98554		-static void plrStep(PLReader *pReader){
98555		- int i, n;
98556		-
98557		- assert( !plrAtEnd(pReader) );
98558		-
98559		- if( pReader->nData==0 ){
98560		- pReader->pData = NULL;
98561		- return;
98562		- }
98563		-
98564		- n = fts3GetVarint32(pReader->pData, &i);
98565		- if( i==POS_COLUMN ){
98566		- n += fts3GetVarint32(pReader->pData+n, &pReader->iColumn);
98567		- pReader->iPosition = 0;
98568		- pReader->iStartOffset = 0;
98569		- n += fts3GetVarint32(pReader->pData+n, &i);
98570		- }
98571		- /* Should never see adjacent column changes. */
98572		- assert( i!=POS_COLUMN );
98573		-
98574		- if( i==POS_END ){
98575		- pReader->nData = 0;
98576		- pReader->pData = NULL;
98577		- return;
98578		- }
98579		-
98580		- pReader->iPosition += i-POS_BASE;
98581		- if( pReader->iType==DL_POSITIONS_OFFSETS ){
98582		- n += fts3GetVarint32(pReader->pData+n, &i);
98583		- pReader->iStartOffset += i;
98584		- n += fts3GetVarint32(pReader->pData+n, &i);
98585		- pReader->iEndOffset = pReader->iStartOffset+i;
98586		- }
98587		- assert( n<=pReader->nData );
98588		- pReader->pData += n;
98589		- pReader->nData -= n;
98590		-}
98591		-
98592		-static void plrInit(PLReader pReader, DLReader pDLReader){
98593		- pReader->pData = dlrPosData(pDLReader);
98594		- pReader->nData = dlrPosDataLen(pDLReader);
98595		- pReader->iType = pDLReader->iType;
98596		- pReader->iColumn = 0;
98597		- pReader->iPosition = 0;
98598		- pReader->iStartOffset = 0;
98599		- pReader->iEndOffset = 0;
98600		- plrStep(pReader);
98601		-}
98602		-static void plrDestroy(PLReader *pReader){
98603		- SCRAMBLE(pReader);
98604		-}
98605		-
98606		-/*******************************************************************/
98607		-/* PLWriter is used in constructing a document's position list. As a
98608		-** convenience, if iType is DL_DOCIDS, PLWriter becomes a no-op.
98609		-** PLWriter writes to the associated DLWriter's buffer.
98610		-**
98611		-** plwInit - init for writing a document's poslist.
98612		-** plwDestroy - clear a writer.
98613		-** plwAdd - append position and offset information.
98614		-** plwCopy - copy next position's data from reader to writer.
98615		-** plwTerminate - add any necessary doclist terminator.
98616		-**
98617		-** Calling plwAdd() after plwTerminate() may result in a corrupt
98618		-** doclist.
98619		-*/
98620		-/* TODO(shess) Until we've written the second item, we can cache the
98621		-** first item's information. Then we'd have three states:
98622		-**
98623		-** - initialized with docid, no positions.
98624		-** - docid and one position.
98625		-** - docid and multiple positions.
98626		-**
98627		-** Only the last state needs to actually write to dlw->b, which would
98628		-** be an improvement in the DLCollector case.
98629		-*/
98630		-typedef struct PLWriter {
98631		- DLWriter *dlw;
98632		-
98633		- int iColumn; /* the last column written */
98634		- int iPos; /* the last position written */
98635		- int iOffset; /* the last start offset written */
98636		-} PLWriter;
98637		-
98638		-/* TODO(shess) In the case where the parent is reading these values
98639		-** from a PLReader, we could optimize to a copy if that PLReader has
98640		-** the same type as pWriter.
98641		-*/
98642		-static void plwAdd(PLWriter *pWriter, int iColumn, int iPos,
98643		- int iStartOffset, int iEndOffset){
98644		- /* Worst-case space for POS_COLUMN, iColumn, iPosDelta,
98645		- ** iStartOffsetDelta, and iEndOffsetDelta.
98646		- */
98647		- char c[5*VARINT_MAX];
98648		- int n = 0;
98649		-
98650		- /* Ban plwAdd() after plwTerminate(). */
98651		- assert( pWriter->iPos!=-1 );
98652		-
98653		- if( pWriter->dlw->iType==DL_DOCIDS ) return;
98654		-
98655		- if( iColumn!=pWriter->iColumn ){
98656		- n += fts3PutVarint(c+n, POS_COLUMN);
98657		- n += fts3PutVarint(c+n, iColumn);
98658		- pWriter->iColumn = iColumn;
98659		- pWriter->iPos = 0;
98660		- pWriter->iOffset = 0;
98661		- }
98662		- assert( iPos>=pWriter->iPos );
98663		- n += fts3PutVarint(c+n, POS_BASE+(iPos-pWriter->iPos));
98664		- pWriter->iPos = iPos;
98665		- if( pWriter->dlw->iType==DL_POSITIONS_OFFSETS ){
98666		- assert( iStartOffset>=pWriter->iOffset );
98667		- n += fts3PutVarint(c+n, iStartOffset-pWriter->iOffset);
98668		- pWriter->iOffset = iStartOffset;
98669		- assert( iEndOffset>=iStartOffset );
98670		- n += fts3PutVarint(c+n, iEndOffset-iStartOffset);
98671		- }
98672		- dataBufferAppend(pWriter->dlw->b, c, n);
98673		-}
98674		-static void plwCopy(PLWriter pWriter, PLReader pReader){
98675		- plwAdd(pWriter, plrColumn(pReader), plrPosition(pReader),
98676		- plrStartOffset(pReader), plrEndOffset(pReader));
98677		-}
98678		-static void plwInit(PLWriter pWriter, DLWriter dlw, sqlite_int64 iDocid){
98679		- char c[VARINT_MAX];
98680		- int n;
98681		-
98682		- pWriter->dlw = dlw;
98683		-
98684		- /* Docids must ascend. */
98685		- assert( !pWriter->dlw->has_iPrevDocid \|\| iDocid>pWriter->dlw->iPrevDocid );
98686		- n = fts3PutVarint(c, iDocid-pWriter->dlw->iPrevDocid);
98687		- dataBufferAppend(pWriter->dlw->b, c, n);
98688		- pWriter->dlw->iPrevDocid = iDocid;
98689		-#ifndef NDEBUG
98690		- pWriter->dlw->has_iPrevDocid = 1;
98691		-#endif
98692		-
98693		- pWriter->iColumn = 0;
98694		- pWriter->iPos = 0;
98695		- pWriter->iOffset = 0;
98696		-}
98697		-/* TODO(shess) Should plwDestroy() also terminate the doclist? But
98698		-** then plwDestroy() would no longer be just a destructor, it would
98699		-** also be doing work, which isn't consistent with the overall idiom.
98700		-** Another option would be for plwAdd() to always append any necessary
98701		-** terminator, so that the output is always correct. But that would
98702		-** add incremental work to the common case with the only benefit being
98703		-** API elegance. Punt for now.
98704		-*/
98705		-static void plwTerminate(PLWriter *pWriter){
98706		- if( pWriter->dlw->iType>DL_DOCIDS ){
98707		- char c[VARINT_MAX];
98708		- int n = fts3PutVarint(c, POS_END);
98709		- dataBufferAppend(pWriter->dlw->b, c, n);
98710		- }
98711		-#ifndef NDEBUG
98712		- /* Mark as terminated for assert in plwAdd(). */
98713		- pWriter->iPos = -1;
98714		-#endif
98715		-}
98716		-static void plwDestroy(PLWriter *pWriter){
98717		- SCRAMBLE(pWriter);
98718		-}
98719		-
98720		-/*******************************************************************/
98721		-/* DLCollector wraps PLWriter and DLWriter to provide a
98722		-** dynamically-allocated doclist area to use during tokenization.
98723		-**
98724		-** dlcNew - malloc up and initialize a collector.
98725		-** dlcDelete - destroy a collector and all contained items.
98726		-** dlcAddPos - append position and offset information.
98727		-** dlcAddDoclist - add the collected doclist to the given buffer.
98728		-** dlcNext - terminate the current document and open another.
98729		-*/
98730		-typedef struct DLCollector {
98731		- DataBuffer b;
98732		- DLWriter dlw;
98733		- PLWriter plw;
98734		-} DLCollector;
98735		-
98736		-/* TODO(shess) This could also be done by calling plwTerminate() and
98737		-** dataBufferAppend(). I tried that, expecting nominal performance
98738		-** differences, but it seemed to pretty reliably be worth 1% to code
98739		-** it this way. I suspect it is the incremental malloc overhead (some
98740		-** percentage of the plwTerminate() calls will cause a realloc), so
98741		-** this might be worth revisiting if the DataBuffer implementation
98742		-** changes.
98743		-*/
98744		-static void dlcAddDoclist(DLCollector pCollector, DataBuffer b){
98745		- if( pCollector->dlw.iType>DL_DOCIDS ){
98746		- char c[VARINT_MAX];
98747		- int n = fts3PutVarint(c, POS_END);
98748		- dataBufferAppend2(b, pCollector->b.pData, pCollector->b.nData, c, n);
98749		- }else{
98750		- dataBufferAppend(b, pCollector->b.pData, pCollector->b.nData);
98751		- }
98752		-}
98753		-static void dlcNext(DLCollector *pCollector, sqlite_int64 iDocid){
98754		- plwTerminate(&pCollector->plw);
98755		- plwDestroy(&pCollector->plw);
98756		- plwInit(&pCollector->plw, &pCollector->dlw, iDocid);
98757		-}
98758		-static void dlcAddPos(DLCollector *pCollector, int iColumn, int iPos,
98759		- int iStartOffset, int iEndOffset){
98760		- plwAdd(&pCollector->plw, iColumn, iPos, iStartOffset, iEndOffset);
98761		-}
98762		-
98763		-static DLCollector *dlcNew(sqlite_int64 iDocid, DocListType iType){
98764		- DLCollector *pCollector = sqlite3_malloc(sizeof(DLCollector));
98765		- dataBufferInit(&pCollector->b, 0);
98766		- dlwInit(&pCollector->dlw, iType, &pCollector->b);
98767		- plwInit(&pCollector->plw, &pCollector->dlw, iDocid);
98768		- return pCollector;
98769		-}
98770		-static void dlcDelete(DLCollector *pCollector){
98771		- plwDestroy(&pCollector->plw);
98772		- dlwDestroy(&pCollector->dlw);
98773		- dataBufferDestroy(&pCollector->b);
98774		- SCRAMBLE(pCollector);
98775		- sqlite3_free(pCollector);
98776		-}
98777		-
98778		-
98779		-/* Copy the doclist data of iType in pData/nData into *out, trimming
98780		-** unnecessary data as we go. Only columns matching iColumn are
98781		-** copied, all columns copied if iColumn is -1. Elements with no
98782		-** matching columns are dropped. The output is an iOutType doclist.
98783		-*/
98784		-/* NOTE(shess) This code is only valid after all doclists are merged.
98785		-** If this is run before merges, then doclist items which represent
98786		-** deletion will be trimmed, and will thus not effect a deletion
98787		-** during the merge.
98788		-*/
98789		-static void docListTrim(DocListType iType, const char *pData, int nData,
98790		- int iColumn, DocListType iOutType, DataBuffer *out){
98791		- DLReader dlReader;
98792		- DLWriter dlWriter;
98793		-
98794		- assert( iOutType<=iType );
98795		-
98796		- dlrInit(&dlReader, iType, pData, nData);
98797		- dlwInit(&dlWriter, iOutType, out);
98798		-
98799		- while( !dlrAtEnd(&dlReader) ){
98800		- PLReader plReader;
98801		- PLWriter plWriter;
98802		- int match = 0;
98803		-
98804		- plrInit(&plReader, &dlReader);
98805		-
98806		- while( !plrAtEnd(&plReader) ){
98807		- if( iColumn==-1 \|\| plrColumn(&plReader)==iColumn ){
98808		- if( !match ){
98809		- plwInit(&plWriter, &dlWriter, dlrDocid(&dlReader));
98810		- match = 1;
98811		- }
98812		- plwAdd(&plWriter, plrColumn(&plReader), plrPosition(&plReader),
98813		- plrStartOffset(&plReader), plrEndOffset(&plReader));
98814		- }
98815		- plrStep(&plReader);
98816		- }
98817		- if( match ){
98818		- plwTerminate(&plWriter);
98819		- plwDestroy(&plWriter);
98820		- }
98821		-
98822		- plrDestroy(&plReader);
98823		- dlrStep(&dlReader);
98824		- }
98825		- dlwDestroy(&dlWriter);
98826		- dlrDestroy(&dlReader);
98827		-}
98828		-
98829		-/* Used by docListMerge() to keep doclists in the ascending order by
98830		-** docid, then ascending order by age (so the newest comes first).
98831		-*/
98832		-typedef struct OrderedDLReader {
98833		- DLReader *pReader;
98834		-
98835		- /* TODO(shess) If we assume that docListMerge pReaders is ordered by
98836		- ** age (which we do), then we could use pReader comparisons to break
98837		- ** ties.
98838		- */
98839		- int idx;
98840		-} OrderedDLReader;
98841		-
98842		-/* Order eof to end, then by docid asc, idx desc. */
98843		-static int orderedDLReaderCmp(OrderedDLReader r1, OrderedDLReader r2){
98844		- if( dlrAtEnd(r1->pReader) ){
98845		- if( dlrAtEnd(r2->pReader) ) return 0; /* Both atEnd(). */
98846		- return 1; /* Only r1 atEnd(). */
98847		- }
98848		- if( dlrAtEnd(r2->pReader) ) return -1; /* Only r2 atEnd(). */
98849		-
98850		- if( dlrDocid(r1->pReader)<dlrDocid(r2->pReader) ) return -1;
98851		- if( dlrDocid(r1->pReader)>dlrDocid(r2->pReader) ) return 1;
98852		-
98853		- /* Descending on idx. */
98854		- return r2->idx-r1->idx;
98855		-}
98856		-
98857		-/* Bubble p[0] to appropriate place in p[1..n-1]. Assumes that
98858		-** p[1..n-1] is already sorted.
98859		-*/
98860		-/* TODO(shess) Is this frequent enough to warrant a binary search?
98861		-** Before implementing that, instrument the code to check. In most
98862		-** current usage, I expect that p[0] will be less than p[1] a very
98863		-** high proportion of the time.
98864		-*/
98865		-static void orderedDLReaderReorder(OrderedDLReader *p, int n){
98866		- while( n>1 && orderedDLReaderCmp(p, p+1)>0 ){
98867		- OrderedDLReader tmp = p[0];
98868		- p[0] = p[1];
98869		- p[1] = tmp;
98870		- n--;
98871		- p++;
98872		- }
98873		-}
98874		-
98875		-/* Given an array of doclist readers, merge their doclist elements
98876		-** into out in sorted order (by docid), dropping elements from older
98877		-** readers when there is a duplicate docid. pReaders is assumed to be
98878		-** ordered by age, oldest first.
98879		-*/
98880		-/* TODO(shess) nReaders must be <= MERGE_COUNT. This should probably
98881		-** be fixed.
98882		-*/
98883		-static void docListMerge(DataBuffer *out,
98884		- DLReader *pReaders, int nReaders){
98885		- OrderedDLReader readers[MERGE_COUNT];
98886		- DLWriter writer;
98887		- int i, n;
98888		- const char *pStart = 0;
98889		- int nStart = 0;
98890		- sqlite_int64 iFirstDocid = 0, iLastDocid = 0;
98891		-
98892		- assert( nReaders>0 );
98893		- if( nReaders==1 ){
98894		- dataBufferAppend(out, dlrDocData(pReaders), dlrAllDataBytes(pReaders));
98895		- return;
98896		- }
98897		-
98898		- assert( nReaders<=MERGE_COUNT );
98899		- n = 0;
98900		- for(i=0; i<nReaders; i++){
98901		- assert( pReaders[i].iType==pReaders[0].iType );
98902		- readers[i].pReader = pReaders+i;
98903		- readers[i].idx = i;
98904		- n += dlrAllDataBytes(&pReaders[i]);
98905		- }
98906		- /* Conservatively size output to sum of inputs. Output should end
98907		- ** up strictly smaller than input.
98908		- */
98909		- dataBufferExpand(out, n);
98910		-
98911		- /* Get the readers into sorted order. */
98912		- while( i-->0 ){
98913		- orderedDLReaderReorder(readers+i, nReaders-i);
98914		- }
98915		-
98916		- dlwInit(&writer, pReaders[0].iType, out);
98917		- while( !dlrAtEnd(readers[0].pReader) ){
98918		- sqlite_int64 iDocid = dlrDocid(readers[0].pReader);
98919		-
98920		- /* If this is a continuation of the current buffer to copy, extend
98921		- ** that buffer. memcpy() seems to be more efficient if it has a
98922		- ** lots of data to copy.
98923		- */
98924		- if( dlrDocData(readers[0].pReader)==pStart+nStart ){
98925		- nStart += dlrDocDataBytes(readers[0].pReader);
98926		- }else{
98927		- if( pStart!=0 ){
98928		- dlwAppend(&writer, pStart, nStart, iFirstDocid, iLastDocid);
98929		- }
98930		- pStart = dlrDocData(readers[0].pReader);
98931		- nStart = dlrDocDataBytes(readers[0].pReader);
98932		- iFirstDocid = iDocid;
98933		- }
98934		- iLastDocid = iDocid;
98935		- dlrStep(readers[0].pReader);
98936		-
98937		- /* Drop all of the older elements with the same docid. */
98938		- for(i=1; i<nReaders &&
98939		- !dlrAtEnd(readers[i].pReader) &&
98940		- dlrDocid(readers[i].pReader)==iDocid; i++){
98941		- dlrStep(readers[i].pReader);
98942		- }
98943		-
98944		- /* Get the readers back into order. */
98945		- while( i-->0 ){
98946		- orderedDLReaderReorder(readers+i, nReaders-i);
98947		- }
98948		- }
98949		-
98950		- /* Copy over any remaining elements. */
98951		- if( nStart>0 ) dlwAppend(&writer, pStart, nStart, iFirstDocid, iLastDocid);
98952		- dlwDestroy(&writer);
98953		-}
98954		-
98955		-/* Helper function for posListUnion(). Compares the current position
98956		-** between left and right, returning as standard C idiom of <0 if
98957		-** left<right, >0 if left>right, and 0 if left==right. "End" always
98958		-** compares greater.
98959		-*/
98960		-static int posListCmp(PLReader pLeft, PLReader pRight){
98961		- assert( pLeft->iType==pRight->iType );
98962		- if( pLeft->iType==DL_DOCIDS ) return 0;
98963		-
98964		- if( plrAtEnd(pLeft) ) return plrAtEnd(pRight) ? 0 : 1;
98965		- if( plrAtEnd(pRight) ) return -1;
98966		-
98967		- if( plrColumn(pLeft)<plrColumn(pRight) ) return -1;
98968		- if( plrColumn(pLeft)>plrColumn(pRight) ) return 1;
98969		-
98970		- if( plrPosition(pLeft)<plrPosition(pRight) ) return -1;
98971		- if( plrPosition(pLeft)>plrPosition(pRight) ) return 1;
98972		- if( pLeft->iType==DL_POSITIONS ) return 0;
98973		-
98974		- if( plrStartOffset(pLeft)<plrStartOffset(pRight) ) return -1;
98975		- if( plrStartOffset(pLeft)>plrStartOffset(pRight) ) return 1;
98976		-
98977		- if( plrEndOffset(pLeft)<plrEndOffset(pRight) ) return -1;
98978		- if( plrEndOffset(pLeft)>plrEndOffset(pRight) ) return 1;
98979		-
98980		- return 0;
98981		-}
98982		-
98983		-/* Write the union of position lists in pLeft and pRight to pOut.
98984		-** "Union" in this case meaning "All unique position tuples". Should
98985		-** work with any doclist type, though both inputs and the output
98986		-** should be the same type.
98987		-*/
98988		-static void posListUnion(DLReader pLeft, DLReader pRight, DLWriter *pOut){
98989		- PLReader left, right;
98990		- PLWriter writer;
98991		-
98992		- assert( dlrDocid(pLeft)==dlrDocid(pRight) );
98993		- assert( pLeft->iType==pRight->iType );
98994		- assert( pLeft->iType==pOut->iType );
98995		-
98996		- plrInit(&left, pLeft);
98997		- plrInit(&right, pRight);
98998		- plwInit(&writer, pOut, dlrDocid(pLeft));
98999		-
99000		- while( !plrAtEnd(&left) \|\| !plrAtEnd(&right) ){
99001		- int c = posListCmp(&left, &right);
99002		- if( c<0 ){
99003		- plwCopy(&writer, &left);
99004		- plrStep(&left);
99005		- }else if( c>0 ){
99006		- plwCopy(&writer, &right);
99007		- plrStep(&right);
99008		- }else{
99009		- plwCopy(&writer, &left);
99010		- plrStep(&left);
99011		- plrStep(&right);
99012		- }
99013		- }
99014		-
99015		- plwTerminate(&writer);
99016		- plwDestroy(&writer);
99017		- plrDestroy(&left);
99018		- plrDestroy(&right);
99019		-}
99020		-
99021		-/* Write the union of doclists in pLeft and pRight to pOut. For
99022		-** docids in common between the inputs, the union of the position
99023		-** lists is written. Inputs and outputs are always type DL_DEFAULT.
99024		-*/
99025		-static void docListUnion(
99026		- const char *pLeft, int nLeft,
99027		- const char *pRight, int nRight,
99028		- DataBuffer pOut / Write the combined doclist here */
99029		-){
99030		- DLReader left, right;
99031		- DLWriter writer;
99032		-
99033		- if( nLeft==0 ){
99034		- if( nRight!=0) dataBufferAppend(pOut, pRight, nRight);
99035		- return;
99036		- }
99037		- if( nRight==0 ){
99038		- dataBufferAppend(pOut, pLeft, nLeft);
99039		- return;
99040		- }
99041		-
99042		- dlrInit(&left, DL_DEFAULT, pLeft, nLeft);
99043		- dlrInit(&right, DL_DEFAULT, pRight, nRight);
99044		- dlwInit(&writer, DL_DEFAULT, pOut);
99045		-
99046		- while( !dlrAtEnd(&left) \|\| !dlrAtEnd(&right) ){
99047		- if( dlrAtEnd(&right) ){
99048		- dlwCopy(&writer, &left);
99049		- dlrStep(&left);
99050		- }else if( dlrAtEnd(&left) ){
99051		- dlwCopy(&writer, &right);
99052		- dlrStep(&right);
99053		- }else if( dlrDocid(&left)<dlrDocid(&right) ){
99054		- dlwCopy(&writer, &left);
99055		- dlrStep(&left);
99056		- }else if( dlrDocid(&left)>dlrDocid(&right) ){
99057		- dlwCopy(&writer, &right);
99058		- dlrStep(&right);
99059		- }else{
99060		- posListUnion(&left, &right, &writer);
99061		- dlrStep(&left);
99062		- dlrStep(&right);
99063		- }
99064		- }
99065		-
99066		- dlrDestroy(&left);
99067		- dlrDestroy(&right);
99068		- dlwDestroy(&writer);
99069		-}
99070		-
99071		-/*
99072		-** This function is used as part of the implementation of phrase and
99073		-** NEAR matching.
99074		-**
99075		-** pLeft and pRight are DLReaders positioned to the same docid in
99076		-** lists of type DL_POSITION. This function writes an entry to the
99077		-** DLWriter pOut for each position in pRight that is less than
99078		-** (nNear+1) greater (but not equal to or smaller) than a position
99079		-** in pLeft. For example, if nNear is 0, and the positions contained
99080		-** by pLeft and pRight are:
99081		-**
99082		-** pLeft: 5 10 15 20
99083		-** pRight: 6 9 17 21
99084		-**
99085		-** then the docid is added to pOut. If pOut is of type DL_POSITIONS,
99086		-** then a positionids "6" and "21" are also added to pOut.
99087		-**
99088		-** If boolean argument isSaveLeft is true, then positionids are copied
99089		-** from pLeft instead of pRight. In the example above, the positions "5"
99090		-** and "20" would be added instead of "6" and "21".
99091		-*/
99092		-static void posListPhraseMerge(
99093		- DLReader *pLeft,
99094		- DLReader *pRight,
99095		- int nNear,
99096		- int isSaveLeft,
99097		- DLWriter *pOut
99098		-){
99099		- PLReader left, right;
99100		- PLWriter writer;
99101		- int match = 0;
99102		-
99103		- assert( dlrDocid(pLeft)==dlrDocid(pRight) );
99104		- assert( pOut->iType!=DL_POSITIONS_OFFSETS );
99105		-
99106		- plrInit(&left, pLeft);
99107		- plrInit(&right, pRight);
99108		-
99109		- while( !plrAtEnd(&left) && !plrAtEnd(&right) ){
99110		- if( plrColumn(&left)<plrColumn(&right) ){
99111		- plrStep(&left);
99112		- }else if( plrColumn(&left)>plrColumn(&right) ){
99113		- plrStep(&right);
99114		- }else if( plrPosition(&left)>=plrPosition(&right) ){
99115		- plrStep(&right);
99116		- }else{
99117		- if( (plrPosition(&right)-plrPosition(&left))<=(nNear+1) ){
99118		- if( !match ){
99119		- plwInit(&writer, pOut, dlrDocid(pLeft));
99120		- match = 1;
99121		- }
99122		- if( !isSaveLeft ){
99123		- plwAdd(&writer, plrColumn(&right), plrPosition(&right), 0, 0);
99124		- }else{
99125		- plwAdd(&writer, plrColumn(&left), plrPosition(&left), 0, 0);
99126		- }
99127		- plrStep(&right);
99128		- }else{
99129		- plrStep(&left);
99130		- }
99131		- }
99132		- }
99133		-
99134		- if( match ){
99135		- plwTerminate(&writer);
99136		- plwDestroy(&writer);
99137		- }
99138		-
99139		- plrDestroy(&left);
99140		- plrDestroy(&right);
99141		-}
99142		-
99143		-/*
99144		-** Compare the values pointed to by the PLReaders passed as arguments.
99145		-** Return -1 if the value pointed to by pLeft is considered less than
99146		-** the value pointed to by pRight, +1 if it is considered greater
99147		-** than it, or 0 if it is equal. i.e.
99148		-**
99149		-** (pLeft - pRight)
99150		-**
99151		-** A PLReader that is in the EOF condition is considered greater than
99152		-** any other. If neither argument is in EOF state, the return value of
99153		-** plrColumn() is used. If the plrColumn() values are equal, the
99154		-** comparison is on the basis of plrPosition().
99155		-*/
99156		-static int plrCompare(PLReader pLeft, PLReader pRight){
99157		- assert(!plrAtEnd(pLeft) \|\| !plrAtEnd(pRight));
99158		-
99159		- if( plrAtEnd(pRight) \|\| plrAtEnd(pLeft) ){
99160		- return (plrAtEnd(pRight) ? -1 : 1);
99161		- }
99162		- if( plrColumn(pLeft)!=plrColumn(pRight) ){
99163		- return ((plrColumn(pLeft)<plrColumn(pRight)) ? -1 : 1);
99164		- }
99165		- if( plrPosition(pLeft)!=plrPosition(pRight) ){
99166		- return ((plrPosition(pLeft)<plrPosition(pRight)) ? -1 : 1);
99167		- }
99168		- return 0;
99169		-}
99170		-
99171		-/* We have two doclists with positions: pLeft and pRight. Depending
99172		-** on the value of the nNear parameter, perform either a phrase
99173		-** intersection (if nNear==0) or a NEAR intersection (if nNear>0)
99174		-** and write the results into pOut.
99175		-**
99176		-** A phrase intersection means that two documents only match
99177		-** if pLeft.iPos+1==pRight.iPos.
99178		-**
99179		-** A NEAR intersection means that two documents only match if
99180		-** (abs(pLeft.iPos-pRight.iPos)<nNear).
99181		-**
99182		-** If a NEAR intersection is requested, then the nPhrase argument should
99183		-** be passed the number of tokens in the two operands to the NEAR operator
99184		-** combined. For example:
99185		-**
99186		-** Query syntax nPhrase
99187		-** ------------------------------------
99188		-** "A B C" NEAR "D E" 5
99189		-** A NEAR B 2
99190		-**
99191		-** iType controls the type of data written to pOut. If iType is
99192		-** DL_POSITIONS, the positions are those from pRight.
99193		-*/
99194		-static void docListPhraseMerge(
99195		- const char *pLeft, int nLeft,
99196		- const char *pRight, int nRight,
99197		- int nNear, /* 0 for a phrase merge, non-zero for a NEAR merge */
99198		- int nPhrase, /* Number of tokens in left+right operands to NEAR */
99199		- DocListType iType, /* Type of doclist to write to pOut */
99200		- DataBuffer pOut / Write the combined doclist here */
99201		-){
99202		- DLReader left, right;
99203		- DLWriter writer;
99204		-
99205		- if( nLeft==0 \|\| nRight==0 ) return;
99206		-
99207		- assert( iType!=DL_POSITIONS_OFFSETS );
99208		-
99209		- dlrInit(&left, DL_POSITIONS, pLeft, nLeft);
99210		- dlrInit(&right, DL_POSITIONS, pRight, nRight);
99211		- dlwInit(&writer, iType, pOut);
99212		-
99213		- while( !dlrAtEnd(&left) && !dlrAtEnd(&right) ){
99214		- if( dlrDocid(&left)<dlrDocid(&right) ){
99215		- dlrStep(&left);
99216		- }else if( dlrDocid(&right)<dlrDocid(&left) ){
99217		- dlrStep(&right);
99218		- }else{
99219		- if( nNear==0 ){
99220		- posListPhraseMerge(&left, &right, 0, 0, &writer);
99221		- }else{
99222		- /* This case occurs when two terms (simple terms or phrases) are
99223		- * connected by a NEAR operator, span (nNear+1). i.e.
99224		- *
99225		- * '"terrible company" NEAR widget'
99226		- */
99227		- DataBuffer one = {0, 0, 0};
99228		- DataBuffer two = {0, 0, 0};
99229		-
99230		- DLWriter dlwriter2;
99231		- DLReader dr1 = {0, 0, 0, 0, 0};
99232		- DLReader dr2 = {0, 0, 0, 0, 0};
99233		-
99234		- dlwInit(&dlwriter2, iType, &one);
99235		- posListPhraseMerge(&right, &left, nNear-3+nPhrase, 1, &dlwriter2);
99236		- dlwInit(&dlwriter2, iType, &two);
99237		- posListPhraseMerge(&left, &right, nNear-1, 0, &dlwriter2);
99238		-
99239		- if( one.nData) dlrInit(&dr1, iType, one.pData, one.nData);
99240		- if( two.nData) dlrInit(&dr2, iType, two.pData, two.nData);
99241		-
99242		- if( !dlrAtEnd(&dr1) \|\| !dlrAtEnd(&dr2) ){
99243		- PLReader pr1 = {0};
99244		- PLReader pr2 = {0};
99245		-
99246		- PLWriter plwriter;
99247		- plwInit(&plwriter, &writer, dlrDocid(dlrAtEnd(&dr1)?&dr2:&dr1));
99248		-
99249		- if( one.nData ) plrInit(&pr1, &dr1);
99250		- if( two.nData ) plrInit(&pr2, &dr2);
99251		- while( !plrAtEnd(&pr1) \|\| !plrAtEnd(&pr2) ){
99252		- int iCompare = plrCompare(&pr1, &pr2);
99253		- switch( iCompare ){
99254		- case -1:
99255		- plwCopy(&plwriter, &pr1);
99256		- plrStep(&pr1);
99257		- break;
99258		- case 1:
99259		- plwCopy(&plwriter, &pr2);
99260		- plrStep(&pr2);
99261		- break;
99262		- case 0:
99263		- plwCopy(&plwriter, &pr1);
99264		- plrStep(&pr1);
99265		- plrStep(&pr2);
99266		- break;
99267		- }
99268		- }
99269		- plwTerminate(&plwriter);
99270		- }
99271		- dataBufferDestroy(&one);
99272		- dataBufferDestroy(&two);
99273		- }
99274		- dlrStep(&left);
99275		- dlrStep(&right);
99276		- }
99277		- }
99278		-
99279		- dlrDestroy(&left);
99280		- dlrDestroy(&right);
99281		- dlwDestroy(&writer);
99282		-}
99283		-
99284		-/* We have two DL_DOCIDS doclists: pLeft and pRight.
99285		-** Write the intersection of these two doclists into pOut as a
99286		-** DL_DOCIDS doclist.
99287		-*/
99288		-static void docListAndMerge(
99289		- const char *pLeft, int nLeft,
99290		- const char *pRight, int nRight,
99291		- DataBuffer pOut / Write the combined doclist here */
99292		-){
99293		- DLReader left, right;
99294		- DLWriter writer;
99295		-
99296		- if( nLeft==0 \|\| nRight==0 ) return;
99297		-
99298		- dlrInit(&left, DL_DOCIDS, pLeft, nLeft);
99299		- dlrInit(&right, DL_DOCIDS, pRight, nRight);
99300		- dlwInit(&writer, DL_DOCIDS, pOut);
99301		-
99302		- while( !dlrAtEnd(&left) && !dlrAtEnd(&right) ){
99303		- if( dlrDocid(&left)<dlrDocid(&right) ){
99304		- dlrStep(&left);
99305		- }else if( dlrDocid(&right)<dlrDocid(&left) ){
99306		- dlrStep(&right);
99307		- }else{
99308		- dlwAdd(&writer, dlrDocid(&left));
99309		- dlrStep(&left);
99310		- dlrStep(&right);
99311		- }
99312		- }
99313		-
99314		- dlrDestroy(&left);
99315		- dlrDestroy(&right);
99316		- dlwDestroy(&writer);
99317		-}
99318		-
99319		-/* We have two DL_DOCIDS doclists: pLeft and pRight.
99320		-** Write the union of these two doclists into pOut as a
99321		-** DL_DOCIDS doclist.
99322		-*/
99323		-static void docListOrMerge(
99324		- const char *pLeft, int nLeft,
99325		- const char *pRight, int nRight,
99326		- DataBuffer pOut / Write the combined doclist here */
99327		-){
99328		- DLReader left, right;
99329		- DLWriter writer;
99330		-
99331		- if( nLeft==0 ){
99332		- if( nRight!=0 ) dataBufferAppend(pOut, pRight, nRight);
99333		- return;
99334		- }
99335		- if( nRight==0 ){
99336		- dataBufferAppend(pOut, pLeft, nLeft);
99337		- return;
99338		- }
99339		-
99340		- dlrInit(&left, DL_DOCIDS, pLeft, nLeft);
99341		- dlrInit(&right, DL_DOCIDS, pRight, nRight);
99342		- dlwInit(&writer, DL_DOCIDS, pOut);
99343		-
99344		- while( !dlrAtEnd(&left) \|\| !dlrAtEnd(&right) ){
99345		- if( dlrAtEnd(&right) ){
99346		- dlwAdd(&writer, dlrDocid(&left));
99347		- dlrStep(&left);
99348		- }else if( dlrAtEnd(&left) ){
99349		- dlwAdd(&writer, dlrDocid(&right));
99350		- dlrStep(&right);
99351		- }else if( dlrDocid(&left)<dlrDocid(&right) ){
99352		- dlwAdd(&writer, dlrDocid(&left));
99353		- dlrStep(&left);
99354		- }else if( dlrDocid(&right)<dlrDocid(&left) ){
99355		- dlwAdd(&writer, dlrDocid(&right));
99356		- dlrStep(&right);
99357		- }else{
99358		- dlwAdd(&writer, dlrDocid(&left));
99359		- dlrStep(&left);
99360		- dlrStep(&right);
99361		- }
99362		- }
99363		-
99364		- dlrDestroy(&left);
99365		- dlrDestroy(&right);
99366		- dlwDestroy(&writer);
99367		-}
99368		-
99369		-/* We have two DL_DOCIDS doclists: pLeft and pRight.
99370		-** Write into pOut as DL_DOCIDS doclist containing all documents that
99371		-** occur in pLeft but not in pRight.
99372		-*/
99373		-static void docListExceptMerge(
99374		- const char *pLeft, int nLeft,
99375		- const char *pRight, int nRight,
99376		- DataBuffer pOut / Write the combined doclist here */
99377		-){
99378		- DLReader left, right;
99379		- DLWriter writer;
99380		-
99381		- if( nLeft==0 ) return;
99382		- if( nRight==0 ){
99383		- dataBufferAppend(pOut, pLeft, nLeft);
99384		- return;
99385		- }
99386		-
99387		- dlrInit(&left, DL_DOCIDS, pLeft, nLeft);
99388		- dlrInit(&right, DL_DOCIDS, pRight, nRight);
99389		- dlwInit(&writer, DL_DOCIDS, pOut);
99390		-
99391		- while( !dlrAtEnd(&left) ){
99392		- while( !dlrAtEnd(&right) && dlrDocid(&right)<dlrDocid(&left) ){
99393		- dlrStep(&right);
99394		- }
99395		- if( dlrAtEnd(&right) \|\| dlrDocid(&left)<dlrDocid(&right) ){
99396		- dlwAdd(&writer, dlrDocid(&left));
99397		- }
99398		- dlrStep(&left);
99399		- }
99400		-
99401		- dlrDestroy(&left);
99402		- dlrDestroy(&right);
99403		- dlwDestroy(&writer);
99404		-}
99405		-
99406		-static char string_dup_n(const char s, int n){
99407		- char *str = sqlite3_malloc(n + 1);
99408		- memcpy(str, s, n);
99409		- str[n] = '\0';
99410		- return str;
99411		-}
99412		-
99413		-/* Duplicate a string; the caller must free() the returned string.
99414		- * (We don't use strdup() since it is not part of the standard C library and
99415		- * may not be available everywhere.) */
99416		-static char string_dup(const char s){
99417		- return string_dup_n(s, strlen(s));
99418		-}
99419		-
99420		-/* Format a string, replacing each occurrence of the % character with
99421		- * zDb.zName. This may be more convenient than sqlite_mprintf()
99422		- * when one string is used repeatedly in a format string.
99423		- * The caller must free() the returned string. */
99424		-static char string_format(const char zFormat,
99425		- const char zDb, const char zName){
99426		- const char *p;
99427		- size_t len = 0;
99428		- size_t nDb = strlen(zDb);
99429		- size_t nName = strlen(zName);
99430		- size_t nFullTableName = nDb+1+nName;
99431		- char *result;
99432		- char *r;
99433		-
99434		- /* first compute length needed */
99435		- for(p = zFormat ; *p ; ++p){
99436		- len += (*p=='%' ? nFullTableName : 1);
99437		- }
99438		- len += 1; /* for null terminator */
99439		-
99440		- r = result = sqlite3_malloc(len);
99441		- for(p = zFormat; *p; ++p){
99442		- if( *p=='%' ){
99443		- memcpy(r, zDb, nDb);
99444		- r += nDb;
99445		- *r++ = '.';
99446		- memcpy(r, zName, nName);
99447		- r += nName;
99448		- } else {
99449		- r++ = p;
99450		- }
99451		- }
99452		- *r++ = '\0';
99453		- assert( r == result + len );
99454		- return result;
99455		-}
99456		-
99457		-static int sql_exec(sqlite3 db, const char zDb, const char *zName,
99458		- const char *zFormat){
99459		- char *zCommand = string_format(zFormat, zDb, zName);
99460		- int rc;
99461		- FTSTRACE(("FTS3 sql: %s\n", zCommand));
99462		- rc = sqlite3_exec(db, zCommand, NULL, 0, NULL);
99463		- sqlite3_free(zCommand);
99464		- return rc;
99465		-}
99466		-
99467		-static int sql_prepare(sqlite3 db, const char zDb, const char *zName,
99468		- sqlite3_stmt *ppStmt, const char zFormat){
99469		- char *zCommand = string_format(zFormat, zDb, zName);
99470		- int rc;
99471		- FTSTRACE(("FTS3 prepare: %s\n", zCommand));
99472		- rc = sqlite3_prepare_v2(db, zCommand, -1, ppStmt, NULL);
99473		- sqlite3_free(zCommand);
99474		- return rc;
99475		-}
99476		-
99477		-/* end utility functions */
99478		-
99479		-/* Forward reference */
99480		-typedef struct fulltext_vtab fulltext_vtab;
99481		-
99482		-/*
99483		-** An instance of the following structure keeps track of generated
99484		-** matching-word offset information and snippets.
99485		-*/
99486		-typedef struct Snippet {
99487		- int nMatch; /* Total number of matches */
99488		- int nAlloc; /* Space allocated for aMatch[] */
99489		- struct snippetMatch { /* One entry for each matching term */
99490		- char snStatus; /* Status flag for use while constructing snippets */
99491		- short int iCol; /* The column that contains the match */
99492		- short int iTerm; /* The index in Query.pTerms[] of the matching term */
99493		- int iToken; /* The index of the matching document token */
99494		- short int nByte; /* Number of bytes in the term */
99495		- int iStart; /* The offset to the first character of the term */
99496		- } aMatch; / Points to space obtained from malloc */
99497		- char zOffset; / Text rendering of aMatch[] */
99498		- int nOffset; /* strlen(zOffset) */
99499		- char zSnippet; / Snippet text */
99500		- int nSnippet; /* strlen(zSnippet) */
99501		-} Snippet;
99502		-
99503		-
99504		-typedef enum QueryType {
99505		- QUERY_GENERIC, /* table scan */
99506		- QUERY_DOCID, /* lookup by docid */
99507		- QUERY_FULLTEXT /* QUERY_FULLTEXT + [i] is a full-text search for column i*/
99508		-} QueryType;
99509		-
99510		-typedef enum fulltext_statement {
99511		- CONTENT_INSERT_STMT,
99512		- CONTENT_SELECT_STMT,
99513		- CONTENT_UPDATE_STMT,
99514		- CONTENT_DELETE_STMT,
99515		- CONTENT_EXISTS_STMT,
99516		-
99517		- BLOCK_INSERT_STMT,
99518		- BLOCK_SELECT_STMT,
99519		- BLOCK_DELETE_STMT,
99520		- BLOCK_DELETE_ALL_STMT,
99521		-
99522		- SEGDIR_MAX_INDEX_STMT,
99523		- SEGDIR_SET_STMT,
99524		- SEGDIR_SELECT_LEVEL_STMT,
99525		- SEGDIR_SPAN_STMT,
99526		- SEGDIR_DELETE_STMT,
99527		- SEGDIR_SELECT_SEGMENT_STMT,
99528		- SEGDIR_SELECT_ALL_STMT,
99529		- SEGDIR_DELETE_ALL_STMT,
99530		- SEGDIR_COUNT_STMT,
99531		-
99532		- MAX_STMT /* Always at end! */
99533		-} fulltext_statement;
99534		-
99535		-/* These must exactly match the enum above. */
99536		-/* TODO(shess): Is there some risk that a statement will be used in two
99537		-** cursors at once, e.g. if a query joins a virtual table to itself?
99538		-** If so perhaps we should move some of these to the cursor object.
99539		-*/
99540		-static const char *const fulltext_zStatement[MAX_STMT] = {
99541		- /* CONTENT_INSERT / NULL, / generated in contentInsertStatement() */
99542		- /* CONTENT_SELECT / NULL, / generated in contentSelectStatement() */
99543		- /* CONTENT_UPDATE / NULL, / generated in contentUpdateStatement() */
99544		- /* CONTENT_DELETE */ "delete from %_content where docid = ?",
99545		- /* CONTENT_EXISTS */ "select docid from %_content limit 1",
99546		-
99547		- /* BLOCK_INSERT */
99548		- "insert into %_segments (blockid, block) values (null, ?)",
99549		- /* BLOCK_SELECT */ "select block from %_segments where blockid = ?",
99550		- /* BLOCK_DELETE */ "delete from %_segments where blockid between ? and ?",
99551		- /* BLOCK_DELETE_ALL */ "delete from %_segments",
99552		-
99553		- /* SEGDIR_MAX_INDEX */ "select max(idx) from %_segdir where level = ?",
99554		- /* SEGDIR_SET */ "insert into %_segdir values (?, ?, ?, ?, ?, ?)",
99555		- /* SEGDIR_SELECT_LEVEL */
99556		- "select start_block, leaves_end_block, root from %_segdir "
99557		- " where level = ? order by idx",
99558		- /* SEGDIR_SPAN */
99559		- "select min(start_block), max(end_block) from %_segdir "
99560		- " where level = ? and start_block <> 0",
99561		- /* SEGDIR_DELETE */ "delete from %_segdir where level = ?",
99562		-
99563		- /* NOTE(shess): The first three results of the following two
99564		- ** statements must match.
99565		- */
99566		- /* SEGDIR_SELECT_SEGMENT */
99567		- "select start_block, leaves_end_block, root from %_segdir "
99568		- " where level = ? and idx = ?",
99569		- /* SEGDIR_SELECT_ALL */
99570		- "select start_block, leaves_end_block, root from %_segdir "
99571		- " order by level desc, idx asc",
99572		- /* SEGDIR_DELETE_ALL */ "delete from %_segdir",
99573		- /* SEGDIR_COUNT / "select count(), ifnull(max(level),0) from %_segdir",
99574		-};
99575		-
99576		-/*
99577		-** A connection to a fulltext index is an instance of the following
99578		-** structure. The xCreate and xConnect methods create an instance
99579		-** of this structure and xDestroy and xDisconnect free that instance.
99580		-** All other methods receive a pointer to the structure as one of their
99581		-** arguments.
99582		-*/
99583		-struct fulltext_vtab {
99584		- sqlite3_vtab base; /* Base class used by SQLite core */
99585		- sqlite3 db; / The database connection */
99586		- const char zDb; / logical database name */
99587		- const char zName; / virtual table name */
99588		- int nColumn; /* number of columns in virtual table */
99589		- char *azColumn; / column names. malloced */
99590		- char *azContentColumn; / column names in content table; malloced */
99591		- sqlite3_tokenizer pTokenizer; / tokenizer for inserts and queries */
99592		-
99593		- /* Precompiled statements which we keep as long as the table is
99594		- ** open.
99595		- */
99596		- sqlite3_stmt *pFulltextStatements[MAX_STMT];
99597		-
99598		- /* Precompiled statements used for segment merges. We run a
99599		- ** separate select across the leaf level of each tree being merged.
99600		- */
99601		- sqlite3_stmt *pLeafSelectStmts[MERGE_COUNT];
99602		- /* The statement used to prepare pLeafSelectStmts. */
99603		-#define LEAF_SELECT \
99604		- "select block from %_segments where blockid between ? and ? order by blockid"
99605		-
99606		- /* These buffer pending index updates during transactions.
99607		- ** nPendingData estimates the memory size of the pending data. It
99608		- ** doesn't include the hash-bucket overhead, nor any malloc
99609		- ** overhead. When nPendingData exceeds kPendingThreshold, the
99610		- ** buffer is flushed even before the transaction closes.
99611		- ** pendingTerms stores the data, and is only valid when nPendingData
99612		- ** is >=0 (nPendingData<0 means pendingTerms has not been
99613		- ** initialized). iPrevDocid is the last docid written, used to make
99614		- ** certain we're inserting in sorted order.
99615		- */
99616		- int nPendingData;
99617		-#define kPendingThreshold (110241024)
99618		- sqlite_int64 iPrevDocid;
99619		- fts3Hash pendingTerms;
99620		-};
99621		-
99622		-/*
99623		-** When the core wants to do a query, it create a cursor using a
99624		-** call to xOpen. This structure is an instance of a cursor. It
99625		-** is destroyed by xClose.
99626		-*/
99627		-typedef struct fulltext_cursor {
99628		- sqlite3_vtab_cursor base; /* Base class used by SQLite core */
99629		- QueryType iCursorType; /* Copy of sqlite3_index_info.idxNum */
99630		- sqlite3_stmt pStmt; / Prepared statement in use by the cursor */
99631		- int eof; /* True if at End Of Results */
99632		- Fts3Expr pExpr; / Parsed MATCH query string */
99633		- Snippet snippet; /* Cached snippet for the current row */
99634		- int iColumn; /* Column being searched */
99635		- DataBuffer result; /* Doclist results from fulltextQuery */
99636		- DLReader reader; /* Result reader if result not empty */
99637		-} fulltext_cursor;
99638		-
99639		-static fulltext_vtab cursor_vtab(fulltext_cursor c){
99640		- return (fulltext_vtab *) c->base.pVtab;
99641		-}
99642		-
99643		-static const sqlite3_module fts3Module; /* forward declaration */
99644		-
99645		-/* Return a dynamically generated statement of the form
99646		- * insert into %_content (docid, ...) values (?, ...)
99647		- */
99648		-static const char contentInsertStatement(fulltext_vtab v){
99649		- StringBuffer sb;
99650		- int i;
99651		-
99652		- initStringBuffer(&sb);
99653		- append(&sb, "insert into %_content (docid, ");
99654		- appendList(&sb, v->nColumn, v->azContentColumn);
99655		- append(&sb, ") values (?");
99656		- for(i=0; i<v->nColumn; ++i)
99657		- append(&sb, ", ?");
99658		- append(&sb, ")");
99659		- return stringBufferData(&sb);
99660		-}
99661		-
99662		-/* Return a dynamically generated statement of the form
99663		- * select <content columns> from %_content where docid = ?
99664		- */
99665		-static const char contentSelectStatement(fulltext_vtab v){
99666		- StringBuffer sb;
99667		- initStringBuffer(&sb);
99668		- append(&sb, "SELECT ");
99669		- appendList(&sb, v->nColumn, v->azContentColumn);
99670		- append(&sb, " FROM %_content WHERE docid = ?");
99671		- return stringBufferData(&sb);
99672		-}
99673		-
99674		-/* Return a dynamically generated statement of the form
99675		- * update %_content set [col_0] = ?, [col_1] = ?, ...
99676		- * where docid = ?
99677		- */
99678		-static const char contentUpdateStatement(fulltext_vtab v){
99679		- StringBuffer sb;
99680		- int i;
99681		-
99682		- initStringBuffer(&sb);
99683		- append(&sb, "update %_content set ");
99684		- for(i=0; i<v->nColumn; ++i) {
99685		- if( i>0 ){
99686		- append(&sb, ", ");
99687		- }
99688		- append(&sb, v->azContentColumn[i]);
99689		- append(&sb, " = ?");
99690		- }
99691		- append(&sb, " where docid = ?");
99692		- return stringBufferData(&sb);
99693		-}
99694		-
99695		-/* Puts a freshly-prepared statement determined by iStmt in *ppStmt.
99696		-** If the indicated statement has never been prepared, it is prepared
99697		-** and cached, otherwise the cached version is reset.
99698		-*/
99699		-static int sql_get_statement(fulltext_vtab *v, fulltext_statement iStmt,
99700		- sqlite3_stmt **ppStmt){
99701		- assert( iStmt<MAX_STMT );
99702		- if( v->pFulltextStatements[iStmt]==NULL ){
99703		- const char *zStmt;
99704		- int rc;
99705		- switch( iStmt ){
99706		- case CONTENT_INSERT_STMT:
99707		- zStmt = contentInsertStatement(v); break;
99708		- case CONTENT_SELECT_STMT:
99709		- zStmt = contentSelectStatement(v); break;
99710		- case CONTENT_UPDATE_STMT:
99711		- zStmt = contentUpdateStatement(v); break;
99712		- default:
99713		- zStmt = fulltext_zStatement[iStmt];
99714		- }
99715		- rc = sql_prepare(v->db, v->zDb, v->zName, &v->pFulltextStatements[iStmt],
99716		- zStmt);
99717		- if( zStmt != fulltext_zStatement[iStmt]) sqlite3_free((void *) zStmt);
99718		- if( rc!=SQLITE_OK ) return rc;
99719		- } else {
99720		- int rc = sqlite3_reset(v->pFulltextStatements[iStmt]);
99721		- if( rc!=SQLITE_OK ) return rc;
99722		- }
99723		-
99724		- *ppStmt = v->pFulltextStatements[iStmt];
99725		- return SQLITE_OK;
99726		-}
99727		-
99728		-/* Like sqlite3_step(), but convert SQLITE_DONE to SQLITE_OK and
99729		-** SQLITE_ROW to SQLITE_ERROR. Useful for statements like UPDATE,
99730		-** where we expect no results.
99731		-*/
99732		-static int sql_single_step(sqlite3_stmt *s){
99733		- int rc = sqlite3_step(s);
99734		- return (rc==SQLITE_DONE) ? SQLITE_OK : rc;
99735		-}
99736		-
99737		-/* Like sql_get_statement(), but for special replicated LEAF_SELECT
99738		-** statements. idx -1 is a special case for an uncached version of
99739		-** the statement (used in the optimize implementation).
99740		-*/
99741		-/* TODO(shess) Write version for generic statements and then share
99742		-** that between the cached-statement functions.
99743		-*/
99744		-static int sql_get_leaf_statement(fulltext_vtab *v, int idx,
99745		- sqlite3_stmt **ppStmt){
99746		- assert( idx>=-1 && idx<MERGE_COUNT );
99747		- if( idx==-1 ){
99748		- return sql_prepare(v->db, v->zDb, v->zName, ppStmt, LEAF_SELECT);
99749		- }else if( v->pLeafSelectStmts[idx]==NULL ){
99750		- int rc = sql_prepare(v->db, v->zDb, v->zName, &v->pLeafSelectStmts[idx],
99751		- LEAF_SELECT);
99752		- if( rc!=SQLITE_OK ) return rc;
99753		- }else{
99754		- int rc = sqlite3_reset(v->pLeafSelectStmts[idx]);
99755		- if( rc!=SQLITE_OK ) return rc;
99756		- }
99757		-
99758		- *ppStmt = v->pLeafSelectStmts[idx];
99759		- return SQLITE_OK;
99760		-}
99761		-
99762		-/* insert into %_content (docid, ...) values ([docid], [pValues])
99763		-** If the docid contains SQL NULL, then a unique docid will be
99764		-** generated.
99765		-*/
99766		-static int content_insert(fulltext_vtab v, sqlite3_value docid,
99767		- sqlite3_value **pValues){
99768		- sqlite3_stmt *s;
99769		- int i;
99770		- int rc = sql_get_statement(v, CONTENT_INSERT_STMT, &s);
99771		- if( rc!=SQLITE_OK ) return rc;
99772		-
99773		- rc = sqlite3_bind_value(s, 1, docid);
99774		- if( rc!=SQLITE_OK ) return rc;
99775		-
99776		- for(i=0; i<v->nColumn; ++i){
99777		- rc = sqlite3_bind_value(s, 2+i, pValues[i]);
99778		- if( rc!=SQLITE_OK ) return rc;
99779		- }
99780		-
99781		- return sql_single_step(s);
99782		-}
99783		-
99784		-/* update %_content set col0 = pValues[0], col1 = pValues[1], ...
99785		- * where docid = [iDocid] */
99786		-static int content_update(fulltext_vtab v, sqlite3_value *pValues,
99787		- sqlite_int64 iDocid){
99788		- sqlite3_stmt *s;
99789		- int i;
99790		- int rc = sql_get_statement(v, CONTENT_UPDATE_STMT, &s);
99791		- if( rc!=SQLITE_OK ) return rc;
99792		-
99793		- for(i=0; i<v->nColumn; ++i){
99794		- rc = sqlite3_bind_value(s, 1+i, pValues[i]);
99795		- if( rc!=SQLITE_OK ) return rc;
99796		- }
99797		-
99798		- rc = sqlite3_bind_int64(s, 1+v->nColumn, iDocid);
99799		- if( rc!=SQLITE_OK ) return rc;
99800		-
99801		- return sql_single_step(s);
99802		-}
99803		-
99804		-static void freeStringArray(int nString, const char **pString){
99805		- int i;
99806		-
99807		- for (i=0 ; i < nString ; ++i) {
99808		- if( pString[i]!=NULL ) sqlite3_free((void *) pString[i]);
99809		- }
99810		- sqlite3_free((void *) pString);
99811		-}
99812		-
99813		-/* select * from %_content where docid = [iDocid]
99814		- * The caller must delete the returned array and all strings in it.
99815		- * null fields will be NULL in the returned array.
99816		- *
99817		- * TODO: Perhaps we should return pointer/length strings here for consistency
99818		- * with other code which uses pointer/length. */
99819		-static int content_select(fulltext_vtab *v, sqlite_int64 iDocid,
99820		- const char ***pValues){
99821		- sqlite3_stmt *s;
99822		- const char **values;
99823		- int i;
99824		- int rc;
99825		-
99826		- *pValues = NULL;
99827		-
99828		- rc = sql_get_statement(v, CONTENT_SELECT_STMT, &s);
99829		- if( rc!=SQLITE_OK ) return rc;
99830		-
99831		- rc = sqlite3_bind_int64(s, 1, iDocid);
99832		- if( rc!=SQLITE_OK ) return rc;
99833		-
99834		- rc = sqlite3_step(s);
99835		- if( rc!=SQLITE_ROW ) return rc;
99836		-
99837		- values = (const char *) sqlite3_malloc(v->nColumn sizeof(const char *));
99838		- for(i=0; i<v->nColumn; ++i){
99839		- if( sqlite3_column_type(s, i)==SQLITE_NULL ){
99840		- values[i] = NULL;
99841		- }else{
99842		- values[i] = string_dup((char*)sqlite3_column_text(s, i));
99843		- }
99844		- }
99845		-
99846		- /* We expect only one row. We must execute another sqlite3_step()
99847		- * to complete the iteration; otherwise the table will remain locked. */
99848		- rc = sqlite3_step(s);
99849		- if( rc==SQLITE_DONE ){
99850		- *pValues = values;
99851		- return SQLITE_OK;
99852		- }
99853		-
99854		- freeStringArray(v->nColumn, values);
99855		- return rc;
99856		-}
99857		-
99858		-/* delete from %_content where docid = [iDocid ] */
99859		-static int content_delete(fulltext_vtab *v, sqlite_int64 iDocid){
99860		- sqlite3_stmt *s;
99861		- int rc = sql_get_statement(v, CONTENT_DELETE_STMT, &s);
99862		- if( rc!=SQLITE_OK ) return rc;
99863		-
99864		- rc = sqlite3_bind_int64(s, 1, iDocid);
99865		- if( rc!=SQLITE_OK ) return rc;
99866		-
99867		- return sql_single_step(s);
99868		-}
99869		-
99870		-/* Returns SQLITE_ROW if any rows exist in %_content, SQLITE_DONE if
99871		-** no rows exist, and any error in case of failure.
99872		-*/
99873		-static int content_exists(fulltext_vtab *v){
99874		- sqlite3_stmt *s;
99875		- int rc = sql_get_statement(v, CONTENT_EXISTS_STMT, &s);
99876		- if( rc!=SQLITE_OK ) return rc;
99877		-
99878		- rc = sqlite3_step(s);
99879		- if( rc!=SQLITE_ROW ) return rc;
99880		-
99881		- /* We expect only one row. We must execute another sqlite3_step()
99882		- * to complete the iteration; otherwise the table will remain locked. */
99883		- rc = sqlite3_step(s);
99884		- if( rc==SQLITE_DONE ) return SQLITE_ROW;
99885		- if( rc==SQLITE_ROW ) return SQLITE_ERROR;
99886		- return rc;
99887		-}
99888		-
99889		-/* insert into %_segments values ([pData])
99890		-** returns assigned blockid in *piBlockid
99891		-*/
99892		-static int block_insert(fulltext_vtab v, const char pData, int nData,
99893		- sqlite_int64 *piBlockid){
99894		- sqlite3_stmt *s;
99895		- int rc = sql_get_statement(v, BLOCK_INSERT_STMT, &s);
99896		- if( rc!=SQLITE_OK ) return rc;
99897		-
99898		- rc = sqlite3_bind_blob(s, 1, pData, nData, SQLITE_STATIC);
99899		- if( rc!=SQLITE_OK ) return rc;
99900		-
99901		- rc = sqlite3_step(s);
99902		- if( rc==SQLITE_ROW ) return SQLITE_ERROR;
99903		- if( rc!=SQLITE_DONE ) return rc;
99904		-
99905		- /* blockid column is an alias for rowid. */
99906		- *piBlockid = sqlite3_last_insert_rowid(v->db);
99907		- return SQLITE_OK;
99908		-}
99909		-
99910		-/* delete from %_segments
99911		-** where blockid between [iStartBlockid] and [iEndBlockid]
99912		-**
99913		-** Deletes the range of blocks, inclusive, used to delete the blocks
99914		-** which form a segment.
99915		-*/
99916		-static int block_delete(fulltext_vtab *v,
99917		- sqlite_int64 iStartBlockid, sqlite_int64 iEndBlockid){
99918		- sqlite3_stmt *s;
99919		- int rc = sql_get_statement(v, BLOCK_DELETE_STMT, &s);
99920		- if( rc!=SQLITE_OK ) return rc;
99921		-
99922		- rc = sqlite3_bind_int64(s, 1, iStartBlockid);
99923		- if( rc!=SQLITE_OK ) return rc;
99924		-
99925		- rc = sqlite3_bind_int64(s, 2, iEndBlockid);
99926		- if( rc!=SQLITE_OK ) return rc;
99927		-
99928		- return sql_single_step(s);
99929		-}
99930		-
99931		-/* Returns SQLITE_ROW with *pidx set to the maximum segment idx found
99932		-** at iLevel. Returns SQLITE_DONE if there are no segments at
99933		-** iLevel. Otherwise returns an error.
99934		-*/
99935		-static int segdir_max_index(fulltext_vtab v, int iLevel, int pidx){
99936		- sqlite3_stmt *s;
99937		- int rc = sql_get_statement(v, SEGDIR_MAX_INDEX_STMT, &s);
99938		- if( rc!=SQLITE_OK ) return rc;
99939		-
99940		- rc = sqlite3_bind_int(s, 1, iLevel);
99941		- if( rc!=SQLITE_OK ) return rc;
99942		-
99943		- rc = sqlite3_step(s);
99944		- /* Should always get at least one row due to how max() works. */
99945		- if( rc==SQLITE_DONE ) return SQLITE_DONE;
99946		- if( rc!=SQLITE_ROW ) return rc;
99947		-
99948		- /* NULL means that there were no inputs to max(). */
99949		- if( SQLITE_NULL==sqlite3_column_type(s, 0) ){
99950		- rc = sqlite3_step(s);
99951		- if( rc==SQLITE_ROW ) return SQLITE_ERROR;
99952		- return rc;
99953		- }
99954		-
99955		- *pidx = sqlite3_column_int(s, 0);
99956		-
99957		- /* We expect only one row. We must execute another sqlite3_step()
99958		- * to complete the iteration; otherwise the table will remain locked. */
99959		- rc = sqlite3_step(s);
99960		- if( rc==SQLITE_ROW ) return SQLITE_ERROR;
99961		- if( rc!=SQLITE_DONE ) return rc;
99962		- return SQLITE_ROW;
99963		-}
99964		-
99965		-/* insert into %_segdir values (
99966		-** [iLevel], [idx],
99967		-** [iStartBlockid], [iLeavesEndBlockid], [iEndBlockid],
99968		-** [pRootData]
99969		-** )
99970		-*/
99971		-static int segdir_set(fulltext_vtab *v, int iLevel, int idx,
99972		- sqlite_int64 iStartBlockid,
99973		- sqlite_int64 iLeavesEndBlockid,
99974		- sqlite_int64 iEndBlockid,
99975		- const char *pRootData, int nRootData){
99976		- sqlite3_stmt *s;
99977		- int rc = sql_get_statement(v, SEGDIR_SET_STMT, &s);
99978		- if( rc!=SQLITE_OK ) return rc;
99979		-
99980		- rc = sqlite3_bind_int(s, 1, iLevel);
99981		- if( rc!=SQLITE_OK ) return rc;
99982		-
99983		- rc = sqlite3_bind_int(s, 2, idx);
99984		- if( rc!=SQLITE_OK ) return rc;
99985		-
99986		- rc = sqlite3_bind_int64(s, 3, iStartBlockid);
99987		- if( rc!=SQLITE_OK ) return rc;
99988		-
99989		- rc = sqlite3_bind_int64(s, 4, iLeavesEndBlockid);
99990		- if( rc!=SQLITE_OK ) return rc;
99991		-
99992		- rc = sqlite3_bind_int64(s, 5, iEndBlockid);
99993		- if( rc!=SQLITE_OK ) return rc;
99994		-
99995		- rc = sqlite3_bind_blob(s, 6, pRootData, nRootData, SQLITE_STATIC);
99996		- if( rc!=SQLITE_OK ) return rc;
99997		-
99998		- return sql_single_step(s);
99999		-}
100000		-
100001		-/* Queries %_segdir for the block span of the segments in level
100002		-** iLevel. Returns SQLITE_DONE if there are no blocks for iLevel,
100003		-** SQLITE_ROW if there are blocks, else an error.
100004		-*/
100005		-static int segdir_span(fulltext_vtab *v, int iLevel,
100006		- sqlite_int64 *piStartBlockid,
100007		- sqlite_int64 *piEndBlockid){
100008		- sqlite3_stmt *s;
100009		- int rc = sql_get_statement(v, SEGDIR_SPAN_STMT, &s);
100010		- if( rc!=SQLITE_OK ) return rc;
100011		-
100012		- rc = sqlite3_bind_int(s, 1, iLevel);
100013		- if( rc!=SQLITE_OK ) return rc;
100014		-
100015		- rc = sqlite3_step(s);
100016		- if( rc==SQLITE_DONE ) return SQLITE_DONE; /* Should never happen */
100017		- if( rc!=SQLITE_ROW ) return rc;
100018		-
100019		- /* This happens if all segments at this level are entirely inline. */
100020		- if( SQLITE_NULL==sqlite3_column_type(s, 0) ){
100021		- /* We expect only one row. We must execute another sqlite3_step()
100022		- * to complete the iteration; otherwise the table will remain locked. */
100023		- int rc2 = sqlite3_step(s);
100024		- if( rc2==SQLITE_ROW ) return SQLITE_ERROR;
100025		- return rc2;
100026		- }
100027		-
100028		- *piStartBlockid = sqlite3_column_int64(s, 0);
100029		- *piEndBlockid = sqlite3_column_int64(s, 1);
100030		-
100031		- /* We expect only one row. We must execute another sqlite3_step()
100032		- * to complete the iteration; otherwise the table will remain locked. */
100033		- rc = sqlite3_step(s);
100034		- if( rc==SQLITE_ROW ) return SQLITE_ERROR;
100035		- if( rc!=SQLITE_DONE ) return rc;
100036		- return SQLITE_ROW;
100037		-}
100038		-
100039		-/* Delete the segment blocks and segment directory records for all
100040		-** segments at iLevel.
100041		-*/
100042		-static int segdir_delete(fulltext_vtab *v, int iLevel){
100043		- sqlite3_stmt *s;
100044		- sqlite_int64 iStartBlockid, iEndBlockid;
100045		- int rc = segdir_span(v, iLevel, &iStartBlockid, &iEndBlockid);
100046		- if( rc!=SQLITE_ROW && rc!=SQLITE_DONE ) return rc;
100047		-
100048		- if( rc==SQLITE_ROW ){
100049		- rc = block_delete(v, iStartBlockid, iEndBlockid);
100050		- if( rc!=SQLITE_OK ) return rc;
100051		- }
100052		-
100053		- /* Delete the segment directory itself. */
100054		- rc = sql_get_statement(v, SEGDIR_DELETE_STMT, &s);
100055		- if( rc!=SQLITE_OK ) return rc;
100056		-
100057		- rc = sqlite3_bind_int64(s, 1, iLevel);
100058		- if( rc!=SQLITE_OK ) return rc;
100059		-
100060		- return sql_single_step(s);
100061		-}
100062		-
100063		-/* Delete entire fts index, SQLITE_OK on success, relevant error on
100064		-** failure.
100065		-*/
100066		-static int segdir_delete_all(fulltext_vtab *v){
100067		- sqlite3_stmt *s;
100068		- int rc = sql_get_statement(v, SEGDIR_DELETE_ALL_STMT, &s);
100069		- if( rc!=SQLITE_OK ) return rc;
100070		-
100071		- rc = sql_single_step(s);
100072		- if( rc!=SQLITE_OK ) return rc;
100073		-
100074		- rc = sql_get_statement(v, BLOCK_DELETE_ALL_STMT, &s);
100075		- if( rc!=SQLITE_OK ) return rc;
100076		-
100077		- return sql_single_step(s);
100078		-}
100079		-
100080		-/* Returns SQLITE_OK with *pnSegments set to the number of entries in
100081		-** %_segdir and *piMaxLevel set to the highest level which has a
100082		-** segment. Otherwise returns the SQLite error which caused failure.
100083		-*/
100084		-static int segdir_count(fulltext_vtab v, int pnSegments, int *piMaxLevel){
100085		- sqlite3_stmt *s;
100086		- int rc = sql_get_statement(v, SEGDIR_COUNT_STMT, &s);
100087		- if( rc!=SQLITE_OK ) return rc;
100088		-
100089		- rc = sqlite3_step(s);
100090		- /* TODO(shess): This case should not be possible? Should stronger
100091		- ** measures be taken if it happens?
100092		- */
100093		- if( rc==SQLITE_DONE ){
100094		- *pnSegments = 0;
100095		- *piMaxLevel = 0;
100096		- return SQLITE_OK;
100097		- }
100098		- if( rc!=SQLITE_ROW ) return rc;
100099		-
100100		- *pnSegments = sqlite3_column_int(s, 0);
100101		- *piMaxLevel = sqlite3_column_int(s, 1);
100102		-
100103		- /* We expect only one row. We must execute another sqlite3_step()
100104		- * to complete the iteration; otherwise the table will remain locked. */
100105		- rc = sqlite3_step(s);
100106		- if( rc==SQLITE_DONE ) return SQLITE_OK;
100107		- if( rc==SQLITE_ROW ) return SQLITE_ERROR;
100108		- return rc;
100109		-}
100110		-
100111		-/* TODO(shess) clearPendingTerms() is far down the file because
100112		-** writeZeroSegment() is far down the file because LeafWriter is far
100113		-** down the file. Consider refactoring the code to move the non-vtab
100114		-** code above the vtab code so that we don't need this forward
100115		-** reference.
100116		-*/
100117		-static int clearPendingTerms(fulltext_vtab *v);
100118		-
100119		-/*
100120		-** Free the memory used to contain a fulltext_vtab structure.
100121		-*/
100122		-static void fulltext_vtab_destroy(fulltext_vtab *v){
100123		- int iStmt, i;
100124		-
100125		- FTSTRACE(("FTS3 Destroy %p\n", v));
100126		- for( iStmt=0; iStmt<MAX_STMT; iStmt++ ){
100127		- if( v->pFulltextStatements[iStmt]!=NULL ){
100128		- sqlite3_finalize(v->pFulltextStatements[iStmt]);
100129		- v->pFulltextStatements[iStmt] = NULL;
100130		- }
100131		- }
100132		-
100133		- for( i=0; i<MERGE_COUNT; i++ ){
100134		- if( v->pLeafSelectStmts[i]!=NULL ){
100135		- sqlite3_finalize(v->pLeafSelectStmts[i]);
100136		- v->pLeafSelectStmts[i] = NULL;
100137		- }
100138		- }
100139		-
100140		- if( v->pTokenizer!=NULL ){
100141		- v->pTokenizer->pModule->xDestroy(v->pTokenizer);
100142		- v->pTokenizer = NULL;
100143		- }
100144		-
100145		- clearPendingTerms(v);
100146		-
100147		- sqlite3_free(v->azColumn);
100148		- for(i = 0; i < v->nColumn; ++i) {
100149		- sqlite3_free(v->azContentColumn[i]);
100150		- }
100151		- sqlite3_free(v->azContentColumn);
100152		- sqlite3_free(v);
100153		-}
100154		-
100155		-/*
100156		-** Token types for parsing the arguments to xConnect or xCreate.
100157		-*/
100158		-#define TOKEN_EOF 0 /* End of file */
100159		-#define TOKEN_SPACE 1 /* Any kind of whitespace */
100160		-#define TOKEN_ID 2 /* An identifier */
100161		-#define TOKEN_STRING 3 /* A string literal */
100162		-#define TOKEN_PUNCT 4 /* A single punctuation character */
100163		-
100164		-/*
100165		-** If X is a character that can be used in an identifier then
100166		-** ftsIdChar(X) will be true. Otherwise it is false.
100167		-**
100168		-** For ASCII, any character with the high-order bit set is
100169		-** allowed in an identifier. For 7-bit characters,
100170		-** isFtsIdChar[X] must be 1.
100171		-**
100172		-** Ticket #1066. the SQL standard does not allow '$' in the
100173		-** middle of identfiers. But many SQL implementations do.
100174		-** SQLite will allow '$' in identifiers for compatibility.
100175		-** But the feature is undocumented.
100176		-*/
100177		-static const char isFtsIdChar[] = {
100178		-/* x0 x1 x2 x3 x4 x5 x6 x7 x8 x9 xA xB xC xD xE xF */
100179		- 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 2x */
100180		- 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, /* 3x */
100181		- 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* 4x */
100182		- 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 1, /* 5x */
100183		- 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* 6x */
100184		- 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, /* 7x */
100185		-};
100186		-#define ftsIdChar(C) (((c=C)&0x80)!=0 \|\| (c>0x1f && isFtsIdChar[c-0x20]))
100187		-
100188		-
100189		-/*
100190		-** Return the length of the token that begins at z[0].
100191		-** Store the token type in *tokenType before returning.
100192		-*/
100193		-static int ftsGetToken(const char z, int tokenType){
100194		- int i, c;
100195		- switch( *z ){
100196		- case 0: {
100197		- *tokenType = TOKEN_EOF;
100198		- return 0;
100199		- }
100200		- case ' ': case '\t': case '\n': case '\f': case '\r': {
100201		- for(i=1; safe_isspace(z[i]); i++){}
100202		- *tokenType = TOKEN_SPACE;
100203		- return i;
100204		- }
100205		- case '`':
100206		- case '\'':
100207		- case '"': {
100208		- int delim = z[0];
100209		- for(i=1; (c=z[i])!=0; i++){
100210		- if( c==delim ){
100211		- if( z[i+1]==delim ){
100212		- i++;
100213		- }else{
100214		- break;
100215		- }
100216		- }
100217		- }
100218		- *tokenType = TOKEN_STRING;
100219		- return i + (c!=0);
100220		- }
100221		- case '[': {
100222		- for(i=1, c=z[0]; c!=']' && (c=z[i])!=0; i++){}
100223		- *tokenType = TOKEN_ID;
100224		- return i;
100225		- }
100226		- default: {
100227		- if( !ftsIdChar(*z) ){
100228		- break;
100229		- }
100230		- for(i=1; ftsIdChar(z[i]); i++){}
100231		- *tokenType = TOKEN_ID;
100232		- return i;
100233		- }
100234		- }
100235		- *tokenType = TOKEN_PUNCT;
100236		- return 1;
100237		-}
100238		-
100239		-/*
100240		-** A token extracted from a string is an instance of the following
100241		-** structure.
100242		-*/
100243		-typedef struct FtsToken {
100244		- const char z; / Pointer to token text. Not '\000' terminated */
100245		- short int n; /* Length of the token text in bytes. */
100246		-} FtsToken;
100247		-
100248		-/*
100249		-** Given a input string (which is really one of the argv[] parameters
100250		-** passed into xConnect or xCreate) split the string up into tokens.
100251		-** Return an array of pointers to '\000' terminated strings, one string
100252		-** for each non-whitespace token.
100253		-**
100254		-** The returned array is terminated by a single NULL pointer.
100255		-**
100256		-** Space to hold the returned array is obtained from a single
100257		-** malloc and should be freed by passing the return value to free().
100258		-** The individual strings within the token list are all a part of
100259		-** the single memory allocation and will all be freed at once.
100260		-*/
100261		-static char *tokenizeString(const char z, int *pnToken){
100262		- int nToken = 0;
100263		- FtsToken aToken = sqlite3_malloc( strlen(z) sizeof(aToken[0]) );
100264		- int n = 1;
100265		- int e, i;
100266		- int totalSize = 0;
100267		- char **azToken;
100268		- char *zCopy;
100269		- while( n>0 ){
100270		- n = ftsGetToken(z, &e);
100271		- if( e!=TOKEN_SPACE ){
100272		- aToken[nToken].z = z;
100273		- aToken[nToken].n = n;
100274		- nToken++;
100275		- totalSize += n+1;
100276		- }
100277		- z += n;
100278		- }
100279		- azToken = (char*)sqlite3_malloc( nTokensizeof(char*) + totalSize );
100280		- zCopy = (char*)&azToken[nToken];
100281		- nToken--;
100282		- for(i=0; i<nToken; i++){
100283		- azToken[i] = zCopy;
100284		- n = aToken[i].n;
100285		- memcpy(zCopy, aToken[i].z, n);
100286		- zCopy[n] = 0;
100287		- zCopy += n+1;
100288		- }
100289		- azToken[nToken] = 0;
100290		- sqlite3_free(aToken);
100291		- *pnToken = nToken;
100292		- return azToken;
	97897	+/*
	97898	+** Return the number of bytes required to store the value passed as the
	97899	+** first argument in varint form.
	97900	+*/
	97901	+SQLITE_PRIVATE int sqlite3Fts3VarintLen(sqlite3_uint64 v){
	97902	+ int i = 0;
	97903	+ do{
	97904	+ i++;
	97905	+ v >>= 7;
	97906	+ }while( v!=0 );
	97907	+ return i;
100293	97908	}
100294	97909
100295	97910	/*
100296	97911	** Convert an SQL-style quoted string into a normal string by removing
100297	97912	** the quote characters. The conversion is done in-place. If the
		@@ -100303,14 +97918,14 @@
100303	97918	** "abc" becomes abc
100304	97919	** 'xyz' becomes xyz
100305	97920	** [pqr] becomes pqr
100306	97921	** `mno` becomes mno
100307	97922	*/
100308		-static void dequoteString(char *z){
	97923	+SQLITE_PRIVATE void sqlite3Fts3Dequote(char *z){
100309	97924	int quote;
100310	97925	int i, j;
100311		- if( z==0 ) return;
	97926	+
100312	97927	quote = z[0];
100313	97928	switch( quote ){
100314	97929	case '\'': break;
100315	97930	case '"': break;
100316	97931	case '`': break; /* For MySQL compatibility */
		@@ -100330,1363 +97945,1413 @@
100330	97945	z[j++] = z[i];
100331	97946	}
100332	97947	}
100333	97948	}
100334	97949
100335		-/*
100336		-** The input azIn is a NULL-terminated list of tokens. Remove the first
100337		-** token and all punctuation tokens. Remove the quotes from
100338		-** around string literal tokens.
100339		-**
100340		-** Example:
100341		-**
100342		-** input: tokenize chinese ( 'simplifed' , 'mixed' )
100343		-** output: chinese simplifed mixed
100344		-**
100345		-** Another example:
100346		-**
100347		-** input: delimiters ( '[' , ']' , '...' )
100348		-** output: [ ] ...
100349		-*/
100350		-static void tokenListToIdList(char **azIn){
100351		- int i, j;
100352		- if( azIn ){
100353		- for(i=0, j=-1; azIn[i]; i++){
100354		- if( safe_isalnum(azIn[i][0]) \|\| azIn[i][1] ){
100355		- dequoteString(azIn[i]);
100356		- if( j>=0 ){
100357		- azIn[j] = azIn[i];
100358		- }
100359		- j++;
100360		- }
100361		- }
100362		- azIn[j] = 0;
100363		- }
100364		-}
100365		-
100366		-
100367		-/*
100368		-** Find the first alphanumeric token in the string zIn. Null-terminate
100369		-** this token. Remove any quotation marks. And return a pointer to
100370		-** the result.
100371		-*/
100372		-static char firstToken(char zIn, char **pzTail){
100373		- int n, ttype;
100374		- while(1){
100375		- n = ftsGetToken(zIn, &ttype);
100376		- if( ttype==TOKEN_SPACE ){
100377		- zIn += n;
100378		- }else if( ttype==TOKEN_EOF ){
100379		- *pzTail = zIn;
100380		- return 0;
100381		- }else{
100382		- zIn[n] = 0;
100383		- *pzTail = &zIn[1];
100384		- dequoteString(zIn);
100385		- return zIn;
100386		- }
100387		- }
100388		- /NOTREACHED/
100389		-}
100390		-
100391		-/* Return true if...
100392		-**
100393		-** * s begins with the string t, ignoring case
100394		-** * s is longer than t
100395		-** * The first character of s beyond t is not a alphanumeric
100396		-**
100397		-** Ignore leading space in *s.
100398		-**
100399		-** To put it another way, return true if the first token of
100400		-** s[] is t[].
100401		-*/
100402		-static int startsWith(const char s, const char t){
100403		- while( safe_isspace(*s) ){ s++; }
100404		- while( *t ){
100405		- if( safe_tolower(s++)!=safe_tolower(t++) ) return 0;
100406		- }
100407		- return s!='_' && !safe_isalnum(s);
100408		-}
100409		-
100410		-/*
100411		-** An instance of this structure defines the "spec" of a
100412		-** full text index. This structure is populated by parseSpec
100413		-** and use by fulltextConnect and fulltextCreate.
100414		-*/
100415		-typedef struct TableSpec {
100416		- const char zDb; / Logical database name */
100417		- const char zName; / Name of the full-text index */
100418		- int nColumn; /* Number of columns to be indexed */
100419		- char *azColumn; / Original names of columns to be indexed */
100420		- char *azContentColumn; / Column names for %_content */
100421		- char *azTokenizer; / Name of tokenizer and its arguments */
100422		-} TableSpec;
100423		-
100424		-/*
100425		-** Reclaim all of the memory used by a TableSpec
100426		-*/
100427		-static void clearTableSpec(TableSpec *p) {
100428		- sqlite3_free(p->azColumn);
100429		- sqlite3_free(p->azContentColumn);
100430		- sqlite3_free(p->azTokenizer);
100431		-}
100432		-
100433		-/* Parse a CREATE VIRTUAL TABLE statement, which looks like this:
100434		- *
100435		- * CREATE VIRTUAL TABLE email
100436		- * USING fts3(subject, body, tokenize mytokenizer(myarg))
100437		- *
100438		- * We return parsed information in a TableSpec structure.
100439		- *
100440		- */
100441		-static int parseSpec(TableSpec pSpec, int argc, const char const*argv,
100442		- char**pzErr){
100443		- int i, n;
100444		- char z, zDummy;
100445		- char **azArg;
100446		- const char zTokenizer = 0; / argv[] entry describing the tokenizer */
100447		-
100448		- assert( argc>=3 );
100449		- /* Current interface:
100450		- ** argv[0] - module name
100451		- ** argv[1] - database name
100452		- ** argv[2] - table name
100453		- ** argv[3..] - columns, optionally followed by tokenizer specification
100454		- ** and snippet delimiters specification.
100455		- */
100456		-
100457		- /* Make a copy of the complete argv[][] array in a single allocation.
100458		- ** The argv[][] array is read-only and transient. We can write to the
100459		- ** copy in order to modify things and the copy is persistent.
100460		- */
100461		- CLEAR(pSpec);
100462		- for(i=n=0; i<argc; i++){
100463		- n += strlen(argv[i]) + 1;
100464		- }
100465		- azArg = sqlite3_malloc( sizeof(char)argc + n );
100466		- if( azArg==0 ){
100467		- return SQLITE_NOMEM;
100468		- }
100469		- z = (char*)&azArg[argc];
100470		- for(i=0; i<argc; i++){
100471		- azArg[i] = z;
100472		- strcpy(z, argv[i]);
100473		- z += strlen(z)+1;
100474		- }
100475		-
100476		- /* Identify the column names and the tokenizer and delimiter arguments
100477		- ** in the argv[][] array.
100478		- */
100479		- pSpec->zDb = azArg[1];
100480		- pSpec->zName = azArg[2];
100481		- pSpec->nColumn = 0;
100482		- pSpec->azColumn = azArg;
100483		- zTokenizer = "tokenize simple";
100484		- for(i=3; i<argc; ++i){
100485		- if( startsWith(azArg[i],"tokenize") ){
100486		- zTokenizer = azArg[i];
100487		- }else{
100488		- z = azArg[pSpec->nColumn] = firstToken(azArg[i], &zDummy);
100489		- pSpec->nColumn++;
100490		- }
100491		- }
100492		- if( pSpec->nColumn==0 ){
100493		- azArg[0] = "content";
100494		- pSpec->nColumn = 1;
100495		- }
100496		-
100497		- /*
100498		- ** Construct the list of content column names.
100499		- **
100500		- ** Each content column name will be of the form cNNAAAA
100501		- ** where NN is the column number and AAAA is the sanitized
100502		- ** column name. "sanitized" means that special characters are
100503		- ** converted to "_". The cNN prefix guarantees that all column
100504		- ** names are unique.
100505		- **
100506		- ** The AAAA suffix is not strictly necessary. It is included
100507		- ** for the convenience of people who might examine the generated
100508		- ** %_content table and wonder what the columns are used for.
100509		- */
100510		- pSpec->azContentColumn = sqlite3_malloc( pSpec->nColumn * sizeof(char *) );
100511		- if( pSpec->azContentColumn==0 ){
100512		- clearTableSpec(pSpec);
100513		- return SQLITE_NOMEM;
100514		- }
100515		- for(i=0; i<pSpec->nColumn; i++){
100516		- char *p;
100517		- pSpec->azContentColumn[i] = sqlite3_mprintf("c%d%s", i, azArg[i]);
100518		- for (p = pSpec->azContentColumn[i]; *p ; ++p) {
100519		- if( !safe_isalnum(p) ) p = '_';
100520		- }
100521		- }
100522		-
100523		- /*
100524		- ** Parse the tokenizer specification string.
100525		- */
100526		- pSpec->azTokenizer = tokenizeString(zTokenizer, &n);
100527		- tokenListToIdList(pSpec->azTokenizer);
100528		-
100529		- return SQLITE_OK;
100530		-}
100531		-
100532		-/*
100533		-** Generate a CREATE TABLE statement that describes the schema of
100534		-** the virtual table. Return a pointer to this schema string.
100535		-**
100536		-** Space is obtained from sqlite3_mprintf() and should be freed
100537		-** using sqlite3_free().
100538		-*/
100539		-static char *fulltextSchema(
100540		- int nColumn, /* Number of columns */
100541		- const char const azColumn, /* List of columns */
100542		- const char zTableName / Name of the table */
100543		-){
100544		- int i;
100545		- char zSchema, zNext;
100546		- const char *zSep = "(";
100547		- zSchema = sqlite3_mprintf("CREATE TABLE x");
100548		- for(i=0; i<nColumn; i++){
100549		- zNext = sqlite3_mprintf("%s%s%Q", zSchema, zSep, azColumn[i]);
100550		- sqlite3_free(zSchema);
100551		- zSchema = zNext;
100552		- zSep = ",";
100553		- }
100554		- zNext = sqlite3_mprintf("%s,%Q HIDDEN", zSchema, zTableName);
100555		- sqlite3_free(zSchema);
100556		- zSchema = zNext;
100557		- zNext = sqlite3_mprintf("%s,docid HIDDEN)", zSchema);
100558		- sqlite3_free(zSchema);
100559		- return zNext;
100560		-}
100561		-
100562		-/*
100563		-** Build a new sqlite3_vtab structure that will describe the
100564		-** fulltext index defined by spec.
100565		-*/
100566		-static int constructVtab(
100567		- sqlite3 db, / The SQLite database connection */
100568		- fts3Hash pHash, / Hash table containing tokenizers */
100569		- TableSpec spec, / Parsed spec information from parseSpec() */
100570		- sqlite3_vtab *ppVTab, / Write the resulting vtab structure here */
100571		- char *pzErr / Write any error message here */
100572		-){
100573		- int rc;
100574		- int n;
100575		- fulltext_vtab *v = 0;
100576		- const sqlite3_tokenizer_module *m = NULL;
100577		- char *schema;
100578		-
100579		- char const zTok; / Name of tokenizer to use for this fts table */
100580		- int nTok; /* Length of zTok, including nul terminator */
100581		-
100582		- v = (fulltext_vtab *) sqlite3_malloc(sizeof(fulltext_vtab));
100583		- if( v==0 ) return SQLITE_NOMEM;
100584		- CLEAR(v);
100585		- /* sqlite will initialize v->base */
100586		- v->db = db;
100587		- v->zDb = spec->zDb; /* Freed when azColumn is freed */
100588		- v->zName = spec->zName; /* Freed when azColumn is freed */
100589		- v->nColumn = spec->nColumn;
100590		- v->azContentColumn = spec->azContentColumn;
100591		- spec->azContentColumn = 0;
100592		- v->azColumn = spec->azColumn;
100593		- spec->azColumn = 0;
100594		-
100595		- if( spec->azTokenizer==0 ){
100596		- return SQLITE_NOMEM;
100597		- }
100598		-
100599		- zTok = spec->azTokenizer[0];
100600		- if( !zTok ){
100601		- zTok = "simple";
100602		- }
100603		- nTok = strlen(zTok)+1;
100604		-
100605		- m = (sqlite3_tokenizer_module *)sqlite3Fts3HashFind(pHash, zTok, nTok);
100606		- if( !m ){
100607		- *pzErr = sqlite3_mprintf("unknown tokenizer: %s", spec->azTokenizer[0]);
100608		- rc = SQLITE_ERROR;
100609		- goto err;
100610		- }
100611		-
100612		- for(n=0; spec->azTokenizer[n]; n++){}
100613		- if( n ){
100614		- rc = m->xCreate(n-1, (const charconst)&spec->azTokenizer[1],
100615		- &v->pTokenizer);
100616		- }else{
100617		- rc = m->xCreate(0, 0, &v->pTokenizer);
100618		- }
100619		- if( rc!=SQLITE_OK ) goto err;
100620		- v->pTokenizer->pModule = m;
100621		-
100622		- /* TODO: verify the existence of backing tables foo_content, foo_term */
100623		-
100624		- schema = fulltextSchema(v->nColumn, (const charconst)v->azColumn,
100625		- spec->zName);
100626		- rc = sqlite3_declare_vtab(db, schema);
100627		- sqlite3_free(schema);
100628		- if( rc!=SQLITE_OK ) goto err;
100629		-
100630		- memset(v->pFulltextStatements, 0, sizeof(v->pFulltextStatements));
100631		-
100632		- /* Indicate that the buffer is not live. */
100633		- v->nPendingData = -1;
100634		-
100635		- *ppVTab = &v->base;
100636		- FTSTRACE(("FTS3 Connect %p\n", v));
100637		-
100638		- return rc;
100639		-
100640		-err:
100641		- fulltext_vtab_destroy(v);
100642		- return rc;
100643		-}
100644		-
100645		-static int fulltextConnect(
100646		- sqlite3 *db,
100647		- void *pAux,
100648		- int argc, const char constargv,
100649		- sqlite3_vtab **ppVTab,
100650		- char **pzErr
100651		-){
100652		- TableSpec spec;
100653		- int rc = parseSpec(&spec, argc, argv, pzErr);
100654		- if( rc!=SQLITE_OK ) return rc;
100655		-
100656		- rc = constructVtab(db, (fts3Hash *)pAux, &spec, ppVTab, pzErr);
100657		- clearTableSpec(&spec);
100658		- return rc;
100659		-}
100660		-
100661		-/* The %_content table holds the text of each document, with
100662		-** the docid column exposed as the SQLite rowid for the table.
100663		-*/
100664		-/* TODO(shess) This comment needs elaboration to match the updated
100665		-** code. Work it into the top-of-file comment at that time.
100666		-*/
100667		-static int fulltextCreate(sqlite3 db, void pAux,
100668		- int argc, const char * const *argv,
100669		- sqlite3_vtab ppVTab, char pzErr){
100670		- int rc;
100671		- TableSpec spec;
100672		- StringBuffer schema;
100673		- FTSTRACE(("FTS3 Create\n"));
100674		-
100675		- rc = parseSpec(&spec, argc, argv, pzErr);
100676		- if( rc!=SQLITE_OK ) return rc;
100677		-
100678		- initStringBuffer(&schema);
100679		- append(&schema, "CREATE TABLE %_content(");
100680		- append(&schema, " docid INTEGER PRIMARY KEY,");
100681		- appendList(&schema, spec.nColumn, spec.azContentColumn);
100682		- append(&schema, ")");
100683		- rc = sql_exec(db, spec.zDb, spec.zName, stringBufferData(&schema));
100684		- stringBufferDestroy(&schema);
100685		- if( rc!=SQLITE_OK ) goto out;
100686		-
100687		- rc = sql_exec(db, spec.zDb, spec.zName,
100688		- "create table %_segments("
100689		- " blockid INTEGER PRIMARY KEY,"
100690		- " block blob"
100691		- ");"
100692		- );
100693		- if( rc!=SQLITE_OK ) goto out;
100694		-
100695		- rc = sql_exec(db, spec.zDb, spec.zName,
100696		- "create table %_segdir("
100697		- " level integer,"
100698		- " idx integer,"
100699		- " start_block integer,"
100700		- " leaves_end_block integer,"
100701		- " end_block integer,"
100702		- " root blob,"
100703		- " primary key(level, idx)"
100704		- ");");
100705		- if( rc!=SQLITE_OK ) goto out;
100706		-
100707		- rc = constructVtab(db, (fts3Hash *)pAux, &spec, ppVTab, pzErr);
100708		-
100709		-out:
100710		- clearTableSpec(&spec);
100711		- return rc;
100712		-}
100713		-
100714		-/* Decide how to handle an SQL query. */
100715		-static int fulltextBestIndex(sqlite3_vtab pVTab, sqlite3_index_info pInfo){
100716		- fulltext_vtab v = (fulltext_vtab )pVTab;
100717		- int i;
100718		- FTSTRACE(("FTS3 BestIndex\n"));
100719		-
100720		- for(i=0; i<pInfo->nConstraint; ++i){
100721		- const struct sqlite3_index_constraint *pConstraint;
100722		- pConstraint = &pInfo->aConstraint[i];
100723		- if( pConstraint->usable ) {
100724		- if( (pConstraint->iColumn==-1 \|\| pConstraint->iColumn==v->nColumn+1) &&
100725		- pConstraint->op==SQLITE_INDEX_CONSTRAINT_EQ ){
100726		- pInfo->idxNum = QUERY_DOCID; /* lookup by docid */
100727		- FTSTRACE(("FTS3 QUERY_DOCID\n"));
100728		- } else if( pConstraint->iColumn>=0 && pConstraint->iColumn<=v->nColumn &&
100729		- pConstraint->op==SQLITE_INDEX_CONSTRAINT_MATCH ){
100730		- /* full-text search */
100731		- pInfo->idxNum = QUERY_FULLTEXT + pConstraint->iColumn;
100732		- FTSTRACE(("FTS3 QUERY_FULLTEXT %d\n", pConstraint->iColumn));
100733		- } else continue;
100734		-
100735		- pInfo->aConstraintUsage[i].argvIndex = 1;
100736		- pInfo->aConstraintUsage[i].omit = 1;
100737		-
100738		- /* An arbitrary value for now.
100739		- * TODO: Perhaps docid matches should be considered cheaper than
100740		- * full-text searches. */
100741		- pInfo->estimatedCost = 1.0;
100742		-
100743		- return SQLITE_OK;
100744		- }
100745		- }
100746		- pInfo->idxNum = QUERY_GENERIC;
100747		- return SQLITE_OK;
100748		-}
100749		-
100750		-static int fulltextDisconnect(sqlite3_vtab *pVTab){
100751		- FTSTRACE(("FTS3 Disconnect %p\n", pVTab));
100752		- fulltext_vtab_destroy((fulltext_vtab *)pVTab);
100753		- return SQLITE_OK;
100754		-}
100755		-
100756		-static int fulltextDestroy(sqlite3_vtab *pVTab){
100757		- fulltext_vtab v = (fulltext_vtab )pVTab;
100758		- int rc;
100759		-
100760		- FTSTRACE(("FTS3 Destroy %p\n", pVTab));
100761		- rc = sql_exec(v->db, v->zDb, v->zName,
100762		- "drop table if exists %_content;"
100763		- "drop table if exists %_segments;"
100764		- "drop table if exists %_segdir;"
100765		- );
100766		- if( rc!=SQLITE_OK ) return rc;
100767		-
100768		- fulltext_vtab_destroy((fulltext_vtab *)pVTab);
100769		- return SQLITE_OK;
100770		-}
100771		-
100772		-static int fulltextOpen(sqlite3_vtab pVTab, sqlite3_vtab_cursor *ppCursor){
100773		- fulltext_cursor *c;
100774		-
100775		- c = (fulltext_cursor *) sqlite3_malloc(sizeof(fulltext_cursor));
100776		- if( c ){
100777		- memset(c, 0, sizeof(fulltext_cursor));
100778		- /* sqlite will initialize c->base */
100779		- *ppCursor = &c->base;
100780		- FTSTRACE(("FTS3 Open %p: %p\n", pVTab, c));
100781		- return SQLITE_OK;
100782		- }else{
100783		- return SQLITE_NOMEM;
100784		- }
100785		-}
100786		-
100787		-/* Free all of the dynamically allocated memory held by the
100788		-** Snippet
100789		-*/
100790		-static void snippetClear(Snippet *p){
100791		- sqlite3_free(p->aMatch);
100792		- sqlite3_free(p->zOffset);
100793		- sqlite3_free(p->zSnippet);
100794		- CLEAR(p);
100795		-}
100796		-
100797		-/*
100798		-** Append a single entry to the p->aMatch[] log.
100799		-*/
100800		-static void snippetAppendMatch(
100801		- Snippet p, / Append the entry to this snippet */
100802		- int iCol, int iTerm, /* The column and query term */
100803		- int iToken, /* Matching token in document */
100804		- int iStart, int nByte /* Offset and size of the match */
100805		-){
100806		- int i;
100807		- struct snippetMatch *pMatch;
100808		- if( p->nMatch+1>=p->nAlloc ){
100809		- p->nAlloc = p->nAlloc*2 + 10;
100810		- p->aMatch = sqlite3_realloc(p->aMatch, p->nAlloc*sizeof(p->aMatch[0]) );
100811		- if( p->aMatch==0 ){
100812		- p->nMatch = 0;
100813		- p->nAlloc = 0;
100814		- return;
100815		- }
100816		- }
100817		- i = p->nMatch++;
100818		- pMatch = &p->aMatch[i];
100819		- pMatch->iCol = iCol;
100820		- pMatch->iTerm = iTerm;
100821		- pMatch->iToken = iToken;
100822		- pMatch->iStart = iStart;
100823		- pMatch->nByte = nByte;
100824		-}
100825		-
100826		-/*
100827		-** Sizing information for the circular buffer used in snippetOffsetsOfColumn()
100828		-*/
100829		-#define FTS3_ROTOR_SZ (32)
100830		-#define FTS3_ROTOR_MASK (FTS3_ROTOR_SZ-1)
100831		-
100832		-/*
100833		-** Function to iterate through the tokens of a compiled expression.
100834		-**
100835		-** Except, skip all tokens on the right-hand side of a NOT operator.
100836		-** This function is used to find tokens as part of snippet and offset
100837		-** generation and we do nt want snippets and offsets to report matches
100838		-** for tokens on the RHS of a NOT.
100839		-*/
100840		-static int fts3NextExprToken(Fts3Expr *ppExpr, int piToken){
100841		- Fts3Expr p = ppExpr;
100842		- int iToken = *piToken;
100843		- if( iToken<0 ){
100844		- /* In this case the expression p is the root of an expression tree.
100845		- ** Move to the first token in the expression tree.
100846		- */
100847		- while( p->pLeft ){
100848		- p = p->pLeft;
100849		- }
100850		- iToken = 0;
100851		- }else{
100852		- assert(p && p->eType==FTSQUERY_PHRASE );
100853		- if( iToken<(p->pPhrase->nToken-1) ){
100854		- iToken++;
100855		- }else{
100856		- iToken = 0;
100857		- while( p->pParent && p->pParent->pLeft!=p ){
100858		- assert( p->pParent->pRight==p );
100859		- p = p->pParent;
100860		- }
100861		- p = p->pParent;
100862		- if( p ){
100863		- assert( p->pRight!=0 );
100864		- p = p->pRight;
100865		- while( p->pLeft ){
100866		- p = p->pLeft;
100867		- }
100868		- }
100869		- }
100870		- }
100871		-
100872		- *ppExpr = p;
100873		- *piToken = iToken;
100874		- return p?1:0;
100875		-}
100876		-
100877		-/*
100878		-** Return TRUE if the expression node pExpr is located beneath the
100879		-** RHS of a NOT operator.
100880		-*/
100881		-static int fts3ExprBeneathNot(Fts3Expr *p){
100882		- Fts3Expr *pParent;
100883		- while( p ){
100884		- pParent = p->pParent;
100885		- if( pParent && pParent->eType==FTSQUERY_NOT && pParent->pRight==p ){
100886		- return 1;
100887		- }
100888		- p = pParent;
100889		- }
100890		- return 0;
100891		-}
100892		-
100893		-/*
100894		-** Add entries to pSnippet->aMatch[] for every match that occurs against
100895		-** document zDoc[0..nDoc-1] which is stored in column iColumn.
100896		-*/
100897		-static void snippetOffsetsOfColumn(
100898		- fulltext_cursor pCur, / The fulltest search cursor */
100899		- Snippet pSnippet, / The Snippet object to be filled in */
100900		- int iColumn, /* Index of fulltext table column */
100901		- const char zDoc, / Text of the fulltext table column */
100902		- int nDoc /* Length of zDoc in bytes */
100903		-){
100904		- const sqlite3_tokenizer_module pTModule; / The tokenizer module */
100905		- sqlite3_tokenizer pTokenizer; / The specific tokenizer */
100906		- sqlite3_tokenizer_cursor pTCursor; / Tokenizer cursor */
100907		- fulltext_vtab pVtab; / The full text index */
100908		- int nColumn; /* Number of columns in the index */
100909		- int i, j; /* Loop counters */
100910		- int rc; /* Return code */
100911		- unsigned int match, prevMatch; /* Phrase search bitmasks */
100912		- const char zToken; / Next token from the tokenizer */
100913		- int nToken; /* Size of zToken */
100914		- int iBegin, iEnd, iPos; /* Offsets of beginning and end */
100915		-
100916		- /* The following variables keep a circular buffer of the last
100917		- ** few tokens */
100918		- unsigned int iRotor = 0; /* Index of current token */
100919		- int iRotorBegin[FTS3_ROTOR_SZ]; /* Beginning offset of token */
100920		- int iRotorLen[FTS3_ROTOR_SZ]; /* Length of token */
100921		-
100922		- pVtab = cursor_vtab(pCur);
100923		- nColumn = pVtab->nColumn;
100924		- pTokenizer = pVtab->pTokenizer;
100925		- pTModule = pTokenizer->pModule;
100926		- rc = pTModule->xOpen(pTokenizer, zDoc, nDoc, &pTCursor);
100927		- if( rc ) return;
100928		- pTCursor->pTokenizer = pTokenizer;
100929		-
100930		- prevMatch = 0;
100931		- while( !pTModule->xNext(pTCursor, &zToken, &nToken, &iBegin, &iEnd, &iPos) ){
100932		- Fts3Expr *pIter = pCur->pExpr;
100933		- int iIter = -1;
100934		- iRotorBegin[iRotor&FTS3_ROTOR_MASK] = iBegin;
100935		- iRotorLen[iRotor&FTS3_ROTOR_MASK] = iEnd-iBegin;
100936		- match = 0;
100937		- for(i=0; i<(FTS3_ROTOR_SZ-1) && fts3NextExprToken(&pIter, &iIter); i++){
100938		- int nPhrase; /* Number of tokens in current phrase */
100939		- struct PhraseToken pToken; / Current token */
100940		- int iCol; /* Column index */
100941		-
100942		- if( fts3ExprBeneathNot(pIter) ) continue;
100943		- nPhrase = pIter->pPhrase->nToken;
100944		- pToken = &pIter->pPhrase->aToken[iIter];
100945		- iCol = pIter->pPhrase->iColumn;
100946		- if( iCol>=0 && iCol<nColumn && iCol!=iColumn ) continue;
100947		- if( pToken->n>nToken ) continue;
100948		- if( !pToken->isPrefix && pToken->n<nToken ) continue;
100949		- assert( pToken->n<=nToken );
100950		- if( memcmp(pToken->z, zToken, pToken->n) ) continue;
100951		- if( iIter>0 && (prevMatch & (1<<i))==0 ) continue;
100952		- match \|= 1<<i;
100953		- if( i==(FTS3_ROTOR_SZ-2) \|\| nPhrase==iIter+1 ){
100954		- for(j=nPhrase-1; j>=0; j--){
100955		- int k = (iRotor-j) & FTS3_ROTOR_MASK;
100956		- snippetAppendMatch(pSnippet, iColumn, i-j, iPos-j,
100957		- iRotorBegin[k], iRotorLen[k]);
100958		- }
100959		- }
100960		- }
100961		- prevMatch = match<<1;
100962		- iRotor++;
100963		- }
100964		- pTModule->xClose(pTCursor);
100965		-}
100966		-
100967		-/*
100968		-** Remove entries from the pSnippet structure to account for the NEAR
100969		-** operator. When this is called, pSnippet contains the list of token
100970		-** offsets produced by treating all NEAR operators as AND operators.
100971		-** This function removes any entries that should not be present after
100972		-** accounting for the NEAR restriction. For example, if the queried
100973		-** document is:
100974		-**
100975		-** "A B C D E A"
100976		-**
100977		-** and the query is:
100978		-**
100979		-** A NEAR/0 E
100980		-**
100981		-** then when this function is called the Snippet contains token offsets
100982		-** 0, 4 and 5. This function removes the "0" entry (because the first A
100983		-** is not near enough to an E).
100984		-**
100985		-** When this function is called, the value pointed to by parameter piLeft is
100986		-** the integer id of the left-most token in the expression tree headed by
100987		-** pExpr. This function increments *piLeft by the total number of tokens
100988		-** in the expression tree headed by pExpr.
100989		-**
100990		-** Return 1 if any trimming occurs. Return 0 if no trimming is required.
100991		-*/
100992		-static int trimSnippetOffsets(
100993		- Fts3Expr pExpr, / The search expression */
100994		- Snippet pSnippet, / The set of snippet offsets to be trimmed */
100995		- int piLeft / Index of left-most token in pExpr */
100996		-){
100997		- if( pExpr ){
100998		- if( trimSnippetOffsets(pExpr->pLeft, pSnippet, piLeft) ){
100999		- return 1;
101000		- }
101001		-
101002		- switch( pExpr->eType ){
101003		- case FTSQUERY_PHRASE:
101004		- *piLeft += pExpr->pPhrase->nToken;
101005		- break;
101006		- case FTSQUERY_NEAR: {
101007		- /* The right-hand-side of a NEAR operator is always a phrase. The
101008		- ** left-hand-side is either a phrase or an expression tree that is
101009		- ** itself headed by a NEAR operator. The following initializations
101010		- ** set local variable iLeft to the token number of the left-most
101011		- ** token in the right-hand phrase, and iRight to the right most
101012		- ** token in the same phrase. For example, if we had:
101013		- **
101014		- ** <col> MATCH '"abc def" NEAR/2 "ghi jkl"'
101015		- **
101016		- ** then iLeft will be set to 2 (token number of ghi) and nToken will
101017		- ** be set to 4.
101018		- */
101019		- Fts3Expr *pLeft = pExpr->pLeft;
101020		- Fts3Expr *pRight = pExpr->pRight;
101021		- int iLeft = *piLeft;
101022		- int nNear = pExpr->nNear;
101023		- int nToken = pRight->pPhrase->nToken;
101024		- int jj, ii;
101025		- if( pLeft->eType==FTSQUERY_NEAR ){
101026		- pLeft = pLeft->pRight;
101027		- }
101028		- assert( pRight->eType==FTSQUERY_PHRASE );
101029		- assert( pLeft->eType==FTSQUERY_PHRASE );
101030		- nToken += pLeft->pPhrase->nToken;
101031		-
101032		- for(ii=0; ii<pSnippet->nMatch; ii++){
101033		- struct snippetMatch *p = &pSnippet->aMatch[ii];
101034		- if( p->iTerm==iLeft ){
101035		- int isOk = 0;
101036		- /* Snippet ii is an occurence of query term iLeft in the document.
101037		- ** It occurs at position (p->iToken) of the document. We now
101038		- ** search for an instance of token (iLeft-1) somewhere in the
101039		- ** range (p->iToken - nNear)...(p->iToken + nNear + nToken) within
101040		- ** the set of snippetMatch structures. If one is found, proceed.
101041		- ** If one cannot be found, then remove snippets ii..(ii+N-1)
101042		- ** from the matching snippets, where N is the number of tokens
101043		- ** in phrase pRight->pPhrase.
101044		- */
101045		- for(jj=0; isOk==0 && jj<pSnippet->nMatch; jj++){
101046		- struct snippetMatch *p2 = &pSnippet->aMatch[jj];
101047		- if( p2->iTerm==(iLeft-1) ){
101048		- if( p2->iToken>=(p->iToken-nNear-1)
101049		- && p2->iToken<(p->iToken+nNear+nToken)
101050		- ){
101051		- isOk = 1;
101052		- }
101053		- }
101054		- }
101055		- if( !isOk ){
101056		- int kk;
101057		- for(kk=0; kk<pRight->pPhrase->nToken; kk++){
101058		- pSnippet->aMatch[kk+ii].iTerm = -2;
101059		- }
101060		- return 1;
101061		- }
101062		- }
101063		- if( p->iTerm==(iLeft-1) ){
101064		- int isOk = 0;
101065		- for(jj=0; isOk==0 && jj<pSnippet->nMatch; jj++){
101066		- struct snippetMatch *p2 = &pSnippet->aMatch[jj];
101067		- if( p2->iTerm==iLeft ){
101068		- if( p2->iToken<=(p->iToken+nNear+1)
101069		- && p2->iToken>(p->iToken-nNear-nToken)
101070		- ){
101071		- isOk = 1;
101072		- }
101073		- }
101074		- }
101075		- if( !isOk ){
101076		- int kk;
101077		- for(kk=0; kk<pLeft->pPhrase->nToken; kk++){
101078		- pSnippet->aMatch[ii-kk].iTerm = -2;
101079		- }
101080		- return 1;
101081		- }
101082		- }
101083		- }
101084		- break;
101085		- }
101086		- }
101087		-
101088		- if( trimSnippetOffsets(pExpr->pRight, pSnippet, piLeft) ){
101089		- return 1;
101090		- }
101091		- }
101092		- return 0;
101093		-}
101094		-
101095		-/*
101096		-** Compute all offsets for the current row of the query.
101097		-** If the offsets have already been computed, this routine is a no-op.
101098		-*/
101099		-static void snippetAllOffsets(fulltext_cursor *p){
101100		- int nColumn;
101101		- int iColumn, i;
101102		- int iFirst, iLast;
101103		- int iTerm = 0;
101104		- fulltext_vtab *pFts = cursor_vtab(p);
101105		-
101106		- if( p->snippet.nMatch \|\| p->pExpr==0 ){
101107		- return;
101108		- }
101109		- nColumn = pFts->nColumn;
101110		- iColumn = (p->iCursorType - QUERY_FULLTEXT);
101111		- if( iColumn<0 \|\| iColumn>=nColumn ){
101112		- /* Look for matches over all columns of the full-text index */
101113		- iFirst = 0;
101114		- iLast = nColumn-1;
101115		- }else{
101116		- /* Look for matches in the iColumn-th column of the index only */
101117		- iFirst = iColumn;
101118		- iLast = iColumn;
101119		- }
101120		- for(i=iFirst; i<=iLast; i++){
101121		- const char *zDoc;
101122		- int nDoc;
101123		- zDoc = (const char*)sqlite3_column_text(p->pStmt, i+1);
101124		- nDoc = sqlite3_column_bytes(p->pStmt, i+1);
101125		- snippetOffsetsOfColumn(p, &p->snippet, i, zDoc, nDoc);
101126		- }
101127		-
101128		- while( trimSnippetOffsets(p->pExpr, &p->snippet, &iTerm) ){
101129		- iTerm = 0;
101130		- }
101131		-}
101132		-
101133		-/*
101134		-** Convert the information in the aMatch[] array of the snippet
101135		-** into the string zOffset[0..nOffset-1]. This string is used as
101136		-** the return of the SQL offsets() function.
101137		-*/
101138		-static void snippetOffsetText(Snippet *p){
101139		- int i;
101140		- int cnt = 0;
101141		- StringBuffer sb;
101142		- char zBuf[200];
101143		- if( p->zOffset ) return;
101144		- initStringBuffer(&sb);
101145		- for(i=0; i<p->nMatch; i++){
101146		- struct snippetMatch *pMatch = &p->aMatch[i];
101147		- if( pMatch->iTerm>=0 ){
101148		- /* If snippetMatch.iTerm is less than 0, then the match was
101149		- ** discarded as part of processing the NEAR operator (see the
101150		- ** trimSnippetOffsetsForNear() function for details). Ignore
101151		- ** it in this case
101152		- */
101153		- zBuf[0] = ' ';
101154		- sqlite3_snprintf(sizeof(zBuf)-1, &zBuf[cnt>0], "%d %d %d %d",
101155		- pMatch->iCol, pMatch->iTerm, pMatch->iStart, pMatch->nByte);
101156		- append(&sb, zBuf);
101157		- cnt++;
101158		- }
101159		- }
101160		- p->zOffset = stringBufferData(&sb);
101161		- p->nOffset = stringBufferLength(&sb);
101162		-}
101163		-
101164		-/*
101165		-** zDoc[0..nDoc-1] is phrase of text. aMatch[0..nMatch-1] are a set
101166		-** of matching words some of which might be in zDoc. zDoc is column
101167		-** number iCol.
101168		-**
101169		-** iBreak is suggested spot in zDoc where we could begin or end an
101170		-** excerpt. Return a value similar to iBreak but possibly adjusted
101171		-** to be a little left or right so that the break point is better.
101172		-*/
101173		-static int wordBoundary(
101174		- int iBreak, /* The suggested break point */
101175		- const char zDoc, / Document text */
101176		- int nDoc, /* Number of bytes in zDoc[] */
101177		- struct snippetMatch aMatch, / Matching words */
101178		- int nMatch, /* Number of entries in aMatch[] */
101179		- int iCol /* The column number for zDoc[] */
101180		-){
101181		- int i;
101182		- if( iBreak<=10 ){
101183		- return 0;
101184		- }
101185		- if( iBreak>=nDoc-10 ){
101186		- return nDoc;
101187		- }
101188		- for(i=0; i<nMatch && aMatch[i].iCol<iCol; i++){}
101189		- while( i<nMatch && aMatch[i].iStart+aMatch[i].nByte<iBreak ){ i++; }
101190		- if( i<nMatch ){
101191		- if( aMatch[i].iStart<iBreak+10 ){
101192		- return aMatch[i].iStart;
101193		- }
101194		- if( i>0 && aMatch[i-1].iStart+aMatch[i-1].nByte>=iBreak ){
101195		- return aMatch[i-1].iStart;
101196		- }
101197		- }
101198		- for(i=1; i<=10; i++){
101199		- if( safe_isspace(zDoc[iBreak-i]) ){
101200		- return iBreak - i + 1;
101201		- }
101202		- if( safe_isspace(zDoc[iBreak+i]) ){
101203		- return iBreak + i + 1;
101204		- }
101205		- }
101206		- return iBreak;
101207		-}
101208		-
101209		-
101210		-
101211		-/*
101212		-** Allowed values for Snippet.aMatch[].snStatus
101213		-*/
101214		-#define SNIPPET_IGNORE 0 /* It is ok to omit this match from the snippet */
101215		-#define SNIPPET_DESIRED 1 /* We want to include this match in the snippet */
101216		-
101217		-/*
101218		-** Generate the text of a snippet.
101219		-*/
101220		-static void snippetText(
101221		- fulltext_cursor pCursor, / The cursor we need the snippet for */
101222		- const char zStartMark, / Markup to appear before each match */
101223		- const char zEndMark, / Markup to appear after each match */
101224		- const char zEllipsis / Ellipsis mark */
101225		-){
101226		- int i, j;
101227		- struct snippetMatch *aMatch;
101228		- int nMatch;
101229		- int nDesired;
101230		- StringBuffer sb;
101231		- int tailCol;
101232		- int tailOffset;
	97950	+static void fts3GetDeltaVarint(char *pp, sqlite3_int64 pVal){
	97951	+ sqlite3_int64 iVal;
	97952	+ pp += sqlite3Fts3GetVarint(pp, &iVal);
	97953	+ *pVal += iVal;
	97954	+}
	97955	+
	97956	+static void fts3GetDeltaVarint2(char *pp, char pEnd, sqlite3_int64 *pVal){
	97957	+ if( *pp>=pEnd ){
	97958	+ *pp = 0;
	97959	+ }else{
	97960	+ fts3GetDeltaVarint(pp, pVal);
	97961	+ }
	97962	+}
	97963	+
	97964	+
	97965	+/*
	97966	+** The Fts3Cursor.eType member is always set to one of the following.
	97967	+*/
	97968	+#define FTS3_FULLSCAN_SEARCH 0 /* Linear scan of %_content table */
	97969	+#define FTS3_DOCID_SEARCH 1 /* Lookup by rowid on %_content table */
	97970	+#define FTS3_FULLTEXT_SEARCH 2 /* Full-text index search */
	97971	+
	97972	+static Fts3Table cursor_vtab(Fts3Cursor c){
	97973	+ return (Fts3Table *) c->base.pVtab;
	97974	+}
	97975	+
	97976	+/*
	97977	+** The xDisconnect() virtual table method.
	97978	+*/
	97979	+static int fts3DisconnectMethod(sqlite3_vtab *pVtab){
	97980	+ Fts3Table p = (Fts3Table )pVtab;
	97981	+ int i;
	97982	+
	97983	+ assert( p->nPendingData==0 );
	97984	+
	97985	+ /* Free any prepared statements held */
	97986	+ for(i=0; i<SizeofArray(p->aStmt); i++){
	97987	+ sqlite3_finalize(p->aStmt[i]);
	97988	+ }
	97989	+ for(i=0; i<p->nLeavesStmt; i++){
	97990	+ sqlite3_finalize(p->aLeavesStmt[i]);
	97991	+ }
	97992	+ sqlite3_free(p->zSelectLeaves);
	97993	+ sqlite3_free(p->aLeavesStmt);
	97994	+
	97995	+ /* Invoke the tokenizer destructor to free the tokenizer. */
	97996	+ p->pTokenizer->pModule->xDestroy(p->pTokenizer);
	97997	+
	97998	+ sqlite3_free(p);
	97999	+ return SQLITE_OK;
	98000	+}
	98001	+
	98002	+/*
	98003	+** The xDestroy() virtual table method.
	98004	+*/
	98005	+static int fts3DestroyMethod(sqlite3_vtab *pVtab){
	98006	+ int rc; /* Return code */
	98007	+ Fts3Table p = (Fts3Table )pVtab;
	98008	+
	98009	+ /* Create a script to drop the underlying three storage tables. */
	98010	+ char *zSql = sqlite3_mprintf(
	98011	+ "DROP TABLE IF EXISTS %Q.'%q_content';"
	98012	+ "DROP TABLE IF EXISTS %Q.'%q_segments';"
	98013	+ "DROP TABLE IF EXISTS %Q.'%q_segdir';",
	98014	+ p->zDb, p->zName, p->zDb, p->zName, p->zDb, p->zName
	98015	+ );
	98016	+
	98017	+ /* If malloc has failed, set rc to SQLITE_NOMEM. Otherwise, try to
	98018	+ ** execute the SQL script created above.
	98019	+ */
	98020	+ if( zSql ){
	98021	+ rc = sqlite3_exec(p->db, zSql, 0, 0, 0);
	98022	+ sqlite3_free(zSql);
	98023	+ }else{
	98024	+ rc = SQLITE_NOMEM;
	98025	+ }
	98026	+
	98027	+ /* If everything has worked, invoke fts3DisconnectMethod() to free the
	98028	+ ** memory associated with the Fts3Table structure and return SQLITE_OK.
	98029	+ ** Otherwise, return an SQLite error code.
	98030	+ */
	98031	+ return (rc==SQLITE_OK ? fts3DisconnectMethod(pVtab) : rc);
	98032	+}
	98033	+
	98034	+
	98035	+/*
	98036	+** Invoke sqlite3_declare_vtab() to declare the schema for the FTS3 table
	98037	+** passed as the first argument. This is done as part of the xConnect()
	98038	+** and xCreate() methods.
	98039	+*/
	98040	+static int fts3DeclareVtab(Fts3Table *p){
	98041	+ int i; /* Iterator variable */
	98042	+ int rc; /* Return code */
	98043	+ char zSql; / SQL statement passed to declare_vtab() */
	98044	+ char zCols; / List of user defined columns */
	98045	+
	98046	+ /* Create a list of user columns for the virtual table */
	98047	+ zCols = sqlite3_mprintf("%Q, ", p->azColumn[0]);
	98048	+ for(i=1; zCols && i<p->nColumn; i++){
	98049	+ zCols = sqlite3_mprintf("%z%Q, ", zCols, p->azColumn[i]);
	98050	+ }
	98051	+
	98052	+ /* Create the whole "CREATE TABLE" statement to pass to SQLite */
	98053	+ zSql = sqlite3_mprintf(
	98054	+ "CREATE TABLE x(%s %Q HIDDEN, docid HIDDEN)", zCols, p->zName
	98055	+ );
	98056	+
	98057	+ if( !zCols \|\| !zSql ){
	98058	+ rc = SQLITE_NOMEM;
	98059	+ }else{
	98060	+ rc = sqlite3_declare_vtab(p->db, zSql);
	98061	+ }
	98062	+
	98063	+ sqlite3_free(zSql);
	98064	+ sqlite3_free(zCols);
	98065	+ return rc;
	98066	+}
	98067	+
	98068	+/*
	98069	+** Create the backing store tables (%_content, %_segments and %_segdir)
	98070	+** required by the FTS3 table passed as the only argument. This is done
	98071	+** as part of the vtab xCreate() method.
	98072	+*/
	98073	+static int fts3CreateTables(Fts3Table *p){
	98074	+ int rc; /* Return code */
	98075	+ int i; /* Iterator variable */
	98076	+ char zContentCols; / Columns of %_content table */
	98077	+ char zSql; / SQL script to create required tables */
	98078	+
	98079	+ /* Create a list of user columns for the content table */
	98080	+ zContentCols = sqlite3_mprintf("docid INTEGER PRIMARY KEY");
	98081	+ for(i=0; zContentCols && i<p->nColumn; i++){
	98082	+ char *z = p->azColumn[i];
	98083	+ zContentCols = sqlite3_mprintf("%z, 'c%d%q'", zContentCols, i, z);
	98084	+ }
	98085	+
	98086	+ /* Create the whole SQL script */
	98087	+ zSql = sqlite3_mprintf(
	98088	+ "CREATE TABLE %Q.'%q_content'(%s);"
	98089	+ "CREATE TABLE %Q.'%q_segments'(blockid INTEGER PRIMARY KEY, block BLOB);"
	98090	+ "CREATE TABLE %Q.'%q_segdir'("
	98091	+ "level INTEGER,"
	98092	+ "idx INTEGER,"
	98093	+ "start_block INTEGER,"
	98094	+ "leaves_end_block INTEGER,"
	98095	+ "end_block INTEGER,"
	98096	+ "root BLOB,"
	98097	+ "PRIMARY KEY(level, idx)"
	98098	+ ");",
	98099	+ p->zDb, p->zName, zContentCols, p->zDb, p->zName, p->zDb, p->zName
	98100	+ );
	98101	+
	98102	+ /* Unless a malloc() failure has occurred, execute the SQL script to
	98103	+ ** create the tables used to store data for this FTS3 virtual table.
	98104	+ */
	98105	+ if( zContentCols==0 \|\| zSql==0 ){
	98106	+ rc = SQLITE_NOMEM;
	98107	+ }else{
	98108	+ rc = sqlite3_exec(p->db, zSql, 0, 0, 0);
	98109	+ }
	98110	+
	98111	+ sqlite3_free(zSql);
	98112	+ sqlite3_free(zContentCols);
	98113	+ return rc;
	98114	+}
	98115	+
	98116	+/*
	98117	+** This function is the implementation of both the xConnect and xCreate
	98118	+** methods of the FTS3 virtual table.
	98119	+**
	98120	+** The argv[] array contains the following:
	98121	+**
	98122	+** argv[0] -> module name
	98123	+** argv[1] -> database name
	98124	+** argv[2] -> table name
	98125	+** argv[...] -> "column name" fields...
	98126	+*/
	98127	+int fts3InitVtab(
	98128	+ int isCreate, /* True for xCreate, false for xConnect */
	98129	+ sqlite3 db, / The SQLite database connection */
	98130	+ void pAux, / Hash table containing tokenizers */
	98131	+ int argc, /* Number of elements in argv array */
	98132	+ const char * const argv, / xCreate/xConnect argument array */
	98133	+ sqlite3_vtab *ppVTab, / Write the resulting vtab structure here */
	98134	+ char *pzErr / Write any error message here */
	98135	+){
	98136	+ Fts3Hash pHash = (Fts3Hash )pAux;
	98137	+ Fts3Table p; / Pointer to allocated vtab */
	98138	+ int rc; /* Return code */
	98139	+ int i;
	98140	+ int nByte;
101233	98141	int iCol;
101234		- int nDoc;
101235		- const char *zDoc;
101236		- int iStart, iEnd;
101237		- int tailEllipsis = 0;
101238		- int iMatch;
101239		-
101240		-
101241		- sqlite3_free(pCursor->snippet.zSnippet);
101242		- pCursor->snippet.zSnippet = 0;
101243		- aMatch = pCursor->snippet.aMatch;
101244		- nMatch = pCursor->snippet.nMatch;
101245		- initStringBuffer(&sb);
101246		-
101247		- for(i=0; i<nMatch; i++){
101248		- aMatch[i].snStatus = SNIPPET_IGNORE;
101249		- }
101250		- nDesired = 0;
101251		- for(i=0; i<FTS3_ROTOR_SZ; i++){
101252		- for(j=0; j<nMatch; j++){
101253		- if( aMatch[j].iTerm==i ){
101254		- aMatch[j].snStatus = SNIPPET_DESIRED;
101255		- nDesired++;
101256		- break;
101257		- }
101258		- }
101259		- }
101260		-
101261		- iMatch = 0;
101262		- tailCol = -1;
101263		- tailOffset = 0;
101264		- for(i=0; i<nMatch && nDesired>0; i++){
101265		- if( aMatch[i].snStatus!=SNIPPET_DESIRED ) continue;
101266		- nDesired--;
101267		- iCol = aMatch[i].iCol;
101268		- zDoc = (const char*)sqlite3_column_text(pCursor->pStmt, iCol+1);
101269		- nDoc = sqlite3_column_bytes(pCursor->pStmt, iCol+1);
101270		- iStart = aMatch[i].iStart - 40;
101271		- iStart = wordBoundary(iStart, zDoc, nDoc, aMatch, nMatch, iCol);
101272		- if( iStart<=10 ){
101273		- iStart = 0;
101274		- }
101275		- if( iCol==tailCol && iStart<=tailOffset+20 ){
101276		- iStart = tailOffset;
101277		- }
101278		- if( (iCol!=tailCol && tailCol>=0) \|\| iStart!=tailOffset ){
101279		- trimWhiteSpace(&sb);
101280		- appendWhiteSpace(&sb);
101281		- append(&sb, zEllipsis);
101282		- appendWhiteSpace(&sb);
101283		- }
101284		- iEnd = aMatch[i].iStart + aMatch[i].nByte + 40;
101285		- iEnd = wordBoundary(iEnd, zDoc, nDoc, aMatch, nMatch, iCol);
101286		- if( iEnd>=nDoc-10 ){
101287		- iEnd = nDoc;
101288		- tailEllipsis = 0;
101289		- }else{
101290		- tailEllipsis = 1;
101291		- }
101292		- while( iMatch<nMatch && aMatch[iMatch].iCol<iCol ){ iMatch++; }
101293		- while( iStart<iEnd ){
101294		- while( iMatch<nMatch && aMatch[iMatch].iStart<iStart
101295		- && aMatch[iMatch].iCol<=iCol ){
101296		- iMatch++;
101297		- }
101298		- if( iMatch<nMatch && aMatch[iMatch].iStart<iEnd
101299		- && aMatch[iMatch].iCol==iCol ){
101300		- nappend(&sb, &zDoc[iStart], aMatch[iMatch].iStart - iStart);
101301		- iStart = aMatch[iMatch].iStart;
101302		- append(&sb, zStartMark);
101303		- nappend(&sb, &zDoc[iStart], aMatch[iMatch].nByte);
101304		- append(&sb, zEndMark);
101305		- iStart += aMatch[iMatch].nByte;
101306		- for(j=iMatch+1; j<nMatch; j++){
101307		- if( aMatch[j].iTerm==aMatch[iMatch].iTerm
101308		- && aMatch[j].snStatus==SNIPPET_DESIRED ){
101309		- nDesired--;
101310		- aMatch[j].snStatus = SNIPPET_IGNORE;
101311		- }
101312		- }
101313		- }else{
101314		- nappend(&sb, &zDoc[iStart], iEnd - iStart);
101315		- iStart = iEnd;
101316		- }
101317		- }
101318		- tailCol = iCol;
101319		- tailOffset = iEnd;
101320		- }
101321		- trimWhiteSpace(&sb);
101322		- if( tailEllipsis ){
101323		- appendWhiteSpace(&sb);
101324		- append(&sb, zEllipsis);
101325		- }
101326		- pCursor->snippet.zSnippet = stringBufferData(&sb);
101327		- pCursor->snippet.nSnippet = stringBufferLength(&sb);
101328		-}
	98142	+ int nString = 0;
	98143	+ int nCol = 0;
	98144	+ char *zCsr;
	98145	+ int nDb;
	98146	+ int nName;
	98147	+
	98148	+ const char *zTokenizer = 0;
	98149	+ sqlite3_tokenizer pTokenizer; / Tokenizer for this table */
	98150	+
	98151	+ nDb = strlen(argv[1]) + 1;
	98152	+ nName = strlen(argv[2]) + 1;
	98153	+ for(i=3; i<argc; i++){
	98154	+ char const *z = argv[i];
	98155	+ rc = sqlite3Fts3InitTokenizer(pHash, z, &pTokenizer, &zTokenizer, pzErr);
	98156	+ if( rc!=SQLITE_OK ){
	98157	+ return rc;
	98158	+ }
	98159	+ if( z!=zTokenizer ){
	98160	+ nString += strlen(z) + 1;
	98161	+ }
	98162	+ }
	98163	+ nCol = argc - 3 - (zTokenizer!=0);
	98164	+ if( zTokenizer==0 ){
	98165	+ rc = sqlite3Fts3InitTokenizer(pHash, 0, &pTokenizer, 0, pzErr);
	98166	+ if( rc!=SQLITE_OK ){
	98167	+ return rc;
	98168	+ }
	98169	+ assert( pTokenizer );
	98170	+ }
	98171	+
	98172	+ /* Allocate and populate the Fts3Table structure. */
	98173	+ nByte = sizeof(Fts3Table) + /* Fts3Table */
	98174	+ nCol * sizeof(char ) + / azColumn */
	98175	+ nName + /* zName */
	98176	+ nDb + /* zDb */
	98177	+ nString; /* Space for azColumn strings */
	98178	+ p = (Fts3Table*)sqlite3_malloc(nByte);
	98179	+ if( p==0 ){
	98180	+ rc = SQLITE_NOMEM;
	98181	+ goto fts3_init_out;
	98182	+ }
	98183	+ memset(p, 0, nByte);
	98184	+
	98185	+ p->db = db;
	98186	+ p->nColumn = nCol;
	98187	+ p->nPendingData = 0;
	98188	+ p->azColumn = (char **)&p[1];
	98189	+ p->pTokenizer = pTokenizer;
	98190	+ zCsr = (char *)&p->azColumn[nCol];
	98191	+
	98192	+ fts3HashInit(&p->pendingTerms, FTS3_HASH_STRING, 1);
	98193	+
	98194	+ /* Fill in the zName and zDb fields of the vtab structure. */
	98195	+ p->zName = zCsr;
	98196	+ memcpy(zCsr, argv[2], nName);
	98197	+ zCsr += nName;
	98198	+ p->zDb = zCsr;
	98199	+ memcpy(zCsr, argv[1], nDb);
	98200	+ zCsr += nDb;
	98201	+
	98202	+ /* Fill in the azColumn array */
	98203	+ iCol = 0;
	98204	+ for(i=3; i<argc; i++){
	98205	+ if( argv[i]!=zTokenizer ){
	98206	+ char *z;
	98207	+ int n;
	98208	+ z = (char *)sqlite3Fts3NextToken(argv[i], &n);
	98209	+ memcpy(zCsr, z, n);
	98210	+ zCsr[n] = '\0';
	98211	+ sqlite3Fts3Dequote(zCsr);
	98212	+ p->azColumn[iCol++] = zCsr;
	98213	+ zCsr += n+1;
	98214	+ assert( zCsr <= &((char *)p)[nByte] );
	98215	+ }
	98216	+ }
	98217	+
	98218	+ /* If this is an xCreate call, create the underlying tables in the
	98219	+ ** database. TODO: For xConnect(), it could verify that said tables exist.
	98220	+ */
	98221	+ if( isCreate ){
	98222	+ rc = fts3CreateTables(p);
	98223	+ if( rc!=SQLITE_OK ) goto fts3_init_out;
	98224	+ }
	98225	+
	98226	+ rc = fts3DeclareVtab(p);
	98227	+ if( rc!=SQLITE_OK ) goto fts3_init_out;
	98228	+
	98229	+ *ppVTab = &p->base;
	98230	+
	98231	+fts3_init_out:
	98232	+ if( rc!=SQLITE_OK ){
	98233	+ if( p ) fts3DisconnectMethod((sqlite3_vtab *)p);
	98234	+ else if( pTokenizer ) pTokenizer->pModule->xDestroy(pTokenizer);
	98235	+ }
	98236	+ return rc;
	98237	+}
	98238	+
	98239	+/*
	98240	+** The xConnect() and xCreate() methods for the virtual table. All the
	98241	+** work is done in function fts3InitVtab().
	98242	+*/
	98243	+static int fts3ConnectMethod(
	98244	+ sqlite3 db, / Database connection */
	98245	+ void pAux, / Pointer to tokenizer hash table */
	98246	+ int argc, /* Number of elements in argv array */
	98247	+ const char * const argv, / xCreate/xConnect argument array */
	98248	+ sqlite3_vtab *ppVtab, / OUT: New sqlite3_vtab object */
	98249	+ char *pzErr / OUT: sqlite3_malloc'd error message */
	98250	+){
	98251	+ return fts3InitVtab(0, db, pAux, argc, argv, ppVtab, pzErr);
	98252	+}
	98253	+static int fts3CreateMethod(
	98254	+ sqlite3 db, / Database connection */
	98255	+ void pAux, / Pointer to tokenizer hash table */
	98256	+ int argc, /* Number of elements in argv array */
	98257	+ const char * const argv, / xCreate/xConnect argument array */
	98258	+ sqlite3_vtab *ppVtab, / OUT: New sqlite3_vtab object */
	98259	+ char *pzErr / OUT: sqlite3_malloc'd error message */
	98260	+){
	98261	+ return fts3InitVtab(1, db, pAux, argc, argv, ppVtab, pzErr);
	98262	+}
	98263	+
	98264	+/*
	98265	+** Implementation of the xBestIndex method for FTS3 tables. There
	98266	+** are three possible strategies, in order of preference:
	98267	+**
	98268	+** 1. Direct lookup by rowid or docid.
	98269	+** 2. Full-text search using a MATCH operator on a non-docid column.
	98270	+** 3. Linear scan of %_content table.
	98271	+*/
	98272	+static int fts3BestIndexMethod(sqlite3_vtab pVTab, sqlite3_index_info pInfo){
	98273	+ Fts3Table p = (Fts3Table )pVTab;
	98274	+ int i; /* Iterator variable */
	98275	+ int iCons = -1; /* Index of constraint to use */
	98276	+
	98277	+ /* By default use a full table scan. This is an expensive option,
	98278	+ ** so search through the constraints to see if a more efficient
	98279	+ ** strategy is possible.
	98280	+ */
	98281	+ pInfo->idxNum = FTS3_FULLSCAN_SEARCH;
	98282	+ pInfo->estimatedCost = 500000;
	98283	+ for(i=0; i<pInfo->nConstraint; i++){
	98284	+ struct sqlite3_index_constraint *pCons = &pInfo->aConstraint[i];
	98285	+ if( pCons->usable==0 ) continue;
	98286	+
	98287	+ /* A direct lookup on the rowid or docid column. This is the best
	98288	+ ** strategy in all cases. Assign a cost of 1.0 and return early.
	98289	+ */
	98290	+ if( pCons->op==SQLITE_INDEX_CONSTRAINT_EQ
	98291	+ && (pCons->iColumn<0 \|\| pCons->iColumn==p->nColumn+1 )
	98292	+ ){
	98293	+ pInfo->idxNum = FTS3_DOCID_SEARCH;
	98294	+ pInfo->estimatedCost = 1.0;
	98295	+ iCons = i;
	98296	+ break;
	98297	+ }
	98298	+
	98299	+ /* A MATCH constraint. Use a full-text search.
	98300	+ **
	98301	+ ** If there is more than one MATCH constraint available, use the first
	98302	+ ** one encountered. If there is both a MATCH constraint and a direct
	98303	+ ** rowid/docid lookup, prefer the rowid/docid strategy.
	98304	+ */
	98305	+ if( iCons<0
	98306	+ && pCons->op==SQLITE_INDEX_CONSTRAINT_MATCH
	98307	+ && pCons->iColumn>=0 && pCons->iColumn<=p->nColumn
	98308	+ ){
	98309	+ pInfo->idxNum = FTS3_FULLTEXT_SEARCH + pCons->iColumn;
	98310	+ pInfo->estimatedCost = 2.0;
	98311	+ iCons = i;
	98312	+ }
	98313	+ }
	98314	+
	98315	+ if( iCons>=0 ){
	98316	+ pInfo->aConstraintUsage[iCons].argvIndex = 1;
	98317	+ pInfo->aConstraintUsage[iCons].omit = 1;
	98318	+ }
	98319	+ return SQLITE_OK;
	98320	+}
	98321	+
	98322	+/*
	98323	+** Implementation of xOpen method.
	98324	+*/
	98325	+static int fts3OpenMethod(sqlite3_vtab pVTab, sqlite3_vtab_cursor *ppCsr){
	98326	+ sqlite3_vtab_cursor pCsr; / Allocated cursor */
	98327	+
	98328	+ /* Allocate a buffer large enough for an Fts3Cursor structure. If the
	98329	+ ** allocation succeeds, zero it and return SQLITE_OK. Otherwise,
	98330	+ ** if the allocation fails, return SQLITE_NOMEM.
	98331	+ */
	98332	+ ppCsr = pCsr = (sqlite3_vtab_cursor )sqlite3_malloc(sizeof(Fts3Cursor));
	98333	+ if( !pCsr ){
	98334	+ return SQLITE_NOMEM;
	98335	+ }
	98336	+ memset(pCsr, 0, sizeof(Fts3Cursor));
	98337	+ return SQLITE_OK;
	98338	+}
	98339	+
	98340	+/****************************************************************/
	98341	+/****************************************************************/
	98342	+/****************************************************************/
	98343	+/****************************************************************/
101329	98344
101330	98345
101331	98346	/*
101332	98347	** Close the cursor. For additional information see the documentation
101333	98348	** on the xClose method of the virtual table interface.
101334	98349	*/
101335	98350	static int fulltextClose(sqlite3_vtab_cursor *pCursor){
101336		- fulltext_cursor c = (fulltext_cursor ) pCursor;
101337		- FTSTRACE(("FTS3 Close %p\n", c));
101338		- sqlite3_finalize(c->pStmt);
101339		- sqlite3Fts3ExprFree(c->pExpr);
101340		- snippetClear(&c->snippet);
101341		- if( c->result.nData!=0 ){
101342		- dlrDestroy(&c->reader);
101343		- }
101344		- dataBufferDestroy(&c->result);
101345		- sqlite3_free(c);
101346		- return SQLITE_OK;
101347		-}
101348		-
101349		-static int fulltextNext(sqlite3_vtab_cursor *pCursor){
101350		- fulltext_cursor c = (fulltext_cursor ) pCursor;
101351		- int rc;
101352		-
101353		- FTSTRACE(("FTS3 Next %p\n", pCursor));
101354		- snippetClear(&c->snippet);
101355		- if( c->iCursorType < QUERY_FULLTEXT ){
101356		- /* TODO(shess) Handle SQLITE_SCHEMA AND SQLITE_BUSY. */
101357		- rc = sqlite3_step(c->pStmt);
101358		- switch( rc ){
101359		- case SQLITE_ROW:
101360		- c->eof = 0;
101361		- return SQLITE_OK;
101362		- case SQLITE_DONE:
101363		- c->eof = 1;
101364		- return SQLITE_OK;
101365		- default:
101366		- c->eof = 1;
101367		- return rc;
101368		- }
101369		- } else { /* full-text query */
101370		- rc = sqlite3_reset(c->pStmt);
101371		- if( rc!=SQLITE_OK ) return rc;
101372		-
101373		- if( c->result.nData==0 \|\| dlrAtEnd(&c->reader) ){
101374		- c->eof = 1;
101375		- return SQLITE_OK;
101376		- }
101377		- rc = sqlite3_bind_int64(c->pStmt, 1, dlrDocid(&c->reader));
101378		- dlrStep(&c->reader);
101379		- if( rc!=SQLITE_OK ) return rc;
101380		- /* TODO(shess) Handle SQLITE_SCHEMA AND SQLITE_BUSY. */
101381		- rc = sqlite3_step(c->pStmt);
101382		- if( rc==SQLITE_ROW ){ /* the case we expect */
101383		- c->eof = 0;
101384		- return SQLITE_OK;
101385		- }
101386		- /* an error occurred; abort */
101387		- return rc==SQLITE_DONE ? SQLITE_ERROR : rc;
101388		- }
101389		-}
101390		-
101391		-
101392		-/* TODO(shess) If we pushed LeafReader to the top of the file, or to
101393		-** another file, term_select() could be pushed above
101394		-** docListOfTerm().
101395		-*/
101396		-static int termSelect(fulltext_vtab *v, int iColumn,
101397		- const char *pTerm, int nTerm, int isPrefix,
101398		- DocListType iType, DataBuffer *out);
	98351	+ Fts3Cursor pCsr = (Fts3Cursor )pCursor;
	98352	+ sqlite3_finalize(pCsr->pStmt);
	98353	+ sqlite3Fts3ExprFree(pCsr->pExpr);
	98354	+ sqlite3_free(pCsr->aDoclist);
	98355	+ sqlite3_free(pCsr);
	98356	+ return SQLITE_OK;
	98357	+}
	98358	+
	98359	+static int fts3CursorSeek(Fts3Cursor *pCsr){
	98360	+ if( pCsr->isRequireSeek ){
	98361	+ pCsr->isRequireSeek = 0;
	98362	+ sqlite3_bind_int64(pCsr->pStmt, 1, pCsr->iPrevId);
	98363	+ if( SQLITE_ROW==sqlite3_step(pCsr->pStmt) ){
	98364	+ return SQLITE_OK;
	98365	+ }else{
	98366	+ int rc;
	98367	+ pCsr->isEof = 1;
	98368	+ if( SQLITE_OK==(rc = sqlite3_reset(pCsr->pStmt)) ){
	98369	+ rc = SQLITE_ERROR;
	98370	+ }
	98371	+ return rc;
	98372	+ }
	98373	+ }else{
	98374	+ return SQLITE_OK;
	98375	+ }
	98376	+}
	98377	+
	98378	+static int fts3NextMethod(sqlite3_vtab_cursor *pCursor){
	98379	+ int rc = SQLITE_OK; /* Return code */
	98380	+ Fts3Cursor pCsr = (Fts3Cursor )pCursor;
	98381	+
	98382	+ if( pCsr->aDoclist==0 ){
	98383	+ if( SQLITE_ROW!=sqlite3_step(pCsr->pStmt) ){
	98384	+ pCsr->isEof = 1;
	98385	+ rc = sqlite3_reset(pCsr->pStmt);
	98386	+ }
	98387	+ }else if( pCsr->pNextId>=&pCsr->aDoclist[pCsr->nDoclist] ){
	98388	+ pCsr->isEof = 1;
	98389	+ }else{
	98390	+ sqlite3_reset(pCsr->pStmt);
	98391	+ fts3GetDeltaVarint(&pCsr->pNextId, &pCsr->iPrevId);
	98392	+ pCsr->isRequireSeek = 1;
	98393	+ }
	98394	+ return rc;
	98395	+}
	98396	+
	98397	+
	98398	+/*
	98399	+** The buffer pointed to by argument zNode (size nNode bytes) contains the
	98400	+** root node of a b-tree segment. The segment is guaranteed to be at least
	98401	+** one level high (i.e. the root node is not also a leaf). If successful,
	98402	+** this function locates the leaf node of the segment that may contain the
	98403	+** term specified by arguments zTerm and nTerm and writes its block number
	98404	+** to *piLeaf.
	98405	+**
	98406	+** It is possible that the returned leaf node does not contain the specified
	98407	+** term. However, if the segment does contain said term, it is stored on
	98408	+** the identified leaf node. Because this function only inspects interior
	98409	+** segment nodes (and never loads leaf nodes into memory), it is not possible
	98410	+** to be sure.
	98411	+**
	98412	+** If an error occurs, an error code other than SQLITE_OK is returned.
	98413	+*/
	98414	+static int fts3SelectLeaf(
	98415	+ Fts3Table p, / Virtual table handle */
	98416	+ const char zTerm, / Term to select leaves for */
	98417	+ int nTerm, /* Size of term zTerm in bytes */
	98418	+ const char zNode, / Buffer containing segment interior node */
	98419	+ int nNode, /* Size of buffer at zNode */
	98420	+ sqlite3_int64 piLeaf / Selected leaf node */
	98421	+){
	98422	+ int rc = SQLITE_OK; /* Return code */
	98423	+ const char zCsr = zNode; / Cursor to iterate through node */
	98424	+ const char zEnd = &zCsr[nNode];/ End of interior node buffer */
	98425	+ char zBuffer = 0; / Buffer to load terms into */
	98426	+ int nAlloc = 0; /* Size of allocated buffer */
	98427	+
	98428	+ while( 1 ){
	98429	+ int isFirstTerm = 1; /* True when processing first term on page */
	98430	+ int iHeight; /* Height of this node in tree */
	98431	+ sqlite3_int64 iChild; /* Block id of child node to descend to */
	98432	+ int nBlock; /* Size of child node in bytes */
	98433	+
	98434	+ zCsr += sqlite3Fts3GetVarint32(zCsr, &iHeight);
	98435	+ zCsr += sqlite3Fts3GetVarint(zCsr, &iChild);
	98436	+
	98437	+ while( zCsr<zEnd ){
	98438	+ int cmp; /* memcmp() result */
	98439	+ int nSuffix; /* Size of term suffix */
	98440	+ int nPrefix = 0; /* Size of term prefix */
	98441	+ int nBuffer; /* Total term size */
	98442	+
	98443	+ /* Load the next term on the node into zBuffer */
	98444	+ if( !isFirstTerm ){
	98445	+ zCsr += sqlite3Fts3GetVarint32(zCsr, &nPrefix);
	98446	+ }
	98447	+ isFirstTerm = 0;
	98448	+ zCsr += sqlite3Fts3GetVarint32(zCsr, &nSuffix);
	98449	+ if( nPrefix+nSuffix>nAlloc ){
	98450	+ char *zNew;
	98451	+ nAlloc = (nPrefix+nSuffix) * 2;
	98452	+ zNew = (char *)sqlite3_realloc(zBuffer, nAlloc);
	98453	+ if( !zNew ){
	98454	+ sqlite3_free(zBuffer);
	98455	+ return SQLITE_NOMEM;
	98456	+ }
	98457	+ zBuffer = zNew;
	98458	+ }
	98459	+ memcpy(&zBuffer[nPrefix], zCsr, nSuffix);
	98460	+ nBuffer = nPrefix + nSuffix;
	98461	+ zCsr += nSuffix;
	98462	+
	98463	+ /* Compare the term we are searching for with the term just loaded from
	98464	+ ** the interior node. If the specified term is greater than or equal
	98465	+ ** to the term from the interior node, then all terms on the sub-tree
	98466	+ ** headed by node iChild are smaller than zTerm. No need to search
	98467	+ ** iChild.
	98468	+ **
	98469	+ ** If the interior node term is larger than the specified term, then
	98470	+ ** the tree headed by iChild may contain the specified term.
	98471	+ */
	98472	+ cmp = memcmp(zTerm, zBuffer, (nBuffer>nTerm ? nTerm : nBuffer));
	98473	+ if( cmp<0 \|\| (cmp==0 && nBuffer>nTerm) ) break;
	98474	+ iChild++;
	98475	+ };
	98476	+
	98477	+ /* If (iHeight==1), the children of this interior node are leaves. The
	98478	+ ** specified term may be present on leaf node iChild.
	98479	+ */
	98480	+ if( iHeight==1 ){
	98481	+ *piLeaf = iChild;
	98482	+ break;
	98483	+ }
	98484	+
	98485	+ /* Descend to interior node iChild. */
	98486	+ rc = sqlite3Fts3ReadBlock(p, iChild, &zCsr, &nBlock);
	98487	+ if( rc!=SQLITE_OK ) break;
	98488	+ zEnd = &zCsr[nBlock];
	98489	+ }
	98490	+ sqlite3_free(zBuffer);
	98491	+ return rc;
	98492	+}
	98493	+
	98494	+static void fts3PutDeltaVarint(
	98495	+ char **pp,
	98496	+ sqlite3_int64 *piPrev,
	98497	+ sqlite3_int64 iVal
	98498	+){
	98499	+ assert( iVal-*piPrev > 0 );
	98500	+ pp += sqlite3Fts3PutVarint(pp, iVal-*piPrev);
	98501	+ *piPrev = iVal;
	98502	+}
	98503	+
	98504	+static void fts3PoslistCopy(char pp, char ppPoslist){
	98505	+ char pEnd = ppPoslist;
	98506	+ char c = 0;
	98507	+ while( pEnd \| c ) c = pEnd++ & 0x80;
	98508	+ pEnd++;
	98509	+ if( pp ){
	98510	+ int n = pEnd - *ppPoslist;
	98511	+ char p = pp;
	98512	+ memcpy(p, *ppPoslist, n);
	98513	+ p += n;
	98514	+ *pp = p;
	98515	+ }
	98516	+ *ppPoslist = pEnd;
	98517	+}
	98518	+
	98519	+static void fts3ColumnlistCopy(char pp, char ppPoslist){
	98520	+ char pEnd = ppPoslist;
	98521	+ char c = 0;
	98522	+ while( 0xFE & (pEnd \| c) ) c = pEnd++ & 0x80;
	98523	+ if( pp ){
	98524	+ int n = pEnd - *ppPoslist;
	98525	+ char p = pp;
	98526	+ memcpy(p, *ppPoslist, n);
	98527	+ p += n;
	98528	+ *pp = p;
	98529	+ }
	98530	+ *ppPoslist = pEnd;
	98531	+}
	98532	+
	98533	+/*
	98534	+**
	98535	+*/
	98536	+static void fts3PoslistMerge(
	98537	+ char *pp, / Output buffer */
	98538	+ char *pp1, / Left input list */
	98539	+ char *pp2 / Right input list */
	98540	+){
	98541	+ char p = pp;
	98542	+ char p1 = pp1;
	98543	+ char p2 = pp2;
	98544	+
	98545	+ while( p1 && p2 ){
	98546	+ int iCol1 = 0;
	98547	+ int iCol2 = 0;
	98548	+ if( *p1==0x01 ) sqlite3Fts3GetVarint32(&p1[1], &iCol1);
	98549	+ if( *p2==0x01 ) sqlite3Fts3GetVarint32(&p2[1], &iCol2);
	98550	+
	98551	+ if( iCol1==iCol2 ){
	98552	+ sqlite3_int64 i1 = 0;
	98553	+ sqlite3_int64 i2 = 0;
	98554	+ sqlite3_int64 iPrev = 0;
	98555	+ if( iCol1!=0 ){
	98556	+ int n;
	98557	+ *p++ = 0x01;
	98558	+ n = sqlite3Fts3PutVarint(p, iCol1);
	98559	+ p += n;
	98560	+ p1 += 1 + n;
	98561	+ p2 += 1 + n;
	98562	+ }
	98563	+ while( (p1&0xFE) \|\| (p2&0xFE) ){
	98564	+ if( i1==i2 ){
	98565	+ fts3GetDeltaVarint(&p1, &i1); i1 -= 2;
	98566	+ fts3GetDeltaVarint(&p2, &i2); i2 -= 2;
	98567	+ }else if( i1<i2 ){
	98568	+ fts3GetDeltaVarint(&p1, &i1); i1 -= 2;
	98569	+ }else{
	98570	+ fts3GetDeltaVarint(&p2, &i2); i2 -= 2;
	98571	+ }
	98572	+ fts3PutDeltaVarint(&p, &iPrev, (i1<i2 ? i1 : i2) + 2); iPrev -= 2;
	98573	+ if( 0==(*p1&0xFE) ) i1 = 0x7FFFFFFF;
	98574	+ if( 0==(*p2&0xFE) ) i2 = 0x7FFFFFFF;
	98575	+ }
	98576	+ }else if( iCol1<iCol2 ){
	98577	+ fts3ColumnlistCopy(&p, &p1);
	98578	+ }else{
	98579	+ fts3ColumnlistCopy(&p, &p2);
	98580	+ }
	98581	+ }
	98582	+
	98583	+ *p++ = '\0';
	98584	+ *pp = p;
	98585	+ *pp1 = p1 + 1;
	98586	+ *pp2 = p2 + 1;
	98587	+}
	98588	+
	98589	+/*
	98590	+** nToken==1 searches for adjacent positions.
	98591	+*/
	98592	+static int fts3PoslistPhraseMerge(
	98593	+ char *pp, / Output buffer */
	98594	+ int nToken, /* Maximum difference in token positions */
	98595	+ int isSaveLeft, /* Save the left position */
	98596	+ char *pp1, / Left input list */
	98597	+ char *pp2 / Right input list */
	98598	+){
	98599	+ char p = (pp ? pp : 0);
	98600	+ char p1 = pp1;
	98601	+ char p2 = pp2;
	98602	+
	98603	+ int iCol1 = 0;
	98604	+ int iCol2 = 0;
	98605	+ assert( p1!=0 && p2!=0 );
	98606	+ if( *p1==0x01 ){
	98607	+ p1++;
	98608	+ p1 += sqlite3Fts3GetVarint32(p1, &iCol1);
	98609	+ }
	98610	+ if( *p2==0x01 ){
	98611	+ p2++;
	98612	+ p2 += sqlite3Fts3GetVarint32(p2, &iCol2);
	98613	+ }
	98614	+
	98615	+ while( 1 ){
	98616	+ if( iCol1==iCol2 ){
	98617	+ char *pSave = p;
	98618	+ sqlite3_int64 iPrev = 0;
	98619	+ sqlite3_int64 iPos1 = 0;
	98620	+ sqlite3_int64 iPos2 = 0;
	98621	+
	98622	+ if( pp && iCol1 ){
	98623	+ *p++ = 0x01;
	98624	+ p += sqlite3Fts3PutVarint(p, iCol1);
	98625	+ }
	98626	+
	98627	+ assert( p1!=0x00 && p2!=0x00 && p1!=0x01 && p2!=0x01 );
	98628	+ fts3GetDeltaVarint(&p1, &iPos1); iPos1 -= 2;
	98629	+ fts3GetDeltaVarint(&p2, &iPos2); iPos2 -= 2;
	98630	+
	98631	+ while( 1 ){
	98632	+ if( iPos2>iPos1 && iPos2<=iPos1+nToken ){
	98633	+ sqlite3_int64 iSave;
	98634	+ if( !pp ){
	98635	+ fts3PoslistCopy(0, &p2);
	98636	+ fts3PoslistCopy(0, &p1);
	98637	+ *pp1 = p1;
	98638	+ *pp2 = p2;
	98639	+ return 1;
	98640	+ }
	98641	+ iSave = isSaveLeft ? iPos1 : iPos2;
	98642	+ fts3PutDeltaVarint(&p, &iPrev, iSave+2); iPrev -= 2;
	98643	+ pSave = 0;
	98644	+ }
	98645	+ if( iPos2<=iPos1 ){
	98646	+ if( (*p2&0xFE)==0 ) break;
	98647	+ fts3GetDeltaVarint(&p2, &iPos2); iPos2 -= 2;
	98648	+ }else{
	98649	+ if( (*p1&0xFE)==0 ) break;
	98650	+ fts3GetDeltaVarint(&p1, &iPos1); iPos1 -= 2;
	98651	+ }
	98652	+ }
	98653	+ if( pSave && pp ){
	98654	+ p = pSave;
	98655	+ }
	98656	+
	98657	+ fts3ColumnlistCopy(0, &p1);
	98658	+ fts3ColumnlistCopy(0, &p2);
	98659	+ assert( (p1&0xFE)==0 && (p2&0xFE)==0 );
	98660	+ if( 0==p1 \|\| 0==p2 ) break;
	98661	+
	98662	+ p1++;
	98663	+ p1 += sqlite3Fts3GetVarint32(p1, &iCol1);
	98664	+ p2++;
	98665	+ p2 += sqlite3Fts3GetVarint32(p2, &iCol2);
	98666	+ }
	98667	+
	98668	+ /* Advance pointer p1 or p2 (whichever corresponds to the smaller of
	98669	+ ** iCol1 and iCol2) so that it points to either the 0x00 that marks the
	98670	+ ** end of the position list, or the 0x01 that precedes the next
	98671	+ ** column-number in the position list.
	98672	+ */
	98673	+ else if( iCol1<iCol2 ){
	98674	+ fts3ColumnlistCopy(0, &p1);
	98675	+ if( 0==*p1 ) break;
	98676	+ p1++;
	98677	+ p1 += sqlite3Fts3GetVarint32(p1, &iCol1);
	98678	+ }else{
	98679	+ fts3ColumnlistCopy(0, &p2);
	98680	+ if( 0==*p2 ) break;
	98681	+ p2++;
	98682	+ p2 += sqlite3Fts3GetVarint32(p2, &iCol2);
	98683	+ }
	98684	+ }
	98685	+
	98686	+ fts3PoslistCopy(0, &p2);
	98687	+ fts3PoslistCopy(0, &p1);
	98688	+ *pp1 = p1;
	98689	+ *pp2 = p2;
	98690	+ if( !pp \|\| *pp==p ){
	98691	+ return 0;
	98692	+ }
	98693	+ *p++ = 0x00;
	98694	+ *pp = p;
	98695	+ return 1;
	98696	+}
	98697	+
	98698	+static int fts3PoslistNearMerge(
	98699	+ char *pp, / Output buffer */
	98700	+ char aTmp, / Temporary buffer space */
	98701	+ int nRight, /* Maximum difference in token positions */
	98702	+ int nLeft, /* Maximum difference in token positions */
	98703	+ char *pp1, / Left input list */
	98704	+ char *pp2 / Right input list */
	98705	+){
	98706	+ char p1 = pp1;
	98707	+ char p2 = pp2;
	98708	+
	98709	+ if( !pp ){
	98710	+ if( fts3PoslistPhraseMerge(0, nRight, 0, pp1, pp2) ) return 1;
	98711	+ *pp1 = p1;
	98712	+ *pp2 = p2;
	98713	+ return fts3PoslistPhraseMerge(0, nLeft, 0, pp2, pp1);
	98714	+ }else{
	98715	+ char *pTmp1 = aTmp;
	98716	+ char *pTmp2;
	98717	+ char *aTmp2;
	98718	+ int res = 1;
	98719	+
	98720	+ fts3PoslistPhraseMerge(&pTmp1, nRight, 0, pp1, pp2);
	98721	+ aTmp2 = pTmp2 = pTmp1;
	98722	+ *pp1 = p1;
	98723	+ *pp2 = p2;
	98724	+ fts3PoslistPhraseMerge(&pTmp2, nLeft, 1, pp2, pp1);
	98725	+ if( pTmp1!=aTmp && pTmp2!=aTmp2 ){
	98726	+ fts3PoslistMerge(pp, &aTmp, &aTmp2);
	98727	+ }else if( pTmp1!=aTmp ){
	98728	+ fts3PoslistCopy(pp, &aTmp);
	98729	+ }else if( pTmp2!=aTmp2 ){
	98730	+ fts3PoslistCopy(pp, &aTmp2);
	98731	+ }else{
	98732	+ res = 0;
	98733	+ }
	98734	+
	98735	+ return res;
	98736	+ }
	98737	+}
	98738	+
	98739	+/*
	98740	+** Values that may be used as the first parameter to fts3DoclistMerge().
	98741	+*/
	98742	+#define MERGE_NOT 2 /* D + D -> D */
	98743	+#define MERGE_AND 3 /* D + D -> D */
	98744	+#define MERGE_OR 4 /* D + D -> D */
	98745	+#define MERGE_POS_OR 5 /* P + P -> P */
	98746	+#define MERGE_PHRASE 6 /* P + P -> D */
	98747	+#define MERGE_POS_PHRASE 7 /* P + P -> P */
	98748	+#define MERGE_NEAR 8 /* P + P -> D */
	98749	+#define MERGE_POS_NEAR 9 /* P + P -> P */
	98750	+
	98751	+/*
	98752	+** Merge the two doclists passed in buffer a1 (size n1 bytes) and a2
	98753	+** (size n2 bytes). The output is written to pre-allocated buffer aBuffer,
	98754	+** which is guaranteed to be large enough to hold the results. The number
	98755	+** of bytes written to aBuffer is stored in *pnBuffer before returning.
	98756	+**
	98757	+** If successful, SQLITE_OK is returned. Otherwise, if a malloc error
	98758	+** occurs while allocating a temporary buffer as part of the merge operation,
	98759	+** SQLITE_NOMEM is returned.
	98760	+*/
	98761	+static int fts3DoclistMerge(
	98762	+ int mergetype, /* One of the MERGE_XXX constants */
	98763	+ int nParam1, /* Used by MERGE_NEAR and MERGE_POS_NEAR */
	98764	+ int nParam2, /* Used by MERGE_NEAR and MERGE_POS_NEAR */
	98765	+ char aBuffer, / Pre-allocated output buffer */
	98766	+ int pnBuffer, / OUT: Bytes written to aBuffer */
	98767	+ char a1, / Buffer containing first doclist */
	98768	+ int n1, /* Size of buffer a1 */
	98769	+ char a2, / Buffer containing second doclist */
	98770	+ int n2 /* Size of buffer a2 */
	98771	+){
	98772	+ sqlite3_int64 i1 = 0;
	98773	+ sqlite3_int64 i2 = 0;
	98774	+ sqlite3_int64 iPrev = 0;
	98775	+
	98776	+ char *p = aBuffer;
	98777	+ char *p1 = a1;
	98778	+ char *p2 = a2;
	98779	+ char *pEnd1 = &a1[n1];
	98780	+ char *pEnd2 = &a2[n2];
	98781	+
	98782	+ assert( mergetype==MERGE_OR \|\| mergetype==MERGE_POS_OR
	98783	+ \|\| mergetype==MERGE_AND \|\| mergetype==MERGE_NOT
	98784	+ \|\| mergetype==MERGE_PHRASE \|\| mergetype==MERGE_POS_PHRASE
	98785	+ \|\| mergetype==MERGE_NEAR \|\| mergetype==MERGE_POS_NEAR
	98786	+ );
	98787	+
	98788	+ if( !aBuffer ){
	98789	+ return SQLITE_NOMEM;
	98790	+ }
	98791	+
	98792	+ /* Read the first docid from each doclist */
	98793	+ fts3GetDeltaVarint2(&p1, pEnd1, &i1);
	98794	+ fts3GetDeltaVarint2(&p2, pEnd2, &i2);
	98795	+
	98796	+ switch( mergetype ){
	98797	+ case MERGE_OR:
	98798	+ case MERGE_POS_OR:
	98799	+ while( p1 \|\| p2 ){
	98800	+ if( p2 && p1 && i1==i2 ){
	98801	+ fts3PutDeltaVarint(&p, &iPrev, i1);
	98802	+ if( mergetype==MERGE_POS_OR ) fts3PoslistMerge(&p, &p1, &p2);
	98803	+ fts3GetDeltaVarint2(&p1, pEnd1, &i1);
	98804	+ fts3GetDeltaVarint2(&p2, pEnd2, &i2);
	98805	+ }else if( !p2 \|\| (p1 && i1<i2) ){
	98806	+ fts3PutDeltaVarint(&p, &iPrev, i1);
	98807	+ if( mergetype==MERGE_POS_OR ) fts3PoslistCopy(&p, &p1);
	98808	+ fts3GetDeltaVarint2(&p1, pEnd1, &i1);
	98809	+ }else{
	98810	+ fts3PutDeltaVarint(&p, &iPrev, i2);
	98811	+ if( mergetype==MERGE_POS_OR ) fts3PoslistCopy(&p, &p2);
	98812	+ fts3GetDeltaVarint2(&p2, pEnd2, &i2);
	98813	+ }
	98814	+ }
	98815	+ break;
	98816	+
	98817	+ case MERGE_AND:
	98818	+ while( p1 && p2 ){
	98819	+ if( i1==i2 ){
	98820	+ fts3PutDeltaVarint(&p, &iPrev, i1);
	98821	+ fts3GetDeltaVarint2(&p1, pEnd1, &i1);
	98822	+ fts3GetDeltaVarint2(&p2, pEnd2, &i2);
	98823	+ }else if( i1<i2 ){
	98824	+ fts3GetDeltaVarint2(&p1, pEnd1, &i1);
	98825	+ }else{
	98826	+ fts3GetDeltaVarint2(&p2, pEnd2, &i2);
	98827	+ }
	98828	+ }
	98829	+ break;
	98830	+
	98831	+ case MERGE_NOT:
	98832	+ while( p1 ){
	98833	+ if( p2 && i1==i2 ){
	98834	+ fts3GetDeltaVarint2(&p1, pEnd1, &i1);
	98835	+ fts3GetDeltaVarint2(&p2, pEnd2, &i2);
	98836	+ }else if( !p2 \|\| i1<i2 ){
	98837	+ fts3PutDeltaVarint(&p, &iPrev, i1);
	98838	+ fts3GetDeltaVarint2(&p1, pEnd1, &i1);
	98839	+ }else{
	98840	+ fts3GetDeltaVarint2(&p2, pEnd2, &i2);
	98841	+ }
	98842	+ }
	98843	+ break;
	98844	+
	98845	+ case MERGE_POS_PHRASE:
	98846	+ case MERGE_PHRASE: {
	98847	+ char **ppPos = (mergetype==MERGE_PHRASE ? 0 : &p);
	98848	+ while( p1 && p2 ){
	98849	+ if( i1==i2 ){
	98850	+ char *pSave = p;
	98851	+ sqlite3_int64 iPrevSave = iPrev;
	98852	+ fts3PutDeltaVarint(&p, &iPrev, i1);
	98853	+ if( 0==fts3PoslistPhraseMerge(ppPos, 1, 0, &p1, &p2) ){
	98854	+ p = pSave;
	98855	+ iPrev = iPrevSave;
	98856	+ }
	98857	+ fts3GetDeltaVarint2(&p1, pEnd1, &i1);
	98858	+ fts3GetDeltaVarint2(&p2, pEnd2, &i2);
	98859	+ }else if( i1<i2 ){
	98860	+ fts3PoslistCopy(0, &p1);
	98861	+ fts3GetDeltaVarint2(&p1, pEnd1, &i1);
	98862	+ }else{
	98863	+ fts3PoslistCopy(0, &p2);
	98864	+ fts3GetDeltaVarint2(&p2, pEnd2, &i2);
	98865	+ }
	98866	+ }
	98867	+ break;
	98868	+ }
	98869	+
	98870	+ case MERGE_POS_NEAR:
	98871	+ case MERGE_NEAR: {
	98872	+ char *aTmp = 0;
	98873	+ char **ppPos = 0;
	98874	+ if( mergetype==MERGE_POS_NEAR ){
	98875	+ ppPos = &p;
	98876	+ aTmp = sqlite3_malloc(2*(n1+n2));
	98877	+ if( !aTmp ){
	98878	+ return SQLITE_NOMEM;
	98879	+ }
	98880	+ }
	98881	+
	98882	+ while( p1 && p2 ){
	98883	+ if( i1==i2 ){
	98884	+ char *pSave = p;
	98885	+ sqlite3_int64 iPrevSave = iPrev;
	98886	+ fts3PutDeltaVarint(&p, &iPrev, i1);
	98887	+
	98888	+ if( !fts3PoslistNearMerge(ppPos, aTmp, nParam1, nParam2, &p1, &p2) ){
	98889	+ iPrev = iPrevSave;
	98890	+ p = pSave;
	98891	+ }
	98892	+
	98893	+ fts3GetDeltaVarint2(&p1, pEnd1, &i1);
	98894	+ fts3GetDeltaVarint2(&p2, pEnd2, &i2);
	98895	+ }else if( i1<i2 ){
	98896	+ fts3PoslistCopy(0, &p1);
	98897	+ fts3GetDeltaVarint2(&p1, pEnd1, &i1);
	98898	+ }else{
	98899	+ fts3PoslistCopy(0, &p2);
	98900	+ fts3GetDeltaVarint2(&p2, pEnd2, &i2);
	98901	+ }
	98902	+ }
	98903	+ sqlite3_free(aTmp);
	98904	+ break;
	98905	+ }
	98906	+
	98907	+ default:
	98908	+ assert(!"Invalid mergetype value passed to fts3DoclistMerge()");
	98909	+ }
	98910	+
	98911	+ *pnBuffer = (p-aBuffer);
	98912	+ return SQLITE_OK;
	98913	+}
	98914	+
	98915	+/*
	98916	+** A pointer to an instance of this structure is used as the context
	98917	+** argument to sqlite3Fts3SegReaderIterate()
	98918	+*/
	98919	+typedef struct TermSelect TermSelect;
	98920	+struct TermSelect {
	98921	+ int isReqPos;
	98922	+ char aOutput; / Malloc'd output buffer */
	98923	+ int nOutput; /* Size of output in bytes */
	98924	+};
	98925	+
	98926	+/*
	98927	+** This function is used as the sqlite3Fts3SegReaderIterate() callback when
	98928	+** querying the full-text index for a doclist associated with a term or
	98929	+** term-prefix.
	98930	+*/
	98931	+static int fts3TermSelectCb(
	98932	+ Fts3Table p, / Virtual table object */
	98933	+ void pContext, / Pointer to TermSelect structure */
	98934	+ char *zTerm,
	98935	+ int nTerm,
	98936	+ char *aDoclist,
	98937	+ int nDoclist
	98938	+){
	98939	+ TermSelect pTS = (TermSelect )pContext;
	98940	+ int nNew = pTS->nOutput + nDoclist;
	98941	+ char *aNew = sqlite3_malloc(nNew);
	98942	+
	98943	+ if( !aNew ){
	98944	+ return SQLITE_NOMEM;
	98945	+ }
	98946	+
	98947	+ if( pTS->nOutput==0 ){
	98948	+ /* If this is the first term selected, copy the doclist to the output
	98949	+ ** buffer using memcpy(). TODO: Add a way to transfer control of the
	98950	+ ** aDoclist buffer from the caller so as to avoid the memcpy().
	98951	+ */
	98952	+ memcpy(aNew, aDoclist, nDoclist);
	98953	+ }else{
	98954	+ /* The output buffer is not empty. Merge doclist aDoclist with the
	98955	+ ** existing output. This can only happen with prefix-searches (as
	98956	+ ** searches for exact terms return exactly one doclist).
	98957	+ */
	98958	+ int mergetype = (pTS->isReqPos ? MERGE_POS_OR : MERGE_OR);
	98959	+ fts3DoclistMerge(mergetype, 0, 0,
	98960	+ aNew, &nNew, pTS->aOutput, pTS->nOutput, aDoclist, nDoclist
	98961	+ );
	98962	+ }
	98963	+
	98964	+ sqlite3_free(pTS->aOutput);
	98965	+ pTS->aOutput = aNew;
	98966	+ pTS->nOutput = nNew;
	98967	+
	98968	+ return SQLITE_OK;
	98969	+}
	98970	+
	98971	+/*
	98972	+** This function retreives the doclist for the specified term (or term
	98973	+** prefix) from the database.
	98974	+**
	98975	+** The returned doclist may be in one of two formats, depending on the
	98976	+** value of parameter isReqPos. If isReqPos is zero, then the doclist is
	98977	+** a sorted list of delta-compressed docids. If isReqPos is non-zero,
	98978	+** then the returned list is in the same format as is stored in the
	98979	+** database without the found length specifier at the start of on-disk
	98980	+** doclists.
	98981	+*/
	98982	+static int fts3TermSelect(
	98983	+ Fts3Table p, / Virtual table handle */
	98984	+ int iColumn, /* Column to query (or -ve for all columns) */
	98985	+ const char zTerm, / Term to query for */
	98986	+ int nTerm, /* Size of zTerm in bytes */
	98987	+ int isPrefix, /* True for a prefix search */
	98988	+ int isReqPos, /* True to include position lists in output */
	98989	+ int pnOut, / OUT: Size of buffer at ppOut /
	98990	+ char *ppOut / OUT: Malloced result buffer */
	98991	+){
	98992	+ int i;
	98993	+ TermSelect tsc;
	98994	+ Fts3SegFilter filter; /* Segment term filter configuration */
	98995	+ Fts3SegReader *apSegment = 0; / Array of segments to read data from */
	98996	+ int nSegment = 0; /* Size of apSegment array */
	98997	+ int nAlloc = 0; /* Allocated size of segment array */
	98998	+ int rc; /* Return code */
	98999	+ sqlite3_stmt pStmt; / SQL statement to scan %_segdir table */
	99000	+ int iAge = 0; /* Used to assign ages to segments */
	99001	+
	99002	+ /* Loop through the entire %_segdir table. For each segment, create a
	99003	+ ** Fts3SegReader to iterate through the subset of the segment leaves
	99004	+ ** that may contain a term that matches zTerm/nTerm. For non-prefix
	99005	+ ** searches, this is always a single leaf. For prefix searches, this
	99006	+ ** may be a contiguous block of leaves.
	99007	+ **
	99008	+ ** The code in this loop does not actually load any leaves into memory
	99009	+ ** (unless the root node happens to be a leaf). It simply examines the
	99010	+ ** b-tree structure to determine which leaves need to be inspected.
	99011	+ */
	99012	+ rc = sqlite3Fts3AllSegdirs(p, &pStmt);
	99013	+ while( rc==SQLITE_OK && SQLITE_ROW==(rc = sqlite3_step(pStmt)) ){
	99014	+ Fts3SegReader *pNew = 0;
	99015	+ int nRoot = sqlite3_column_bytes(pStmt, 4);
	99016	+ char const *zRoot = sqlite3_column_blob(pStmt, 4);
	99017	+ if( sqlite3_column_int64(pStmt, 1)==0 ){
	99018	+ /* The entire segment is stored on the root node (which must be a
	99019	+ ** leaf). Do not bother inspecting any data in this case, just
	99020	+ ** create a Fts3SegReader to scan the single leaf.
	99021	+ */
	99022	+ rc = sqlite3Fts3SegReaderNew(p, iAge, 0, 0, 0, zRoot, nRoot, &pNew);
	99023	+ }else{
	99024	+ sqlite3_int64 i1;
	99025	+ rc = fts3SelectLeaf(p, zTerm, nTerm, zRoot, nRoot, &i1);
	99026	+ if( rc==SQLITE_OK ){
	99027	+ sqlite3_int64 i2 = sqlite3_column_int64(pStmt, 2);
	99028	+ rc = sqlite3Fts3SegReaderNew(p, iAge, i1, i2, 0, 0, 0, &pNew);
	99029	+ }
	99030	+ }
	99031	+ iAge++;
	99032	+
	99033	+ /* If a new Fts3SegReader was allocated, add it to the apSegment array. */
	99034	+ assert( (rc==SQLITE_OK)==(pNew!=0) );
	99035	+ if( pNew ){
	99036	+ if( nSegment==nAlloc ){
	99037	+ nAlloc += 16;
	99038	+ Fts3SegReader pArray = (Fts3SegReader )sqlite3_realloc(
	99039	+ apSegment, nAllocsizeof(Fts3SegReader )
	99040	+ );
	99041	+ if( !pArray ){
	99042	+ sqlite3Fts3SegReaderFree(p, pNew);
	99043	+ rc = SQLITE_NOMEM;
	99044	+ goto finished;
	99045	+ }
	99046	+ apSegment = pArray;
	99047	+ }
	99048	+ apSegment[nSegment++] = pNew;
	99049	+ }
	99050	+ }
	99051	+ if( rc!=SQLITE_DONE ){
	99052	+ assert( rc!=SQLITE_OK );
	99053	+ goto finished;
	99054	+ }
	99055	+
	99056	+ memset(&tsc, 0, sizeof(TermSelect));
	99057	+ tsc.isReqPos = isReqPos;
	99058	+
	99059	+ filter.flags = FTS3_SEGMENT_IGNORE_EMPTY
	99060	+ \| (isPrefix ? FTS3_SEGMENT_PREFIX : 0)
	99061	+ \| (isReqPos ? FTS3_SEGMENT_REQUIRE_POS : 0)
	99062	+ \| (iColumn<p->nColumn ? FTS3_SEGMENT_COLUMN_FILTER : 0);
	99063	+ filter.iCol = iColumn;
	99064	+ filter.zTerm = zTerm;
	99065	+ filter.nTerm = nTerm;
	99066	+
	99067	+ rc = sqlite3Fts3SegReaderIterate(p, apSegment, nSegment, &filter,
	99068	+ fts3TermSelectCb, (void *)&tsc
	99069	+ );
	99070	+
	99071	+ if( rc==SQLITE_OK ){
	99072	+ *ppOut = tsc.aOutput;
	99073	+ *pnOut = tsc.nOutput;
	99074	+ }else{
	99075	+ sqlite3_free(tsc.aOutput);
	99076	+ }
	99077	+
	99078	+finished:
	99079	+ sqlite3_reset(pStmt);
	99080	+ for(i=0; i<nSegment; i++){
	99081	+ sqlite3Fts3SegReaderFree(p, apSegment[i]);
	99082	+ }
	99083	+ sqlite3_free(apSegment);
	99084	+ return rc;
	99085	+}
	99086	+
101399	99087
101400	99088	/*
101401	99089	** Return a DocList corresponding to the phrase *pPhrase.
101402		-**
101403		-** The resulting DL_DOCIDS doclist is stored in pResult, which is
101404		-** overwritten.
101405	99090	*/
101406		-static int docListOfPhrase(
101407		- fulltext_vtab pTab, / The full text index */
101408		- Fts3Phrase pPhrase, / Phrase to return a doclist corresponding to */
101409		- DocListType eListType, /* Either DL_DOCIDS or DL_POSITIONS */
101410		- DataBuffer pResult / Write the result here */
	99091	+static int fts3PhraseSelect(
	99092	+ Fts3Table p, / Virtual table handle */
	99093	+ Fts3Phrase pPhrase, / Phrase to return a doclist for */
	99094	+ int isReqPos, /* True if output should contain positions */
	99095	+ char *paOut, / OUT: Pointer to malloc'd result buffer */
	99096	+ int pnOut / OUT: Size of buffer at paOut /
101411	99097	){
101412		- int ii;
	99098	+ char *pOut = 0;
	99099	+ int nOut = 0;
101413	99100	int rc = SQLITE_OK;
	99101	+ int ii;
101414	99102	int iCol = pPhrase->iColumn;
101415		- DocListType eType = eListType;
101416		- assert( eType==DL_POSITIONS \|\| eType==DL_DOCIDS );
101417		- if( pPhrase->nToken>1 ){
101418		- eType = DL_POSITIONS;
101419		- }
101420		-
101421		- /* This code should never be called with buffered updates. */
101422		- assert( pTab->nPendingData<0 );
101423		-
101424		- for(ii=0; rc==SQLITE_OK && ii<pPhrase->nToken; ii++){
101425		- DataBuffer tmp;
101426		- struct PhraseToken *p = &pPhrase->aToken[ii];
101427		- rc = termSelect(pTab, iCol, p->z, p->n, p->isPrefix, eType, &tmp);
101428		- if( rc==SQLITE_OK ){
101429		- if( ii==0 ){
101430		- *pResult = tmp;
101431		- }else{
101432		- DataBuffer res = *pResult;
101433		- dataBufferInit(pResult, 0);
101434		- if( ii==(pPhrase->nToken-1) ){
101435		- eType = eListType;
101436		- }
101437		- docListPhraseMerge(
101438		- res.pData, res.nData, tmp.pData, tmp.nData, 0, 0, eType, pResult
101439		- );
101440		- dataBufferDestroy(&res);
101441		- dataBufferDestroy(&tmp);
101442		- }
101443		- }
101444		- }
101445		-
	99103	+ int isTermPos = (pPhrase->nToken>1 \|\| isReqPos);
	99104	+
	99105	+ assert( p->nPendingData==0 );
	99106	+
	99107	+ for(ii=0; ii<pPhrase->nToken; ii++){
	99108	+ struct PhraseToken *pTok = &pPhrase->aToken[ii];
	99109	+ char z = pTok->z; / Next token of the phrase */
	99110	+ int n = pTok->n; /* Size of z in bytes */
	99111	+ int isPrefix = pTok->isPrefix;/* True if token is a prefix */
	99112	+ char pList; / Pointer to token doclist */
	99113	+ int nList; /* Size of buffer at pList */
	99114	+
	99115	+ rc = fts3TermSelect(p, iCol, z, n, isPrefix, isTermPos, &nList, &pList);
	99116	+ if( rc!=SQLITE_OK ) break;
	99117	+
	99118	+ if( ii==0 ){
	99119	+ pOut = pList;
	99120	+ nOut = nList;
	99121	+ }else{
	99122	+ /* Merge the new term list and the current output. If this is the
	99123	+ ** last term in the phrase, and positions are not required in the
	99124	+ ** output of this function, the positions can be dropped as part
	99125	+ ** of this merge. Either way, the result of this merge will be
	99126	+ ** smaller than nList bytes. The code in fts3DoclistMerge() is written
	99127	+ ** so that it is safe to use pList as the output as well as an input
	99128	+ ** in this case.
	99129	+ */
	99130	+ int mergetype = MERGE_POS_PHRASE;
	99131	+ if( ii==pPhrase->nToken-1 && !isReqPos ){
	99132	+ mergetype = MERGE_PHRASE;
	99133	+ }
	99134	+ fts3DoclistMerge(mergetype, 0, 0, pList, &nOut, pOut, nOut, pList, nList);
	99135	+ sqlite3_free(pOut);
	99136	+ pOut = pList;
	99137	+ }
	99138	+ }
	99139	+
	99140	+ if( rc==SQLITE_OK ){
	99141	+ *paOut = pOut;
	99142	+ *pnOut = nOut;
	99143	+ }else{
	99144	+ sqlite3_free(pOut);
	99145	+ }
101446	99146	return rc;
101447	99147	}
101448	99148
101449	99149	/*
101450		-** Evaluate the full-text expression pExpr against fts3 table pTab. Write
101451		-** the results into pRes.
	99150	+** Evaluate the full-text expression pExpr against fts3 table pTab. Store
	99151	+** the resulting doclist in paOut and pnOut.
101452	99152	*/
101453	99153	static int evalFts3Expr(
101454		- fulltext_vtab pTab, / Fts3 Virtual table object */
101455		- Fts3Expr pExpr, / Parsed fts3 expression */
101456		- DataBuffer pRes / OUT: Write results of the expression here */
	99154	+ Fts3Table p, / Virtual table handle */
	99155	+ Fts3Expr pExpr, / Parsed fts3 expression */
	99156	+ char *paOut, / OUT: Pointer to malloc'd result buffer */
	99157	+ int pnOut / OUT: Size of buffer at paOut /
101457	99158	){
101458		- int rc = SQLITE_OK;
	99159	+ int rc = SQLITE_OK; /* Return code */
101459	99160
101460		- /* Initialize the output buffer. If this is an empty query (pExpr==0),
101461		- ** this is all that needs to be done. Empty queries produce empty
101462		- ** result sets.
101463		- */
101464		- dataBufferInit(pRes, 0);
	99161	+ /* Zero the output parameters. */
	99162	+ *paOut = 0;
	99163	+ *pnOut = 0;
101465	99164
101466	99165	if( pExpr ){
101467	99166	if( pExpr->eType==FTSQUERY_PHRASE ){
101468		- DocListType eType = DL_DOCIDS;
101469		- if( pExpr->pParent && pExpr->pParent->eType==FTSQUERY_NEAR ){
101470		- eType = DL_POSITIONS;
101471		- }
101472		- rc = docListOfPhrase(pTab, pExpr->pPhrase, eType, pRes);
	99167	+ int isReqPos = (pExpr->pParent && pExpr->pParent->eType==FTSQUERY_NEAR);
	99168	+ rc = fts3PhraseSelect(p, pExpr->pPhrase, isReqPos, paOut, pnOut);
101473	99169	}else{
101474		- DataBuffer lhs;
101475		- DataBuffer rhs;
	99170	+ char *aLeft;
	99171	+ char *aRight;
	99172	+ int nLeft;
	99173	+ int nRight;
101476	99174
101477		- dataBufferInit(&rhs, 0);
101478		- if( SQLITE_OK==(rc = evalFts3Expr(pTab, pExpr->pLeft, &lhs))
101479		- && SQLITE_OK==(rc = evalFts3Expr(pTab, pExpr->pRight, &rhs))
	99175	+ if( SQLITE_OK==(rc = evalFts3Expr(p, pExpr->pRight, &aRight, &nRight))
	99176	+ && SQLITE_OK==(rc = evalFts3Expr(p, pExpr->pLeft, &aLeft, &nLeft))
101480	99177	){
101481	99178	switch( pExpr->eType ){
101482	99179	case FTSQUERY_NEAR: {
101483		- int nToken;
101484	99180	Fts3Expr *pLeft;
101485		- DocListType eType = DL_DOCIDS;
	99181	+ Fts3Expr *pRight;
	99182	+ int mergetype = MERGE_NEAR;
	99183	+ int nParam1;
	99184	+ int nParam2;
	99185	+ char *aBuffer;
	99186	+
101486	99187	if( pExpr->pParent && pExpr->pParent->eType==FTSQUERY_NEAR ){
101487		- eType = DL_POSITIONS;
	99188	+ mergetype = MERGE_POS_NEAR;
101488	99189	}
101489	99190	pLeft = pExpr->pLeft;
101490	99191	while( pLeft->eType==FTSQUERY_NEAR ){
101491	99192	pLeft=pLeft->pRight;
101492	99193	}
101493		- assert( pExpr->pRight->eType==FTSQUERY_PHRASE );
101494		- assert( pLeft->eType==FTSQUERY_PHRASE );
101495		- nToken = pLeft->pPhrase->nToken + pExpr->pRight->pPhrase->nToken;
101496		- docListPhraseMerge(lhs.pData, lhs.nData, rhs.pData, rhs.nData,
101497		- pExpr->nNear+1, nToken, eType, pRes
101498		- );
101499		- break;
101500		- }
101501		- case FTSQUERY_NOT: {
101502		- docListExceptMerge(lhs.pData, lhs.nData, rhs.pData, rhs.nData,pRes);
101503		- break;
101504		- }
101505		- case FTSQUERY_AND: {
101506		- docListAndMerge(lhs.pData, lhs.nData, rhs.pData, rhs.nData, pRes);
101507		- break;
101508		- }
101509		- case FTSQUERY_OR: {
101510		- docListOrMerge(lhs.pData, lhs.nData, rhs.pData, rhs.nData, pRes);
101511		- break;
101512		- }
101513		- }
101514		- }
101515		- dataBufferDestroy(&lhs);
101516		- dataBufferDestroy(&rhs);
101517		- }
101518		- }
101519		-
101520		- return rc;
101521		-}
101522		-
101523		-/* TODO(shess) Refactor the code to remove this forward decl. */
101524		-static int flushPendingTerms(fulltext_vtab *v);
101525		-
101526		-/* Perform a full-text query using the search expression in
101527		-** zInput[0..nInput-1]. Return a list of matching documents
101528		-** in pResult.
101529		-**
101530		-** Queries must match column iColumn. Or if iColumn>=nColumn
101531		-** they are allowed to match against any column.
101532		-*/
101533		-static int fulltextQuery(
101534		- fulltext_vtab v, / The full text index */
101535		- int iColumn, /* Match against this column by default */
101536		- const char zInput, / The query string */
101537		- int nInput, /* Number of bytes in zInput[] */
101538		- DataBuffer pResult, / Write the result doclist here */
101539		- Fts3Expr *ppExpr / Put parsed query string here */
101540		-){
101541		- int rc;
101542		-
101543		- /* TODO(shess) Instead of flushing pendingTerms, we could query for
101544		- ** the relevant term and merge the doclist into what we receive from
101545		- ** the database. Wait and see if this is a common issue, first.
101546		- **
101547		- ** A good reason not to flush is to not generate update-related
101548		- ** error codes from here.
101549		- */
101550		-
101551		- /* Flush any buffered updates before executing the query. */
101552		- rc = flushPendingTerms(v);
101553		- if( rc!=SQLITE_OK ){
101554		- return rc;
101555		- }
101556		-
101557		- /* Parse the query passed to the MATCH operator. */
101558		- rc = sqlite3Fts3ExprParse(v->pTokenizer,
101559		- v->azColumn, v->nColumn, iColumn, zInput, nInput, ppExpr
101560		- );
101561		- if( rc!=SQLITE_OK ){
101562		- assert( 0==(*ppExpr) );
101563		- return rc;
101564		- }
101565		-
101566		- return evalFts3Expr(v, *ppExpr, pResult);
	99194	+ pRight = pExpr->pRight;
	99195	+ assert( pRight->eType==FTSQUERY_PHRASE );
	99196	+ assert( pLeft->eType==FTSQUERY_PHRASE );
	99197	+
	99198	+ nParam1 = pExpr->nNear+1;
	99199	+ nParam2 = nParam1+pLeft->pPhrase->nToken+pRight->pPhrase->nToken-2;
	99200	+ aBuffer = sqlite3_malloc(nLeft+nRight+1);
	99201	+ rc = fts3DoclistMerge(mergetype, nParam1, nParam2, aBuffer,
	99202	+ pnOut, aLeft, nLeft, aRight, nRight
	99203	+ );
	99204	+ if( rc!=SQLITE_OK ){
	99205	+ sqlite3_free(aBuffer);
	99206	+ }else{
	99207	+ *paOut = aBuffer;
	99208	+ }
	99209	+ sqlite3_free(aLeft);
	99210	+ break;
	99211	+ }
	99212	+
	99213	+ case FTSQUERY_OR: {
	99214	+ /* Allocate a buffer for the output. The maximum size is the
	99215	+ ** sum of the sizes of the two input buffers. The +1 term is
	99216	+ ** so that a buffer of zero bytes is never allocated - this can
	99217	+ ** cause fts3DoclistMerge() to incorrectly return SQLITE_NOMEM.
	99218	+ */
	99219	+ char *aBuffer = sqlite3_malloc(nRight+nLeft+1);
	99220	+ rc = fts3DoclistMerge(MERGE_OR, 0, 0, aBuffer, pnOut,
	99221	+ aLeft, nLeft, aRight, nRight
	99222	+ );
	99223	+ *paOut = aBuffer;
	99224	+ sqlite3_free(aLeft);
	99225	+ break;
	99226	+ }
	99227	+
	99228	+ case FTSQUERY_AND:
	99229	+ case FTSQUERY_NOT: {
	99230	+ assert( FTSQUERY_NOT==MERGE_NOT && FTSQUERY_AND==MERGE_AND );
	99231	+ fts3DoclistMerge(pExpr->eType, 0, 0, aLeft, pnOut,
	99232	+ aLeft, nLeft, aRight, nRight
	99233	+ );
	99234	+ *paOut = aLeft;
	99235	+ break;
	99236	+ }
	99237	+ }
	99238	+ }
	99239	+ sqlite3_free(aRight);
	99240	+ }
	99241	+ }
	99242	+
	99243	+ return rc;
101567	99244	}
101568	99245
101569	99246	/*
101570	99247	** This is the xFilter interface for the virtual table. See
101571	99248	** the virtual table xFilter method documentation for additional
101572	99249	** information.
101573	99250	**
101574		-** If idxNum==QUERY_GENERIC then do a full table scan against
	99251	+** If idxNum==FTS3_FULLSCAN_SEARCH then do a full table scan against
101575	99252	** the %_content table.
101576	99253	**
101577		-** If idxNum==QUERY_DOCID then do a docid lookup for a single entry
	99254	+** If idxNum==FTS3_DOCID_SEARCH then do a docid lookup for a single entry
101578	99255	** in the %_content table.
101579	99256	**
101580		-** If idxNum>=QUERY_FULLTEXT then use the full text index. The
	99257	+** If idxNum>=FTS3_FULLTEXT_SEARCH then use the full text index. The
101581	99258	** column on the left-hand side of the MATCH operator is column
101582		-** number idxNum-QUERY_FULLTEXT, 0 indexed. argv[0] is the right-hand
	99259	+** number idxNum-FTS3_FULLTEXT_SEARCH, 0 indexed. argv[0] is the right-hand
101583	99260	** side of the MATCH operator.
101584	99261	*/
101585	99262	/* TODO(shess) Upgrade the cursor initialization and destruction to
101586		-** account for fulltextFilter() being called multiple times on the
101587		-** same cursor. The current solution is very fragile. Apply fix to
	99263	+** account for fts3FilterMethod() being called multiple times on the
	99264	+** same cursor. The current solution is very fragile. Apply fix to
101588	99265	** fts3 as appropriate.
101589	99266	*/
101590		-static int fulltextFilter(
101591		- sqlite3_vtab_cursor pCursor, / The cursor used for this query */
101592		- int idxNum, const char idxStr, / Which indexing scheme to use */
101593		- int argc, sqlite3_value *argv / Arguments for the indexing scheme */
101594		-){
101595		- fulltext_cursor c = (fulltext_cursor ) pCursor;
101596		- fulltext_vtab *v = cursor_vtab(c);
101597		- int rc;
101598		-
101599		- FTSTRACE(("FTS3 Filter %p\n",pCursor));
101600		-
101601		- /* If the cursor has a statement that was not prepared according to
101602		- ** idxNum, clear it. I believe all calls to fulltextFilter with a
101603		- ** given cursor will have the same idxNum , but in this case it's
101604		- ** easy to be safe.
101605		- */
101606		- if( c->pStmt && c->iCursorType!=idxNum ){
101607		- sqlite3_finalize(c->pStmt);
101608		- c->pStmt = NULL;
101609		- }
101610		-
101611		- /* Get a fresh statement appropriate to idxNum. */
101612		- /* TODO(shess): Add a prepared-statement cache in the vt structure.
101613		- ** The cache must handle multiple open cursors. Easier to cache the
101614		- ** statement variants at the vt to reduce malloc/realloc/free here.
101615		- ** Or we could have a StringBuffer variant which allowed stack
101616		- ** construction for small values.
101617		- */
101618		- if( !c->pStmt ){
101619		- StringBuffer sb;
101620		- initStringBuffer(&sb);
101621		- append(&sb, "SELECT docid, ");
101622		- appendList(&sb, v->nColumn, v->azContentColumn);
101623		- append(&sb, " FROM %_content");
101624		- if( idxNum!=QUERY_GENERIC ) append(&sb, " WHERE docid = ?");
101625		- rc = sql_prepare(v->db, v->zDb, v->zName, &c->pStmt,
101626		- stringBufferData(&sb));
101627		- stringBufferDestroy(&sb);
101628		- if( rc!=SQLITE_OK ) return rc;
101629		- c->iCursorType = idxNum;
101630		- }else{
101631		- sqlite3_reset(c->pStmt);
101632		- assert( c->iCursorType==idxNum );
101633		- }
101634		-
101635		- switch( idxNum ){
101636		- case QUERY_GENERIC:
101637		- break;
101638		-
101639		- case QUERY_DOCID:
101640		- rc = sqlite3_bind_int64(c->pStmt, 1, sqlite3_value_int64(argv[0]));
101641		- if( rc!=SQLITE_OK ) return rc;
101642		- break;
101643		-
101644		- default: /* full-text search */
101645		- {
101646		- int iCol = idxNum-QUERY_FULLTEXT;
101647		- const char zQuery = (const char )sqlite3_value_text(argv[0]);
101648		- assert( idxNum<=QUERY_FULLTEXT+v->nColumn);
101649		- assert( argc==1 );
101650		- if( c->result.nData!=0 ){
101651		- /* This case happens if the same cursor is used repeatedly. */
101652		- dlrDestroy(&c->reader);
101653		- dataBufferReset(&c->result);
101654		- }else{
101655		- dataBufferInit(&c->result, 0);
101656		- }
101657		- rc = fulltextQuery(v, iCol, zQuery, -1, &c->result, &c->pExpr);
101658		- if( rc!=SQLITE_OK ) return rc;
101659		- if( c->result.nData!=0 ){
101660		- dlrInit(&c->reader, DL_DOCIDS, c->result.pData, c->result.nData);
101661		- }
101662		- break;
101663		- }
101664		- }
101665		-
101666		- return fulltextNext(pCursor);
101667		-}
101668		-
101669		-/* This is the xEof method of the virtual table. The SQLite core
101670		-** calls this routine to find out if it has reached the end of
101671		-** a query's results set.
101672		-*/
101673		-static int fulltextEof(sqlite3_vtab_cursor *pCursor){
101674		- fulltext_cursor c = (fulltext_cursor ) pCursor;
101675		- return c->eof;
101676		-}
101677		-
101678		-/* This is the xColumn method of the virtual table. The SQLite
	99267	+static int fts3FilterMethod(
	99268	+ sqlite3_vtab_cursor pCursor, / The cursor used for this query */
	99269	+ int idxNum, /* Strategy index */
	99270	+ const char idxStr, / Unused */
	99271	+ int nVal, /* Number of elements in apVal */
	99272	+ sqlite3_value *apVal / Arguments for the indexing scheme */
	99273	+){
	99274	+ const char *azSql[] = {
	99275	+ "SELECT * FROM %Q.'%q_content' WHERE docid = ?", /* non-full-table-scan */
	99276	+ "SELECT * FROM %Q.'%q_content'", /* full-table-scan */
	99277	+ };
	99278	+ int rc; /* Return code */
	99279	+ char zSql; / SQL statement used to access %_content */
	99280	+ Fts3Table p = (Fts3Table )pCursor->pVtab;
	99281	+ Fts3Cursor pCsr = (Fts3Cursor )pCursor;
	99282	+
	99283	+ assert( idxNum>=0 && idxNum<=(FTS3_FULLTEXT_SEARCH+p->nColumn) );
	99284	+ assert( nVal==0 \|\| nVal==1 );
	99285	+ assert( (nVal==0)==(idxNum==FTS3_FULLSCAN_SEARCH) );
	99286	+
	99287	+ /* In case the cursor has been used before, clear it now. */
	99288	+ sqlite3_finalize(pCsr->pStmt);
	99289	+ sqlite3_free(pCsr->aDoclist);
	99290	+ memset(&pCursor[1], 0, sizeof(Fts3Cursor)-sizeof(sqlite3_vtab_cursor));
	99291	+
	99292	+ /* Compile a SELECT statement for this cursor. For a full-table-scan, the
	99293	+ ** statement loops through all rows of the %_content table. For a
	99294	+ ** full-text query or docid lookup, the statement retrieves a single
	99295	+ ** row by docid.
	99296	+ */
	99297	+ zSql = sqlite3_mprintf(azSql[idxNum==FTS3_FULLSCAN_SEARCH], p->zDb, p->zName);
	99298	+ if( !zSql ){
	99299	+ rc = SQLITE_NOMEM;
	99300	+ }else{
	99301	+ rc = sqlite3_prepare_v2(p->db, zSql, -1, &pCsr->pStmt, 0);
	99302	+ sqlite3_free(zSql);
	99303	+ }
	99304	+ if( rc!=SQLITE_OK ) return rc;
	99305	+ pCsr->eType = idxNum;
	99306	+
	99307	+ if( idxNum==FTS3_DOCID_SEARCH ){
	99308	+ rc = sqlite3_bind_value(pCsr->pStmt, 1, apVal[0]);
	99309	+ }else if( idxNum!=FTS3_FULLSCAN_SEARCH ){
	99310	+ int iCol = idxNum-FTS3_FULLTEXT_SEARCH;
	99311	+ const char zQuery = (const char )sqlite3_value_text(apVal[0]);
	99312	+
	99313	+ rc = sqlite3Fts3PendingTermsFlush(p);
	99314	+ if( rc!=SQLITE_OK ) return rc;
	99315	+
	99316	+ rc = sqlite3Fts3ExprParse(p->pTokenizer, p->azColumn, p->nColumn,
	99317	+ iCol, zQuery, -1, &pCsr->pExpr
	99318	+ );
	99319	+ if( rc!=SQLITE_OK ) return rc;
	99320	+
	99321	+ rc = evalFts3Expr(p, pCsr->pExpr, &pCsr->aDoclist, &pCsr->nDoclist);
	99322	+ pCsr->pNextId = pCsr->aDoclist;
	99323	+ pCsr->iPrevId = 0;
	99324	+ }
	99325	+
	99326	+ if( rc!=SQLITE_OK ) return rc;
	99327	+ return fts3NextMethod(pCursor);
	99328	+}
	99329	+
	99330	+/*
	99331	+** This is the xEof method of the virtual table. SQLite calls this
	99332	+** routine to find out if it has reached the end of a result set.
	99333	+*/
	99334	+static int fts3EofMethod(sqlite3_vtab_cursor *pCursor){
	99335	+ return ((Fts3Cursor *)pCursor)->isEof;
	99336	+}
	99337	+
	99338	+/*
	99339	+** This is the xColumn method of the virtual table. The SQLite
101679	99340	** core calls this method during a query when it needs the value
101680	99341	** of a column from the virtual table. This method needs to use
101681	99342	** one of the sqlite3_result_*() routines to store the requested
101682	99343	** value back in the pContext.
101683	99344	*/
101684		-static int fulltextColumn(sqlite3_vtab_cursor *pCursor,
	99345	+static int fts3ColumnMethod(sqlite3_vtab_cursor *pCursor,
101685	99346	sqlite3_context *pContext, int idxCol){
101686		- fulltext_cursor c = (fulltext_cursor ) pCursor;
101687		- fulltext_vtab *v = cursor_vtab(c);
	99347	+ Fts3Cursor c = (Fts3Cursor ) pCursor;
	99348	+ Fts3Table *v = cursor_vtab(c);
	99349	+ int rc = fts3CursorSeek(c);
	99350	+ if( rc!=SQLITE_OK ){
	99351	+ return rc;
	99352	+ }
101688	99353
101689	99354	if( idxCol<v->nColumn ){
101690	99355	sqlite3_value *pVal = sqlite3_column_value(c->pStmt, idxCol+1);
101691	99356	sqlite3_result_value(pContext, pVal);
101692	99357	}else if( idxCol==v->nColumn ){
		@@ -101700,2884 +99365,227 @@
101700	99365	sqlite3_result_value(pContext, pVal);
101701	99366	}
101702	99367	return SQLITE_OK;
101703	99368	}
101704	99369
101705		-/* This is the xRowid method. The SQLite core calls this routine to
101706		-** retrieve the rowid for the current row of the result set. fts3
101707		-** exposes %_content.docid as the rowid for the virtual table. The
	99370	+/*
	99371	+** This is the xRowid method. The SQLite core calls this routine to
	99372	+** retrieve the rowid for the current row of the result set. fts3
	99373	+** exposes %_content.docid as the rowid for the virtual table. The
101708	99374	** rowid should be written to *pRowid.
101709	99375	*/
101710		-static int fulltextRowid(sqlite3_vtab_cursor pCursor, sqlite_int64 pRowid){
101711		- fulltext_cursor c = (fulltext_cursor ) pCursor;
101712		-
101713		- *pRowid = sqlite3_column_int64(c->pStmt, 0);
101714		- return SQLITE_OK;
101715		-}
101716		-
101717		-/* Add all terms in [zText] to pendingTerms table. If [iColumn] > 0,
101718		-** we also store positions and offsets in the hash table using that
101719		-** column number.
101720		-*/
101721		-static int buildTerms(fulltext_vtab *v, sqlite_int64 iDocid,
101722		- const char *zText, int iColumn){
101723		- sqlite3_tokenizer *pTokenizer = v->pTokenizer;
101724		- sqlite3_tokenizer_cursor *pCursor;
101725		- const char *pToken;
101726		- int nTokenBytes;
101727		- int iStartOffset, iEndOffset, iPosition;
101728		- int rc;
101729		-
101730		- rc = pTokenizer->pModule->xOpen(pTokenizer, zText, -1, &pCursor);
101731		- if( rc!=SQLITE_OK ) return rc;
101732		-
101733		- pCursor->pTokenizer = pTokenizer;
101734		- while( SQLITE_OK==(rc=pTokenizer->pModule->xNext(pCursor,
101735		- &pToken, &nTokenBytes,
101736		- &iStartOffset, &iEndOffset,
101737		- &iPosition)) ){
101738		- DLCollector *p;
101739		- int nData; /* Size of doclist before our update. */
101740		-
101741		- /* Positions can't be negative; we use -1 as a terminator
101742		- * internally. Token can't be NULL or empty. */
101743		- if( iPosition<0 \|\| pToken == NULL \|\| nTokenBytes == 0 ){
101744		- rc = SQLITE_ERROR;
101745		- break;
101746		- }
101747		-
101748		- p = fts3HashFind(&v->pendingTerms, pToken, nTokenBytes);
101749		- if( p==NULL ){
101750		- nData = 0;
101751		- p = dlcNew(iDocid, DL_DEFAULT);
101752		- fts3HashInsert(&v->pendingTerms, pToken, nTokenBytes, p);
101753		-
101754		- /* Overhead for our hash table entry, the key, and the value. */
101755		- v->nPendingData += sizeof(struct fts3HashElem)+sizeof(*p)+nTokenBytes;
101756		- }else{
101757		- nData = p->b.nData;
101758		- if( p->dlw.iPrevDocid!=iDocid ) dlcNext(p, iDocid);
101759		- }
101760		- if( iColumn>=0 ){
101761		- dlcAddPos(p, iColumn, iPosition, iStartOffset, iEndOffset);
101762		- }
101763		-
101764		- /* Accumulate data added by dlcNew or dlcNext, and dlcAddPos. */
101765		- v->nPendingData += p->b.nData-nData;
101766		- }
101767		-
101768		- /* TODO(shess) Check return? Should this be able to cause errors at
101769		- ** this point? Actually, same question about sqlite3_finalize(),
101770		- ** though one could argue that failure there means that the data is
101771		- ** not durable. ponder
101772		- */
101773		- pTokenizer->pModule->xClose(pCursor);
101774		- if( SQLITE_DONE == rc ) return SQLITE_OK;
101775		- return rc;
101776		-}
101777		-
101778		-/* Add doclists for all terms in [pValues] to pendingTerms table. */
101779		-static int insertTerms(fulltext_vtab *v, sqlite_int64 iDocid,
101780		- sqlite3_value **pValues){
101781		- int i;
101782		- for(i = 0; i < v->nColumn ; ++i){
101783		- char zText = (char)sqlite3_value_text(pValues[i]);
101784		- int rc = buildTerms(v, iDocid, zText, i);
101785		- if( rc!=SQLITE_OK ) return rc;
101786		- }
101787		- return SQLITE_OK;
101788		-}
101789		-
101790		-/* Add empty doclists for all terms in the given row's content to
101791		-** pendingTerms.
101792		-*/
101793		-static int deleteTerms(fulltext_vtab *v, sqlite_int64 iDocid){
101794		- const char **pValues;
101795		- int i, rc;
101796		-
101797		- /* TODO(shess) Should we allow such tables at all? */
101798		- if( DL_DEFAULT==DL_DOCIDS ) return SQLITE_ERROR;
101799		-
101800		- rc = content_select(v, iDocid, &pValues);
101801		- if( rc!=SQLITE_OK ) return rc;
101802		-
101803		- for(i = 0 ; i < v->nColumn; ++i) {
101804		- rc = buildTerms(v, iDocid, pValues[i], -1);
101805		- if( rc!=SQLITE_OK ) break;
101806		- }
101807		-
101808		- freeStringArray(v->nColumn, pValues);
101809		- return SQLITE_OK;
101810		-}
101811		-
101812		-/* TODO(shess) Refactor the code to remove this forward decl. */
101813		-static int initPendingTerms(fulltext_vtab *v, sqlite_int64 iDocid);
101814		-
101815		-/* Insert a row into the %_content table; set *piDocid to be the ID of the
101816		-** new row. Add doclists for terms to pendingTerms.
101817		-*/
101818		-static int index_insert(fulltext_vtab v, sqlite3_value pRequestDocid,
101819		- sqlite3_value *pValues, sqlite_int64 piDocid){
101820		- int rc;
101821		-
101822		- rc = content_insert(v, pRequestDocid, pValues); /* execute an SQL INSERT */
101823		- if( rc!=SQLITE_OK ) return rc;
101824		-
101825		- /* docid column is an alias for rowid. */
101826		- *piDocid = sqlite3_last_insert_rowid(v->db);
101827		- rc = initPendingTerms(v, *piDocid);
101828		- if( rc!=SQLITE_OK ) return rc;
101829		-
101830		- return insertTerms(v, *piDocid, pValues);
101831		-}
101832		-
101833		-/* Delete a row from the %_content table; add empty doclists for terms
101834		-** to pendingTerms.
101835		-*/
101836		-static int index_delete(fulltext_vtab *v, sqlite_int64 iRow){
101837		- int rc = initPendingTerms(v, iRow);
101838		- if( rc!=SQLITE_OK ) return rc;
101839		-
101840		- rc = deleteTerms(v, iRow);
101841		- if( rc!=SQLITE_OK ) return rc;
101842		-
101843		- return content_delete(v, iRow); /* execute an SQL DELETE */
101844		-}
101845		-
101846		-/* Update a row in the %_content table; add delete doclists to
101847		-** pendingTerms for old terms not in the new data, add insert doclists
101848		-** to pendingTerms for terms in the new data.
101849		-*/
101850		-static int index_update(fulltext_vtab *v, sqlite_int64 iRow,
101851		- sqlite3_value **pValues){
101852		- int rc = initPendingTerms(v, iRow);
101853		- if( rc!=SQLITE_OK ) return rc;
101854		-
101855		- /* Generate an empty doclist for each term that previously appeared in this
101856		- * row. */
101857		- rc = deleteTerms(v, iRow);
101858		- if( rc!=SQLITE_OK ) return rc;
101859		-
101860		- rc = content_update(v, pValues, iRow); /* execute an SQL UPDATE */
101861		- if( rc!=SQLITE_OK ) return rc;
101862		-
101863		- /* Now add positions for terms which appear in the updated row. */
101864		- return insertTerms(v, iRow, pValues);
101865		-}
101866		-
101867		-/*******************************************************************/
101868		-/* InteriorWriter is used to collect terms and block references into
101869		-** interior nodes in %_segments. See commentary at top of file for
101870		-** format.
101871		-*/
101872		-
101873		-/* How large interior nodes can grow. */
101874		-#define INTERIOR_MAX 2048
101875		-
101876		-/* Minimum number of terms per interior node (except the root). This
101877		-** prevents large terms from making the tree too skinny - must be >0
101878		-** so that the tree always makes progress. Note that the min tree
101879		-** fanout will be INTERIOR_MIN_TERMS+1.
101880		-*/
101881		-#define INTERIOR_MIN_TERMS 7
101882		-#if INTERIOR_MIN_TERMS<1
101883		-# error INTERIOR_MIN_TERMS must be greater than 0.
101884		-#endif
101885		-
101886		-/* ROOT_MAX controls how much data is stored inline in the segment
101887		-** directory.
101888		-*/
101889		-/* TODO(shess) Push ROOT_MAX down to whoever is writing things. It's
101890		-** only here so that interiorWriterRootInfo() and leafWriterRootInfo()
101891		-** can both see it, but if the caller passed it in, we wouldn't even
101892		-** need a define.
101893		-*/
101894		-#define ROOT_MAX 1024
101895		-#if ROOT_MAX<VARINT_MAX*2
101896		-# error ROOT_MAX must have enough space for a header.
101897		-#endif
101898		-
101899		-/* InteriorBlock stores a linked-list of interior blocks while a lower
101900		-** layer is being constructed.
101901		-*/
101902		-typedef struct InteriorBlock {
101903		- DataBuffer term; /* Leftmost term in block's subtree. */
101904		- DataBuffer data; /* Accumulated data for the block. */
101905		- struct InteriorBlock *next;
101906		-} InteriorBlock;
101907		-
101908		-static InteriorBlock *interiorBlockNew(int iHeight, sqlite_int64 iChildBlock,
101909		- const char *pTerm, int nTerm){
101910		- InteriorBlock *block = sqlite3_malloc(sizeof(InteriorBlock));
101911		- char c[VARINT_MAX+VARINT_MAX];
101912		- int n;
101913		-
101914		- if( block ){
101915		- memset(block, 0, sizeof(*block));
101916		- dataBufferInit(&block->term, 0);
101917		- dataBufferReplace(&block->term, pTerm, nTerm);
101918		-
101919		- n = fts3PutVarint(c, iHeight);
101920		- n += fts3PutVarint(c+n, iChildBlock);
101921		- dataBufferInit(&block->data, INTERIOR_MAX);
101922		- dataBufferReplace(&block->data, c, n);
101923		- }
101924		- return block;
101925		-}
101926		-
101927		-#ifndef NDEBUG
101928		-/* Verify that the data is readable as an interior node. */
101929		-static void interiorBlockValidate(InteriorBlock *pBlock){
101930		- const char *pData = pBlock->data.pData;
101931		- int nData = pBlock->data.nData;
101932		- int n, iDummy;
101933		- sqlite_int64 iBlockid;
101934		-
101935		- assert( nData>0 );
101936		- assert( pData!=0 );
101937		- assert( pData+nData>pData );
101938		-
101939		- /* Must lead with height of node as a varint(n), n>0 */
101940		- n = fts3GetVarint32(pData, &iDummy);
101941		- assert( n>0 );
101942		- assert( iDummy>0 );
101943		- assert( n<nData );
101944		- pData += n;
101945		- nData -= n;
101946		-
101947		- /* Must contain iBlockid. */
101948		- n = fts3GetVarint(pData, &iBlockid);
101949		- assert( n>0 );
101950		- assert( n<=nData );
101951		- pData += n;
101952		- nData -= n;
101953		-
101954		- /* Zero or more terms of positive length */
101955		- if( nData!=0 ){
101956		- /* First term is not delta-encoded. */
101957		- n = fts3GetVarint32(pData, &iDummy);
101958		- assert( n>0 );
101959		- assert( iDummy>0 );
101960		- assert( n+iDummy>0);
101961		- assert( n+iDummy<=nData );
101962		- pData += n+iDummy;
101963		- nData -= n+iDummy;
101964		-
101965		- /* Following terms delta-encoded. */
101966		- while( nData!=0 ){
101967		- /* Length of shared prefix. */
101968		- n = fts3GetVarint32(pData, &iDummy);
101969		- assert( n>0 );
101970		- assert( iDummy>=0 );
101971		- assert( n<nData );
101972		- pData += n;
101973		- nData -= n;
101974		-
101975		- /* Length and data of distinct suffix. */
101976		- n = fts3GetVarint32(pData, &iDummy);
101977		- assert( n>0 );
101978		- assert( iDummy>0 );
101979		- assert( n+iDummy>0);
101980		- assert( n+iDummy<=nData );
101981		- pData += n+iDummy;
101982		- nData -= n+iDummy;
101983		- }
101984		- }
101985		-}
101986		-#define ASSERT_VALID_INTERIOR_BLOCK(x) interiorBlockValidate(x)
101987		-#else
101988		-#define ASSERT_VALID_INTERIOR_BLOCK(x) assert( 1 )
101989		-#endif
101990		-
101991		-typedef struct InteriorWriter {
101992		- int iHeight; /* from 0 at leaves. */
101993		- InteriorBlock first, last;
101994		- struct InteriorWriter *parentWriter;
101995		-
101996		- DataBuffer term; /* Last term written to block "last". */
101997		- sqlite_int64 iOpeningChildBlock; /* First child block in block "last". */
101998		-#ifndef NDEBUG
101999		- sqlite_int64 iLastChildBlock; /* for consistency checks. */
102000		-#endif
102001		-} InteriorWriter;
102002		-
102003		-/* Initialize an interior node where pTerm[nTerm] marks the leftmost
102004		-** term in the tree. iChildBlock is the leftmost child block at the
102005		-** next level down the tree.
102006		-*/
102007		-static void interiorWriterInit(int iHeight, const char *pTerm, int nTerm,
102008		- sqlite_int64 iChildBlock,
102009		- InteriorWriter *pWriter){
102010		- InteriorBlock *block;
102011		- assert( iHeight>0 );
102012		- CLEAR(pWriter);
102013		-
102014		- pWriter->iHeight = iHeight;
102015		- pWriter->iOpeningChildBlock = iChildBlock;
102016		-#ifndef NDEBUG
102017		- pWriter->iLastChildBlock = iChildBlock;
102018		-#endif
102019		- block = interiorBlockNew(iHeight, iChildBlock, pTerm, nTerm);
102020		- pWriter->last = pWriter->first = block;
102021		- ASSERT_VALID_INTERIOR_BLOCK(pWriter->last);
102022		- dataBufferInit(&pWriter->term, 0);
102023		-}
102024		-
102025		-/* Append the child node rooted at iChildBlock to the interior node,
102026		-** with pTerm[nTerm] as the leftmost term in iChildBlock's subtree.
102027		-*/
102028		-static void interiorWriterAppend(InteriorWriter *pWriter,
102029		- const char *pTerm, int nTerm,
102030		- sqlite_int64 iChildBlock){
102031		- char c[VARINT_MAX+VARINT_MAX];
102032		- int n, nPrefix = 0;
102033		-
102034		- ASSERT_VALID_INTERIOR_BLOCK(pWriter->last);
102035		-
102036		- /* The first term written into an interior node is actually
102037		- ** associated with the second child added (the first child was added
102038		- ** in interiorWriterInit, or in the if clause at the bottom of this
102039		- ** function). That term gets encoded straight up, with nPrefix left
102040		- ** at 0.
102041		- */
102042		- if( pWriter->term.nData==0 ){
102043		- n = fts3PutVarint(c, nTerm);
102044		- }else{
102045		- while( nPrefix<pWriter->term.nData &&
102046		- pTerm[nPrefix]==pWriter->term.pData[nPrefix] ){
102047		- nPrefix++;
102048		- }
102049		-
102050		- n = fts3PutVarint(c, nPrefix);
102051		- n += fts3PutVarint(c+n, nTerm-nPrefix);
102052		- }
102053		-
102054		-#ifndef NDEBUG
102055		- pWriter->iLastChildBlock++;
102056		-#endif
102057		- assert( pWriter->iLastChildBlock==iChildBlock );
102058		-
102059		- /* Overflow to a new block if the new term makes the current block
102060		- ** too big, and the current block already has enough terms.
102061		- */
102062		- if( pWriter->last->data.nData+n+nTerm-nPrefix>INTERIOR_MAX &&
102063		- iChildBlock-pWriter->iOpeningChildBlock>INTERIOR_MIN_TERMS ){
102064		- pWriter->last->next = interiorBlockNew(pWriter->iHeight, iChildBlock,
102065		- pTerm, nTerm);
102066		- pWriter->last = pWriter->last->next;
102067		- pWriter->iOpeningChildBlock = iChildBlock;
102068		- dataBufferReset(&pWriter->term);
102069		- }else{
102070		- dataBufferAppend2(&pWriter->last->data, c, n,
102071		- pTerm+nPrefix, nTerm-nPrefix);
102072		- dataBufferReplace(&pWriter->term, pTerm, nTerm);
102073		- }
102074		- ASSERT_VALID_INTERIOR_BLOCK(pWriter->last);
102075		-}
102076		-
102077		-/* Free the space used by pWriter, including the linked-list of
102078		-** InteriorBlocks, and parentWriter, if present.
102079		-*/
102080		-static int interiorWriterDestroy(InteriorWriter *pWriter){
102081		- InteriorBlock *block = pWriter->first;
102082		-
102083		- while( block!=NULL ){
102084		- InteriorBlock *b = block;
102085		- block = block->next;
102086		- dataBufferDestroy(&b->term);
102087		- dataBufferDestroy(&b->data);
102088		- sqlite3_free(b);
102089		- }
102090		- if( pWriter->parentWriter!=NULL ){
102091		- interiorWriterDestroy(pWriter->parentWriter);
102092		- sqlite3_free(pWriter->parentWriter);
102093		- }
102094		- dataBufferDestroy(&pWriter->term);
102095		- SCRAMBLE(pWriter);
102096		- return SQLITE_OK;
102097		-}
102098		-
102099		-/* If pWriter can fit entirely in ROOT_MAX, return it as the root info
102100		-** directly, leaving *piEndBlockid unchanged. Otherwise, flush
102101		-** pWriter to %_segments, building a new layer of interior nodes, and
102102		-** recursively ask for their root into.
102103		-*/
102104		-static int interiorWriterRootInfo(fulltext_vtab v, InteriorWriter pWriter,
102105		- char *ppRootInfo, int pnRootInfo,
102106		- sqlite_int64 *piEndBlockid){
102107		- InteriorBlock *block = pWriter->first;
102108		- sqlite_int64 iBlockid = 0;
102109		- int rc;
102110		-
102111		- /* If we can fit the segment inline */
102112		- if( block==pWriter->last && block->data.nData<ROOT_MAX ){
102113		- *ppRootInfo = block->data.pData;
102114		- *pnRootInfo = block->data.nData;
102115		- return SQLITE_OK;
102116		- }
102117		-
102118		- /* Flush the first block to %_segments, and create a new level of
102119		- ** interior node.
102120		- */
102121		- ASSERT_VALID_INTERIOR_BLOCK(block);
102122		- rc = block_insert(v, block->data.pData, block->data.nData, &iBlockid);
102123		- if( rc!=SQLITE_OK ) return rc;
102124		- *piEndBlockid = iBlockid;
102125		-
102126		- pWriter->parentWriter = sqlite3_malloc(sizeof(*pWriter->parentWriter));
102127		- interiorWriterInit(pWriter->iHeight+1,
102128		- block->term.pData, block->term.nData,
102129		- iBlockid, pWriter->parentWriter);
102130		-
102131		- /* Flush additional blocks and append to the higher interior
102132		- ** node.
102133		- */
102134		- for(block=block->next; block!=NULL; block=block->next){
102135		- ASSERT_VALID_INTERIOR_BLOCK(block);
102136		- rc = block_insert(v, block->data.pData, block->data.nData, &iBlockid);
102137		- if( rc!=SQLITE_OK ) return rc;
102138		- *piEndBlockid = iBlockid;
102139		-
102140		- interiorWriterAppend(pWriter->parentWriter,
102141		- block->term.pData, block->term.nData, iBlockid);
102142		- }
102143		-
102144		- /* Parent node gets the chance to be the root. */
102145		- return interiorWriterRootInfo(v, pWriter->parentWriter,
102146		- ppRootInfo, pnRootInfo, piEndBlockid);
102147		-}
102148		-
102149		-/****************************************************************/
102150		-/* InteriorReader is used to read off the data from an interior node
102151		-** (see comment at top of file for the format).
102152		-*/
102153		-typedef struct InteriorReader {
102154		- const char *pData;
102155		- int nData;
102156		-
102157		- DataBuffer term; /* previous term, for decoding term delta. */
102158		-
102159		- sqlite_int64 iBlockid;
102160		-} InteriorReader;
102161		-
102162		-static void interiorReaderDestroy(InteriorReader *pReader){
102163		- dataBufferDestroy(&pReader->term);
102164		- SCRAMBLE(pReader);
102165		-}
102166		-
102167		-/* TODO(shess) The assertions are great, but what if we're in NDEBUG
102168		-** and the blob is empty or otherwise contains suspect data?
102169		-*/
102170		-static void interiorReaderInit(const char *pData, int nData,
102171		- InteriorReader *pReader){
102172		- int n, nTerm;
102173		-
102174		- /* Require at least the leading flag byte */
102175		- assert( nData>0 );
102176		- assert( pData[0]!='\0' );
102177		-
102178		- CLEAR(pReader);
102179		-
102180		- /* Decode the base blockid, and set the cursor to the first term. */
102181		- n = fts3GetVarint(pData+1, &pReader->iBlockid);
102182		- assert( 1+n<=nData );
102183		- pReader->pData = pData+1+n;
102184		- pReader->nData = nData-(1+n);
102185		-
102186		- /* A single-child interior node (such as when a leaf node was too
102187		- ** large for the segment directory) won't have any terms.
102188		- ** Otherwise, decode the first term.
102189		- */
102190		- if( pReader->nData==0 ){
102191		- dataBufferInit(&pReader->term, 0);
102192		- }else{
102193		- n = fts3GetVarint32(pReader->pData, &nTerm);
102194		- dataBufferInit(&pReader->term, nTerm);
102195		- dataBufferReplace(&pReader->term, pReader->pData+n, nTerm);
102196		- assert( n+nTerm<=pReader->nData );
102197		- pReader->pData += n+nTerm;
102198		- pReader->nData -= n+nTerm;
102199		- }
102200		-}
102201		-
102202		-static int interiorReaderAtEnd(InteriorReader *pReader){
102203		- return pReader->term.nData==0;
102204		-}
102205		-
102206		-static sqlite_int64 interiorReaderCurrentBlockid(InteriorReader *pReader){
102207		- return pReader->iBlockid;
102208		-}
102209		-
102210		-static int interiorReaderTermBytes(InteriorReader *pReader){
102211		- assert( !interiorReaderAtEnd(pReader) );
102212		- return pReader->term.nData;
102213		-}
102214		-static const char interiorReaderTerm(InteriorReader pReader){
102215		- assert( !interiorReaderAtEnd(pReader) );
102216		- return pReader->term.pData;
102217		-}
102218		-
102219		-/* Step forward to the next term in the node. */
102220		-static void interiorReaderStep(InteriorReader *pReader){
102221		- assert( !interiorReaderAtEnd(pReader) );
102222		-
102223		- /* If the last term has been read, signal eof, else construct the
102224		- ** next term.
102225		- */
102226		- if( pReader->nData==0 ){
102227		- dataBufferReset(&pReader->term);
102228		- }else{
102229		- int n, nPrefix, nSuffix;
102230		-
102231		- n = fts3GetVarint32(pReader->pData, &nPrefix);
102232		- n += fts3GetVarint32(pReader->pData+n, &nSuffix);
102233		-
102234		- /* Truncate the current term and append suffix data. */
102235		- pReader->term.nData = nPrefix;
102236		- dataBufferAppend(&pReader->term, pReader->pData+n, nSuffix);
102237		-
102238		- assert( n+nSuffix<=pReader->nData );
102239		- pReader->pData += n+nSuffix;
102240		- pReader->nData -= n+nSuffix;
102241		- }
102242		- pReader->iBlockid++;
102243		-}
102244		-
102245		-/* Compare the current term to pTerm[nTerm], returning strcmp-style
102246		-** results. If isPrefix, equality means equal through nTerm bytes.
102247		-*/
102248		-static int interiorReaderTermCmp(InteriorReader *pReader,
102249		- const char *pTerm, int nTerm, int isPrefix){
102250		- const char *pReaderTerm = interiorReaderTerm(pReader);
102251		- int nReaderTerm = interiorReaderTermBytes(pReader);
102252		- int c, n = nReaderTerm<nTerm ? nReaderTerm : nTerm;
102253		-
102254		- if( n==0 ){
102255		- if( nReaderTerm>0 ) return -1;
102256		- if( nTerm>0 ) return 1;
102257		- return 0;
102258		- }
102259		-
102260		- c = memcmp(pReaderTerm, pTerm, n);
102261		- if( c!=0 ) return c;
102262		- if( isPrefix && n==nTerm ) return 0;
102263		- return nReaderTerm - nTerm;
102264		-}
102265		-
102266		-/****************************************************************/
102267		-/* LeafWriter is used to collect terms and associated doclist data
102268		-** into leaf blocks in %_segments (see top of file for format info).
102269		-** Expected usage is:
102270		-**
102271		-** LeafWriter writer;
102272		-** leafWriterInit(0, 0, &writer);
102273		-** while( sorted_terms_left_to_process ){
102274		-** // data is doclist data for that term.
102275		-** rc = leafWriterStep(v, &writer, pTerm, nTerm, pData, nData);
102276		-** if( rc!=SQLITE_OK ) goto err;
102277		-** }
102278		-** rc = leafWriterFinalize(v, &writer);
102279		-**err:
102280		-** leafWriterDestroy(&writer);
102281		-** return rc;
102282		-**
102283		-** leafWriterStep() may write a collected leaf out to %_segments.
102284		-** leafWriterFinalize() finishes writing any buffered data and stores
102285		-** a root node in %_segdir. leafWriterDestroy() frees all buffers and
102286		-** InteriorWriters allocated as part of writing this segment.
102287		-**
102288		-** TODO(shess) Document leafWriterStepMerge().
102289		-*/
102290		-
102291		-/* Put terms with data this big in their own block. */
102292		-#define STANDALONE_MIN 1024
102293		-
102294		-/* Keep leaf blocks below this size. */
102295		-#define LEAF_MAX 2048
102296		-
102297		-typedef struct LeafWriter {
102298		- int iLevel;
102299		- int idx;
102300		- sqlite_int64 iStartBlockid; /* needed to create the root info */
102301		- sqlite_int64 iEndBlockid; /* when we're done writing. */
102302		-
102303		- DataBuffer term; /* previous encoded term */
102304		- DataBuffer data; /* encoding buffer */
102305		-
102306		- /* bytes of first term in the current node which distinguishes that
102307		- ** term from the last term of the previous node.
102308		- */
102309		- int nTermDistinct;
102310		-
102311		- InteriorWriter parentWriter; /* if we overflow */
102312		- int has_parent;
102313		-} LeafWriter;
102314		-
102315		-static void leafWriterInit(int iLevel, int idx, LeafWriter *pWriter){
102316		- CLEAR(pWriter);
102317		- pWriter->iLevel = iLevel;
102318		- pWriter->idx = idx;
102319		-
102320		- dataBufferInit(&pWriter->term, 32);
102321		-
102322		- /* Start out with a reasonably sized block, though it can grow. */
102323		- dataBufferInit(&pWriter->data, LEAF_MAX);
102324		-}
102325		-
102326		-#ifndef NDEBUG
102327		-/* Verify that the data is readable as a leaf node. */
102328		-static void leafNodeValidate(const char *pData, int nData){
102329		- int n, iDummy;
102330		-
102331		- if( nData==0 ) return;
102332		- assert( nData>0 );
102333		- assert( pData!=0 );
102334		- assert( pData+nData>pData );
102335		-
102336		- /* Must lead with a varint(0) */
102337		- n = fts3GetVarint32(pData, &iDummy);
102338		- assert( iDummy==0 );
102339		- assert( n>0 );
102340		- assert( n<nData );
102341		- pData += n;
102342		- nData -= n;
102343		-
102344		- /* Leading term length and data must fit in buffer. */
102345		- n = fts3GetVarint32(pData, &iDummy);
102346		- assert( n>0 );
102347		- assert( iDummy>0 );
102348		- assert( n+iDummy>0 );
102349		- assert( n+iDummy<nData );
102350		- pData += n+iDummy;
102351		- nData -= n+iDummy;
102352		-
102353		- /* Leading term's doclist length and data must fit. */
102354		- n = fts3GetVarint32(pData, &iDummy);
102355		- assert( n>0 );
102356		- assert( iDummy>0 );
102357		- assert( n+iDummy>0 );
102358		- assert( n+iDummy<=nData );
102359		- ASSERT_VALID_DOCLIST(DL_DEFAULT, pData+n, iDummy, NULL);
102360		- pData += n+iDummy;
102361		- nData -= n+iDummy;
102362		-
102363		- /* Verify that trailing terms and doclists also are readable. */
102364		- while( nData!=0 ){
102365		- n = fts3GetVarint32(pData, &iDummy);
102366		- assert( n>0 );
102367		- assert( iDummy>=0 );
102368		- assert( n<nData );
102369		- pData += n;
102370		- nData -= n;
102371		- n = fts3GetVarint32(pData, &iDummy);
102372		- assert( n>0 );
102373		- assert( iDummy>0 );
102374		- assert( n+iDummy>0 );
102375		- assert( n+iDummy<nData );
102376		- pData += n+iDummy;
102377		- nData -= n+iDummy;
102378		-
102379		- n = fts3GetVarint32(pData, &iDummy);
102380		- assert( n>0 );
102381		- assert( iDummy>0 );
102382		- assert( n+iDummy>0 );
102383		- assert( n+iDummy<=nData );
102384		- ASSERT_VALID_DOCLIST(DL_DEFAULT, pData+n, iDummy, NULL);
102385		- pData += n+iDummy;
102386		- nData -= n+iDummy;
102387		- }
102388		-}
102389		-#define ASSERT_VALID_LEAF_NODE(p, n) leafNodeValidate(p, n)
102390		-#else
102391		-#define ASSERT_VALID_LEAF_NODE(p, n) assert( 1 )
102392		-#endif
102393		-
102394		-/* Flush the current leaf node to %_segments, and adding the resulting
102395		-** blockid and the starting term to the interior node which will
102396		-** contain it.
102397		-*/
102398		-static int leafWriterInternalFlush(fulltext_vtab v, LeafWriter pWriter,
102399		- int iData, int nData){
102400		- sqlite_int64 iBlockid = 0;
102401		- const char *pStartingTerm;
102402		- int nStartingTerm, rc, n;
102403		-
102404		- /* Must have the leading varint(0) flag, plus at least some
102405		- ** valid-looking data.
102406		- */
102407		- assert( nData>2 );
102408		- assert( iData>=0 );
102409		- assert( iData+nData<=pWriter->data.nData );
102410		- ASSERT_VALID_LEAF_NODE(pWriter->data.pData+iData, nData);
102411		-
102412		- rc = block_insert(v, pWriter->data.pData+iData, nData, &iBlockid);
102413		- if( rc!=SQLITE_OK ) return rc;
102414		- assert( iBlockid!=0 );
102415		-
102416		- /* Reconstruct the first term in the leaf for purposes of building
102417		- ** the interior node.
102418		- */
102419		- n = fts3GetVarint32(pWriter->data.pData+iData+1, &nStartingTerm);
102420		- pStartingTerm = pWriter->data.pData+iData+1+n;
102421		- assert( pWriter->data.nData>iData+1+n+nStartingTerm );
102422		- assert( pWriter->nTermDistinct>0 );
102423		- assert( pWriter->nTermDistinct<=nStartingTerm );
102424		- nStartingTerm = pWriter->nTermDistinct;
102425		-
102426		- if( pWriter->has_parent ){
102427		- interiorWriterAppend(&pWriter->parentWriter,
102428		- pStartingTerm, nStartingTerm, iBlockid);
102429		- }else{
102430		- interiorWriterInit(1, pStartingTerm, nStartingTerm, iBlockid,
102431		- &pWriter->parentWriter);
102432		- pWriter->has_parent = 1;
102433		- }
102434		-
102435		- /* Track the span of this segment's leaf nodes. */
102436		- if( pWriter->iEndBlockid==0 ){
102437		- pWriter->iEndBlockid = pWriter->iStartBlockid = iBlockid;
102438		- }else{
102439		- pWriter->iEndBlockid++;
102440		- assert( iBlockid==pWriter->iEndBlockid );
102441		- }
102442		-
102443		- return SQLITE_OK;
102444		-}
102445		-static int leafWriterFlush(fulltext_vtab v, LeafWriter pWriter){
102446		- int rc = leafWriterInternalFlush(v, pWriter, 0, pWriter->data.nData);
102447		- if( rc!=SQLITE_OK ) return rc;
102448		-
102449		- /* Re-initialize the output buffer. */
102450		- dataBufferReset(&pWriter->data);
102451		-
102452		- return SQLITE_OK;
102453		-}
102454		-
102455		-/* Fetch the root info for the segment. If the entire leaf fits
102456		-** within ROOT_MAX, then it will be returned directly, otherwise it
102457		-** will be flushed and the root info will be returned from the
102458		-** interior node. *piEndBlockid is set to the blockid of the last
102459		-** interior or leaf node written to disk (0 if none are written at
102460		-** all).
102461		-*/
102462		-static int leafWriterRootInfo(fulltext_vtab v, LeafWriter pWriter,
102463		- char *ppRootInfo, int pnRootInfo,
102464		- sqlite_int64 *piEndBlockid){
102465		- /* we can fit the segment entirely inline */
102466		- if( !pWriter->has_parent && pWriter->data.nData<ROOT_MAX ){
102467		- *ppRootInfo = pWriter->data.pData;
102468		- *pnRootInfo = pWriter->data.nData;
102469		- *piEndBlockid = 0;
102470		- return SQLITE_OK;
102471		- }
102472		-
102473		- /* Flush remaining leaf data. */
102474		- if( pWriter->data.nData>0 ){
102475		- int rc = leafWriterFlush(v, pWriter);
102476		- if( rc!=SQLITE_OK ) return rc;
102477		- }
102478		-
102479		- /* We must have flushed a leaf at some point. */
102480		- assert( pWriter->has_parent );
102481		-
102482		- /* Tenatively set the end leaf blockid as the end blockid. If the
102483		- ** interior node can be returned inline, this will be the final
102484		- ** blockid, otherwise it will be overwritten by
102485		- ** interiorWriterRootInfo().
102486		- */
102487		- *piEndBlockid = pWriter->iEndBlockid;
102488		-
102489		- return interiorWriterRootInfo(v, &pWriter->parentWriter,
102490		- ppRootInfo, pnRootInfo, piEndBlockid);
102491		-}
102492		-
102493		-/* Collect the rootInfo data and store it into the segment directory.
102494		-** This has the effect of flushing the segment's leaf data to
102495		-** %_segments, and also flushing any interior nodes to %_segments.
102496		-*/
102497		-static int leafWriterFinalize(fulltext_vtab v, LeafWriter pWriter){
102498		- sqlite_int64 iEndBlockid;
102499		- char *pRootInfo;
102500		- int rc, nRootInfo;
102501		-
102502		- rc = leafWriterRootInfo(v, pWriter, &pRootInfo, &nRootInfo, &iEndBlockid);
102503		- if( rc!=SQLITE_OK ) return rc;
102504		-
102505		- /* Don't bother storing an entirely empty segment. */
102506		- if( iEndBlockid==0 && nRootInfo==0 ) return SQLITE_OK;
102507		-
102508		- return segdir_set(v, pWriter->iLevel, pWriter->idx,
102509		- pWriter->iStartBlockid, pWriter->iEndBlockid,
102510		- iEndBlockid, pRootInfo, nRootInfo);
102511		-}
102512		-
102513		-static void leafWriterDestroy(LeafWriter *pWriter){
102514		- if( pWriter->has_parent ) interiorWriterDestroy(&pWriter->parentWriter);
102515		- dataBufferDestroy(&pWriter->term);
102516		- dataBufferDestroy(&pWriter->data);
102517		-}
102518		-
102519		-/* Encode a term into the leafWriter, delta-encoding as appropriate.
102520		-** Returns the length of the new term which distinguishes it from the
102521		-** previous term, which can be used to set nTermDistinct when a node
102522		-** boundary is crossed.
102523		-*/
102524		-static int leafWriterEncodeTerm(LeafWriter *pWriter,
102525		- const char *pTerm, int nTerm){
102526		- char c[VARINT_MAX+VARINT_MAX];
102527		- int n, nPrefix = 0;
102528		-
102529		- assert( nTerm>0 );
102530		- while( nPrefix<pWriter->term.nData &&
102531		- pTerm[nPrefix]==pWriter->term.pData[nPrefix] ){
102532		- nPrefix++;
102533		- /* Failing this implies that the terms weren't in order. */
102534		- assert( nPrefix<nTerm );
102535		- }
102536		-
102537		- if( pWriter->data.nData==0 ){
102538		- /* Encode the node header and leading term as:
102539		- ** varint(0)
102540		- ** varint(nTerm)
102541		- ** char pTerm[nTerm]
102542		- */
102543		- n = fts3PutVarint(c, '\0');
102544		- n += fts3PutVarint(c+n, nTerm);
102545		- dataBufferAppend2(&pWriter->data, c, n, pTerm, nTerm);
102546		- }else{
102547		- /* Delta-encode the term as:
102548		- ** varint(nPrefix)
102549		- ** varint(nSuffix)
102550		- ** char pTermSuffix[nSuffix]
102551		- */
102552		- n = fts3PutVarint(c, nPrefix);
102553		- n += fts3PutVarint(c+n, nTerm-nPrefix);
102554		- dataBufferAppend2(&pWriter->data, c, n, pTerm+nPrefix, nTerm-nPrefix);
102555		- }
102556		- dataBufferReplace(&pWriter->term, pTerm, nTerm);
102557		-
102558		- return nPrefix+1;
102559		-}
102560		-
102561		-/* Used to avoid a memmove when a large amount of doclist data is in
102562		-** the buffer. This constructs a node and term header before
102563		-** iDoclistData and flushes the resulting complete node using
102564		-** leafWriterInternalFlush().
102565		-*/
102566		-static int leafWriterInlineFlush(fulltext_vtab v, LeafWriter pWriter,
102567		- const char *pTerm, int nTerm,
102568		- int iDoclistData){
102569		- char c[VARINT_MAX+VARINT_MAX];
102570		- int iData, n = fts3PutVarint(c, 0);
102571		- n += fts3PutVarint(c+n, nTerm);
102572		-
102573		- /* There should always be room for the header. Even if pTerm shared
102574		- ** a substantial prefix with the previous term, the entire prefix
102575		- ** could be constructed from earlier data in the doclist, so there
102576		- ** should be room.
102577		- */
102578		- assert( iDoclistData>=n+nTerm );
102579		-
102580		- iData = iDoclistData-(n+nTerm);
102581		- memcpy(pWriter->data.pData+iData, c, n);
102582		- memcpy(pWriter->data.pData+iData+n, pTerm, nTerm);
102583		-
102584		- return leafWriterInternalFlush(v, pWriter, iData, pWriter->data.nData-iData);
102585		-}
102586		-
102587		-/* Push pTerm[nTerm] along with the doclist data to the leaf layer of
102588		-** %_segments.
102589		-*/
102590		-static int leafWriterStepMerge(fulltext_vtab v, LeafWriter pWriter,
102591		- const char *pTerm, int nTerm,
102592		- DLReader *pReaders, int nReaders){
102593		- char c[VARINT_MAX+VARINT_MAX];
102594		- int iTermData = pWriter->data.nData, iDoclistData;
102595		- int i, nData, n, nActualData, nActual, rc, nTermDistinct;
102596		-
102597		- ASSERT_VALID_LEAF_NODE(pWriter->data.pData, pWriter->data.nData);
102598		- nTermDistinct = leafWriterEncodeTerm(pWriter, pTerm, nTerm);
102599		-
102600		- /* Remember nTermDistinct if opening a new node. */
102601		- if( iTermData==0 ) pWriter->nTermDistinct = nTermDistinct;
102602		-
102603		- iDoclistData = pWriter->data.nData;
102604		-
102605		- /* Estimate the length of the merged doclist so we can leave space
102606		- ** to encode it.
102607		- */
102608		- for(i=0, nData=0; i<nReaders; i++){
102609		- nData += dlrAllDataBytes(&pReaders[i]);
102610		- }
102611		- n = fts3PutVarint(c, nData);
102612		- dataBufferAppend(&pWriter->data, c, n);
102613		-
102614		- docListMerge(&pWriter->data, pReaders, nReaders);
102615		- ASSERT_VALID_DOCLIST(DL_DEFAULT,
102616		- pWriter->data.pData+iDoclistData+n,
102617		- pWriter->data.nData-iDoclistData-n, NULL);
102618		-
102619		- /* The actual amount of doclist data at this point could be smaller
102620		- ** than the length we encoded. Additionally, the space required to
102621		- ** encode this length could be smaller. For small doclists, this is
102622		- ** not a big deal, we can just use memmove() to adjust things.
102623		- */
102624		- nActualData = pWriter->data.nData-(iDoclistData+n);
102625		- nActual = fts3PutVarint(c, nActualData);
102626		- assert( nActualData<=nData );
102627		- assert( nActual<=n );
102628		-
102629		- /* If the new doclist is big enough for force a standalone leaf
102630		- ** node, we can immediately flush it inline without doing the
102631		- ** memmove().
102632		- */
102633		- /* TODO(shess) This test matches leafWriterStep(), which does this
102634		- ** test before it knows the cost to varint-encode the term and
102635		- ** doclist lengths. At some point, change to
102636		- ** pWriter->data.nData-iTermData>STANDALONE_MIN.
102637		- */
102638		- if( nTerm+nActualData>STANDALONE_MIN ){
102639		- /* Push leaf node from before this term. */
102640		- if( iTermData>0 ){
102641		- rc = leafWriterInternalFlush(v, pWriter, 0, iTermData);
102642		- if( rc!=SQLITE_OK ) return rc;
102643		-
102644		- pWriter->nTermDistinct = nTermDistinct;
102645		- }
102646		-
102647		- /* Fix the encoded doclist length. */
102648		- iDoclistData += n - nActual;
102649		- memcpy(pWriter->data.pData+iDoclistData, c, nActual);
102650		-
102651		- /* Push the standalone leaf node. */
102652		- rc = leafWriterInlineFlush(v, pWriter, pTerm, nTerm, iDoclistData);
102653		- if( rc!=SQLITE_OK ) return rc;
102654		-
102655		- /* Leave the node empty. */
102656		- dataBufferReset(&pWriter->data);
102657		-
102658		- return rc;
102659		- }
102660		-
102661		- /* At this point, we know that the doclist was small, so do the
102662		- ** memmove if indicated.
102663		- */
102664		- if( nActual<n ){
102665		- memmove(pWriter->data.pData+iDoclistData+nActual,
102666		- pWriter->data.pData+iDoclistData+n,
102667		- pWriter->data.nData-(iDoclistData+n));
102668		- pWriter->data.nData -= n-nActual;
102669		- }
102670		-
102671		- /* Replace written length with actual length. */
102672		- memcpy(pWriter->data.pData+iDoclistData, c, nActual);
102673		-
102674		- /* If the node is too large, break things up. */
102675		- /* TODO(shess) This test matches leafWriterStep(), which does this
102676		- ** test before it knows the cost to varint-encode the term and
102677		- ** doclist lengths. At some point, change to
102678		- ** pWriter->data.nData>LEAF_MAX.
102679		- */
102680		- if( iTermData+nTerm+nActualData>LEAF_MAX ){
102681		- /* Flush out the leading data as a node */
102682		- rc = leafWriterInternalFlush(v, pWriter, 0, iTermData);
102683		- if( rc!=SQLITE_OK ) return rc;
102684		-
102685		- pWriter->nTermDistinct = nTermDistinct;
102686		-
102687		- /* Rebuild header using the current term */
102688		- n = fts3PutVarint(pWriter->data.pData, 0);
102689		- n += fts3PutVarint(pWriter->data.pData+n, nTerm);
102690		- memcpy(pWriter->data.pData+n, pTerm, nTerm);
102691		- n += nTerm;
102692		-
102693		- /* There should always be room, because the previous encoding
102694		- ** included all data necessary to construct the term.
102695		- */
102696		- assert( n<iDoclistData );
102697		- /* So long as STANDALONE_MIN is half or less of LEAF_MAX, the
102698		- ** following memcpy() is safe (as opposed to needing a memmove).
102699		- */
102700		- assert( 2*STANDALONE_MIN<=LEAF_MAX );
102701		- assert( n+pWriter->data.nData-iDoclistData<iDoclistData );
102702		- memcpy(pWriter->data.pData+n,
102703		- pWriter->data.pData+iDoclistData,
102704		- pWriter->data.nData-iDoclistData);
102705		- pWriter->data.nData -= iDoclistData-n;
102706		- }
102707		- ASSERT_VALID_LEAF_NODE(pWriter->data.pData, pWriter->data.nData);
102708		-
102709		- return SQLITE_OK;
102710		-}
102711		-
102712		-/* Push pTerm[nTerm] along with the doclist data to the leaf layer of
102713		-** %_segments.
102714		-*/
102715		-/* TODO(shess) Revise writeZeroSegment() so that doclists are
102716		-** constructed directly in pWriter->data.
102717		-*/
102718		-static int leafWriterStep(fulltext_vtab v, LeafWriter pWriter,
102719		- const char *pTerm, int nTerm,
102720		- const char *pData, int nData){
102721		- int rc;
102722		- DLReader reader;
102723		-
102724		- dlrInit(&reader, DL_DEFAULT, pData, nData);
102725		- rc = leafWriterStepMerge(v, pWriter, pTerm, nTerm, &reader, 1);
102726		- dlrDestroy(&reader);
102727		-
102728		- return rc;
102729		-}
102730		-
102731		-
102732		-/****************************************************************/
102733		-/* LeafReader is used to iterate over an individual leaf node. */
102734		-typedef struct LeafReader {
102735		- DataBuffer term; /* copy of current term. */
102736		-
102737		- const char pData; / data for current term. */
102738		- int nData;
102739		-} LeafReader;
102740		-
102741		-static void leafReaderDestroy(LeafReader *pReader){
102742		- dataBufferDestroy(&pReader->term);
102743		- SCRAMBLE(pReader);
102744		-}
102745		-
102746		-static int leafReaderAtEnd(LeafReader *pReader){
102747		- return pReader->nData<=0;
102748		-}
102749		-
102750		-/* Access the current term. */
102751		-static int leafReaderTermBytes(LeafReader *pReader){
102752		- return pReader->term.nData;
102753		-}
102754		-static const char leafReaderTerm(LeafReader pReader){
102755		- assert( pReader->term.nData>0 );
102756		- return pReader->term.pData;
102757		-}
102758		-
102759		-/* Access the doclist data for the current term. */
102760		-static int leafReaderDataBytes(LeafReader *pReader){
102761		- int nData;
102762		- assert( pReader->term.nData>0 );
102763		- fts3GetVarint32(pReader->pData, &nData);
102764		- return nData;
102765		-}
102766		-static const char leafReaderData(LeafReader pReader){
102767		- int n, nData;
102768		- assert( pReader->term.nData>0 );
102769		- n = fts3GetVarint32(pReader->pData, &nData);
102770		- return pReader->pData+n;
102771		-}
102772		-
102773		-static void leafReaderInit(const char *pData, int nData,
102774		- LeafReader *pReader){
102775		- int nTerm, n;
102776		-
102777		- assert( nData>0 );
102778		- assert( pData[0]=='\0' );
102779		-
102780		- CLEAR(pReader);
102781		-
102782		- /* Read the first term, skipping the header byte. */
102783		- n = fts3GetVarint32(pData+1, &nTerm);
102784		- dataBufferInit(&pReader->term, nTerm);
102785		- dataBufferReplace(&pReader->term, pData+1+n, nTerm);
102786		-
102787		- /* Position after the first term. */
102788		- assert( 1+n+nTerm<nData );
102789		- pReader->pData = pData+1+n+nTerm;
102790		- pReader->nData = nData-1-n-nTerm;
102791		-}
102792		-
102793		-/* Step the reader forward to the next term. */
102794		-static void leafReaderStep(LeafReader *pReader){
102795		- int n, nData, nPrefix, nSuffix;
102796		- assert( !leafReaderAtEnd(pReader) );
102797		-
102798		- /* Skip previous entry's data block. */
102799		- n = fts3GetVarint32(pReader->pData, &nData);
102800		- assert( n+nData<=pReader->nData );
102801		- pReader->pData += n+nData;
102802		- pReader->nData -= n+nData;
102803		-
102804		- if( !leafReaderAtEnd(pReader) ){
102805		- /* Construct the new term using a prefix from the old term plus a
102806		- ** suffix from the leaf data.
102807		- */
102808		- n = fts3GetVarint32(pReader->pData, &nPrefix);
102809		- n += fts3GetVarint32(pReader->pData+n, &nSuffix);
102810		- assert( n+nSuffix<pReader->nData );
102811		- pReader->term.nData = nPrefix;
102812		- dataBufferAppend(&pReader->term, pReader->pData+n, nSuffix);
102813		-
102814		- pReader->pData += n+nSuffix;
102815		- pReader->nData -= n+nSuffix;
102816		- }
102817		-}
102818		-
102819		-/* strcmp-style comparison of pReader's current term against pTerm.
102820		-** If isPrefix, equality means equal through nTerm bytes.
102821		-*/
102822		-static int leafReaderTermCmp(LeafReader *pReader,
102823		- const char *pTerm, int nTerm, int isPrefix){
102824		- int c, n = pReader->term.nData<nTerm ? pReader->term.nData : nTerm;
102825		- if( n==0 ){
102826		- if( pReader->term.nData>0 ) return -1;
102827		- if(nTerm>0 ) return 1;
102828		- return 0;
102829		- }
102830		-
102831		- c = memcmp(pReader->term.pData, pTerm, n);
102832		- if( c!=0 ) return c;
102833		- if( isPrefix && n==nTerm ) return 0;
102834		- return pReader->term.nData - nTerm;
102835		-}
102836		-
102837		-
102838		-/****************************************************************/
102839		-/* LeavesReader wraps LeafReader to allow iterating over the entire
102840		-** leaf layer of the tree.
102841		-*/
102842		-typedef struct LeavesReader {
102843		- int idx; /* Index within the segment. */
102844		-
102845		- sqlite3_stmt pStmt; / Statement we're streaming leaves from. */
102846		- int eof; /* we've seen SQLITE_DONE from pStmt. */
102847		-
102848		- LeafReader leafReader; /* reader for the current leaf. */
102849		- DataBuffer rootData; /* root data for inline. */
102850		-} LeavesReader;
102851		-
102852		-/* Access the current term. */
102853		-static int leavesReaderTermBytes(LeavesReader *pReader){
102854		- assert( !pReader->eof );
102855		- return leafReaderTermBytes(&pReader->leafReader);
102856		-}
102857		-static const char leavesReaderTerm(LeavesReader pReader){
102858		- assert( !pReader->eof );
102859		- return leafReaderTerm(&pReader->leafReader);
102860		-}
102861		-
102862		-/* Access the doclist data for the current term. */
102863		-static int leavesReaderDataBytes(LeavesReader *pReader){
102864		- assert( !pReader->eof );
102865		- return leafReaderDataBytes(&pReader->leafReader);
102866		-}
102867		-static const char leavesReaderData(LeavesReader pReader){
102868		- assert( !pReader->eof );
102869		- return leafReaderData(&pReader->leafReader);
102870		-}
102871		-
102872		-static int leavesReaderAtEnd(LeavesReader *pReader){
102873		- return pReader->eof;
102874		-}
102875		-
102876		-/* loadSegmentLeaves() may not read all the way to SQLITE_DONE, thus
102877		-** leaving the statement handle open, which locks the table.
102878		-*/
102879		-/* TODO(shess) This "solution" is not satisfactory. Really, there
102880		-** should be check-in function for all statement handles which
102881		-** arranges to call sqlite3_reset(). This most likely will require
102882		-** modification to control flow all over the place, though, so for now
102883		-** just punt.
102884		-**
102885		-** Note the the current system assumes that segment merges will run to
102886		-** completion, which is why this particular probably hasn't arisen in
102887		-** this case. Probably a brittle assumption.
102888		-*/
102889		-static int leavesReaderReset(LeavesReader *pReader){
102890		- return sqlite3_reset(pReader->pStmt);
102891		-}
102892		-
102893		-static void leavesReaderDestroy(LeavesReader *pReader){
102894		- /* If idx is -1, that means we're using a non-cached statement
102895		- ** handle in the optimize() case, so we need to release it.
102896		- */
102897		- if( pReader->pStmt!=NULL && pReader->idx==-1 ){
102898		- sqlite3_finalize(pReader->pStmt);
102899		- }
102900		- leafReaderDestroy(&pReader->leafReader);
102901		- dataBufferDestroy(&pReader->rootData);
102902		- SCRAMBLE(pReader);
102903		-}
102904		-
102905		-/* Initialize pReader with the given root data (if iStartBlockid==0
102906		-** the leaf data was entirely contained in the root), or from the
102907		-** stream of blocks between iStartBlockid and iEndBlockid, inclusive.
102908		-*/
102909		-static int leavesReaderInit(fulltext_vtab *v,
102910		- int idx,
102911		- sqlite_int64 iStartBlockid,
102912		- sqlite_int64 iEndBlockid,
102913		- const char *pRootData, int nRootData,
102914		- LeavesReader *pReader){
102915		- CLEAR(pReader);
102916		- pReader->idx = idx;
102917		-
102918		- dataBufferInit(&pReader->rootData, 0);
102919		- if( iStartBlockid==0 ){
102920		- /* Entire leaf level fit in root data. */
102921		- dataBufferReplace(&pReader->rootData, pRootData, nRootData);
102922		- leafReaderInit(pReader->rootData.pData, pReader->rootData.nData,
102923		- &pReader->leafReader);
102924		- }else{
102925		- sqlite3_stmt *s;
102926		- int rc = sql_get_leaf_statement(v, idx, &s);
102927		- if( rc!=SQLITE_OK ) return rc;
102928		-
102929		- rc = sqlite3_bind_int64(s, 1, iStartBlockid);
102930		- if( rc!=SQLITE_OK ) return rc;
102931		-
102932		- rc = sqlite3_bind_int64(s, 2, iEndBlockid);
102933		- if( rc!=SQLITE_OK ) return rc;
102934		-
102935		- rc = sqlite3_step(s);
102936		- if( rc==SQLITE_DONE ){
102937		- pReader->eof = 1;
102938		- return SQLITE_OK;
102939		- }
102940		- if( rc!=SQLITE_ROW ) return rc;
102941		-
102942		- pReader->pStmt = s;
102943		- leafReaderInit(sqlite3_column_blob(pReader->pStmt, 0),
102944		- sqlite3_column_bytes(pReader->pStmt, 0),
102945		- &pReader->leafReader);
102946		- }
102947		- return SQLITE_OK;
102948		-}
102949		-
102950		-/* Step the current leaf forward to the next term. If we reach the
102951		-** end of the current leaf, step forward to the next leaf block.
102952		-*/
102953		-static int leavesReaderStep(fulltext_vtab v, LeavesReader pReader){
102954		- assert( !leavesReaderAtEnd(pReader) );
102955		- leafReaderStep(&pReader->leafReader);
102956		-
102957		- if( leafReaderAtEnd(&pReader->leafReader) ){
102958		- int rc;
102959		- if( pReader->rootData.pData ){
102960		- pReader->eof = 1;
102961		- return SQLITE_OK;
102962		- }
102963		- rc = sqlite3_step(pReader->pStmt);
102964		- if( rc!=SQLITE_ROW ){
102965		- pReader->eof = 1;
102966		- return rc==SQLITE_DONE ? SQLITE_OK : rc;
102967		- }
102968		- leafReaderDestroy(&pReader->leafReader);
102969		- leafReaderInit(sqlite3_column_blob(pReader->pStmt, 0),
102970		- sqlite3_column_bytes(pReader->pStmt, 0),
102971		- &pReader->leafReader);
102972		- }
102973		- return SQLITE_OK;
102974		-}
102975		-
102976		-/* Order LeavesReaders by their term, ignoring idx. Readers at eof
102977		-** always sort to the end.
102978		-*/
102979		-static int leavesReaderTermCmp(LeavesReader lr1, LeavesReader lr2){
102980		- if( leavesReaderAtEnd(lr1) ){
102981		- if( leavesReaderAtEnd(lr2) ) return 0;
102982		- return 1;
102983		- }
102984		- if( leavesReaderAtEnd(lr2) ) return -1;
102985		-
102986		- return leafReaderTermCmp(&lr1->leafReader,
102987		- leavesReaderTerm(lr2), leavesReaderTermBytes(lr2),
102988		- 0);
102989		-}
102990		-
102991		-/* Similar to leavesReaderTermCmp(), with additional ordering by idx
102992		-** so that older segments sort before newer segments.
102993		-*/
102994		-static int leavesReaderCmp(LeavesReader lr1, LeavesReader lr2){
102995		- int c = leavesReaderTermCmp(lr1, lr2);
102996		- if( c!=0 ) return c;
102997		- return lr1->idx-lr2->idx;
102998		-}
102999		-
103000		-/* Assume that pLr[1]..pLr[nLr] are sorted. Bubble pLr[0] into its
103001		-** sorted position.
103002		-*/
103003		-static void leavesReaderReorder(LeavesReader *pLr, int nLr){
103004		- while( nLr>1 && leavesReaderCmp(pLr, pLr+1)>0 ){
103005		- LeavesReader tmp = pLr[0];
103006		- pLr[0] = pLr[1];
103007		- pLr[1] = tmp;
103008		- nLr--;
103009		- pLr++;
103010		- }
103011		-}
103012		-
103013		-/* Initializes pReaders with the segments from level iLevel, returning
103014		-** the number of segments in *piReaders. Leaves pReaders in sorted
103015		-** order.
103016		-*/
103017		-static int leavesReadersInit(fulltext_vtab *v, int iLevel,
103018		- LeavesReader pReaders, int piReaders){
103019		- sqlite3_stmt *s;
103020		- int i, rc = sql_get_statement(v, SEGDIR_SELECT_LEVEL_STMT, &s);
103021		- if( rc!=SQLITE_OK ) return rc;
103022		-
103023		- rc = sqlite3_bind_int(s, 1, iLevel);
103024		- if( rc!=SQLITE_OK ) return rc;
103025		-
103026		- i = 0;
103027		- while( (rc = sqlite3_step(s))==SQLITE_ROW ){
103028		- sqlite_int64 iStart = sqlite3_column_int64(s, 0);
103029		- sqlite_int64 iEnd = sqlite3_column_int64(s, 1);
103030		- const char *pRootData = sqlite3_column_blob(s, 2);
103031		- int nRootData = sqlite3_column_bytes(s, 2);
103032		-
103033		- assert( i<MERGE_COUNT );
103034		- rc = leavesReaderInit(v, i, iStart, iEnd, pRootData, nRootData,
103035		- &pReaders[i]);
103036		- if( rc!=SQLITE_OK ) break;
103037		-
103038		- i++;
103039		- }
103040		- if( rc!=SQLITE_DONE ){
103041		- while( i-->0 ){
103042		- leavesReaderDestroy(&pReaders[i]);
103043		- }
103044		- return rc;
103045		- }
103046		-
103047		- *piReaders = i;
103048		-
103049		- /* Leave our results sorted by term, then age. */
103050		- while( i-- ){
103051		- leavesReaderReorder(pReaders+i, *piReaders-i);
103052		- }
103053		- return SQLITE_OK;
103054		-}
103055		-
103056		-/* Merge doclists from pReaders[nReaders] into a single doclist, which
103057		-** is written to pWriter. Assumes pReaders is ordered oldest to
103058		-** newest.
103059		-*/
103060		-/* TODO(shess) Consider putting this inline in segmentMerge(). */
103061		-static int leavesReadersMerge(fulltext_vtab *v,
103062		- LeavesReader *pReaders, int nReaders,
103063		- LeafWriter *pWriter){
103064		- DLReader dlReaders[MERGE_COUNT];
103065		- const char *pTerm = leavesReaderTerm(pReaders);
103066		- int i, nTerm = leavesReaderTermBytes(pReaders);
103067		-
103068		- assert( nReaders<=MERGE_COUNT );
103069		-
103070		- for(i=0; i<nReaders; i++){
103071		- dlrInit(&dlReaders[i], DL_DEFAULT,
103072		- leavesReaderData(pReaders+i),
103073		- leavesReaderDataBytes(pReaders+i));
103074		- }
103075		-
103076		- return leafWriterStepMerge(v, pWriter, pTerm, nTerm, dlReaders, nReaders);
103077		-}
103078		-
103079		-/* Forward ref due to mutual recursion with segdirNextIndex(). */
103080		-static int segmentMerge(fulltext_vtab *v, int iLevel);
103081		-
103082		-/* Put the next available index at iLevel into *pidx. If iLevel
103083		-** already has MERGE_COUNT segments, they are merged to a higher
103084		-** level to make room.
103085		-*/
103086		-static int segdirNextIndex(fulltext_vtab v, int iLevel, int pidx){
103087		- int rc = segdir_max_index(v, iLevel, pidx);
103088		- if( rc==SQLITE_DONE ){ /* No segments at iLevel. */
103089		- *pidx = 0;
103090		- }else if( rc==SQLITE_ROW ){
103091		- if( *pidx==(MERGE_COUNT-1) ){
103092		- rc = segmentMerge(v, iLevel);
103093		- if( rc!=SQLITE_OK ) return rc;
103094		- *pidx = 0;
103095		- }else{
103096		- (*pidx)++;
103097		- }
103098		- }else{
103099		- return rc;
103100		- }
103101		- return SQLITE_OK;
103102		-}
103103		-
103104		-/* Merge MERGE_COUNT segments at iLevel into a new segment at
103105		-** iLevel+1. If iLevel+1 is already full of segments, those will be
103106		-** merged to make room.
103107		-*/
103108		-static int segmentMerge(fulltext_vtab *v, int iLevel){
103109		- LeafWriter writer;
103110		- LeavesReader lrs[MERGE_COUNT];
103111		- int i, rc, idx = 0;
103112		-
103113		- /* Determine the next available segment index at the next level,
103114		- ** merging as necessary.
103115		- */
103116		- rc = segdirNextIndex(v, iLevel+1, &idx);
103117		- if( rc!=SQLITE_OK ) return rc;
103118		-
103119		- /* TODO(shess) This assumes that we'll always see exactly
103120		- ** MERGE_COUNT segments to merge at a given level. That will be
103121		- ** broken if we allow the developer to request preemptive or
103122		- ** deferred merging.
103123		- */
103124		- memset(&lrs, '\0', sizeof(lrs));
103125		- rc = leavesReadersInit(v, iLevel, lrs, &i);
103126		- if( rc!=SQLITE_OK ) return rc;
103127		- assert( i==MERGE_COUNT );
103128		-
103129		- leafWriterInit(iLevel+1, idx, &writer);
103130		-
103131		- /* Since leavesReaderReorder() pushes readers at eof to the end,
103132		- ** when the first reader is empty, all will be empty.
103133		- */
103134		- while( !leavesReaderAtEnd(lrs) ){
103135		- /* Figure out how many readers share their next term. */
103136		- for(i=1; i<MERGE_COUNT && !leavesReaderAtEnd(lrs+i); i++){
103137		- if( 0!=leavesReaderTermCmp(lrs, lrs+i) ) break;
103138		- }
103139		-
103140		- rc = leavesReadersMerge(v, lrs, i, &writer);
103141		- if( rc!=SQLITE_OK ) goto err;
103142		-
103143		- /* Step forward those that were merged. */
103144		- while( i-->0 ){
103145		- rc = leavesReaderStep(v, lrs+i);
103146		- if( rc!=SQLITE_OK ) goto err;
103147		-
103148		- /* Reorder by term, then by age. */
103149		- leavesReaderReorder(lrs+i, MERGE_COUNT-i);
103150		- }
103151		- }
103152		-
103153		- for(i=0; i<MERGE_COUNT; i++){
103154		- leavesReaderDestroy(&lrs[i]);
103155		- }
103156		-
103157		- rc = leafWriterFinalize(v, &writer);
103158		- leafWriterDestroy(&writer);
103159		- if( rc!=SQLITE_OK ) return rc;
103160		-
103161		- /* Delete the merged segment data. */
103162		- return segdir_delete(v, iLevel);
103163		-
103164		- err:
103165		- for(i=0; i<MERGE_COUNT; i++){
103166		- leavesReaderDestroy(&lrs[i]);
103167		- }
103168		- leafWriterDestroy(&writer);
103169		- return rc;
103170		-}
103171		-
103172		-/* Accumulate the union of acc and pData into acc. /
103173		-static void docListAccumulateUnion(DataBuffer *acc,
103174		- const char *pData, int nData) {
103175		- DataBuffer tmp = *acc;
103176		- dataBufferInit(acc, tmp.nData+nData);
103177		- docListUnion(tmp.pData, tmp.nData, pData, nData, acc);
103178		- dataBufferDestroy(&tmp);
103179		-}
103180		-
103181		-/* TODO(shess) It might be interesting to explore different merge
103182		-** strategies, here. For instance, since this is a sorted merge, we
103183		-** could easily merge many doclists in parallel. With some
103184		-** comprehension of the storage format, we could merge all of the
103185		-** doclists within a leaf node directly from the leaf node's storage.
103186		-** It may be worthwhile to merge smaller doclists before larger
103187		-** doclists, since they can be traversed more quickly - but the
103188		-** results may have less overlap, making them more expensive in a
103189		-** different way.
103190		-*/
103191		-
103192		-/* Scan pReader for pTerm/nTerm, and merge the term's doclist over
103193		-** out (any doclists with duplicate docids overwrite those in out).
103194		-** Internal function for loadSegmentLeaf().
103195		-*/
103196		-static int loadSegmentLeavesInt(fulltext_vtab v, LeavesReader pReader,
103197		- const char *pTerm, int nTerm, int isPrefix,
103198		- DataBuffer *out){
103199		- /* doclist data is accumulated into pBuffers similar to how one does
103200		- ** increment in binary arithmetic. If index 0 is empty, the data is
103201		- ** stored there. If there is data there, it is merged and the
103202		- ** results carried into position 1, with further merge-and-carry
103203		- ** until an empty position is found.
103204		- */
103205		- DataBuffer *pBuffers = NULL;
103206		- int nBuffers = 0, nMaxBuffers = 0, rc;
103207		-
103208		- assert( nTerm>0 );
103209		-
103210		- for(rc=SQLITE_OK; rc==SQLITE_OK && !leavesReaderAtEnd(pReader);
103211		- rc=leavesReaderStep(v, pReader)){
103212		- /* TODO(shess) Really want leavesReaderTermCmp(), but that name is
103213		- ** already taken to compare the terms of two LeavesReaders. Think
103214		- ** on a better name. [Meanwhile, break encapsulation rather than
103215		- ** use a confusing name.]
103216		- */
103217		- int c = leafReaderTermCmp(&pReader->leafReader, pTerm, nTerm, isPrefix);
103218		- if( c>0 ) break; /* Past any possible matches. */
103219		- if( c==0 ){
103220		- const char *pData = leavesReaderData(pReader);
103221		- int iBuffer, nData = leavesReaderDataBytes(pReader);
103222		-
103223		- /* Find the first empty buffer. */
103224		- for(iBuffer=0; iBuffer<nBuffers; ++iBuffer){
103225		- if( 0==pBuffers[iBuffer].nData ) break;
103226		- }
103227		-
103228		- /* Out of buffers, add an empty one. */
103229		- if( iBuffer==nBuffers ){
103230		- if( nBuffers==nMaxBuffers ){
103231		- DataBuffer *p;
103232		- nMaxBuffers += 20;
103233		-
103234		- /* Manual realloc so we can handle NULL appropriately. */
103235		- p = sqlite3_malloc(nMaxBufferssizeof(pBuffers));
103236		- if( p==NULL ){
103237		- rc = SQLITE_NOMEM;
103238		- break;
103239		- }
103240		-
103241		- if( nBuffers>0 ){
103242		- assert(pBuffers!=NULL);
103243		- memcpy(p, pBuffers, nBufferssizeof(pBuffers));
103244		- sqlite3_free(pBuffers);
103245		- }
103246		- pBuffers = p;
103247		- }
103248		- dataBufferInit(&(pBuffers[nBuffers]), 0);
103249		- nBuffers++;
103250		- }
103251		-
103252		- /* At this point, must have an empty at iBuffer. */
103253		- assert(iBuffer<nBuffers && pBuffers[iBuffer].nData==0);
103254		-
103255		- /* If empty was first buffer, no need for merge logic. */
103256		- if( iBuffer==0 ){
103257		- dataBufferReplace(&(pBuffers[0]), pData, nData);
103258		- }else{
103259		- /* pAcc is the empty buffer the merged data will end up in. */
103260		- DataBuffer *pAcc = &(pBuffers[iBuffer]);
103261		- DataBuffer *p = &(pBuffers[0]);
103262		-
103263		- /* Handle position 0 specially to avoid need to prime pAcc
103264		- ** with pData/nData.
103265		- */
103266		- dataBufferSwap(p, pAcc);
103267		- docListAccumulateUnion(pAcc, pData, nData);
103268		-
103269		- /* Accumulate remaining doclists into pAcc. */
103270		- for(++p; p<pAcc; ++p){
103271		- docListAccumulateUnion(pAcc, p->pData, p->nData);
103272		-
103273		- /* dataBufferReset() could allow a large doclist to blow up
103274		- ** our memory requirements.
103275		- */
103276		- if( p->nCapacity<1024 ){
103277		- dataBufferReset(p);
103278		- }else{
103279		- dataBufferDestroy(p);
103280		- dataBufferInit(p, 0);
103281		- }
103282		- }
103283		- }
103284		- }
103285		- }
103286		-
103287		- /* Union all the doclists together into out. /
103288		- /* TODO(shess) What if out is big? Sigh. /
103289		- if( rc==SQLITE_OK && nBuffers>0 ){
103290		- int iBuffer;
103291		- for(iBuffer=0; iBuffer<nBuffers; ++iBuffer){
103292		- if( pBuffers[iBuffer].nData>0 ){
103293		- if( out->nData==0 ){
103294		- dataBufferSwap(out, &(pBuffers[iBuffer]));
103295		- }else{
103296		- docListAccumulateUnion(out, pBuffers[iBuffer].pData,
103297		- pBuffers[iBuffer].nData);
103298		- }
103299		- }
103300		- }
103301		- }
103302		-
103303		- while( nBuffers-- ){
103304		- dataBufferDestroy(&(pBuffers[nBuffers]));
103305		- }
103306		- if( pBuffers!=NULL ) sqlite3_free(pBuffers);
103307		-
103308		- return rc;
103309		-}
103310		-
103311		-/* Call loadSegmentLeavesInt() with pData/nData as input. */
103312		-static int loadSegmentLeaf(fulltext_vtab v, const char pData, int nData,
103313		- const char *pTerm, int nTerm, int isPrefix,
103314		- DataBuffer *out){
103315		- LeavesReader reader;
103316		- int rc;
103317		-
103318		- assert( nData>1 );
103319		- assert( *pData=='\0' );
103320		- rc = leavesReaderInit(v, 0, 0, 0, pData, nData, &reader);
103321		- if( rc!=SQLITE_OK ) return rc;
103322		-
103323		- rc = loadSegmentLeavesInt(v, &reader, pTerm, nTerm, isPrefix, out);
103324		- leavesReaderReset(&reader);
103325		- leavesReaderDestroy(&reader);
103326		- return rc;
103327		-}
103328		-
103329		-/* Call loadSegmentLeavesInt() with the leaf nodes from iStartLeaf to
103330		-** iEndLeaf (inclusive) as input, and merge the resulting doclist into
103331		-** out.
103332		-*/
103333		-static int loadSegmentLeaves(fulltext_vtab *v,
103334		- sqlite_int64 iStartLeaf, sqlite_int64 iEndLeaf,
103335		- const char *pTerm, int nTerm, int isPrefix,
103336		- DataBuffer *out){
103337		- int rc;
103338		- LeavesReader reader;
103339		-
103340		- assert( iStartLeaf<=iEndLeaf );
103341		- rc = leavesReaderInit(v, 0, iStartLeaf, iEndLeaf, NULL, 0, &reader);
103342		- if( rc!=SQLITE_OK ) return rc;
103343		-
103344		- rc = loadSegmentLeavesInt(v, &reader, pTerm, nTerm, isPrefix, out);
103345		- leavesReaderReset(&reader);
103346		- leavesReaderDestroy(&reader);
103347		- return rc;
103348		-}
103349		-
103350		-/* Taking pData/nData as an interior node, find the sequence of child
103351		-** nodes which could include pTerm/nTerm/isPrefix. Note that the
103352		-** interior node terms logically come between the blocks, so there is
103353		-** one more blockid than there are terms (that block contains terms >=
103354		-** the last interior-node term).
103355		-*/
103356		-/* TODO(shess) The calling code may already know that the end child is
103357		-** not worth calculating, because the end may be in a later sibling
103358		-** node. Consider whether breaking symmetry is worthwhile. I suspect
103359		-** it is not worthwhile.
103360		-*/
103361		-static void getChildrenContaining(const char *pData, int nData,
103362		- const char *pTerm, int nTerm, int isPrefix,
103363		- sqlite_int64 *piStartChild,
103364		- sqlite_int64 *piEndChild){
103365		- InteriorReader reader;
103366		-
103367		- assert( nData>1 );
103368		- assert( *pData!='\0' );
103369		- interiorReaderInit(pData, nData, &reader);
103370		-
103371		- /* Scan for the first child which could contain pTerm/nTerm. */
103372		- while( !interiorReaderAtEnd(&reader) ){
103373		- if( interiorReaderTermCmp(&reader, pTerm, nTerm, 0)>0 ) break;
103374		- interiorReaderStep(&reader);
103375		- }
103376		- *piStartChild = interiorReaderCurrentBlockid(&reader);
103377		-
103378		- /* Keep scanning to find a term greater than our term, using prefix
103379		- ** comparison if indicated. If isPrefix is false, this will be the
103380		- ** same blockid as the starting block.
103381		- */
103382		- while( !interiorReaderAtEnd(&reader) ){
103383		- if( interiorReaderTermCmp(&reader, pTerm, nTerm, isPrefix)>0 ) break;
103384		- interiorReaderStep(&reader);
103385		- }
103386		- *piEndChild = interiorReaderCurrentBlockid(&reader);
103387		-
103388		- interiorReaderDestroy(&reader);
103389		-
103390		- /* Children must ascend, and if !prefix, both must be the same. */
103391		- assert( piEndChild>=piStartChild );
103392		- assert( isPrefix \|\| piStartChild==piEndChild );
103393		-}
103394		-
103395		-/* Read block at iBlockid and pass it with other params to
103396		-** getChildrenContaining().
103397		-*/
103398		-static int loadAndGetChildrenContaining(
103399		- fulltext_vtab *v,
103400		- sqlite_int64 iBlockid,
103401		- const char *pTerm, int nTerm, int isPrefix,
103402		- sqlite_int64 piStartChild, sqlite_int64 piEndChild
103403		-){
103404		- sqlite3_stmt *s = NULL;
103405		- int rc;
103406		-
103407		- assert( iBlockid!=0 );
103408		- assert( pTerm!=NULL );
103409		- assert( nTerm!=0 ); /* TODO(shess) Why not allow this? */
103410		- assert( piStartChild!=NULL );
103411		- assert( piEndChild!=NULL );
103412		-
103413		- rc = sql_get_statement(v, BLOCK_SELECT_STMT, &s);
103414		- if( rc!=SQLITE_OK ) return rc;
103415		-
103416		- rc = sqlite3_bind_int64(s, 1, iBlockid);
103417		- if( rc!=SQLITE_OK ) return rc;
103418		-
103419		- rc = sqlite3_step(s);
103420		- if( rc==SQLITE_DONE ) return SQLITE_ERROR;
103421		- if( rc!=SQLITE_ROW ) return rc;
103422		-
103423		- getChildrenContaining(sqlite3_column_blob(s, 0), sqlite3_column_bytes(s, 0),
103424		- pTerm, nTerm, isPrefix, piStartChild, piEndChild);
103425		-
103426		- /* We expect only one row. We must execute another sqlite3_step()
103427		- * to complete the iteration; otherwise the table will remain
103428		- * locked. */
103429		- rc = sqlite3_step(s);
103430		- if( rc==SQLITE_ROW ) return SQLITE_ERROR;
103431		- if( rc!=SQLITE_DONE ) return rc;
103432		-
103433		- return SQLITE_OK;
103434		-}
103435		-
103436		-/* Traverse the tree represented by pData[nData] looking for
103437		-** pTerm[nTerm], placing its doclist into *out. This is internal to
103438		-** loadSegment() to make error-handling cleaner.
103439		-*/
103440		-static int loadSegmentInt(fulltext_vtab v, const char pData, int nData,
103441		- sqlite_int64 iLeavesEnd,
103442		- const char *pTerm, int nTerm, int isPrefix,
103443		- DataBuffer *out){
103444		- /* Special case where root is a leaf. */
103445		- if( *pData=='\0' ){
103446		- return loadSegmentLeaf(v, pData, nData, pTerm, nTerm, isPrefix, out);
103447		- }else{
103448		- int rc;
103449		- sqlite_int64 iStartChild, iEndChild;
103450		-
103451		- /* Process pData as an interior node, then loop down the tree
103452		- ** until we find the set of leaf nodes to scan for the term.
103453		- */
103454		- getChildrenContaining(pData, nData, pTerm, nTerm, isPrefix,
103455		- &iStartChild, &iEndChild);
103456		- while( iStartChild>iLeavesEnd ){
103457		- sqlite_int64 iNextStart, iNextEnd;
103458		- rc = loadAndGetChildrenContaining(v, iStartChild, pTerm, nTerm, isPrefix,
103459		- &iNextStart, &iNextEnd);
103460		- if( rc!=SQLITE_OK ) return rc;
103461		-
103462		- /* If we've branched, follow the end branch, too. */
103463		- if( iStartChild!=iEndChild ){
103464		- sqlite_int64 iDummy;
103465		- rc = loadAndGetChildrenContaining(v, iEndChild, pTerm, nTerm, isPrefix,
103466		- &iDummy, &iNextEnd);
103467		- if( rc!=SQLITE_OK ) return rc;
103468		- }
103469		-
103470		- assert( iNextStart<=iNextEnd );
103471		- iStartChild = iNextStart;
103472		- iEndChild = iNextEnd;
103473		- }
103474		- assert( iStartChild<=iLeavesEnd );
103475		- assert( iEndChild<=iLeavesEnd );
103476		-
103477		- /* Scan through the leaf segments for doclists. */
103478		- return loadSegmentLeaves(v, iStartChild, iEndChild,
103479		- pTerm, nTerm, isPrefix, out);
103480		- }
103481		-}
103482		-
103483		-/* Call loadSegmentInt() to collect the doclist for pTerm/nTerm, then
103484		-** merge its doclist over *out (any duplicate doclists read from the
103485		-** segment rooted at pData will overwrite those in *out).
103486		-*/
103487		-/* TODO(shess) Consider changing this to determine the depth of the
103488		-** leaves using either the first characters of interior nodes (when
103489		-** ==1, we're one level above the leaves), or the first character of
103490		-** the root (which will describe the height of the tree directly).
103491		-** Either feels somewhat tricky to me.
103492		-*/
103493		-/* TODO(shess) The current merge is likely to be slow for large
103494		-** doclists (though it should process from newest/smallest to
103495		-** oldest/largest, so it may not be that bad). It might be useful to
103496		-** modify things to allow for N-way merging. This could either be
103497		-** within a segment, with pairwise merges across segments, or across
103498		-** all segments at once.
103499		-*/
103500		-static int loadSegment(fulltext_vtab v, const char pData, int nData,
103501		- sqlite_int64 iLeavesEnd,
103502		- const char *pTerm, int nTerm, int isPrefix,
103503		- DataBuffer *out){
103504		- DataBuffer result;
103505		- int rc;
103506		-
103507		- assert( nData>1 );
103508		-
103509		- /* This code should never be called with buffered updates. */
103510		- assert( v->nPendingData<0 );
103511		-
103512		- dataBufferInit(&result, 0);
103513		- rc = loadSegmentInt(v, pData, nData, iLeavesEnd,
103514		- pTerm, nTerm, isPrefix, &result);
103515		- if( rc==SQLITE_OK && result.nData>0 ){
103516		- if( out->nData==0 ){
103517		- DataBuffer tmp = *out;
103518		- *out = result;
103519		- result = tmp;
103520		- }else{
103521		- DataBuffer merged;
103522		- DLReader readers[2];
103523		-
103524		- dlrInit(&readers[0], DL_DEFAULT, out->pData, out->nData);
103525		- dlrInit(&readers[1], DL_DEFAULT, result.pData, result.nData);
103526		- dataBufferInit(&merged, out->nData+result.nData);
103527		- docListMerge(&merged, readers, 2);
103528		- dataBufferDestroy(out);
103529		- *out = merged;
103530		- dlrDestroy(&readers[0]);
103531		- dlrDestroy(&readers[1]);
103532		- }
103533		- }
103534		- dataBufferDestroy(&result);
103535		- return rc;
103536		-}
103537		-
103538		-/* Scan the database and merge together the posting lists for the term
103539		-** into *out.
103540		-*/
103541		-static int termSelect(
103542		- fulltext_vtab *v,
103543		- int iColumn,
103544		- const char pTerm, int nTerm, / Term to query for */
103545		- int isPrefix, /* True for a prefix search */
103546		- DocListType iType,
103547		- DataBuffer out / Write results here */
103548		-){
103549		- DataBuffer doclist;
103550		- sqlite3_stmt *s;
103551		- int rc = sql_get_statement(v, SEGDIR_SELECT_ALL_STMT, &s);
103552		- if( rc!=SQLITE_OK ) return rc;
103553		-
103554		- /* This code should never be called with buffered updates. */
103555		- assert( v->nPendingData<0 );
103556		-
103557		- dataBufferInit(&doclist, 0);
103558		- dataBufferInit(out, 0);
103559		-
103560		- /* Traverse the segments from oldest to newest so that newer doclist
103561		- ** elements for given docids overwrite older elements.
103562		- */
103563		- while( (rc = sqlite3_step(s))==SQLITE_ROW ){
103564		- const char *pData = sqlite3_column_blob(s, 2);
103565		- const int nData = sqlite3_column_bytes(s, 2);
103566		- const sqlite_int64 iLeavesEnd = sqlite3_column_int64(s, 1);
103567		- rc = loadSegment(v, pData, nData, iLeavesEnd, pTerm, nTerm, isPrefix,
103568		- &doclist);
103569		- if( rc!=SQLITE_OK ) goto err;
103570		- }
103571		- if( rc==SQLITE_DONE ){
103572		- if( doclist.nData!=0 ){
103573		- /* TODO(shess) The old term_select_all() code applied the column
103574		- ** restrict as we merged segments, leading to smaller buffers.
103575		- ** This is probably worthwhile to bring back, once the new storage
103576		- ** system is checked in.
103577		- */
103578		- if( iColumn==v->nColumn) iColumn = -1;
103579		- docListTrim(DL_DEFAULT, doclist.pData, doclist.nData,
103580		- iColumn, iType, out);
103581		- }
103582		- rc = SQLITE_OK;
103583		- }
103584		-
103585		- err:
103586		- dataBufferDestroy(&doclist);
103587		- return rc;
103588		-}
103589		-
103590		-/****************************************************************/
103591		-/* Used to hold hashtable data for sorting. */
103592		-typedef struct TermData {
103593		- const char *pTerm;
103594		- int nTerm;
103595		- DLCollector *pCollector;
103596		-} TermData;
103597		-
103598		-/* Orders TermData elements in strcmp fashion ( <0 for less-than, 0
103599		-** for equal, >0 for greater-than).
103600		-*/
103601		-static int termDataCmp(const void av, const void bv){
103602		- const TermData a = (const TermData )av;
103603		- const TermData b = (const TermData )bv;
103604		- int n = a->nTerm<b->nTerm ? a->nTerm : b->nTerm;
103605		- int c = memcmp(a->pTerm, b->pTerm, n);
103606		- if( c!=0 ) return c;
103607		- return a->nTerm-b->nTerm;
103608		-}
103609		-
103610		-/* Order pTerms data by term, then write a new level 0 segment using
103611		-** LeafWriter.
103612		-*/
103613		-static int writeZeroSegment(fulltext_vtab v, fts3Hash pTerms){
103614		- fts3HashElem *e;
103615		- int idx, rc, i, n;
103616		- TermData *pData;
103617		- LeafWriter writer;
103618		- DataBuffer dl;
103619		-
103620		- /* Determine the next index at level 0, merging as necessary. */
103621		- rc = segdirNextIndex(v, 0, &idx);
103622		- if( rc!=SQLITE_OK ) return rc;
103623		-
103624		- n = fts3HashCount(pTerms);
103625		- pData = sqlite3_malloc(n*sizeof(TermData));
103626		-
103627		- for(i = 0, e = fts3HashFirst(pTerms); e; i++, e = fts3HashNext(e)){
103628		- assert( i<n );
103629		- pData[i].pTerm = fts3HashKey(e);
103630		- pData[i].nTerm = fts3HashKeysize(e);
103631		- pData[i].pCollector = fts3HashData(e);
103632		- }
103633		- assert( i==n );
103634		-
103635		- /* TODO(shess) Should we allow user-defined collation sequences,
103636		- ** here? I think we only need that once we support prefix searches.
103637		- */
103638		- if( n>1 ) qsort(pData, n, sizeof(*pData), termDataCmp);
103639		-
103640		- /* TODO(shess) Refactor so that we can write directly to the segment
103641		- ** DataBuffer, as happens for segment merges.
103642		- */
103643		- leafWriterInit(0, idx, &writer);
103644		- dataBufferInit(&dl, 0);
103645		- for(i=0; i<n; i++){
103646		- dataBufferReset(&dl);
103647		- dlcAddDoclist(pData[i].pCollector, &dl);
103648		- rc = leafWriterStep(v, &writer,
103649		- pData[i].pTerm, pData[i].nTerm, dl.pData, dl.nData);
103650		- if( rc!=SQLITE_OK ) goto err;
103651		- }
103652		- rc = leafWriterFinalize(v, &writer);
103653		-
103654		- err:
103655		- dataBufferDestroy(&dl);
103656		- sqlite3_free(pData);
103657		- leafWriterDestroy(&writer);
103658		- return rc;
103659		-}
103660		-
103661		-/* If pendingTerms has data, free it. */
103662		-static int clearPendingTerms(fulltext_vtab *v){
103663		- if( v->nPendingData>=0 ){
103664		- fts3HashElem *e;
103665		- for(e=fts3HashFirst(&v->pendingTerms); e; e=fts3HashNext(e)){
103666		- dlcDelete(fts3HashData(e));
103667		- }
103668		- fts3HashClear(&v->pendingTerms);
103669		- v->nPendingData = -1;
103670		- }
103671		- return SQLITE_OK;
103672		-}
103673		-
103674		-/* If pendingTerms has data, flush it to a level-zero segment, and
103675		-** free it.
103676		-*/
103677		-static int flushPendingTerms(fulltext_vtab *v){
103678		- if( v->nPendingData>=0 ){
103679		- int rc = writeZeroSegment(v, &v->pendingTerms);
103680		- if( rc==SQLITE_OK ) clearPendingTerms(v);
103681		- return rc;
103682		- }
103683		- return SQLITE_OK;
103684		-}
103685		-
103686		-/* If pendingTerms is "too big", or docid is out of order, flush it.
103687		-** Regardless, be certain that pendingTerms is initialized for use.
103688		-*/
103689		-static int initPendingTerms(fulltext_vtab *v, sqlite_int64 iDocid){
103690		- /* TODO(shess) Explore whether partially flushing the buffer on
103691		- ** forced-flush would provide better performance. I suspect that if
103692		- ** we ordered the doclists by size and flushed the largest until the
103693		- ** buffer was half empty, that would let the less frequent terms
103694		- ** generate longer doclists.
103695		- */
103696		- if( iDocid<=v->iPrevDocid \|\| v->nPendingData>kPendingThreshold ){
103697		- int rc = flushPendingTerms(v);
103698		- if( rc!=SQLITE_OK ) return rc;
103699		- }
103700		- if( v->nPendingData<0 ){
103701		- fts3HashInit(&v->pendingTerms, FTS3_HASH_STRING, 1);
103702		- v->nPendingData = 0;
103703		- }
103704		- v->iPrevDocid = iDocid;
103705		- return SQLITE_OK;
103706		-}
103707		-
103708		-/* This function implements the xUpdate callback; it is the top-level entry
103709		- * point for inserting, deleting or updating a row in a full-text table. */
103710		-static int fulltextUpdate(sqlite3_vtab pVtab, int nArg, sqlite3_value *ppArg,
103711		- sqlite_int64 *pRowid){
103712		- fulltext_vtab v = (fulltext_vtab ) pVtab;
103713		- int rc;
103714		-
103715		- FTSTRACE(("FTS3 Update %p\n", pVtab));
103716		-
103717		- if( nArg<2 ){
103718		- rc = index_delete(v, sqlite3_value_int64(ppArg[0]));
103719		- if( rc==SQLITE_OK ){
103720		- /* If we just deleted the last row in the table, clear out the
103721		- ** index data.
103722		- */
103723		- rc = content_exists(v);
103724		- if( rc==SQLITE_ROW ){
103725		- rc = SQLITE_OK;
103726		- }else if( rc==SQLITE_DONE ){
103727		- /* Clear the pending terms so we don't flush a useless level-0
103728		- ** segment when the transaction closes.
103729		- */
103730		- rc = clearPendingTerms(v);
103731		- if( rc==SQLITE_OK ){
103732		- rc = segdir_delete_all(v);
103733		- }
103734		- }
103735		- }
103736		- } else if( sqlite3_value_type(ppArg[0]) != SQLITE_NULL ){
103737		- /* An update:
103738		- * ppArg[0] = old rowid
103739		- * ppArg[1] = new rowid
103740		- * ppArg[2..2+v->nColumn-1] = values
103741		- * ppArg[2+v->nColumn] = value for magic column (we ignore this)
103742		- * ppArg[2+v->nColumn+1] = value for docid
103743		- */
103744		- sqlite_int64 rowid = sqlite3_value_int64(ppArg[0]);
103745		- if( sqlite3_value_type(ppArg[1]) != SQLITE_INTEGER \|\|
103746		- sqlite3_value_int64(ppArg[1]) != rowid ){
103747		- rc = SQLITE_ERROR; /* we don't allow changing the rowid */
103748		- }else if( sqlite3_value_type(ppArg[2+v->nColumn+1]) != SQLITE_INTEGER \|\|
103749		- sqlite3_value_int64(ppArg[2+v->nColumn+1]) != rowid ){
103750		- rc = SQLITE_ERROR; /* we don't allow changing the docid */
103751		- }else{
103752		- assert( nArg==2+v->nColumn+2);
103753		- rc = index_update(v, rowid, &ppArg[2]);
103754		- }
103755		- } else {
103756		- /* An insert:
103757		- * ppArg[1] = requested rowid
103758		- * ppArg[2..2+v->nColumn-1] = values
103759		- * ppArg[2+v->nColumn] = value for magic column (we ignore this)
103760		- * ppArg[2+v->nColumn+1] = value for docid
103761		- */
103762		- sqlite3_value *pRequestDocid = ppArg[2+v->nColumn+1];
103763		- assert( nArg==2+v->nColumn+2);
103764		- if( SQLITE_NULL != sqlite3_value_type(pRequestDocid) &&
103765		- SQLITE_NULL != sqlite3_value_type(ppArg[1]) ){
103766		- /* TODO(shess) Consider allowing this to work if the values are
103767		- ** identical. I'm inclined to discourage that usage, though,
103768		- ** given that both rowid and docid are special columns. Better
103769		- ** would be to define one or the other as the default winner,
103770		- ** but should it be fts3-centric (docid) or SQLite-centric
103771		- ** (rowid)?
103772		- */
103773		- rc = SQLITE_ERROR;
103774		- }else{
103775		- if( SQLITE_NULL == sqlite3_value_type(pRequestDocid) ){
103776		- pRequestDocid = ppArg[1];
103777		- }
103778		- rc = index_insert(v, pRequestDocid, &ppArg[2], pRowid);
103779		- }
103780		- }
103781		-
103782		- return rc;
103783		-}
103784		-
103785		-static int fulltextSync(sqlite3_vtab *pVtab){
103786		- FTSTRACE(("FTS3 xSync()\n"));
103787		- return flushPendingTerms((fulltext_vtab *)pVtab);
103788		-}
103789		-
103790		-static int fulltextBegin(sqlite3_vtab *pVtab){
103791		- fulltext_vtab v = (fulltext_vtab ) pVtab;
103792		- FTSTRACE(("FTS3 xBegin()\n"));
103793		-
103794		- /* Any buffered updates should have been cleared by the previous
103795		- ** transaction.
103796		- */
103797		- assert( v->nPendingData<0 );
103798		- return clearPendingTerms(v);
103799		-}
103800		-
103801		-static int fulltextCommit(sqlite3_vtab *pVtab){
103802		- fulltext_vtab v = (fulltext_vtab ) pVtab;
103803		- FTSTRACE(("FTS3 xCommit()\n"));
103804		-
103805		- /* Buffered updates should have been cleared by fulltextSync(). */
103806		- assert( v->nPendingData<0 );
103807		- return clearPendingTerms(v);
103808		-}
103809		-
103810		-static int fulltextRollback(sqlite3_vtab *pVtab){
103811		- FTSTRACE(("FTS3 xRollback()\n"));
103812		- return clearPendingTerms((fulltext_vtab *)pVtab);
	99376	+static int fts3RowidMethod(sqlite3_vtab_cursor pCursor, sqlite_int64 pRowid){
	99377	+ Fts3Cursor pCsr = (Fts3Cursor ) pCursor;
	99378	+ if( pCsr->aDoclist ){
	99379	+ *pRowid = pCsr->iPrevId;
	99380	+ }else{
	99381	+ *pRowid = sqlite3_column_int64(pCsr->pStmt, 0);
	99382	+ }
	99383	+ return SQLITE_OK;
	99384	+}
	99385	+
	99386	+/*
	99387	+** This function is the implementation of the xUpdate callback used by
	99388	+** FTS3 virtual tables. It is invoked by SQLite each time a row is to be
	99389	+** inserted, updated or deleted.
	99390	+*/
	99391	+static int fts3UpdateMethod(
	99392	+ sqlite3_vtab pVtab, / Virtual table handle */
	99393	+ int nArg, /* Size of argument array */
	99394	+ sqlite3_value *apVal, / Array of arguments */
	99395	+ sqlite_int64 pRowid / OUT: The affected (or effected) rowid */
	99396	+){
	99397	+ return sqlite3Fts3UpdateMethod(pVtab, nArg, apVal, pRowid);
	99398	+}
	99399	+
	99400	+/*
	99401	+** Implementation of xSync() method. Flush the contents of the pending-terms
	99402	+** hash-table to the database.
	99403	+*/
	99404	+static int fts3SyncMethod(sqlite3_vtab *pVtab){
	99405	+ return sqlite3Fts3PendingTermsFlush((Fts3Table *)pVtab);
	99406	+}
	99407	+
	99408	+/*
	99409	+** Implementation of xBegin() method. This is a no-op.
	99410	+*/
	99411	+static int fts3BeginMethod(sqlite3_vtab *pVtab){
	99412	+ assert( ((Fts3Table *)pVtab)->nPendingData==0 );
	99413	+ return SQLITE_OK;
	99414	+}
	99415	+
	99416	+/*
	99417	+** Implementation of xCommit() method. This is a no-op. The contents of
	99418	+** the pending-terms hash-table have already been flushed into the database
	99419	+** by fts3SyncMethod().
	99420	+*/
	99421	+static int fts3CommitMethod(sqlite3_vtab *pVtab){
	99422	+ assert( ((Fts3Table *)pVtab)->nPendingData==0 );
	99423	+ return SQLITE_OK;
	99424	+}
	99425	+
	99426	+/*
	99427	+** Implementation of xRollback(). Discard the contents of the pending-terms
	99428	+** hash-table. Any changes made to the database are reverted by SQLite.
	99429	+*/
	99430	+static int fts3RollbackMethod(sqlite3_vtab *pVtab){
	99431	+ sqlite3Fts3PendingTermsClear((Fts3Table *)pVtab);
	99432	+ return SQLITE_OK;
	99433	+}
	99434	+
	99435	+/*
	99436	+** Helper function used by the implementation of the overloaded snippet(),
	99437	+** offsets() and optimize() SQL functions.
	99438	+**
	99439	+** If the value passed as the third argument is a blob of size
	99440	+** sizeof(Fts3Cursor*), then the blob contents are copied to the
	99441	+** output variable *ppCsr and SQLITE_OK is returned. Otherwise, an error
	99442	+** message is written to context pContext and SQLITE_ERROR returned. The
	99443	+** string passed via zFunc is used as part of the error message.
	99444	+*/
	99445	+static int fts3FunctionArg(
	99446	+ sqlite3_context pContext, / SQL function call context */
	99447	+ const char zFunc, / Function name */
	99448	+ sqlite3_value pVal, / argv[0] passed to function */
	99449	+ Fts3Cursor *ppCsr / OUT: Store cursor handle here */
	99450	+){
	99451	+ Fts3Cursor *pRet;
	99452	+ if( sqlite3_value_type(pVal)!=SQLITE_BLOB
	99453	+ && sqlite3_value_bytes(pVal)!=sizeof(Fts3Cursor *)
	99454	+ ){
	99455	+ char *zErr = sqlite3_mprintf("illegal first argument to %s", zFunc);
	99456	+ sqlite3_result_error(pContext, zErr, -1);
	99457	+ sqlite3_free(zErr);
	99458	+ return SQLITE_ERROR;
	99459	+ }
	99460	+ memcpy(&pRet, sqlite3_value_blob(pVal), sizeof(Fts3Cursor *));
	99461	+ *ppCsr = pRet;
	99462	+ return SQLITE_OK;
103813	99463	}
103814	99464
103815	99465	/*
103816	99466	** Implementation of the snippet() function for FTS3
103817	99467	*/
103818		-static void snippetFunc(
	99468	+static void fts3SnippetFunc(
103819	99469	sqlite3_context *pContext,
103820	99470	int argc,
103821	99471	sqlite3_value **argv
103822	99472	){
103823		- fulltext_cursor *pCursor;
103824		- if( argc<1 ) return;
103825		- if( sqlite3_value_type(argv[0])!=SQLITE_BLOB \|\|
103826		- sqlite3_value_bytes(argv[0])!=sizeof(pCursor) ){
103827		- sqlite3_result_error(pContext, "illegal first argument to html_snippet",-1);
103828		- }else{
103829		- const char *zStart = "<b>";
103830		- const char *zEnd = "</b>";
103831		- const char *zEllipsis = "<b>...</b>";
103832		- memcpy(&pCursor, sqlite3_value_blob(argv[0]), sizeof(pCursor));
103833		- if( argc>=2 ){
103834		- zStart = (const char*)sqlite3_value_text(argv[1]);
103835		- if( argc>=3 ){
103836		- zEnd = (const char*)sqlite3_value_text(argv[2]);
103837		- if( argc>=4 ){
103838		- zEllipsis = (const char*)sqlite3_value_text(argv[3]);
103839		- }
103840		- }
103841		- }
103842		- snippetAllOffsets(pCursor);
103843		- snippetText(pCursor, zStart, zEnd, zEllipsis);
103844		- sqlite3_result_text(pContext, pCursor->snippet.zSnippet,
103845		- pCursor->snippet.nSnippet, SQLITE_STATIC);
103846		- }
	99473	+ Fts3Cursor pCsr; / Cursor handle passed through apVal[0] */
	99474	+ const char *zStart = "<b>";
	99475	+ const char *zEnd = "</b>";
	99476	+ const char *zEllipsis = "<b>...</b>";
	99477	+
	99478	+ if( argc<1 \|\| argc>4 ) return;
	99479	+ if( fts3FunctionArg(pContext, "snippet", argv[0], &pCsr) ) return;
	99480	+
	99481	+ switch( argc ){
	99482	+ case 4: zEllipsis = (const char*)sqlite3_value_text(argv[3]);
	99483	+ case 3: zEnd = (const char*)sqlite3_value_text(argv[2]);
	99484	+ case 2: zStart = (const char*)sqlite3_value_text(argv[1]);
	99485	+ }
	99486	+
	99487	+ sqlite3Fts3Snippet(pContext, pCsr, zStart, zEnd, zEllipsis);
103847	99488	}
103848	99489
103849	99490	/*
103850	99491	** Implementation of the offsets() function for FTS3
103851	99492	*/
103852		-static void snippetOffsetsFunc(
103853		- sqlite3_context *pContext,
103854		- int argc,
103855		- sqlite3_value **argv
103856		-){
103857		- fulltext_cursor *pCursor;
103858		- if( argc<1 ) return;
103859		- if( sqlite3_value_type(argv[0])!=SQLITE_BLOB \|\|
103860		- sqlite3_value_bytes(argv[0])!=sizeof(pCursor) ){
103861		- sqlite3_result_error(pContext, "illegal first argument to offsets",-1);
103862		- }else{
103863		- memcpy(&pCursor, sqlite3_value_blob(argv[0]), sizeof(pCursor));
103864		- snippetAllOffsets(pCursor);
103865		- snippetOffsetText(&pCursor->snippet);
103866		- sqlite3_result_text(pContext,
103867		- pCursor->snippet.zOffset, pCursor->snippet.nOffset,
103868		- SQLITE_STATIC);
103869		- }
103870		-}
103871		-
103872		-/* OptLeavesReader is nearly identical to LeavesReader, except that
103873		-** where LeavesReader is geared towards the merging of complete
103874		-** segment levels (with exactly MERGE_COUNT segments), OptLeavesReader
103875		-** is geared towards implementation of the optimize() function, and
103876		-** can merge all segments simultaneously. This version may be
103877		-** somewhat less efficient than LeavesReader because it merges into an
103878		-** accumulator rather than doing an N-way merge, but since segment
103879		-** size grows exponentially (so segment count logrithmically) this is
103880		-** probably not an immediate problem.
103881		-*/
103882		-/* TODO(shess): Prove that assertion, or extend the merge code to
103883		-** merge tree fashion (like the prefix-searching code does).
103884		-*/
103885		-/* TODO(shess): OptLeavesReader and LeavesReader could probably be
103886		-** merged with little or no loss of performance for LeavesReader. The
103887		-** merged code would need to handle >MERGE_COUNT segments, and would
103888		-** also need to be able to optionally optimize away deletes.
103889		-*/
103890		-typedef struct OptLeavesReader {
103891		- /* Segment number, to order readers by age. */
103892		- int segment;
103893		- LeavesReader reader;
103894		-} OptLeavesReader;
103895		-
103896		-static int optLeavesReaderAtEnd(OptLeavesReader *pReader){
103897		- return leavesReaderAtEnd(&pReader->reader);
103898		-}
103899		-static int optLeavesReaderTermBytes(OptLeavesReader *pReader){
103900		- return leavesReaderTermBytes(&pReader->reader);
103901		-}
103902		-static const char optLeavesReaderData(OptLeavesReader pReader){
103903		- return leavesReaderData(&pReader->reader);
103904		-}
103905		-static int optLeavesReaderDataBytes(OptLeavesReader *pReader){
103906		- return leavesReaderDataBytes(&pReader->reader);
103907		-}
103908		-static const char optLeavesReaderTerm(OptLeavesReader pReader){
103909		- return leavesReaderTerm(&pReader->reader);
103910		-}
103911		-static int optLeavesReaderStep(fulltext_vtab v, OptLeavesReader pReader){
103912		- return leavesReaderStep(v, &pReader->reader);
103913		-}
103914		-static int optLeavesReaderTermCmp(OptLeavesReader lr1, OptLeavesReader lr2){
103915		- return leavesReaderTermCmp(&lr1->reader, &lr2->reader);
103916		-}
103917		-/* Order by term ascending, segment ascending (oldest to newest), with
103918		-** exhausted readers to the end.
103919		-*/
103920		-static int optLeavesReaderCmp(OptLeavesReader lr1, OptLeavesReader lr2){
103921		- int c = optLeavesReaderTermCmp(lr1, lr2);
103922		- if( c!=0 ) return c;
103923		- return lr1->segment-lr2->segment;
103924		-}
103925		-/* Bubble pLr[0] to appropriate place in pLr[1..nLr-1]. Assumes that
103926		-** pLr[1..nLr-1] is already sorted.
103927		-*/
103928		-static void optLeavesReaderReorder(OptLeavesReader *pLr, int nLr){
103929		- while( nLr>1 && optLeavesReaderCmp(pLr, pLr+1)>0 ){
103930		- OptLeavesReader tmp = pLr[0];
103931		- pLr[0] = pLr[1];
103932		- pLr[1] = tmp;
103933		- nLr--;
103934		- pLr++;
103935		- }
103936		-}
103937		-
103938		-/* optimize() helper function. Put the readers in order and iterate
103939		-** through them, merging doclists for matching terms into pWriter.
103940		-** Returns SQLITE_OK on success, or the SQLite error code which
103941		-** prevented success.
103942		-*/
103943		-static int optimizeInternal(fulltext_vtab *v,
103944		- OptLeavesReader *readers, int nReaders,
103945		- LeafWriter *pWriter){
103946		- int i, rc = SQLITE_OK;
103947		- DataBuffer doclist, merged, tmp;
103948		-
103949		- /* Order the readers. */
103950		- i = nReaders;
103951		- while( i-- > 0 ){
103952		- optLeavesReaderReorder(&readers[i], nReaders-i);
103953		- }
103954		-
103955		- dataBufferInit(&doclist, LEAF_MAX);
103956		- dataBufferInit(&merged, LEAF_MAX);
103957		-
103958		- /* Exhausted readers bubble to the end, so when the first reader is
103959		- ** at eof, all are at eof.
103960		- */
103961		- while( !optLeavesReaderAtEnd(&readers[0]) ){
103962		-
103963		- /* Figure out how many readers share the next term. */
103964		- for(i=1; i<nReaders && !optLeavesReaderAtEnd(&readers[i]); i++){
103965		- if( 0!=optLeavesReaderTermCmp(&readers[0], &readers[i]) ) break;
103966		- }
103967		-
103968		- /* Special-case for no merge. */
103969		- if( i==1 ){
103970		- /* Trim deletions from the doclist. */
103971		- dataBufferReset(&merged);
103972		- docListTrim(DL_DEFAULT,
103973		- optLeavesReaderData(&readers[0]),
103974		- optLeavesReaderDataBytes(&readers[0]),
103975		- -1, DL_DEFAULT, &merged);
103976		- }else{
103977		- DLReader dlReaders[MERGE_COUNT];
103978		- int iReader, nReaders;
103979		-
103980		- /* Prime the pipeline with the first reader's doclist. After
103981		- ** one pass index 0 will reference the accumulated doclist.
103982		- */
103983		- dlrInit(&dlReaders[0], DL_DEFAULT,
103984		- optLeavesReaderData(&readers[0]),
103985		- optLeavesReaderDataBytes(&readers[0]));
103986		- iReader = 1;
103987		-
103988		- assert( iReader<i ); /* Must execute the loop at least once. */
103989		- while( iReader<i ){
103990		- /* Merge 16 inputs per pass. */
103991		- for( nReaders=1; iReader<i && nReaders<MERGE_COUNT;
103992		- iReader++, nReaders++ ){
103993		- dlrInit(&dlReaders[nReaders], DL_DEFAULT,
103994		- optLeavesReaderData(&readers[iReader]),
103995		- optLeavesReaderDataBytes(&readers[iReader]));
103996		- }
103997		-
103998		- /* Merge doclists and swap result into accumulator. */
103999		- dataBufferReset(&merged);
104000		- docListMerge(&merged, dlReaders, nReaders);
104001		- tmp = merged;
104002		- merged = doclist;
104003		- doclist = tmp;
104004		-
104005		- while( nReaders-- > 0 ){
104006		- dlrDestroy(&dlReaders[nReaders]);
104007		- }
104008		-
104009		- /* Accumulated doclist to reader 0 for next pass. */
104010		- dlrInit(&dlReaders[0], DL_DEFAULT, doclist.pData, doclist.nData);
104011		- }
104012		-
104013		- /* Destroy reader that was left in the pipeline. */
104014		- dlrDestroy(&dlReaders[0]);
104015		-
104016		- /* Trim deletions from the doclist. */
104017		- dataBufferReset(&merged);
104018		- docListTrim(DL_DEFAULT, doclist.pData, doclist.nData,
104019		- -1, DL_DEFAULT, &merged);
104020		- }
104021		-
104022		- /* Only pass doclists with hits (skip if all hits deleted). */
104023		- if( merged.nData>0 ){
104024		- rc = leafWriterStep(v, pWriter,
104025		- optLeavesReaderTerm(&readers[0]),
104026		- optLeavesReaderTermBytes(&readers[0]),
104027		- merged.pData, merged.nData);
104028		- if( rc!=SQLITE_OK ) goto err;
104029		- }
104030		-
104031		- /* Step merged readers to next term and reorder. */
104032		- while( i-- > 0 ){
104033		- rc = optLeavesReaderStep(v, &readers[i]);
104034		- if( rc!=SQLITE_OK ) goto err;
104035		-
104036		- optLeavesReaderReorder(&readers[i], nReaders-i);
104037		- }
104038		- }
104039		-
104040		- err:
104041		- dataBufferDestroy(&doclist);
104042		- dataBufferDestroy(&merged);
104043		- return rc;
104044		-}
104045		-
104046		-/* Implement optimize() function for FTS3. optimize(t) merges all
104047		-** segments in the fts index into a single segment. 't' is the magic
104048		-** table-named column.
104049		-*/
104050		-static void optimizeFunc(sqlite3_context *pContext,
104051		- int argc, sqlite3_value **argv){
104052		- fulltext_cursor *pCursor;
104053		- if( argc>1 ){
104054		- sqlite3_result_error(pContext, "excess arguments to optimize()",-1);
104055		- }else if( sqlite3_value_type(argv[0])!=SQLITE_BLOB \|\|
104056		- sqlite3_value_bytes(argv[0])!=sizeof(pCursor) ){
104057		- sqlite3_result_error(pContext, "illegal first argument to optimize",-1);
104058		- }else{
104059		- fulltext_vtab *v;
104060		- int i, rc, iMaxLevel;
104061		- OptLeavesReader *readers;
104062		- int nReaders;
104063		- LeafWriter writer;
104064		- sqlite3_stmt *s;
104065		-
104066		- memcpy(&pCursor, sqlite3_value_blob(argv[0]), sizeof(pCursor));
104067		- v = cursor_vtab(pCursor);
104068		-
104069		- /* Flush any buffered updates before optimizing. */
104070		- rc = flushPendingTerms(v);
104071		- if( rc!=SQLITE_OK ) goto err;
104072		-
104073		- rc = segdir_count(v, &nReaders, &iMaxLevel);
104074		- if( rc!=SQLITE_OK ) goto err;
104075		- if( nReaders==0 \|\| nReaders==1 ){
104076		- sqlite3_result_text(pContext, "Index already optimal", -1,
104077		- SQLITE_STATIC);
104078		- return;
104079		- }
104080		-
104081		- rc = sql_get_statement(v, SEGDIR_SELECT_ALL_STMT, &s);
104082		- if( rc!=SQLITE_OK ) goto err;
104083		-
104084		- readers = sqlite3_malloc(nReaders*sizeof(readers[0]));
104085		- if( readers==NULL ) goto err;
104086		-
104087		- /* Note that there will already be a segment at this position
104088		- ** until we call segdir_delete() on iMaxLevel.
104089		- */
104090		- leafWriterInit(iMaxLevel, 0, &writer);
104091		-
104092		- i = 0;
104093		- while( (rc = sqlite3_step(s))==SQLITE_ROW ){
104094		- sqlite_int64 iStart = sqlite3_column_int64(s, 0);
104095		- sqlite_int64 iEnd = sqlite3_column_int64(s, 1);
104096		- const char *pRootData = sqlite3_column_blob(s, 2);
104097		- int nRootData = sqlite3_column_bytes(s, 2);
104098		-
104099		- assert( i<nReaders );
104100		- rc = leavesReaderInit(v, -1, iStart, iEnd, pRootData, nRootData,
104101		- &readers[i].reader);
104102		- if( rc!=SQLITE_OK ) break;
104103		-
104104		- readers[i].segment = i;
104105		- i++;
104106		- }
104107		-
104108		- /* If we managed to successfully read them all, optimize them. */
104109		- if( rc==SQLITE_DONE ){
104110		- assert( i==nReaders );
104111		- rc = optimizeInternal(v, readers, nReaders, &writer);
104112		- }
104113		-
104114		- while( i-- > 0 ){
104115		- leavesReaderDestroy(&readers[i].reader);
104116		- }
104117		- sqlite3_free(readers);
104118		-
104119		- /* If we've successfully gotten to here, delete the old segments
104120		- ** and flush the interior structure of the new segment.
104121		- */
104122		- if( rc==SQLITE_OK ){
104123		- for( i=0; i<=iMaxLevel; i++ ){
104124		- rc = segdir_delete(v, i);
104125		- if( rc!=SQLITE_OK ) break;
104126		- }
104127		-
104128		- if( rc==SQLITE_OK ) rc = leafWriterFinalize(v, &writer);
104129		- }
104130		-
104131		- leafWriterDestroy(&writer);
104132		-
104133		- if( rc!=SQLITE_OK ) goto err;
104134		-
104135		- sqlite3_result_text(pContext, "Index optimized", -1, SQLITE_STATIC);
104136		- return;
104137		-
104138		- /* TODO(shess): Error-handling needs to be improved along the
104139		- ** lines of the dump_ functions.
104140		- */
104141		- err:
104142		- {
104143		- char buf[512];
104144		- sqlite3_snprintf(sizeof(buf), buf, "Error in optimize: %s",
104145		- sqlite3_errmsg(sqlite3_context_db_handle(pContext)));
104146		- sqlite3_result_error(pContext, buf, -1);
104147		- }
104148		- }
104149		-}
104150		-
104151		-#ifdef SQLITE_TEST
104152		-/* Generate an error of the form "<prefix>: <msg>". If msg is NULL,
104153		-** pull the error from the context's db handle.
104154		-*/
104155		-static void generateError(sqlite3_context *pContext,
104156		- const char prefix, const char msg){
104157		- char buf[512];
104158		- if( msg==NULL ) msg = sqlite3_errmsg(sqlite3_context_db_handle(pContext));
104159		- sqlite3_snprintf(sizeof(buf), buf, "%s: %s", prefix, msg);
104160		- sqlite3_result_error(pContext, buf, -1);
104161		-}
104162		-
104163		-/* Helper function to collect the set of terms in the segment into
104164		-** pTerms. The segment is defined by the leaf nodes between
104165		-** iStartBlockid and iEndBlockid, inclusive, or by the contents of
104166		-** pRootData if iStartBlockid is 0 (in which case the entire segment
104167		-** fit in a leaf).
104168		-*/
104169		-static int collectSegmentTerms(fulltext_vtab v, sqlite3_stmt s,
104170		- fts3Hash *pTerms){
104171		- const sqlite_int64 iStartBlockid = sqlite3_column_int64(s, 0);
104172		- const sqlite_int64 iEndBlockid = sqlite3_column_int64(s, 1);
104173		- const char *pRootData = sqlite3_column_blob(s, 2);
104174		- const int nRootData = sqlite3_column_bytes(s, 2);
104175		- LeavesReader reader;
104176		- int rc = leavesReaderInit(v, 0, iStartBlockid, iEndBlockid,
104177		- pRootData, nRootData, &reader);
104178		- if( rc!=SQLITE_OK ) return rc;
104179		-
104180		- while( rc==SQLITE_OK && !leavesReaderAtEnd(&reader) ){
104181		- const char *pTerm = leavesReaderTerm(&reader);
104182		- const int nTerm = leavesReaderTermBytes(&reader);
104183		- void *oldValue = sqlite3Fts3HashFind(pTerms, pTerm, nTerm);
104184		- void newValue = (void )((char *)oldValue+1);
104185		-
104186		- /* From the comment before sqlite3Fts3HashInsert in fts3_hash.c,
104187		- ** the data value passed is returned in case of malloc failure.
104188		- */
104189		- if( newValue==sqlite3Fts3HashInsert(pTerms, pTerm, nTerm, newValue) ){
104190		- rc = SQLITE_NOMEM;
104191		- }else{
104192		- rc = leavesReaderStep(v, &reader);
104193		- }
104194		- }
104195		-
104196		- leavesReaderDestroy(&reader);
104197		- return rc;
104198		-}
104199		-
104200		-/* Helper function to build the result string for dump_terms(). */
104201		-static int generateTermsResult(sqlite3_context pContext, fts3Hash pTerms){
104202		- int iTerm, nTerms, nResultBytes, iByte;
104203		- char *result;
104204		- TermData *pData;
104205		- fts3HashElem *e;
104206		-
104207		- /* Iterate pTerms to generate an array of terms in pData for
104208		- ** sorting.
104209		- */
104210		- nTerms = fts3HashCount(pTerms);
104211		- assert( nTerms>0 );
104212		- pData = sqlite3_malloc(nTerms*sizeof(TermData));
104213		- if( pData==NULL ) return SQLITE_NOMEM;
104214		-
104215		- nResultBytes = 0;
104216		- for(iTerm = 0, e = fts3HashFirst(pTerms); e; iTerm++, e = fts3HashNext(e)){
104217		- nResultBytes += fts3HashKeysize(e)+1; /* Term plus trailing space */
104218		- assert( iTerm<nTerms );
104219		- pData[iTerm].pTerm = fts3HashKey(e);
104220		- pData[iTerm].nTerm = fts3HashKeysize(e);
104221		- pData[iTerm].pCollector = fts3HashData(e); /* unused */
104222		- }
104223		- assert( iTerm==nTerms );
104224		-
104225		- assert( nResultBytes>0 ); /* nTerms>0, nResultsBytes must be, too. */
104226		- result = sqlite3_malloc(nResultBytes);
104227		- if( result==NULL ){
104228		- sqlite3_free(pData);
104229		- return SQLITE_NOMEM;
104230		- }
104231		-
104232		- if( nTerms>1 ) qsort(pData, nTerms, sizeof(*pData), termDataCmp);
104233		-
104234		- /* Read the terms in order to build the result. */
104235		- iByte = 0;
104236		- for(iTerm=0; iTerm<nTerms; ++iTerm){
104237		- memcpy(result+iByte, pData[iTerm].pTerm, pData[iTerm].nTerm);
104238		- iByte += pData[iTerm].nTerm;
104239		- result[iByte++] = ' ';
104240		- }
104241		- assert( iByte==nResultBytes );
104242		- assert( result[nResultBytes-1]==' ' );
104243		- result[nResultBytes-1] = '\0';
104244		-
104245		- /* Passes away ownership of result. */
104246		- sqlite3_result_text(pContext, result, nResultBytes-1, sqlite3_free);
104247		- sqlite3_free(pData);
104248		- return SQLITE_OK;
104249		-}
104250		-
104251		-/* Implements dump_terms() for use in inspecting the fts3 index from
104252		-** tests. TEXT result containing the ordered list of terms joined by
104253		-** spaces. dump_terms(t, level, idx) dumps the terms for the segment
104254		-** specified by level, idx (in %_segdir), while dump_terms(t) dumps
104255		-** all terms in the index. In both cases t is the fts table's magic
104256		-** table-named column.
104257		-*/
104258		-static void dumpTermsFunc(
104259		- sqlite3_context *pContext,
104260		- int argc, sqlite3_value **argv
104261		-){
104262		- fulltext_cursor *pCursor;
104263		- if( argc!=3 && argc!=1 ){
104264		- generateError(pContext, "dump_terms", "incorrect arguments");
104265		- }else if( sqlite3_value_type(argv[0])!=SQLITE_BLOB \|\|
104266		- sqlite3_value_bytes(argv[0])!=sizeof(pCursor) ){
104267		- generateError(pContext, "dump_terms", "illegal first argument");
104268		- }else{
104269		- fulltext_vtab *v;
104270		- fts3Hash terms;
104271		- sqlite3_stmt *s = NULL;
104272		- int rc;
104273		-
104274		- memcpy(&pCursor, sqlite3_value_blob(argv[0]), sizeof(pCursor));
104275		- v = cursor_vtab(pCursor);
104276		-
104277		- /* If passed only the cursor column, get all segments. Otherwise
104278		- ** get the segment described by the following two arguments.
104279		- */
104280		- if( argc==1 ){
104281		- rc = sql_get_statement(v, SEGDIR_SELECT_ALL_STMT, &s);
104282		- }else{
104283		- rc = sql_get_statement(v, SEGDIR_SELECT_SEGMENT_STMT, &s);
104284		- if( rc==SQLITE_OK ){
104285		- rc = sqlite3_bind_int(s, 1, sqlite3_value_int(argv[1]));
104286		- if( rc==SQLITE_OK ){
104287		- rc = sqlite3_bind_int(s, 2, sqlite3_value_int(argv[2]));
104288		- }
104289		- }
104290		- }
104291		-
104292		- if( rc!=SQLITE_OK ){
104293		- generateError(pContext, "dump_terms", NULL);
104294		- return;
104295		- }
104296		-
104297		- /* Collect the terms for each segment. */
104298		- sqlite3Fts3HashInit(&terms, FTS3_HASH_STRING, 1);
104299		- while( (rc = sqlite3_step(s))==SQLITE_ROW ){
104300		- rc = collectSegmentTerms(v, s, &terms);
104301		- if( rc!=SQLITE_OK ) break;
104302		- }
104303		-
104304		- if( rc!=SQLITE_DONE ){
104305		- sqlite3_reset(s);
104306		- generateError(pContext, "dump_terms", NULL);
104307		- }else{
104308		- const int nTerms = fts3HashCount(&terms);
104309		- if( nTerms>0 ){
104310		- rc = generateTermsResult(pContext, &terms);
104311		- if( rc==SQLITE_NOMEM ){
104312		- generateError(pContext, "dump_terms", "out of memory");
104313		- }else{
104314		- assert( rc==SQLITE_OK );
104315		- }
104316		- }else if( argc==3 ){
104317		- /* The specific segment asked for could not be found. */
104318		- generateError(pContext, "dump_terms", "segment not found");
104319		- }else{
104320		- /* No segments found. */
104321		- /* TODO(shess): It should be impossible to reach this. This
104322		- ** case can only happen for an empty table, in which case
104323		- ** SQLite has no rows to call this function on.
104324		- */
104325		- sqlite3_result_null(pContext);
104326		- }
104327		- }
104328		- sqlite3Fts3HashClear(&terms);
104329		- }
104330		-}
104331		-
104332		-/* Expand the DL_DEFAULT doclist in pData into a text result in
104333		-** pContext.
104334		-*/
104335		-static void createDoclistResult(sqlite3_context *pContext,
104336		- const char *pData, int nData){
104337		- DataBuffer dump;
104338		- DLReader dlReader;
104339		-
104340		- assert( pData!=NULL && nData>0 );
104341		-
104342		- dataBufferInit(&dump, 0);
104343		- dlrInit(&dlReader, DL_DEFAULT, pData, nData);
104344		- for( ; !dlrAtEnd(&dlReader); dlrStep(&dlReader) ){
104345		- char buf[256];
104346		- PLReader plReader;
104347		-
104348		- plrInit(&plReader, &dlReader);
104349		- if( DL_DEFAULT==DL_DOCIDS \|\| plrAtEnd(&plReader) ){
104350		- sqlite3_snprintf(sizeof(buf), buf, "[%lld] ", dlrDocid(&dlReader));
104351		- dataBufferAppend(&dump, buf, strlen(buf));
104352		- }else{
104353		- int iColumn = plrColumn(&plReader);
104354		-
104355		- sqlite3_snprintf(sizeof(buf), buf, "[%lld %d[",
104356		- dlrDocid(&dlReader), iColumn);
104357		- dataBufferAppend(&dump, buf, strlen(buf));
104358		-
104359		- for( ; !plrAtEnd(&plReader); plrStep(&plReader) ){
104360		- if( plrColumn(&plReader)!=iColumn ){
104361		- iColumn = plrColumn(&plReader);
104362		- sqlite3_snprintf(sizeof(buf), buf, "] %d[", iColumn);
104363		- assert( dump.nData>0 );
104364		- dump.nData--; /* Overwrite trailing space. */
104365		- assert( dump.pData[dump.nData]==' ');
104366		- dataBufferAppend(&dump, buf, strlen(buf));
104367		- }
104368		- if( DL_DEFAULT==DL_POSITIONS_OFFSETS ){
104369		- sqlite3_snprintf(sizeof(buf), buf, "%d,%d,%d ",
104370		- plrPosition(&plReader),
104371		- plrStartOffset(&plReader), plrEndOffset(&plReader));
104372		- }else if( DL_DEFAULT==DL_POSITIONS ){
104373		- sqlite3_snprintf(sizeof(buf), buf, "%d ", plrPosition(&plReader));
104374		- }else{
104375		- assert( NULL=="Unhandled DL_DEFAULT value");
104376		- }
104377		- dataBufferAppend(&dump, buf, strlen(buf));
104378		- }
104379		- plrDestroy(&plReader);
104380		-
104381		- assert( dump.nData>0 );
104382		- dump.nData--; /* Overwrite trailing space. */
104383		- assert( dump.pData[dump.nData]==' ');
104384		- dataBufferAppend(&dump, "]] ", 3);
104385		- }
104386		- }
104387		- dlrDestroy(&dlReader);
104388		-
104389		- assert( dump.nData>0 );
104390		- dump.nData--; /* Overwrite trailing space. */
104391		- assert( dump.pData[dump.nData]==' ');
104392		- dump.pData[dump.nData] = '\0';
104393		- assert( dump.nData>0 );
104394		-
104395		- /* Passes ownership of dump's buffer to pContext. */
104396		- sqlite3_result_text(pContext, dump.pData, dump.nData, sqlite3_free);
104397		- dump.pData = NULL;
104398		- dump.nData = dump.nCapacity = 0;
104399		-}
104400		-
104401		-/* Implements dump_doclist() for use in inspecting the fts3 index from
104402		-** tests. TEXT result containing a string representation of the
104403		-** doclist for the indicated term. dump_doclist(t, term, level, idx)
104404		-** dumps the doclist for term from the segment specified by level, idx
104405		-** (in %_segdir), while dump_doclist(t, term) dumps the logical
104406		-** doclist for the term across all segments. The per-segment doclist
104407		-** can contain deletions, while the full-index doclist will not
104408		-** (deletions are omitted).
104409		-**
104410		-** Result formats differ with the setting of DL_DEFAULTS. Examples:
104411		-**
104412		-** DL_DOCIDS: [1] [3] [7]
104413		-** DL_POSITIONS: [1 0[0 4] 1[17]] [3 1[5]]
104414		-** DL_POSITIONS_OFFSETS: [1 0[0,0,3 4,23,26] 1[17,102,105]] [3 1[5,20,23]]
104415		-**
104416		-** In each case the number after the outer '[' is the docid. In the
104417		-** latter two cases, the number before the inner '[' is the column
104418		-** associated with the values within. For DL_POSITIONS the numbers
104419		-** within are the positions, for DL_POSITIONS_OFFSETS they are the
104420		-** position, the start offset, and the end offset.
104421		-*/
104422		-static void dumpDoclistFunc(
104423		- sqlite3_context *pContext,
104424		- int argc, sqlite3_value **argv
104425		-){
104426		- fulltext_cursor *pCursor;
104427		- if( argc!=2 && argc!=4 ){
104428		- generateError(pContext, "dump_doclist", "incorrect arguments");
104429		- }else if( sqlite3_value_type(argv[0])!=SQLITE_BLOB \|\|
104430		- sqlite3_value_bytes(argv[0])!=sizeof(pCursor) ){
104431		- generateError(pContext, "dump_doclist", "illegal first argument");
104432		- }else if( sqlite3_value_text(argv[1])==NULL \|\|
104433		- sqlite3_value_text(argv[1])[0]=='\0' ){
104434		- generateError(pContext, "dump_doclist", "empty second argument");
104435		- }else{
104436		- const char pTerm = (const char )sqlite3_value_text(argv[1]);
104437		- const int nTerm = strlen(pTerm);
104438		- fulltext_vtab *v;
104439		- int rc;
104440		- DataBuffer doclist;
104441		-
104442		- memcpy(&pCursor, sqlite3_value_blob(argv[0]), sizeof(pCursor));
104443		- v = cursor_vtab(pCursor);
104444		-
104445		- dataBufferInit(&doclist, 0);
104446		-
104447		- /* termSelect() yields the same logical doclist that queries are
104448		- ** run against.
104449		- */
104450		- if( argc==2 ){
104451		- rc = termSelect(v, v->nColumn, pTerm, nTerm, 0, DL_DEFAULT, &doclist);
104452		- }else{
104453		- sqlite3_stmt *s = NULL;
104454		-
104455		- /* Get our specific segment's information. */
104456		- rc = sql_get_statement(v, SEGDIR_SELECT_SEGMENT_STMT, &s);
104457		- if( rc==SQLITE_OK ){
104458		- rc = sqlite3_bind_int(s, 1, sqlite3_value_int(argv[2]));
104459		- if( rc==SQLITE_OK ){
104460		- rc = sqlite3_bind_int(s, 2, sqlite3_value_int(argv[3]));
104461		- }
104462		- }
104463		-
104464		- if( rc==SQLITE_OK ){
104465		- rc = sqlite3_step(s);
104466		-
104467		- if( rc==SQLITE_DONE ){
104468		- dataBufferDestroy(&doclist);
104469		- generateError(pContext, "dump_doclist", "segment not found");
104470		- return;
104471		- }
104472		-
104473		- /* Found a segment, load it into doclist. */
104474		- if( rc==SQLITE_ROW ){
104475		- const sqlite_int64 iLeavesEnd = sqlite3_column_int64(s, 1);
104476		- const char *pData = sqlite3_column_blob(s, 2);
104477		- const int nData = sqlite3_column_bytes(s, 2);
104478		-
104479		- /* loadSegment() is used by termSelect() to load each
104480		- ** segment's data.
104481		- */
104482		- rc = loadSegment(v, pData, nData, iLeavesEnd, pTerm, nTerm, 0,
104483		- &doclist);
104484		- if( rc==SQLITE_OK ){
104485		- rc = sqlite3_step(s);
104486		-
104487		- /* Should not have more than one matching segment. */
104488		- if( rc!=SQLITE_DONE ){
104489		- sqlite3_reset(s);
104490		- dataBufferDestroy(&doclist);
104491		- generateError(pContext, "dump_doclist", "invalid segdir");
104492		- return;
104493		- }
104494		- rc = SQLITE_OK;
104495		- }
104496		- }
104497		- }
104498		-
104499		- sqlite3_reset(s);
104500		- }
104501		-
104502		- if( rc==SQLITE_OK ){
104503		- if( doclist.nData>0 ){
104504		- createDoclistResult(pContext, doclist.pData, doclist.nData);
104505		- }else{
104506		- /* TODO(shess): This can happen if the term is not present, or
104507		- ** if all instances of the term have been deleted and this is
104508		- ** an all-index dump. It may be interesting to distinguish
104509		- ** these cases.
104510		- */
104511		- sqlite3_result_text(pContext, "", 0, SQLITE_STATIC);
104512		- }
104513		- }else if( rc==SQLITE_NOMEM ){
104514		- /* Handle out-of-memory cases specially because if they are
104515		- ** generated in fts3 code they may not be reflected in the db
104516		- ** handle.
104517		- */
104518		- /* TODO(shess): Handle this more comprehensively.
104519		- ** sqlite3ErrStr() has what I need, but is internal.
104520		- */
104521		- generateError(pContext, "dump_doclist", "out of memory");
104522		- }else{
104523		- generateError(pContext, "dump_doclist", NULL);
104524		- }
104525		-
104526		- dataBufferDestroy(&doclist);
104527		- }
104528		-}
104529		-#endif
	99493	+static void fts3OffsetsFunc(
	99494	+ sqlite3_context pContext, / SQLite function call context */
	99495	+ int nVal, /* Size of argument array */
	99496	+ sqlite3_value *apVal / Array of arguments */
	99497	+){
	99498	+ Fts3Cursor pCsr; / Cursor handle passed through apVal[0] */
	99499	+
	99500	+ assert( nVal==1 );
	99501	+ if( fts3FunctionArg(pContext, "offsets", apVal[0], &pCsr) ) return;
	99502	+ assert( pCsr );
	99503	+ sqlite3Fts3Offsets(pContext, pCsr);
	99504	+}
	99505	+
	99506	+/*
	99507	+** Implementation of the special optimize() function for FTS3. This
	99508	+** function merges all segments in the database to a single segment.
	99509	+** Example usage is:
	99510	+**
	99511	+** SELECT optimize(t) FROM t LIMIT 1;
	99512	+**
	99513	+** where 't' is the name of an FTS3 table.
	99514	+*/
	99515	+static void fts3OptimizeFunc(
	99516	+ sqlite3_context pContext, / SQLite function call context */
	99517	+ int nVal, /* Size of argument array */
	99518	+ sqlite3_value *apVal / Array of arguments */
	99519	+){
	99520	+ int rc; /* Return code */
	99521	+ Fts3Table p; / Virtual table handle */
	99522	+ Fts3Cursor pCursor; / Cursor handle passed through apVal[0] */
	99523	+
	99524	+ assert( nVal==1 );
	99525	+ if( fts3FunctionArg(pContext, "optimize", apVal[0], &pCursor) ) return;
	99526	+ p = (Fts3Table *)pCursor->base.pVtab;
	99527	+ assert( p );
	99528	+
	99529	+ rc = sqlite3Fts3Optimize(p);
	99530	+
	99531	+ switch( rc ){
	99532	+ case SQLITE_OK:
	99533	+ sqlite3_result_text(pContext, "Index optimized", -1, SQLITE_STATIC);
	99534	+ break;
	99535	+ case SQLITE_DONE:
	99536	+ sqlite3_result_text(pContext, "Index already optimal", -1, SQLITE_STATIC);
	99537	+ break;
	99538	+ default:
	99539	+ sqlite3_result_error_code(pContext, rc);
	99540	+ break;
	99541	+ }
	99542	+}
104530	99543
104531	99544	/*
104532	99545	** This routine implements the xFindFunction method for the FTS3
104533	99546	** virtual table.
104534	99547	*/
104535		-static int fulltextFindFunction(
104536		- sqlite3_vtab *pVtab,
104537		- int nArg,
104538		- const char *zName,
104539		- void (*pxFunc)(sqlite3_context,int,sqlite3_value**),
104540		- void **ppArg
	99548	+static int fts3FindFunctionMethod(
	99549	+ sqlite3_vtab pVtab, / Virtual table handle */
	99550	+ int nArg, /* Number of SQL function arguments */
	99551	+ const char zName, / Name of SQL function */
	99552	+ void (*pxFunc)(sqlite3_context,int,sqlite3_value*), / OUT: Result */
	99553	+ void *ppArg / Unused */
104541	99554	){
104542		- if( strcmp(zName,"snippet")==0 ){
104543		- *pxFunc = snippetFunc;
104544		- return 1;
104545		- }else if( strcmp(zName,"offsets")==0 ){
104546		- *pxFunc = snippetOffsetsFunc;
104547		- return 1;
104548		- }else if( strcmp(zName,"optimize")==0 ){
104549		- *pxFunc = optimizeFunc;
104550		- return 1;
104551		-#ifdef SQLITE_TEST
104552		- /* NOTE(shess): These functions are present only for testing
104553		- ** purposes. No particular effort is made to optimize their
104554		- ** execution or how they build their results.
104555		- */
104556		- }else if( strcmp(zName,"dump_terms")==0 ){
104557		- /* fprintf(stderr, "Found dump_terms\n"); */
104558		- *pxFunc = dumpTermsFunc;
104559		- return 1;
104560		- }else if( strcmp(zName,"dump_doclist")==0 ){
104561		- /* fprintf(stderr, "Found dump_doclist\n"); */
104562		- *pxFunc = dumpDoclistFunc;
104563		- return 1;
104564		-#endif
104565		- }
	99555	+ struct Overloaded {
	99556	+ const char *zName;
	99557	+ void (xFunc)(sqlite3_context,int,sqlite3_value**);
	99558	+ } aOverload[] = {
	99559	+ { "snippet", fts3SnippetFunc },
	99560	+ { "offsets", fts3OffsetsFunc },
	99561	+ { "optimize", fts3OptimizeFunc },
	99562	+ };
	99563	+ int i; /* Iterator variable */
	99564	+ for(i=0; i<SizeofArray(aOverload); i++){
	99565	+ if( strcmp(zName, aOverload[i].zName)==0 ){
	99566	+ *pxFunc = aOverload[i].xFunc;
	99567	+ return 1;
	99568	+ }
	99569	+ }
	99570	+
	99571	+ /* No function of the specified name was found. Return 0. */
104566	99572	return 0;
104567	99573	}
104568	99574
104569	99575	/*
104570		-** Rename an fts3 table.
	99576	+** Implementation of FTS3 xRename method. Rename an fts3 table.
104571	99577	*/
104572		-static int fulltextRename(
104573		- sqlite3_vtab *pVtab,
104574		- const char *zName
	99578	+static int fts3RenameMethod(
	99579	+ sqlite3_vtab pVtab, / Virtual table handle */
	99580	+ const char zName / New name of table */
104575	99581	){
104576		- fulltext_vtab p = (fulltext_vtab )pVtab;
104577		- int rc = SQLITE_NOMEM;
104578		- char *zSql = sqlite3_mprintf(
	99582	+ Fts3Table p = (Fts3Table )pVtab;
	99583	+ int rc = SQLITE_NOMEM; /* Return Code */
	99584	+ char zSql; / SQL script to run to rename tables */
	99585	+
	99586	+ zSql = sqlite3_mprintf(
104579	99587	"ALTER TABLE %Q.'%q_content' RENAME TO '%q_content';"
104580	99588	"ALTER TABLE %Q.'%q_segments' RENAME TO '%q_segments';"
104581	99589	"ALTER TABLE %Q.'%q_segdir' RENAME TO '%q_segdir';"
104582	99590	, p->zDb, p->zName, zName
104583	99591	, p->zDb, p->zName, zName
		@@ -104590,33 +99598,38 @@
104590	99598	return rc;
104591	99599	}
104592	99600
104593	99601	static const sqlite3_module fts3Module = {
104594	99602	/* iVersion */ 0,
104595		- /* xCreate */ fulltextCreate,
104596		- /* xConnect */ fulltextConnect,
104597		- /* xBestIndex */ fulltextBestIndex,
104598		- /* xDisconnect */ fulltextDisconnect,
104599		- /* xDestroy */ fulltextDestroy,
104600		- /* xOpen */ fulltextOpen,
	99603	+ /* xCreate */ fts3CreateMethod,
	99604	+ /* xConnect */ fts3ConnectMethod,
	99605	+ /* xBestIndex */ fts3BestIndexMethod,
	99606	+ /* xDisconnect */ fts3DisconnectMethod,
	99607	+ /* xDestroy */ fts3DestroyMethod,
	99608	+ /* xOpen */ fts3OpenMethod,
104601	99609	/* xClose */ fulltextClose,
104602		- /* xFilter */ fulltextFilter,
104603		- /* xNext */ fulltextNext,
104604		- /* xEof */ fulltextEof,
104605		- /* xColumn */ fulltextColumn,
104606		- /* xRowid */ fulltextRowid,
104607		- /* xUpdate */ fulltextUpdate,
104608		- /* xBegin */ fulltextBegin,
104609		- /* xSync */ fulltextSync,
104610		- /* xCommit */ fulltextCommit,
104611		- /* xRollback */ fulltextRollback,
104612		- /* xFindFunction */ fulltextFindFunction,
104613		- /* xRename */ fulltextRename,
	99610	+ /* xFilter */ fts3FilterMethod,
	99611	+ /* xNext */ fts3NextMethod,
	99612	+ /* xEof */ fts3EofMethod,
	99613	+ /* xColumn */ fts3ColumnMethod,
	99614	+ /* xRowid */ fts3RowidMethod,
	99615	+ /* xUpdate */ fts3UpdateMethod,
	99616	+ /* xBegin */ fts3BeginMethod,
	99617	+ /* xSync */ fts3SyncMethod,
	99618	+ /* xCommit */ fts3CommitMethod,
	99619	+ /* xRollback */ fts3RollbackMethod,
	99620	+ /* xFindFunction */ fts3FindFunctionMethod,
	99621	+ /* xRename */ fts3RenameMethod,
104614	99622	};
104615	99623
	99624	+/*
	99625	+** This function is registered as the module destructor (called when an
	99626	+** FTS3 enabled database connection is closed). It frees the memory
	99627	+** allocated for the tokenizer hash table.
	99628	+*/
104616	99629	static void hashDestroy(void *p){
104617		- fts3Hash pHash = (fts3Hash )p;
	99630	+ Fts3Hash pHash = (Fts3Hash )p;
104618	99631	sqlite3Fts3HashClear(pHash);
104619	99632	sqlite3_free(pHash);
104620	99633	}
104621	99634
104622	99635	/*
		@@ -104632,21 +99645,19 @@
104632	99645	*/
104633	99646	SQLITE_PRIVATE void sqlite3Fts3SimpleTokenizerModule(sqlite3_tokenizer_module const**ppModule);
104634	99647	SQLITE_PRIVATE void sqlite3Fts3PorterTokenizerModule(sqlite3_tokenizer_module const**ppModule);
104635	99648	SQLITE_PRIVATE void sqlite3Fts3IcuTokenizerModule(sqlite3_tokenizer_module const**ppModule);
104636	99649
104637		-SQLITE_PRIVATE int sqlite3Fts3InitHashTable(sqlite3 , fts3Hash , const char *);
104638		-
104639	99650	/*
104640	99651	** Initialise the fts3 extension. If this extension is built as part
104641	99652	** of the sqlite library, then this function is called directly by
104642	99653	** SQLite. If fts3 is built as a dynamically loadable extension, this
104643	99654	** function is called by the sqlite3_extension_init() entry point.
104644	99655	*/
104645	99656	SQLITE_PRIVATE int sqlite3Fts3Init(sqlite3 *db){
104646	99657	int rc = SQLITE_OK;
104647		- fts3Hash *pHash = 0;
	99658	+ Fts3Hash *pHash = 0;
104648	99659	const sqlite3_tokenizer_module *pSimple = 0;
104649	99660	const sqlite3_tokenizer_module *pPorter = 0;
104650	99661	const sqlite3_tokenizer_module *pIcu = 0;
104651	99662
104652	99663	sqlite3Fts3SimpleTokenizerModule(&pSimple);
		@@ -104654,11 +99665,11 @@
104654	99665	#ifdef SQLITE_ENABLE_ICU
104655	99666	sqlite3Fts3IcuTokenizerModule(&pIcu);
104656	99667	#endif
104657	99668
104658	99669	/* Allocate and initialise the hash-table used to store tokenizers. */
104659		- pHash = sqlite3_malloc(sizeof(fts3Hash));
	99670	+ pHash = sqlite3_malloc(sizeof(Fts3Hash));
104660	99671	if( !pHash ){
104661	99672	rc = SQLITE_NOMEM;
104662	99673	}else{
104663	99674	sqlite3Fts3HashInit(pHash, FTS3_HASH_STRING, 1);
104664	99675	}
		@@ -104682,16 +99693,12 @@
104682	99693	** module with sqlite.
104683	99694	*/
104684	99695	if( SQLITE_OK==rc
104685	99696	&& SQLITE_OK==(rc = sqlite3Fts3InitHashTable(db, pHash, "fts3_tokenizer"))
104686	99697	&& SQLITE_OK==(rc = sqlite3_overload_function(db, "snippet", -1))
104687		- && SQLITE_OK==(rc = sqlite3_overload_function(db, "offsets", -1))
104688		- && SQLITE_OK==(rc = sqlite3_overload_function(db, "optimize", -1))
104689		-#ifdef SQLITE_TEST
104690		- && SQLITE_OK==(rc = sqlite3_overload_function(db, "dump_terms", -1))
104691		- && SQLITE_OK==(rc = sqlite3_overload_function(db, "dump_doclist", -1))
104692		-#endif
	99698	+ && SQLITE_OK==(rc = sqlite3_overload_function(db, "offsets", 1))
	99699	+ && SQLITE_OK==(rc = sqlite3_overload_function(db, "optimize", 1))
104693	99700	){
104694	99701	return sqlite3_create_module_v2(
104695	99702	db, "fts3", &fts3Module, (void *)pHash, hashDestroy
104696	99703	);
104697	99704	}
		@@ -104714,11 +99721,11 @@
104714	99721	SQLITE_EXTENSION_INIT2(pApi)
104715	99722	return sqlite3Fts3Init(db);
104716	99723	}
104717	99724	#endif
104718	99725
104719		-#endif /* !defined(SQLITE_CORE) \|\| defined(SQLITE_ENABLE_FTS3) */
	99726	+#endif
104720	99727
104721	99728	/************ End of fts3.c **********************************************/
104722	99729	/************ Begin file fts3_expr.c *************************************/
104723	99730	/*
104724	99731	** 2008 Nov 28
		@@ -104733,12 +99740,11 @@
104733	99740	******************************************************************************
104734	99741	**
104735	99742	** This module contains code that implements a parser for fts3 query strings
104736	99743	** (the right-hand argument to the MATCH operator). Because the supported
104737	99744	** syntax is relatively simple, the whole tokenizer/parser system is
104738		-** hand-coded. The public interface to this module is declared in source
104739		-** code file "fts3_expr.h".
	99745	+** hand-coded.
104740	99746	*/
104741	99747	#if !defined(SQLITE_CORE) \|\| defined(SQLITE_ENABLE_FTS3)
104742	99748
104743	99749	/*
104744	99750	** By default, this module parses the legacy syntax that has been
		@@ -104760,11 +99766,33 @@
104760	99766	** If compiled with SQLITE_TEST defined, then this module exports the
104761	99767	** symbol "int sqlite3_fts3_enable_parentheses". Setting this variable
104762	99768	** to zero causes the module to use the old syntax. If it is set to
104763	99769	** non-zero the new syntax is activated. This is so both syntaxes can
104764	99770	** be tested using a single build of testfixture.
	99771	+**
	99772	+** The following describes the syntax supported by the fts3 MATCH
	99773	+** operator in a similar format to that used by the lemon parser
	99774	+** generator. This module does not use actually lemon, it uses a
	99775	+** custom parser.
	99776	+**
	99777	+** query ::= andexpr (OR andexpr)*.
	99778	+**
	99779	+** andexpr ::= notexpr (AND? notexpr)*.
	99780	+**
	99781	+** notexpr ::= nearexpr (NOT nearexpr\|-TOKEN)*.
	99782	+** notexpr ::= LP query RP.
	99783	+**
	99784	+** nearexpr ::= phrase (NEAR distance_opt nearexpr)*.
	99785	+**
	99786	+** distance_opt ::= .
	99787	+** distance_opt ::= / INTEGER.
	99788	+**
	99789	+** phrase ::= TOKEN.
	99790	+** phrase ::= COLUMN:TOKEN.
	99791	+** phrase ::= "TOKEN TOKEN TOKEN...".
104765	99792	*/
	99793	+
104766	99794	#ifdef SQLITE_TEST
104767	99795	SQLITE_API int sqlite3_fts3_enable_parentheses = 0;
104768	99796	#else
104769	99797	# ifdef SQLITE_ENABLE_FTS3_PARENTHESIS
104770	99798	# define sqlite3_fts3_enable_parentheses 1
		@@ -105069,10 +100097,13 @@
105069	100097	cNext = zInput[nKey];
105070	100098	if( fts3isspace(cNext)
105071	100099	\|\| cNext=='"' \|\| cNext=='(' \|\| cNext==')' \|\| cNext==0
105072	100100	){
105073	100101	pRet = (Fts3Expr *)sqlite3_malloc(sizeof(Fts3Expr));
	100102	+ if( !pRet ){
	100103	+ return SQLITE_NOMEM;
	100104	+ }
105074	100105	memset(pRet, 0, sizeof(Fts3Expr));
105075	100106	pRet->eType = pKey->eType;
105076	100107	pRet->nNear = nNear;
105077	100108	*ppExpr = pRet;
105078	100109	*pnConsumed = (zInput - z) + nKey;
		@@ -105643,10 +100674,125 @@
105643	100674	** SQLite (in which case SQLITE_ENABLE_FTS3 is defined).
105644	100675	*/
105645	100676	#if !defined(SQLITE_CORE) \|\| defined(SQLITE_ENABLE_FTS3)
105646	100677
105647	100678
	100679	+/************ Include fts3_hash.h in the middle of fts3_hash.c ***********/
	100680	+/************ Begin file fts3_hash.h *************************************/
	100681	+/*
	100682	+** 2001 September 22
	100683	+**
	100684	+** The author disclaims copyright to this source code. In place of
	100685	+** a legal notice, here is a blessing:
	100686	+**
	100687	+** May you do good and not evil.
	100688	+** May you find forgiveness for yourself and forgive others.
	100689	+** May you share freely, never taking more than you give.
	100690	+**
	100691	+*************************************************************************
	100692	+** This is the header file for the generic hash-table implemenation
	100693	+** used in SQLite. We've modified it slightly to serve as a standalone
	100694	+** hash table implementation for the full-text indexing module.
	100695	+**
	100696	+*/
	100697	+#ifndef _FTS3_HASH_H_
	100698	+#define _FTS3_HASH_H_
	100699	+
	100700	+/* Forward declarations of structures. */
	100701	+typedef struct Fts3Hash Fts3Hash;
	100702	+typedef struct Fts3HashElem Fts3HashElem;
	100703	+
	100704	+/* A complete hash table is an instance of the following structure.
	100705	+** The internals of this structure are intended to be opaque -- client
	100706	+** code should not attempt to access or modify the fields of this structure
	100707	+** directly. Change this structure only by using the routines below.
	100708	+** However, many of the "procedures" and "functions" for modifying and
	100709	+** accessing this structure are really macros, so we can't really make
	100710	+** this structure opaque.
	100711	+*/
	100712	+struct Fts3Hash {
	100713	+ char keyClass; /* HASH_INT, _POINTER, _STRING, _BINARY */
	100714	+ char copyKey; /* True if copy of key made on insert */
	100715	+ int count; /* Number of entries in this table */
	100716	+ Fts3HashElem first; / The first element of the array */
	100717	+ int htsize; /* Number of buckets in the hash table */
	100718	+ struct _fts3ht { /* the hash table */
	100719	+ int count; /* Number of entries with this hash */
	100720	+ Fts3HashElem chain; / Pointer to first entry with this hash */
	100721	+ } *ht;
	100722	+};
	100723	+
	100724	+/* Each element in the hash table is an instance of the following
	100725	+** structure. All elements are stored on a single doubly-linked list.
	100726	+**
	100727	+** Again, this structure is intended to be opaque, but it can't really
	100728	+** be opaque because it is used by macros.
	100729	+*/
	100730	+struct Fts3HashElem {
	100731	+ Fts3HashElem next, prev; /* Next and previous elements in the table */
	100732	+ void data; / Data associated with this element */
	100733	+ void pKey; int nKey; / Key associated with this element */
	100734	+};
	100735	+
	100736	+/*
	100737	+** There are 2 different modes of operation for a hash table:
	100738	+**
	100739	+** FTS3_HASH_STRING pKey points to a string that is nKey bytes long
	100740	+** (including the null-terminator, if any). Case
	100741	+** is respected in comparisons.
	100742	+**
	100743	+** FTS3_HASH_BINARY pKey points to binary data nKey bytes long.
	100744	+** memcmp() is used to compare keys.
	100745	+**
	100746	+** A copy of the key is made if the copyKey parameter to fts3HashInit is 1.
	100747	+*/
	100748	+#define FTS3_HASH_STRING 1
	100749	+#define FTS3_HASH_BINARY 2
	100750	+
	100751	+/*
	100752	+** Access routines. To delete, insert a NULL pointer.
	100753	+*/
	100754	+SQLITE_PRIVATE void sqlite3Fts3HashInit(Fts3Hash*, int keytype, int copyKey);
	100755	+SQLITE_PRIVATE void sqlite3Fts3HashInsert(Fts3Hash, const void pKey, int nKey, void pData);
	100756	+SQLITE_PRIVATE void sqlite3Fts3HashFind(const Fts3Hash, const void *pKey, int nKey);
	100757	+SQLITE_PRIVATE void sqlite3Fts3HashClear(Fts3Hash*);
	100758	+
	100759	+/*
	100760	+** Shorthand for the functions above
	100761	+*/
	100762	+#define fts3HashInit sqlite3Fts3HashInit
	100763	+#define fts3HashInsert sqlite3Fts3HashInsert
	100764	+#define fts3HashFind sqlite3Fts3HashFind
	100765	+#define fts3HashClear sqlite3Fts3HashClear
	100766	+
	100767	+/*
	100768	+** Macros for looping over all elements of a hash table. The idiom is
	100769	+** like this:
	100770	+**
	100771	+** Fts3Hash h;
	100772	+** Fts3HashElem *p;
	100773	+** ...
	100774	+** for(p=fts3HashFirst(&h); p; p=fts3HashNext(p)){
	100775	+** SomeStructure *pData = fts3HashData(p);
	100776	+** // do something with pData
	100777	+** }
	100778	+*/
	100779	+#define fts3HashFirst(H) ((H)->first)
	100780	+#define fts3HashNext(E) ((E)->next)
	100781	+#define fts3HashData(E) ((E)->data)
	100782	+#define fts3HashKey(E) ((E)->pKey)
	100783	+#define fts3HashKeysize(E) ((E)->nKey)
	100784	+
	100785	+/*
	100786	+** Number of entries in a hash table
	100787	+*/
	100788	+#define fts3HashCount(H) ((H)->count)
	100789	+
	100790	+#endif /* _FTS3_HASH_H_ */
	100791	+
	100792	+/************ End of fts3_hash.h *****************************************/
	100793	+/************ Continuing where we left off in fts3_hash.c ****************/
105648	100794
105649	100795	/*
105650	100796	** Malloc and Free functions
105651	100797	*/
105652	100798	static void *fts3HashMalloc(int n){
		@@ -105668,11 +100814,11 @@
105668	100814	** FTS3_HASH_BINARY or FTS3_HASH_STRING. The value of keyClass
105669	100815	** determines what kind of key the hash table will use. "copyKey" is
105670	100816	** true if the hash table should make its own private copy of keys and
105671	100817	** false if it should just use the supplied pointer.
105672	100818	*/
105673		-SQLITE_PRIVATE void sqlite3Fts3HashInit(fts3Hash *pNew, int keyClass, int copyKey){
	100819	+SQLITE_PRIVATE void sqlite3Fts3HashInit(Fts3Hash *pNew, int keyClass, int copyKey){
105674	100820	assert( pNew!=0 );
105675	100821	assert( keyClass>=FTS3_HASH_STRING && keyClass<=FTS3_HASH_BINARY );
105676	100822	pNew->keyClass = keyClass;
105677	100823	pNew->copyKey = copyKey;
105678	100824	pNew->first = 0;
		@@ -105683,21 +100829,21 @@
105683	100829
105684	100830	/* Remove all entries from a hash table. Reclaim all memory.
105685	100831	** Call this routine to delete a hash table or to reset a hash table
105686	100832	** to the empty state.
105687	100833	*/
105688		-SQLITE_PRIVATE void sqlite3Fts3HashClear(fts3Hash *pH){
105689		- fts3HashElem elem; / For looping over all elements of the table */
	100834	+SQLITE_PRIVATE void sqlite3Fts3HashClear(Fts3Hash *pH){
	100835	+ Fts3HashElem elem; / For looping over all elements of the table */
105690	100836
105691	100837	assert( pH!=0 );
105692	100838	elem = pH->first;
105693	100839	pH->first = 0;
105694	100840	fts3HashFree(pH->ht);
105695	100841	pH->ht = 0;
105696	100842	pH->htsize = 0;
105697	100843	while( elem ){
105698		- fts3HashElem *next_elem = elem->next;
	100844	+ Fts3HashElem *next_elem = elem->next;
105699	100845	if( pH->copyKey && elem->pKey ){
105700	100846	fts3HashFree(elem->pKey);
105701	100847	}
105702	100848	fts3HashFree(elem);
105703	100849	elem = next_elem;
		@@ -105776,15 +100922,15 @@
105776	100922	}
105777	100923
105778	100924	/* Link an element into the hash table
105779	100925	*/
105780	100926	static void fts3HashInsertElement(
105781		- fts3Hash pH, / The complete hash table */
	100927	+ Fts3Hash pH, / The complete hash table */
105782	100928	struct _fts3ht pEntry, / The entry into which pNew is inserted */
105783		- fts3HashElem pNew / The element to be inserted */
	100929	+ Fts3HashElem pNew / The element to be inserted */
105784	100930	){
105785		- fts3HashElem pHead; / First element already in pEntry */
	100931	+ Fts3HashElem pHead; / First element already in pEntry */
105786	100932	pHead = pEntry->chain;
105787	100933	if( pHead ){
105788	100934	pNew->next = pHead;
105789	100935	pNew->prev = pHead->prev;
105790	100936	if( pHead->prev ){ pHead->prev->next = pNew; }
		@@ -105803,13 +100949,13 @@
105803	100949
105804	100950	/* Resize the hash table so that it cantains "new_size" buckets.
105805	100951	** "new_size" must be a power of 2. The hash table might fail
105806	100952	** to resize if sqliteMalloc() fails.
105807	100953	*/
105808		-static void fts3Rehash(fts3Hash *pH, int new_size){
	100954	+static void fts3Rehash(Fts3Hash *pH, int new_size){
105809	100955	struct _fts3ht new_ht; / The new hash table */
105810		- fts3HashElem elem, next_elem; /* For looping over existing elements */
	100956	+ Fts3HashElem elem, next_elem; /* For looping over existing elements */
105811	100957	int (xHash)(const void,int); /* The hash function */
105812	100958
105813	100959	assert( (new_size & (new_size-1))==0 );
105814	100960	new_ht = (struct _fts3ht )fts3HashMalloc( new_sizesizeof(struct _fts3ht) );
105815	100961	if( new_ht==0 ) return;
		@@ -105826,17 +100972,17 @@
105826	100972
105827	100973	/* This function (for internal use only) locates an element in an
105828	100974	** hash table that matches the given key. The hash for this key has
105829	100975	** already been computed and is passed as the 4th parameter.
105830	100976	*/
105831		-static fts3HashElem *fts3FindElementByHash(
105832		- const fts3Hash pH, / The pH to be searched */
	100977	+static Fts3HashElem *fts3FindElementByHash(
	100978	+ const Fts3Hash pH, / The pH to be searched */
105833	100979	const void pKey, / The key we are searching for */
105834	100980	int nKey,
105835	100981	int h /* The hash for this key. */
105836	100982	){
105837		- fts3HashElem elem; / Used to loop thru the element list */
	100983	+ Fts3HashElem elem; / Used to loop thru the element list */
105838	100984	int count; /* Number of elements left to test */
105839	100985	int (xCompare)(const void,int,const void,int); / comparison function */
105840	100986
105841	100987	if( pH->ht ){
105842	100988	struct _fts3ht *pEntry = &pH->ht[h];
		@@ -105855,12 +101001,12 @@
105855	101001
105856	101002	/* Remove a single entry from the hash table given a pointer to that
105857	101003	** element and a hash on the element's key.
105858	101004	*/
105859	101005	static void fts3RemoveElementByHash(
105860		- fts3Hash pH, / The pH containing "elem" */
105861		- fts3HashElem* elem, /* The element to be removed from the pH */
	101006	+ Fts3Hash pH, / The pH containing "elem" */
	101007	+ Fts3HashElem* elem, /* The element to be removed from the pH */
105862	101008	int h /* Hash value for the element */
105863	101009	){
105864	101010	struct _fts3ht *pEntry;
105865	101011	if( elem->prev ){
105866	101012	elem->prev->next = elem->next;
		@@ -105892,13 +101038,13 @@
105892	101038
105893	101039	/* Attempt to locate an element of the hash table pH with a key
105894	101040	** that matches pKey,nKey. Return the data for this element if it is
105895	101041	** found, or NULL if there is no match.
105896	101042	*/
105897		-SQLITE_PRIVATE void sqlite3Fts3HashFind(const fts3Hash pH, const void *pKey, int nKey){
	101043	+SQLITE_PRIVATE void sqlite3Fts3HashFind(const Fts3Hash pH, const void *pKey, int nKey){
105898	101044	int h; /* A hash on key */
105899		- fts3HashElem elem; / The element that matches key */
	101045	+ Fts3HashElem elem; / The element that matches key */
105900	101046	int (xHash)(const void,int); /* The hash function */
105901	101047
105902	101048	if( pH==0 \|\| pH->ht==0 ) return 0;
105903	101049	xHash = ftsHashFunction(pH->keyClass);
105904	101050	assert( xHash!=0 );
		@@ -105922,19 +101068,19 @@
105922	101068	**
105923	101069	** If the "data" parameter to this function is NULL, then the
105924	101070	** element corresponding to "key" is removed from the hash table.
105925	101071	*/
105926	101072	SQLITE_PRIVATE void *sqlite3Fts3HashInsert(
105927		- fts3Hash pH, / The hash table to insert into */
	101073	+ Fts3Hash pH, / The hash table to insert into */
105928	101074	const void pKey, / The key */
105929	101075	int nKey, /* Number of bytes in the key */
105930	101076	void data / The data */
105931	101077	){
105932	101078	int hraw; /* Raw hash value of the key */
105933	101079	int h; /* the hash of the key modulo hash table size */
105934		- fts3HashElem elem; / Used to loop thru the element list */
105935		- fts3HashElem new_elem; / New element added to the pH */
	101080	+ Fts3HashElem elem; / Used to loop thru the element list */
	101081	+ Fts3HashElem new_elem; / New element added to the pH */
105936	101082	int (xHash)(const void,int); /* The hash function */
105937	101083
105938	101084	assert( pH!=0 );
105939	101085	xHash = ftsHashFunction(pH->keyClass);
105940	101086	assert( xHash!=0 );
		@@ -105957,11 +101103,11 @@
105957	101103	if( pH->htsize==0 ){
105958	101104	pH->count = 0;
105959	101105	return data;
105960	101106	}
105961	101107	}
105962		- new_elem = (fts3HashElem*)fts3HashMalloc( sizeof(fts3HashElem) );
	101108	+ new_elem = (Fts3HashElem*)fts3HashMalloc( sizeof(Fts3HashElem) );
105963	101109	if( new_elem==0 ) return data;
105964	101110	if( pH->copyKey && pKey!=0 ){
105965	101111	new_elem->pKey = fts3HashMalloc( nKey );
105966	101112	if( new_elem->pKey==0 ){
105967	101113	fts3HashFree(new_elem);
		@@ -106014,10 +101160,162 @@
106014	101160	*/
106015	101161	#if !defined(SQLITE_CORE) \|\| defined(SQLITE_ENABLE_FTS3)
106016	101162
106017	101163
106018	101164
	101165	+/************ Include fts3_tokenizer.h in the middle of fts3_porter.c ****/
	101166	+/************ Begin file fts3_tokenizer.h ********************************/
	101167	+/*
	101168	+** 2006 July 10
	101169	+**
	101170	+** The author disclaims copyright to this source code.
	101171	+**
	101172	+*************************************************************************
	101173	+** Defines the interface to tokenizers used by fulltext-search. There
	101174	+** are three basic components:
	101175	+**
	101176	+** sqlite3_tokenizer_module is a singleton defining the tokenizer
	101177	+** interface functions. This is essentially the class structure for
	101178	+** tokenizers.
	101179	+**
	101180	+** sqlite3_tokenizer is used to define a particular tokenizer, perhaps
	101181	+** including customization information defined at creation time.
	101182	+**
	101183	+** sqlite3_tokenizer_cursor is generated by a tokenizer to generate
	101184	+** tokens from a particular input.
	101185	+*/
	101186	+#ifndef _FTS3_TOKENIZER_H_
	101187	+#define _FTS3_TOKENIZER_H_
	101188	+
	101189	+/* TODO(shess) Only used for SQLITE_OK and SQLITE_DONE at this time.
	101190	+** If tokenizers are to be allowed to call sqlite3_*() functions, then
	101191	+** we will need a way to register the API consistently.
	101192	+*/
	101193	+
	101194	+/*
	101195	+** Structures used by the tokenizer interface. When a new tokenizer
	101196	+** implementation is registered, the caller provides a pointer to
	101197	+** an sqlite3_tokenizer_module containing pointers to the callback
	101198	+** functions that make up an implementation.
	101199	+**
	101200	+** When an fts3 table is created, it passes any arguments passed to
	101201	+** the tokenizer clause of the CREATE VIRTUAL TABLE statement to the
	101202	+** sqlite3_tokenizer_module.xCreate() function of the requested tokenizer
	101203	+** implementation. The xCreate() function in turn returns an
	101204	+** sqlite3_tokenizer structure representing the specific tokenizer to
	101205	+** be used for the fts3 table (customized by the tokenizer clause arguments).
	101206	+**
	101207	+** To tokenize an input buffer, the sqlite3_tokenizer_module.xOpen()
	101208	+** method is called. It returns an sqlite3_tokenizer_cursor object
	101209	+** that may be used to tokenize a specific input buffer based on
	101210	+** the tokenization rules supplied by a specific sqlite3_tokenizer
	101211	+** object.
	101212	+*/
	101213	+typedef struct sqlite3_tokenizer_module sqlite3_tokenizer_module;
	101214	+typedef struct sqlite3_tokenizer sqlite3_tokenizer;
	101215	+typedef struct sqlite3_tokenizer_cursor sqlite3_tokenizer_cursor;
	101216	+
	101217	+struct sqlite3_tokenizer_module {
	101218	+
	101219	+ /*
	101220	+ ** Structure version. Should always be set to 0.
	101221	+ */
	101222	+ int iVersion;
	101223	+
	101224	+ /*
	101225	+ ** Create a new tokenizer. The values in the argv[] array are the
	101226	+ ** arguments passed to the "tokenizer" clause of the CREATE VIRTUAL
	101227	+ ** TABLE statement that created the fts3 table. For example, if
	101228	+ ** the following SQL is executed:
	101229	+ **
	101230	+ ** CREATE .. USING fts3( ... , tokenizer <tokenizer-name> arg1 arg2)
	101231	+ **
	101232	+ ** then argc is set to 2, and the argv[] array contains pointers
	101233	+ ** to the strings "arg1" and "arg2".
	101234	+ **
	101235	+ ** This method should return either SQLITE_OK (0), or an SQLite error
	101236	+ ** code. If SQLITE_OK is returned, then *ppTokenizer should be set
	101237	+ ** to point at the newly created tokenizer structure. The generic
	101238	+ ** sqlite3_tokenizer.pModule variable should not be initialised by
	101239	+ ** this callback. The caller will do so.
	101240	+ */
	101241	+ int (*xCreate)(
	101242	+ int argc, /* Size of argv array */
	101243	+ const char constargv, /* Tokenizer argument strings */
	101244	+ sqlite3_tokenizer *ppTokenizer / OUT: Created tokenizer */
	101245	+ );
	101246	+
	101247	+ /*
	101248	+ ** Destroy an existing tokenizer. The fts3 module calls this method
	101249	+ ** exactly once for each successful call to xCreate().
	101250	+ */
	101251	+ int (xDestroy)(sqlite3_tokenizer pTokenizer);
	101252	+
	101253	+ /*
	101254	+ ** Create a tokenizer cursor to tokenize an input buffer. The caller
	101255	+ ** is responsible for ensuring that the input buffer remains valid
	101256	+ ** until the cursor is closed (using the xClose() method).
	101257	+ */
	101258	+ int (*xOpen)(
	101259	+ sqlite3_tokenizer pTokenizer, / Tokenizer object */
	101260	+ const char pInput, int nBytes, / Input buffer */
	101261	+ sqlite3_tokenizer_cursor *ppCursor / OUT: Created tokenizer cursor */
	101262	+ );
	101263	+
	101264	+ /*
	101265	+ ** Destroy an existing tokenizer cursor. The fts3 module calls this
	101266	+ ** method exactly once for each successful call to xOpen().
	101267	+ */
	101268	+ int (xClose)(sqlite3_tokenizer_cursor pCursor);
	101269	+
	101270	+ /*
	101271	+ ** Retrieve the next token from the tokenizer cursor pCursor. This
	101272	+ ** method should either return SQLITE_OK and set the values of the
	101273	+ ** "OUT" variables identified below, or SQLITE_DONE to indicate that
	101274	+ ** the end of the buffer has been reached, or an SQLite error code.
	101275	+ **
	101276	+ ** *ppToken should be set to point at a buffer containing the
	101277	+ ** normalized version of the token (i.e. after any case-folding and/or
	101278	+ ** stemming has been performed). *pnBytes should be set to the length
	101279	+ ** of this buffer in bytes. The input text that generated the token is
	101280	+ ** identified by the byte offsets returned in *piStartOffset and
	101281	+ ** piEndOffset. piStartOffset should be set to the index of the first
	101282	+ ** byte of the token in the input buffer. *piEndOffset should be set
	101283	+ ** to the index of the first byte just past the end of the token in
	101284	+ ** the input buffer.
	101285	+ **
	101286	+ ** The buffer *ppToken is set to point at is managed by the tokenizer
	101287	+ ** implementation. It is only required to be valid until the next call
	101288	+ ** to xNext() or xClose().
	101289	+ */
	101290	+ /* TODO(shess) current implementation requires pInput to be
	101291	+ ** nul-terminated. This should either be fixed, or pInput/nBytes
	101292	+ ** should be converted to zInput.
	101293	+ */
	101294	+ int (*xNext)(
	101295	+ sqlite3_tokenizer_cursor pCursor, / Tokenizer cursor */
	101296	+ const char *ppToken, int pnBytes, /* OUT: Normalized text for token */
	101297	+ int piStartOffset, / OUT: Byte offset of token in input buffer */
	101298	+ int piEndOffset, / OUT: Byte offset of end of token in input buffer */
	101299	+ int piPosition / OUT: Number of tokens returned before this one */
	101300	+ );
	101301	+};
	101302	+
	101303	+struct sqlite3_tokenizer {
	101304	+ const sqlite3_tokenizer_module pModule; / The module for this tokenizer */
	101305	+ /* Tokenizer implementations will typically add additional fields */
	101306	+};
	101307	+
	101308	+struct sqlite3_tokenizer_cursor {
	101309	+ sqlite3_tokenizer pTokenizer; / Tokenizer for this cursor. */
	101310	+ /* Tokenizer implementations will typically add additional fields */
	101311	+};
	101312	+
	101313	+#endif /* _FTS3_TOKENIZER_H_ */
	101314	+
	101315	+/************ End of fts3_tokenizer.h ************************************/
	101316	+/************ Continuing where we left off in fts3_porter.c **************/
106019	101317
106020	101318	/*
106021	101319	** Class derived from sqlite3_tokenizer
106022	101320	*/
106023	101321	typedef struct porter_tokenizer {
		@@ -106682,18 +101980,18 @@
106682	101980	static void scalarFunc(
106683	101981	sqlite3_context *context,
106684	101982	int argc,
106685	101983	sqlite3_value **argv
106686	101984	){
106687		- fts3Hash *pHash;
	101985	+ Fts3Hash *pHash;
106688	101986	void *pPtr = 0;
106689	101987	const unsigned char *zName;
106690	101988	int nName;
106691	101989
106692	101990	assert( argc==1 \|\| argc==2 );
106693	101991
106694		- pHash = (fts3Hash *)sqlite3_user_data(context);
	101992	+ pHash = (Fts3Hash *)sqlite3_user_data(context);
106695	101993
106696	101994	zName = sqlite3_value_text(argv[0]);
106697	101995	nName = sqlite3_value_bytes(argv[0])+1;
106698	101996
106699	101997	if( argc==2 ){
		@@ -106719,10 +102017,132 @@
106719	102017	}
106720	102018	}
106721	102019
106722	102020	sqlite3_result_blob(context, (void *)&pPtr, sizeof(pPtr), SQLITE_TRANSIENT);
106723	102021	}
	102022	+
	102023	+static int fts3IsIdChar(char c){
	102024	+ static const char isFtsIdChar[] = {
	102025	+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 0x */
	102026	+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 1x */
	102027	+ 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 2x */
	102028	+ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, /* 3x */
	102029	+ 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* 4x */
	102030	+ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 1, /* 5x */
	102031	+ 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* 6x */
	102032	+ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, /* 7x */
	102033	+ };
	102034	+ return (c&0x80 \|\| isFtsIdChar[(int)(c)]);
	102035	+}
	102036	+
	102037	+SQLITE_PRIVATE const char sqlite3Fts3NextToken(const char zStr, int *pn){
	102038	+ const char *z1;
	102039	+ const char *z2 = 0;
	102040	+
	102041	+ /* Find the start of the next token. */
	102042	+ z1 = zStr;
	102043	+ while( z2==0 ){
	102044	+ switch( *z1 ){
	102045	+ case '\0': return 0; /* No more tokens here */
	102046	+ case '\'':
	102047	+ case '"':
	102048	+ case '`': {
	102049	+ z2 = &z1[1];
	102050	+ while( z2 && (z2[0]!=z1 \|\| z2[1]==*z1) ) z2++;
	102051	+ if( *z2 ) z2++;
	102052	+ break;
	102053	+ }
	102054	+ case '[':
	102055	+ z2 = &z1[1];
	102056	+ while( *z2 && z2[0]!=']' ) z2++;
	102057	+ if( *z2 ) z2++;
	102058	+ break;
	102059	+
	102060	+ default:
	102061	+ if( fts3IsIdChar(*z1) ){
	102062	+ z2 = &z1[1];
	102063	+ while( fts3IsIdChar(*z2) ) z2++;
	102064	+ }else{
	102065	+ z1++;
	102066	+ }
	102067	+ }
	102068	+ }
	102069	+
	102070	+ *pn = (z2-z1);
	102071	+ return z1;
	102072	+}
	102073	+
	102074	+SQLITE_PRIVATE int sqlite3Fts3InitTokenizer(
	102075	+ Fts3Hash pHash, / Tokenizer hash table */
	102076	+ const char zArg, / Possible tokenizer specification */
	102077	+ sqlite3_tokenizer *ppTok, / OUT: Tokenizer (if applicable) */
	102078	+ const char *pzTokenizer, / OUT: Set to zArg if is tokenizer */
	102079	+ char *pzErr / OUT: Set to malloced error message */
	102080	+){
	102081	+ int rc;
	102082	+ char z = (char )zArg;
	102083	+ int n;
	102084	+ char *zCopy;
	102085	+ char zEnd; / Pointer to nul-term of zCopy */
	102086	+ sqlite3_tokenizer_module *m;
	102087	+
	102088	+ if( !z ){
	102089	+ zCopy = sqlite3_mprintf("simple");
	102090	+ }else{
	102091	+ while( (z&0x80) && isspace(z) ) z++;
	102092	+ if( sqlite3_strnicmp(z, "tokenize", 8) \|\| fts3IsIdChar(z[8])){
	102093	+ return SQLITE_OK;
	102094	+ }
	102095	+ zCopy = sqlite3_mprintf("%s", &z[8]);
	102096	+ *pzTokenizer = zArg;
	102097	+ }
	102098	+ if( !zCopy ){
	102099	+ return SQLITE_NOMEM;
	102100	+ }
	102101	+
	102102	+ zEnd = &zCopy[strlen(zCopy)];
	102103	+
	102104	+ z = (char *)sqlite3Fts3NextToken(zCopy, &n);
	102105	+ z[n] = '\0';
	102106	+ sqlite3Fts3Dequote(z);
	102107	+
	102108	+ m = (sqlite3_tokenizer_module *)sqlite3Fts3HashFind(pHash, z, strlen(z)+1);
	102109	+ if( !m ){
	102110	+ *pzErr = sqlite3_mprintf("unknown tokenizer: %s", z);
	102111	+ rc = SQLITE_ERROR;
	102112	+ }else{
	102113	+ char const **aArg = 0;
	102114	+ int iArg = 0;
	102115	+ z = &z[n+1];
	102116	+ while( z<zEnd && (z = (char *)sqlite3Fts3NextToken(z, &n)) ){
	102117	+ int nNew = sizeof(char )(iArg+1);
	102118	+ char const aNew = (const char )sqlite3_realloc(aArg, nNew);
	102119	+ if( !aNew ){
	102120	+ sqlite3_free(zCopy);
	102121	+ sqlite3_free(aArg);
	102122	+ return SQLITE_NOMEM;
	102123	+ }
	102124	+ aArg = aNew;
	102125	+ aArg[iArg++] = z;
	102126	+ z[n] = '\0';
	102127	+ sqlite3Fts3Dequote(z);
	102128	+ z = &z[n+1];
	102129	+ }
	102130	+ rc = m->xCreate(iArg, aArg, ppTok);
	102131	+ assert( rc!=SQLITE_OK \|\| *ppTok );
	102132	+ if( rc!=SQLITE_OK ){
	102133	+ *pzErr = sqlite3_mprintf("unknown tokenizer: %s", z);
	102134	+ }else{
	102135	+ (*ppTok)->pModule = m;
	102136	+ }
	102137	+ sqlite3_free(aArg);
	102138	+ }
	102139	+
	102140	+ sqlite3_free(zCopy);
	102141	+ return rc;
	102142	+}
	102143	+
106724	102144
106725	102145	#ifdef SQLITE_TEST
106726	102146
106727	102147
106728	102148	/*
		@@ -106754,11 +102174,11 @@
106754	102174	static void testFunc(
106755	102175	sqlite3_context *context,
106756	102176	int argc,
106757	102177	sqlite3_value **argv
106758	102178	){
106759		- fts3Hash *pHash;
	102179	+ Fts3Hash *pHash;
106760	102180	sqlite3_tokenizer_module *p;
106761	102181	sqlite3_tokenizer *pTokenizer = 0;
106762	102182	sqlite3_tokenizer_cursor *pCsr = 0;
106763	102183
106764	102184	const char *zErr = 0;
		@@ -106787,11 +102207,11 @@
106787	102207
106788	102208	if( argc==3 ){
106789	102209	zArg = (const char *)sqlite3_value_text(argv[1]);
106790	102210	}
106791	102211
106792		- pHash = (fts3Hash *)sqlite3_user_data(context);
	102212	+ pHash = (Fts3Hash *)sqlite3_user_data(context);
106793	102213	p = (sqlite3_tokenizer_module *)sqlite3Fts3HashFind(pHash, zName, nName+1);
106794	102214
106795	102215	if( !p ){
106796	102216	char *zErr = sqlite3_mprintf("unknown tokenizer: %s", zName);
106797	102217	sqlite3_result_error(context, zErr, -1);
		@@ -106956,11 +102376,11 @@
106956	102376	** The third argument to this function, zName, is used as the name
106957	102377	** of both the scalar and, if created, the virtual table.
106958	102378	*/
106959	102379	SQLITE_PRIVATE int sqlite3Fts3InitHashTable(
106960	102380	sqlite3 *db,
106961		- fts3Hash *pHash,
	102381	+ Fts3Hash *pHash,
106962	102382	const char *zName
106963	102383	){
106964	102384	int rc = SQLITE_OK;
106965	102385	void p = (void )pHash;
106966	102386	const int any = SQLITE_ANY;
		@@ -107219,10 +102639,2911 @@
107219	102639	}
107220	102640
107221	102641	#endif /* !defined(SQLITE_CORE) \|\| defined(SQLITE_ENABLE_FTS3) */
107222	102642
107223	102643	/************ End of fts3_tokenizer1.c ***********************************/
	102644	+/************ Begin file fts3_write.c ************************************/
	102645	+/*
	102646	+** 2009 Oct 23
	102647	+**
	102648	+** The author disclaims copyright to this source code. In place of
	102649	+** a legal notice, here is a blessing:
	102650	+**
	102651	+** May you do good and not evil.
	102652	+** May you find forgiveness for yourself and forgive others.
	102653	+** May you share freely, never taking more than you give.
	102654	+**
	102655	+******************************************************************************
	102656	+**
	102657	+** This file is part of the SQLite FTS3 extension module. Specifically,
	102658	+** this file contains code to insert, update and delete rows from FTS3
	102659	+** tables. It also contains code to merge FTS3 b-tree segments. Some
	102660	+** of the sub-routines used to merge segments are also used by the query
	102661	+** code in fts3.c.
	102662	+*/
	102663	+
	102664	+#if !defined(SQLITE_CORE) \|\| defined(SQLITE_ENABLE_FTS3)
	102665	+
	102666	+
	102667	+#define INTERIOR_MAX 2048 /* Soft limit for segment node size */
	102668	+#define LEAF_MAX 2048 /* Soft limit for segment leaf size */
	102669	+
	102670	+typedef struct PendingList PendingList;
	102671	+typedef struct SegmentNode SegmentNode;
	102672	+typedef struct SegmentWriter SegmentWriter;
	102673	+
	102674	+/*
	102675	+** Data structure used while accumulating terms in the pending-terms hash
	102676	+** table. The hash table entry maps from term (a string) to a malloced
	102677	+** instance of this structure.
	102678	+*/
	102679	+struct PendingList {
	102680	+ int nData;
	102681	+ char *aData;
	102682	+ int nSpace;
	102683	+ sqlite3_int64 iLastDocid;
	102684	+ sqlite3_int64 iLastCol;
	102685	+ sqlite3_int64 iLastPos;
	102686	+};
	102687	+
	102688	+/*
	102689	+** An instance of this structure is used to iterate through the terms on
	102690	+** a contiguous set of segment b-tree leaf nodes. Although the details of
	102691	+** this structure are only manipulated by code in this file, opaque handles
	102692	+** of type Fts3SegReader* are also used by code in fts3.c to iterate through
	102693	+** terms when querying the full-text index. See functions:
	102694	+**
	102695	+** sqlite3Fts3SegReaderNew()
	102696	+** sqlite3Fts3SegReaderFree()
	102697	+** sqlite3Fts3SegReaderIterate()
	102698	+*/
	102699	+struct Fts3SegReader {
	102700	+ int iIdx; /* Index within level */
	102701	+ sqlite3_int64 iStartBlock;
	102702	+ sqlite3_int64 iEndBlock;
	102703	+ sqlite3_stmt pStmt; / SQL Statement to access leaf nodes */
	102704	+ char aNode; / Pointer to node data (or NULL) */
	102705	+ int nNode; /* Size of buffer at aNode (or 0) */
	102706	+ int nTermAlloc; /* Allocated size of zTerm buffer */
	102707	+
	102708	+ /* Variables set by fts3SegReaderNext(). These may be read directly
	102709	+ ** by the caller. They are valid from the time SegmentReaderNew() returns
	102710	+ ** until SegmentReaderNext() returns something other than SQLITE_OK
	102711	+ ** (i.e. SQLITE_DONE).
	102712	+ */
	102713	+ int nTerm; /* Number of bytes in current term */
	102714	+ char zTerm; / Pointer to current term */
	102715	+ char aDoclist; / Pointer to doclist of current entry */
	102716	+ int nDoclist; /* Size of doclist in current entry */
	102717	+
	102718	+ /* The following variables are used to iterate through the current doclist */
	102719	+ char *pOffsetList;
	102720	+ sqlite3_int64 iDocid;
	102721	+};
	102722	+
	102723	+/*
	102724	+** An instance of this structure is used to create a segment b-tree in the
	102725	+** database. The internal details of this type are only accessed by the
	102726	+** following functions:
	102727	+**
	102728	+** fts3SegWriterAdd()
	102729	+** fts3SegWriterFlush()
	102730	+** fts3SegWriterFree()
	102731	+*/
	102732	+struct SegmentWriter {
	102733	+ SegmentNode pTree; / Pointer to interior tree structure */
	102734	+ sqlite3_int64 iFirst; /* First slot in %_segments written */
	102735	+ sqlite3_int64 iFree; /* Next free slot in %_segments */
	102736	+ char zTerm; / Pointer to previous term buffer */
	102737	+ int nTerm; /* Number of bytes in zTerm */
	102738	+ int nMalloc; /* Size of malloc'd buffer at zMalloc */
	102739	+ char zMalloc; / Malloc'd space (possibly) used for zTerm */
	102740	+ int nSize; /* Size of allocation at aData */
	102741	+ int nData; /* Bytes of data in aData */
	102742	+ char aData; / Pointer to block from malloc() */
	102743	+};
	102744	+
	102745	+/*
	102746	+** Type SegmentNode is used by the following three functions to create
	102747	+** the interior part of the segment b+-tree structures (everything except
	102748	+** the leaf nodes). These functions and type are only ever used by code
	102749	+** within the fts3SegWriterXXX() family of functions described above.
	102750	+**
	102751	+** fts3NodeAddTerm()
	102752	+** fts3NodeWrite()
	102753	+** fts3NodeFree()
	102754	+*/
	102755	+struct SegmentNode {
	102756	+ SegmentNode pParent; / Parent node (or NULL for root node) */
	102757	+ SegmentNode pRight; / Pointer to right-sibling */
	102758	+ SegmentNode pLeftmost; / Pointer to left-most node of this depth */
	102759	+ int nEntry; /* Number of terms written to node so far */
	102760	+ char zTerm; / Pointer to previous term buffer */
	102761	+ int nTerm; /* Number of bytes in zTerm */
	102762	+ int nMalloc; /* Size of malloc'd buffer at zMalloc */
	102763	+ char zMalloc; / Malloc'd space (possibly) used for zTerm */
	102764	+ int nData; /* Bytes of valid data so far */
	102765	+ char aData; / Node data */
	102766	+};
	102767	+
	102768	+/*
	102769	+** Valid values for the second argument to fts3SqlStmt().
	102770	+*/
	102771	+#define SQL_DELETE_CONTENT 0
	102772	+#define SQL_IS_EMPTY 1
	102773	+#define SQL_DELETE_ALL_CONTENT 2
	102774	+#define SQL_DELETE_ALL_SEGMENTS 3
	102775	+#define SQL_DELETE_ALL_SEGDIR 4
	102776	+#define SQL_SELECT_CONTENT_BY_ROWID 5
	102777	+#define SQL_NEXT_SEGMENT_INDEX 6
	102778	+#define SQL_INSERT_SEGMENTS 7
	102779	+#define SQL_NEXT_SEGMENTS_ID 8
	102780	+#define SQL_INSERT_SEGDIR 9
	102781	+#define SQL_SELECT_LEVEL 10
	102782	+#define SQL_SELECT_ALL_LEVEL 11
	102783	+#define SQL_SELECT_LEVEL_COUNT 12
	102784	+#define SQL_SELECT_SEGDIR_COUNT_MAX 13
	102785	+#define SQL_DELETE_SEGDIR_BY_LEVEL 14
	102786	+#define SQL_DELETE_SEGMENTS_RANGE 15
	102787	+#define SQL_CONTENT_INSERT 16
	102788	+#define SQL_GET_BLOCK 17
	102789	+
	102790	+/*
	102791	+** This function is used to obtain an SQLite prepared statement handle
	102792	+** for the statement identified by the second argument. If successful,
	102793	+** *pp is set to the requested statement handle and SQLITE_OK returned.
	102794	+** Otherwise, an SQLite error code is returned and *pp is set to 0.
	102795	+**
	102796	+** If argument apVal is not NULL, then it must point to an array with
	102797	+** at least as many entries as the requested statement has bound
	102798	+** parameters. The values are bound to the statements parameters before
	102799	+** returning.
	102800	+*/
	102801	+static int fts3SqlStmt(
	102802	+ Fts3Table p, / Virtual table handle */
	102803	+ int eStmt, /* One of the SQL_XXX constants above */
	102804	+ sqlite3_stmt *pp, / OUT: Statement handle */
	102805	+ sqlite3_value *apVal / Values to bind to statement */
	102806	+){
	102807	+ const char *azSql[] = {
	102808	+/* 0 */ "DELETE FROM %Q.'%q_content' WHERE rowid = ?",
	102809	+/* 1 */ "SELECT NOT EXISTS(SELECT docid FROM %Q.'%q_content' WHERE rowid!=?)",
	102810	+/* 2 */ "DELETE FROM %Q.'%q_content'",
	102811	+/* 3 */ "DELETE FROM %Q.'%q_segments'",
	102812	+/* 4 */ "DELETE FROM %Q.'%q_segdir'",
	102813	+/* 5 / "SELECT FROM %Q.'%q_content' WHERE rowid=?",
	102814	+/* 6 */ "SELECT coalesce(max(idx)+1, 0) FROM %Q.'%q_segdir' WHERE level=?",
	102815	+/* 7 */ "INSERT INTO %Q.'%q_segments'(blockid, block) VALUES(?, ?)",
	102816	+/* 8 */ "SELECT coalesce(max(blockid)+1, 1) FROM %Q.'%q_segments'",
	102817	+/* 9 */ "INSERT INTO %Q.'%q_segdir' VALUES(?,?,?,?,?,?)",
	102818	+
	102819	+ /* Return segments in order from oldest to newest.*/
	102820	+/* 10 */ "SELECT idx, start_block, leaves_end_block, end_block, root "
	102821	+ "FROM %Q.'%q_segdir' WHERE level = ? ORDER BY idx ASC",
	102822	+/* 11 */ "SELECT idx, start_block, leaves_end_block, end_block, root "
	102823	+ "FROM %Q.'%q_segdir' ORDER BY level DESC, idx ASC",
	102824	+
	102825	+/* 12 / "SELECT count() FROM %Q.'%q_segdir' WHERE level = ?",
	102826	+/* 13 / "SELECT count(), max(level) FROM %Q.'%q_segdir'",
	102827	+
	102828	+/* 14 */ "DELETE FROM %Q.'%q_segdir' WHERE level = ?",
	102829	+/* 15 */ "DELETE FROM %Q.'%q_segments' WHERE blockid BETWEEN ? AND ?",
	102830	+/* 16 */ "INSERT INTO %Q.'%q_content' VALUES(%z)",
	102831	+/* 17 */ "SELECT block FROM %Q.'%q_segments' WHERE blockid = ?",
	102832	+ };
	102833	+ int rc = SQLITE_OK;
	102834	+ sqlite3_stmt *pStmt;
	102835	+
	102836	+ assert( SizeofArray(azSql)==SizeofArray(p->aStmt) );
	102837	+ assert( eStmt<SizeofArray(azSql) && eStmt>=0 );
	102838	+
	102839	+ pStmt = p->aStmt[eStmt];
	102840	+ if( !pStmt ){
	102841	+ char *zSql;
	102842	+ if( eStmt==SQL_CONTENT_INSERT ){
	102843	+ int i; /* Iterator variable */
	102844	+ char zVarlist; / The "?, ?, ..." string */
	102845	+ zVarlist = (char )sqlite3_malloc(2p->nColumn+2);
	102846	+ if( !zVarlist ){
	102847	+ *pp = 0;
	102848	+ return SQLITE_NOMEM;
	102849	+ }
	102850	+ zVarlist[0] = '?';
	102851	+ zVarlist[p->nColumn*2+1] = '\0';
	102852	+ for(i=1; i<=p->nColumn; i++){
	102853	+ zVarlist[i*2-1] = ',';
	102854	+ zVarlist[i*2] = '?';
	102855	+ }
	102856	+ zSql = sqlite3_mprintf(azSql[eStmt], p->zDb, p->zName, zVarlist);
	102857	+ }else{
	102858	+ zSql = sqlite3_mprintf(azSql[eStmt], p->zDb, p->zName);
	102859	+ }
	102860	+ if( !zSql ){
	102861	+ rc = SQLITE_NOMEM;
	102862	+ }else{
	102863	+ rc = sqlite3_prepare_v2(p->db, zSql, -1, &pStmt, NULL);
	102864	+ sqlite3_free(zSql);
	102865	+ assert( rc==SQLITE_OK \|\| pStmt==0 );
	102866	+ p->aStmt[eStmt] = pStmt;
	102867	+ }
	102868	+ }
	102869	+ if( apVal ){
	102870	+ int i;
	102871	+ int nParam = sqlite3_bind_parameter_count(pStmt);
	102872	+ for(i=0; rc==SQLITE_OK && i<nParam; i++){
	102873	+ rc = sqlite3_bind_value(pStmt, i+1, apVal[i]);
	102874	+ }
	102875	+ }
	102876	+ *pp = pStmt;
	102877	+ return rc;
	102878	+}
	102879	+
	102880	+/*
	102881	+** Similar to fts3SqlStmt(). Except, after binding the parameters in
	102882	+** array apVal[] to the SQL statement identified by eStmt, the statement
	102883	+** is executed.
	102884	+**
	102885	+** Returns SQLITE_OK if the statement is successfully executed, or an
	102886	+** SQLite error code otherwise.
	102887	+*/
	102888	+static int fts3SqlExec(Fts3Table p, int eStmt, sqlite3_value *apVal){
	102889	+ sqlite3_stmt *pStmt;
	102890	+ int rc = fts3SqlStmt(p, eStmt, &pStmt, apVal);
	102891	+ if( rc==SQLITE_OK ){
	102892	+ sqlite3_step(pStmt);
	102893	+ rc = sqlite3_reset(pStmt);
	102894	+ }
	102895	+ return rc;
	102896	+}
	102897	+
	102898	+
	102899	+/*
	102900	+** Read a single block from the %_segments table. If the specified block
	102901	+** does not exist, return SQLITE_CORRUPT. If some other error (malloc, IO
	102902	+** etc.) occurs, return the appropriate SQLite error code.
	102903	+**
	102904	+** Otherwise, if successful, set *pzBlock to point to a buffer containing
	102905	+** the block read from the database, and *pnBlock to the size of the read
	102906	+** block in bytes.
	102907	+**
	102908	+** WARNING: The returned buffer is only valid until the next call to
	102909	+** sqlite3Fts3ReadBlock().
	102910	+*/
	102911	+SQLITE_PRIVATE int sqlite3Fts3ReadBlock(
	102912	+ Fts3Table *p,
	102913	+ sqlite3_int64 iBlock,
	102914	+ char const **pzBlock,
	102915	+ int *pnBlock
	102916	+){
	102917	+ sqlite3_stmt *pStmt;
	102918	+ int rc = fts3SqlStmt(p, SQL_GET_BLOCK, &pStmt, 0);
	102919	+ if( rc!=SQLITE_OK ) return rc;
	102920	+ sqlite3_reset(pStmt);
	102921	+
	102922	+ sqlite3_bind_int64(pStmt, 1, iBlock);
	102923	+ rc = sqlite3_step(pStmt);
	102924	+ if( rc!=SQLITE_ROW ){
	102925	+ return SQLITE_CORRUPT;
	102926	+ }
	102927	+
	102928	+ *pnBlock = sqlite3_column_bytes(pStmt, 0);
	102929	+ pzBlock = (char )sqlite3_column_blob(pStmt, 0);
	102930	+ if( !*pzBlock ){
	102931	+ return SQLITE_NOMEM;
	102932	+ }
	102933	+ return SQLITE_OK;
	102934	+}
	102935	+
	102936	+/*
	102937	+** Set *ppStmt to a statement handle that may be used to iterate through
	102938	+** all rows in the %_segdir table, from oldest to newest. If successful,
	102939	+** return SQLITE_OK. If an error occurs while preparing the statement,
	102940	+** return an SQLite error code.
	102941	+**
	102942	+** There is only ever one instance of this SQL statement compiled for
	102943	+** each FTS3 table.
	102944	+**
	102945	+** The statement returns the following columns from the %_segdir table:
	102946	+**
	102947	+** 0: idx
	102948	+** 1: start_block
	102949	+** 2: leaves_end_block
	102950	+** 3: end_block
	102951	+** 4: root
	102952	+*/
	102953	+SQLITE_PRIVATE int sqlite3Fts3AllSegdirs(Fts3Table p, sqlite3_stmt *ppStmt){
	102954	+ return fts3SqlStmt(p, SQL_SELECT_ALL_LEVEL, ppStmt, 0);
	102955	+}
	102956	+
	102957	+
	102958	+/*
	102959	+** Append a single varint to a PendingList buffer. SQLITE_OK is returned
	102960	+** if successful, or an SQLite error code otherwise.
	102961	+**
	102962	+** This function also serves to allocate the PendingList structure itself.
	102963	+** For example, to create a new PendingList structure containing two
	102964	+** varints:
	102965	+**
	102966	+** PendingList *p = 0;
	102967	+** fts3PendingListAppendVarint(&p, 1);
	102968	+** fts3PendingListAppendVarint(&p, 2);
	102969	+*/
	102970	+static int fts3PendingListAppendVarint(
	102971	+ PendingList *pp, / IN/OUT: Pointer to PendingList struct */
	102972	+ sqlite3_int64 i /* Value to append to data */
	102973	+){
	102974	+ PendingList p = pp;
	102975	+
	102976	+ /* Allocate or grow the PendingList as required. */
	102977	+ if( !p ){
	102978	+ p = sqlite3_malloc(sizeof(*p) + 100);
	102979	+ if( !p ){
	102980	+ return SQLITE_NOMEM;
	102981	+ }
	102982	+ p->nSpace = 100;
	102983	+ p->aData = (char *)&p[1];
	102984	+ p->nData = 0;
	102985	+ }
	102986	+ else if( p->nData+FTS3_VARINT_MAX+1>p->nSpace ){
	102987	+ int nNew = p->nSpace * 2;
	102988	+ p = sqlite3_realloc(p, sizeof(*p) + nNew);
	102989	+ if( !p ){
	102990	+ sqlite3_free(*pp);
	102991	+ *pp = 0;
	102992	+ return SQLITE_NOMEM;
	102993	+ }
	102994	+ p->nSpace = nNew;
	102995	+ p->aData = (char *)&p[1];
	102996	+ }
	102997	+
	102998	+ /* Append the new serialized varint to the end of the list. */
	102999	+ p->nData += sqlite3Fts3PutVarint(&p->aData[p->nData], i);
	103000	+ p->aData[p->nData] = '\0';
	103001	+ *pp = p;
	103002	+ return SQLITE_OK;
	103003	+}
	103004	+
	103005	+/*
	103006	+** Add a docid/column/position entry to a PendingList structure. Non-zero
	103007	+** is returned if the structure is sqlite3_realloced as part of adding
	103008	+** the entry. Otherwise, zero.
	103009	+**
	103010	+** If an OOM error occurs, *pRc is set to SQLITE_NOMEM before returning.
	103011	+** Zero is always returned in this case. Otherwise, if no OOM error occurs,
	103012	+** it is set to SQLITE_OK.
	103013	+*/
	103014	+static int fts3PendingListAppend(
	103015	+ PendingList *pp, / IN/OUT: PendingList structure */
	103016	+ sqlite3_int64 iDocid, /* Docid for entry to add */
	103017	+ sqlite3_int64 iCol, /* Column for entry to add */
	103018	+ sqlite3_int64 iPos, /* Position of term for entry to add */
	103019	+ int pRc / OUT: Return code */
	103020	+){
	103021	+ PendingList p = pp;
	103022	+ int rc = SQLITE_OK;
	103023	+
	103024	+ assert( !p \|\| p->iLastDocid<=iDocid );
	103025	+
	103026	+ if( !p \|\| p->iLastDocid!=iDocid ){
	103027	+ sqlite3_int64 iDelta = iDocid - (p ? p->iLastDocid : 0);
	103028	+ if( p ){
	103029	+ assert( p->nData<p->nSpace );
	103030	+ assert( p->aData[p->nData]==0 );
	103031	+ p->nData++;
	103032	+ }
	103033	+ if( SQLITE_OK!=(rc = fts3PendingListAppendVarint(&p, iDelta)) ){
	103034	+ goto pendinglistappend_out;
	103035	+ }
	103036	+ p->iLastCol = -1;
	103037	+ p->iLastPos = 0;
	103038	+ p->iLastDocid = iDocid;
	103039	+ }
	103040	+ if( iCol>0 && p->iLastCol!=iCol ){
	103041	+ if( SQLITE_OK!=(rc = fts3PendingListAppendVarint(&p, 1))
	103042	+ \|\| SQLITE_OK!=(rc = fts3PendingListAppendVarint(&p, iCol))
	103043	+ ){
	103044	+ goto pendinglistappend_out;
	103045	+ }
	103046	+ p->iLastCol = iCol;
	103047	+ p->iLastPos = 0;
	103048	+ }
	103049	+ if( iCol>=0 ){
	103050	+ assert( iPos>p->iLastPos \|\| (iPos==0 && p->iLastPos==0) );
	103051	+ rc = fts3PendingListAppendVarint(&p, 2+iPos-p->iLastPos);
	103052	+ p->iLastPos = iPos;
	103053	+ }
	103054	+
	103055	+ pendinglistappend_out:
	103056	+ *pRc = rc;
	103057	+ if( p!=*pp ){
	103058	+ *pp = p;
	103059	+ return 1;
	103060	+ }
	103061	+ return 0;
	103062	+}
	103063	+
	103064	+/*
	103065	+** Tokenize the nul-terminated string zText and add all tokens to the
	103066	+** pending-terms hash-table. The docid used is that currently stored in
	103067	+** p->iPrevDocid, and the column is specified by argument iCol.
	103068	+**
	103069	+** If successful, SQLITE_OK is returned. Otherwise, an SQLite error code.
	103070	+*/
	103071	+static int fts3PendingTermsAdd(Fts3Table p, const char zText, int iCol){
	103072	+ int rc;
	103073	+ int iStart;
	103074	+ int iEnd;
	103075	+ int iPos;
	103076	+
	103077	+ char const *zToken;
	103078	+ int nToken;
	103079	+
	103080	+ sqlite3_tokenizer *pTokenizer = p->pTokenizer;
	103081	+ sqlite3_tokenizer_module const *pModule = pTokenizer->pModule;
	103082	+ sqlite3_tokenizer_cursor *pCsr;
	103083	+ int (xNext)(sqlite3_tokenizer_cursor pCursor,
	103084	+ const char*,int,int,int,int*);
	103085	+
	103086	+ assert( pTokenizer && pModule );
	103087	+
	103088	+ rc = pModule->xOpen(pTokenizer, zText, -1, &pCsr);
	103089	+ if( rc!=SQLITE_OK ){
	103090	+ return rc;
	103091	+ }
	103092	+ pCsr->pTokenizer = pTokenizer;
	103093	+
	103094	+ xNext = pModule->xNext;
	103095	+ while( SQLITE_OK==rc
	103096	+ && SQLITE_OK==(rc = xNext(pCsr, &zToken, &nToken, &iStart, &iEnd, &iPos))
	103097	+ ){
	103098	+ PendingList *pList;
	103099	+
	103100	+ /* Positions cannot be negative; we use -1 as a terminator internally.
	103101	+ ** Tokens must have a non-zero length.
	103102	+ */
	103103	+ if( iPos<0 \|\| !zToken \|\| nToken<=0 ){
	103104	+ rc = SQLITE_ERROR;
	103105	+ break;
	103106	+ }
	103107	+
	103108	+ pList = (PendingList *)fts3HashFind(&p->pendingTerms, zToken, nToken);
	103109	+ if( pList ){
	103110	+ p->nPendingData -= (pList->nData + nToken + sizeof(Fts3HashElem));
	103111	+ }
	103112	+ if( fts3PendingListAppend(&pList, p->iPrevDocid, iCol, iPos, &rc) ){
	103113	+ if( pList==fts3HashInsert(&p->pendingTerms, zToken, nToken, pList) ){
	103114	+ /* Malloc failed while inserting the new entry. This can only
	103115	+ ** happen if there was no previous entry for this token.
	103116	+ */
	103117	+ assert( 0==fts3HashFind(&p->pendingTerms, zToken, nToken) );
	103118	+ sqlite3_free(pList);
	103119	+ rc = SQLITE_NOMEM;
	103120	+ }
	103121	+ }
	103122	+ if( rc==SQLITE_OK ){
	103123	+ p->nPendingData += (pList->nData + nToken + sizeof(Fts3HashElem));
	103124	+ }
	103125	+ }
	103126	+
	103127	+ pModule->xClose(pCsr);
	103128	+ return (rc==SQLITE_DONE ? SQLITE_OK : rc);
	103129	+}
	103130	+
	103131	+/*
	103132	+** Calling this function indicates that subsequent calls to
	103133	+** fts3PendingTermsAdd() are to add term/position-list pairs for the
	103134	+** contents of the document with docid iDocid.
	103135	+*/
	103136	+static int fts3PendingTermsDocid(Fts3Table *p, sqlite_int64 iDocid){
	103137	+ /* TODO(shess) Explore whether partially flushing the buffer on
	103138	+ ** forced-flush would provide better performance. I suspect that if
	103139	+ ** we ordered the doclists by size and flushed the largest until the
	103140	+ ** buffer was half empty, that would let the less frequent terms
	103141	+ ** generate longer doclists.
	103142	+ */
	103143	+ if( iDocid<=p->iPrevDocid \|\| p->nPendingData>FTS3_MAX_PENDING_DATA ){
	103144	+ int rc = sqlite3Fts3PendingTermsFlush(p);
	103145	+ if( rc!=SQLITE_OK ) return rc;
	103146	+ }
	103147	+ p->iPrevDocid = iDocid;
	103148	+ return SQLITE_OK;
	103149	+}
	103150	+
	103151	+SQLITE_PRIVATE void sqlite3Fts3PendingTermsClear(Fts3Table *p){
	103152	+ Fts3HashElem *pElem;
	103153	+ for(pElem=fts3HashFirst(&p->pendingTerms); pElem; pElem=fts3HashNext(pElem)){
	103154	+ sqlite3_free(fts3HashData(pElem));
	103155	+ }
	103156	+ fts3HashClear(&p->pendingTerms);
	103157	+ p->nPendingData = 0;
	103158	+}
	103159	+
	103160	+/*
	103161	+** This function is called by the xUpdate() method as part of an INSERT
	103162	+** operation. It adds entries for each term in the new record to the
	103163	+** pendingTerms hash table.
	103164	+**
	103165	+** Argument apVal is the same as the similarly named argument passed to
	103166	+** fts3InsertData(). Parameter iDocid is the docid of the new row.
	103167	+*/
	103168	+static int fts3InsertTerms(Fts3Table p, sqlite3_value *apVal){
	103169	+ int i; /* Iterator variable */
	103170	+ for(i=2; i<p->nColumn+2; i++){
	103171	+ const char zText = (const char )sqlite3_value_text(apVal[i]);
	103172	+ if( zText ){
	103173	+ int rc = fts3PendingTermsAdd(p, zText, i-2);
	103174	+ if( rc!=SQLITE_OK ){
	103175	+ return rc;
	103176	+ }
	103177	+ }
	103178	+ }
	103179	+ return SQLITE_OK;
	103180	+}
	103181	+
	103182	+/*
	103183	+** This function is called by the xUpdate() method for an INSERT operation.
	103184	+** The apVal parameter is passed a copy of the apVal argument passed by
	103185	+** SQLite to the xUpdate() method. i.e:
	103186	+**
	103187	+** apVal[0] Not used for INSERT.
	103188	+** apVal[1] rowid
	103189	+** apVal[2] Left-most user-defined column
	103190	+** ...
	103191	+** apVal[p->nColumn+1] Right-most user-defined column
	103192	+** apVal[p->nColumn+2] Hidden column with same name as table
	103193	+** apVal[p->nColumn+3] Hidden "docid" column (alias for rowid)
	103194	+*/
	103195	+static int fts3InsertData(
	103196	+ Fts3Table p, / Full-text table */
	103197	+ sqlite3_value *apVal, / Array of values to insert */
	103198	+ sqlite3_int64 piDocid / OUT: Docid for row just inserted */
	103199	+){
	103200	+ int rc; /* Return code */
	103201	+ int i; /* Iterator variable */
	103202	+ sqlite3_stmt pContentInsert; / INSERT INTO %_content VALUES(...) */
	103203	+
	103204	+ /* Locate the statement handle used to insert data into the %_content
	103205	+ ** table. The SQL for this statement is:
	103206	+ **
	103207	+ ** INSERT INTO %_content VALUES(?, ?, ?, ...)
	103208	+ **
	103209	+ ** The statement features N '?' variables, where N is the number of user
	103210	+ ** defined columns in the FTS3 table, plus one for the docid field.
	103211	+ */
	103212	+ rc = fts3SqlStmt(p, SQL_CONTENT_INSERT, &pContentInsert, &apVal[1]);
	103213	+ if( rc!=SQLITE_OK ){
	103214	+ return rc;
	103215	+ }
	103216	+
	103217	+ /* There is a quirk here. The users INSERT statement may have specified
	103218	+ ** a value for the "rowid" field, for the "docid" field, or for both.
	103219	+ ** Which is a problem, since "rowid" and "docid" are aliases for the
	103220	+ ** same value. For example:
	103221	+ **
	103222	+ ** INSERT INTO fts3tbl(rowid, docid) VALUES(1, 2);
	103223	+ **
	103224	+ ** In FTS3, if a non-NULL docid value is specified, it is the value
	103225	+ ** inserted. Otherwise, the rowid value is used.
	103226	+ */
	103227	+ if( SQLITE_NULL!=sqlite3_value_type(apVal[3+p->nColumn]) ){
	103228	+ rc = sqlite3_bind_value(pContentInsert, 1, apVal[3+p->nColumn]);
	103229	+ if( rc!=SQLITE_OK ) return rc;
	103230	+ }
	103231	+
	103232	+ /* Execute the statement to insert the record. Set *piDocid to the
	103233	+ ** new docid value.
	103234	+ */
	103235	+ sqlite3_step(pContentInsert);
	103236	+ rc = sqlite3_reset(pContentInsert);
	103237	+
	103238	+ *piDocid = sqlite3_last_insert_rowid(p->db);
	103239	+ return rc;
	103240	+}
	103241	+
	103242	+
	103243	+
	103244	+/*
	103245	+** Remove all data from the FTS3 table. Clear the hash table containing
	103246	+** pending terms.
	103247	+*/
	103248	+static int fts3DeleteAll(Fts3Table *p){
	103249	+ int rc; /* Return code */
	103250	+
	103251	+ /* Discard the contents of the pending-terms hash table. */
	103252	+ sqlite3Fts3PendingTermsClear(p);
	103253	+
	103254	+ /* Delete everything from the %_content, %_segments and %_segdir tables. */
	103255	+ rc = fts3SqlExec(p, SQL_DELETE_ALL_CONTENT, 0);
	103256	+ if( rc==SQLITE_OK ){
	103257	+ rc = fts3SqlExec(p, SQL_DELETE_ALL_SEGMENTS, 0);
	103258	+ }
	103259	+ if( rc==SQLITE_OK ){
	103260	+ rc = fts3SqlExec(p, SQL_DELETE_ALL_SEGDIR, 0);
	103261	+ }
	103262	+ return rc;
	103263	+}
	103264	+
	103265	+/*
	103266	+** The first element in the apVal[] array is assumed to contain the docid
	103267	+** (an integer) of a row about to be deleted. Remove all terms from the
	103268	+** full-text index.
	103269	+*/
	103270	+static int fts3DeleteTerms(Fts3Table p, sqlite3_value *apVal){
	103271	+ int rc;
	103272	+ sqlite3_stmt *pSelect;
	103273	+
	103274	+ rc = fts3SqlStmt(p, SQL_SELECT_CONTENT_BY_ROWID, &pSelect, apVal);
	103275	+ if( rc==SQLITE_OK ){
	103276	+ if( SQLITE_ROW==sqlite3_step(pSelect) ){
	103277	+ int i;
	103278	+ for(i=1; i<=p->nColumn; i++){
	103279	+ const char zText = (const char )sqlite3_column_text(pSelect, i);
	103280	+ rc = fts3PendingTermsAdd(p, zText, -1);
	103281	+ if( rc!=SQLITE_OK ){
	103282	+ sqlite3_reset(pSelect);
	103283	+ return rc;
	103284	+ }
	103285	+ }
	103286	+ }
	103287	+ }
	103288	+
	103289	+ return sqlite3_reset(pSelect);
	103290	+}
	103291	+
	103292	+/*
	103293	+** Forward declaration to account for the circular dependency between
	103294	+** functions fts3SegmentMerge() and fts3AllocateSegdirIdx().
	103295	+*/
	103296	+static int fts3SegmentMerge(Fts3Table *, int);
	103297	+
	103298	+/*
	103299	+** This function allocates a new level iLevel index in the segdir table.
	103300	+** Usually, indexes are allocated within a level sequentially starting
	103301	+** with 0, so the allocated index is one greater than the value returned
	103302	+** by:
	103303	+**
	103304	+** SELECT max(idx) FROM %_segdir WHERE level = :iLevel
	103305	+**
	103306	+** However, if there are already FTS3_MERGE_COUNT indexes at the requested
	103307	+** level, they are merged into a single level (iLevel+1) segment and the
	103308	+** allocated index is 0.
	103309	+**
	103310	+** If successful, *piIdx is set to the allocated index slot and SQLITE_OK
	103311	+** returned. Otherwise, an SQLite error code is returned.
	103312	+*/
	103313	+static int fts3AllocateSegdirIdx(Fts3Table p, int iLevel, int piIdx){
	103314	+ int rc; /* Return Code */
	103315	+ sqlite3_stmt pNextIdx; / Query for next idx at level iLevel */
	103316	+ int iNext; /* Result of query pNextIdx */
	103317	+
	103318	+ /* Set variable iNext to the next available segdir index at level iLevel. */
	103319	+ rc = fts3SqlStmt(p, SQL_NEXT_SEGMENT_INDEX, &pNextIdx, 0);
	103320	+ if( rc==SQLITE_OK ){
	103321	+ sqlite3_bind_int(pNextIdx, 1, iLevel);
	103322	+ if( SQLITE_ROW==sqlite3_step(pNextIdx) ){
	103323	+ iNext = sqlite3_column_int64(pNextIdx, 0);
	103324	+ }
	103325	+ rc = sqlite3_reset(pNextIdx);
	103326	+ }
	103327	+
	103328	+ if( rc==SQLITE_OK ){
	103329	+ /* If iNext is FTS3_MERGE_COUNT, indicating that level iLevel is already
	103330	+ ** full, merge all segments in level iLevel into a single iLevel+1
	103331	+ ** segment and allocate (newly freed) index 0 at level iLevel. Otherwise,
	103332	+ ** if iNext is less than FTS3_MERGE_COUNT, allocate index iNext.
	103333	+ */
	103334	+ if( iNext>=FTS3_MERGE_COUNT ){
	103335	+ rc = fts3SegmentMerge(p, iLevel);
	103336	+ *piIdx = 0;
	103337	+ }else{
	103338	+ *piIdx = iNext;
	103339	+ }
	103340	+ }
	103341	+
	103342	+ return rc;
	103343	+}
	103344	+
	103345	+/*
	103346	+** Move the iterator passed as the first argument to the next term in the
	103347	+** segment. If successful, SQLITE_OK is returned. If there is no next term,
	103348	+** SQLITE_DONE. Otherwise, an SQLite error code.
	103349	+*/
	103350	+static int fts3SegReaderNext(Fts3SegReader *pReader){
	103351	+ char pNext; / Cursor variable */
	103352	+ int nPrefix; /* Number of bytes in term prefix */
	103353	+ int nSuffix; /* Number of bytes in term suffix */
	103354	+
	103355	+ if( !pReader->aDoclist ){
	103356	+ pNext = pReader->aNode;
	103357	+ }else{
	103358	+ pNext = &pReader->aDoclist[pReader->nDoclist];
	103359	+ }
	103360	+
	103361	+ if( !pNext \|\| pNext>=&pReader->aNode[pReader->nNode] ){
	103362	+ int rc;
	103363	+ if( !pReader->pStmt ){
	103364	+ pReader->aNode = 0;
	103365	+ return SQLITE_OK;
	103366	+ }
	103367	+ rc = sqlite3_step(pReader->pStmt);
	103368	+ if( rc!=SQLITE_ROW ){
	103369	+ pReader->aNode = 0;
	103370	+ return (rc==SQLITE_DONE ? SQLITE_OK : rc);
	103371	+ }
	103372	+ pReader->nNode = sqlite3_column_bytes(pReader->pStmt, 0);
	103373	+ pReader->aNode = (char *)sqlite3_column_blob(pReader->pStmt, 0);
	103374	+ pNext = pReader->aNode;
	103375	+ }
	103376	+
	103377	+ pNext += sqlite3Fts3GetVarint32(pNext, &nPrefix);
	103378	+ pNext += sqlite3Fts3GetVarint32(pNext, &nSuffix);
	103379	+
	103380	+ if( nPrefix+nSuffix>pReader->nTermAlloc ){
	103381	+ int nNew = (nPrefix+nSuffix)*2;
	103382	+ char *zNew = sqlite3_realloc(pReader->zTerm, nNew);
	103383	+ if( !zNew ){
	103384	+ return SQLITE_NOMEM;
	103385	+ }
	103386	+ pReader->zTerm = zNew;
	103387	+ pReader->nTermAlloc = nNew;
	103388	+ }
	103389	+ memcpy(&pReader->zTerm[nPrefix], pNext, nSuffix);
	103390	+ pReader->nTerm = nPrefix+nSuffix;
	103391	+ pNext += nSuffix;
	103392	+ pNext += sqlite3Fts3GetVarint32(pNext, &pReader->nDoclist);
	103393	+ assert( pNext<&pReader->aNode[pReader->nNode] );
	103394	+ pReader->aDoclist = pNext;
	103395	+ pReader->pOffsetList = 0;
	103396	+ return SQLITE_OK;
	103397	+}
	103398	+
	103399	+/*
	103400	+** Set the SegReader to point to the first docid in the doclist associated
	103401	+** with the current term.
	103402	+*/
	103403	+static void fts3SegReaderFirstDocid(Fts3SegReader *pReader){
	103404	+ int n;
	103405	+ assert( pReader->aDoclist );
	103406	+ assert( !pReader->pOffsetList );
	103407	+ n = sqlite3Fts3GetVarint(pReader->aDoclist, &pReader->iDocid);
	103408	+ pReader->pOffsetList = &pReader->aDoclist[n];
	103409	+}
	103410	+
	103411	+/*
	103412	+** Advance the SegReader to point to the next docid in the doclist
	103413	+** associated with the current term.
	103414	+**
	103415	+** If arguments ppOffsetList and pnOffsetList are not NULL, then
	103416	+** *ppOffsetList is set to point to the first column-offset list
	103417	+** in the doclist entry (i.e. immediately past the docid varint).
	103418	+** *pnOffsetList is set to the length of the set of column-offset
	103419	+** lists, not including the nul-terminator byte. For example:
	103420	+*/
	103421	+static void fts3SegReaderNextDocid(
	103422	+ Fts3SegReader *pReader,
	103423	+ char **ppOffsetList,
	103424	+ int *pnOffsetList
	103425	+){
	103426	+ char *p = pReader->pOffsetList;
	103427	+ char c = 0;
	103428	+
	103429	+ /* Pointer p currently points at the first byte of an offset list. The
	103430	+ ** following two lines advance it to point one byte past the end of
	103431	+ ** the same offset list.
	103432	+ */
	103433	+ while( p \| c ) c = p++ & 0x80;
	103434	+ p++;
	103435	+
	103436	+ /* If required, populate the output variables with a pointer to and the
	103437	+ ** size of the previous offset-list.
	103438	+ */
	103439	+ if( ppOffsetList ){
	103440	+ *ppOffsetList = pReader->pOffsetList;
	103441	+ *pnOffsetList = p - pReader->pOffsetList - 1;
	103442	+ }
	103443	+
	103444	+ /* If there are no more entries in the doclist, set pOffsetList to
	103445	+ ** NULL. Otherwise, set Fts3SegReader.iDocid to the next docid and
	103446	+ ** Fts3SegReader.pOffsetList to point to the next offset list before
	103447	+ ** returning.
	103448	+ */
	103449	+ if( p>=&pReader->aDoclist[pReader->nDoclist] ){
	103450	+ pReader->pOffsetList = 0;
	103451	+ }else{
	103452	+ sqlite3_int64 iDelta;
	103453	+ pReader->pOffsetList = p + sqlite3Fts3GetVarint(p, &iDelta);
	103454	+ pReader->iDocid += iDelta;
	103455	+ }
	103456	+}
	103457	+
	103458	+/*
	103459	+** Free all allocations associated with the iterator passed as the
	103460	+** second argument.
	103461	+*/
	103462	+SQLITE_PRIVATE void sqlite3Fts3SegReaderFree(Fts3Table p, Fts3SegReader pReader){
	103463	+ if( pReader ){
	103464	+ if( pReader->pStmt ){
	103465	+ /* Move the leaf-range SELECT statement to the aLeavesStmt[] array,
	103466	+ ** so that it can be reused when required by another query.
	103467	+ */
	103468	+ assert( p->nLeavesStmt<p->nLeavesTotal );
	103469	+ sqlite3_reset(pReader->pStmt);
	103470	+ p->aLeavesStmt[p->nLeavesStmt++] = pReader->pStmt;
	103471	+ }
	103472	+ sqlite3_free(pReader->zTerm);
	103473	+ sqlite3_free(pReader);
	103474	+ }
	103475	+}
	103476	+
	103477	+/*
	103478	+** Allocate a new SegReader object.
	103479	+*/
	103480	+SQLITE_PRIVATE int sqlite3Fts3SegReaderNew(
	103481	+ Fts3Table p, / Virtual table handle */
	103482	+ int iAge, /* Segment "age". */
	103483	+ sqlite3_int64 iStartLeaf, /* First leaf to traverse */
	103484	+ sqlite3_int64 iEndLeaf, /* Final leaf to traverse */
	103485	+ sqlite3_int64 iEndBlock, /* Final block of segment */
	103486	+ const char zRoot, / Buffer containing root node */
	103487	+ int nRoot, /* Size of buffer containing root node */
	103488	+ Fts3SegReader *ppReader / OUT: Allocated Fts3SegReader */
	103489	+){
	103490	+ int rc = SQLITE_OK; /* Return code */
	103491	+ Fts3SegReader pReader; / Newly allocated SegReader object */
	103492	+ int nExtra = 0; /* Bytes to allocate segment root node */
	103493	+
	103494	+ if( iStartLeaf==0 ){
	103495	+ nExtra = nRoot;
	103496	+ }
	103497	+
	103498	+ pReader = (Fts3SegReader *)sqlite3_malloc(sizeof(Fts3SegReader) + nExtra);
	103499	+ if( !pReader ){
	103500	+ return SQLITE_NOMEM;
	103501	+ }
	103502	+ memset(pReader, 0, sizeof(Fts3SegReader));
	103503	+ pReader->iStartBlock = iStartLeaf;
	103504	+ pReader->iIdx = iAge;
	103505	+ pReader->iEndBlock = iEndBlock;
	103506	+
	103507	+ if( nExtra ){
	103508	+ /* The entire segment is stored in the root node. */
	103509	+ pReader->aNode = (char *)&pReader[1];
	103510	+ pReader->nNode = nRoot;
	103511	+ memcpy(pReader->aNode, zRoot, nRoot);
	103512	+ }else{
	103513	+ sqlite3_stmt *pStmt;
	103514	+
	103515	+ /* If the text of the SQL statement to iterate through a contiguous
	103516	+ ** set of entries in the %_segments table has not yet been composed,
	103517	+ ** compose it now.
	103518	+ */
	103519	+ if( !p->zSelectLeaves ){
	103520	+ p->zSelectLeaves = sqlite3_mprintf(
	103521	+ "SELECT block FROM %Q.'%q_segments' WHERE blockid BETWEEN ? AND ? "
	103522	+ "ORDER BY blockid", p->zDb, p->zName
	103523	+ );
	103524	+ if( !p->zSelectLeaves ){
	103525	+ rc = SQLITE_NOMEM;
	103526	+ goto finished;
	103527	+ }
	103528	+ }
	103529	+
	103530	+ /* If there are no free statements in the aLeavesStmt[] array, prepare
	103531	+ ** a new statement now. Otherwise, reuse a prepared statement from
	103532	+ ** aLeavesStmt[].
	103533	+ */
	103534	+ if( p->nLeavesStmt==0 ){
	103535	+ if( p->nLeavesTotal==p->nLeavesAlloc ){
	103536	+ int nNew = p->nLeavesAlloc + 16;
	103537	+ sqlite3_stmt aNew = (sqlite3_stmt )sqlite3_realloc(
	103538	+ p->aLeavesStmt, nNewsizeof(sqlite3_stmt )
	103539	+ );
	103540	+ if( !aNew ){
	103541	+ rc = SQLITE_NOMEM;
	103542	+ goto finished;
	103543	+ }
	103544	+ p->nLeavesAlloc = nNew;
	103545	+ p->aLeavesStmt = aNew;
	103546	+ }
	103547	+ rc = sqlite3_prepare_v2(p->db, p->zSelectLeaves, -1, &pReader->pStmt, 0);
	103548	+ if( rc!=SQLITE_OK ){
	103549	+ goto finished;
	103550	+ }
	103551	+ p->nLeavesTotal++;
	103552	+ }else{
	103553	+ pReader->pStmt = p->aLeavesStmt[--p->nLeavesStmt];
	103554	+ }
	103555	+
	103556	+ /* Bind the start and end leaf blockids to the prepared SQL statement. */
	103557	+ sqlite3_bind_int64(pReader->pStmt, 1, iStartLeaf);
	103558	+ sqlite3_bind_int64(pReader->pStmt, 2, iEndLeaf);
	103559	+ }
	103560	+ rc = fts3SegReaderNext(pReader);
	103561	+
	103562	+ finished:
	103563	+ if( rc==SQLITE_OK ){
	103564	+ *ppReader = pReader;
	103565	+ }else{
	103566	+ sqlite3Fts3SegReaderFree(p, pReader);
	103567	+ }
	103568	+ return rc;
	103569	+}
	103570	+
	103571	+
	103572	+/*
	103573	+** The second argument to this function is expected to be a statement of
	103574	+** the form:
	103575	+**
	103576	+** SELECT
	103577	+** idx, -- col 0
	103578	+** start_block, -- col 1
	103579	+** leaves_end_block, -- col 2
	103580	+** end_block, -- col 3
	103581	+** root -- col 4
	103582	+** FROM %_segdir ...
	103583	+**
	103584	+** This function allocates and initializes a Fts3SegReader structure to
	103585	+** iterate through the terms stored in the segment identified by the
	103586	+** current row that pStmt is pointing to.
	103587	+**
	103588	+** If successful, the Fts3SegReader is left pointing to the first term
	103589	+** in the segment and SQLITE_OK is returned. Otherwise, an SQLite error
	103590	+** code is returned.
	103591	+*/
	103592	+static int fts3SegReaderNew(
	103593	+ Fts3Table p, / Virtual table handle */
	103594	+ sqlite3_stmt pStmt, / See above */
	103595	+ int iAge, /* Segment "age". */
	103596	+ Fts3SegReader *ppReader / OUT: Allocated Fts3SegReader */
	103597	+){
	103598	+ return sqlite3Fts3SegReaderNew(p, iAge,
	103599	+ sqlite3_column_int64(pStmt, 1),
	103600	+ sqlite3_column_int64(pStmt, 2),
	103601	+ sqlite3_column_int64(pStmt, 3),
	103602	+ sqlite3_column_blob(pStmt, 4),
	103603	+ sqlite3_column_bytes(pStmt, 4),
	103604	+ ppReader
	103605	+ );
	103606	+}
	103607	+
	103608	+/*
	103609	+** Compare the entries pointed to by two Fts3SegReader structures.
	103610	+** Comparison is as follows:
	103611	+**
	103612	+** 1) EOF is greater than not EOF.
	103613	+**
	103614	+** 2) The current terms (if any) are compared with memcmp(). If one
	103615	+** term is a prefix of another, the longer term is considered the
	103616	+** larger.
	103617	+**
	103618	+** 3) By segment age. An older segment is considered larger.
	103619	+*/
	103620	+static int fts3SegReaderCmp(Fts3SegReader pLhs, Fts3SegReader pRhs){
	103621	+ int rc;
	103622	+ if( pLhs->aNode && pRhs->aNode ){
	103623	+ int rc2 = pLhs->nTerm - pRhs->nTerm;
	103624	+ if( rc2<0 ){
	103625	+ rc = memcmp(pLhs->zTerm, pRhs->zTerm, pLhs->nTerm);
	103626	+ }else{
	103627	+ rc = memcmp(pLhs->zTerm, pRhs->zTerm, pRhs->nTerm);
	103628	+ }
	103629	+ if( rc==0 ){
	103630	+ rc = rc2;
	103631	+ }
	103632	+ }else{
	103633	+ rc = (pLhs->aNode==0) - (pRhs->aNode==0);
	103634	+ }
	103635	+ if( rc==0 ){
	103636	+ rc = pRhs->iIdx - pLhs->iIdx;
	103637	+ }
	103638	+ assert( rc!=0 );
	103639	+ return rc;
	103640	+}
	103641	+
	103642	+/*
	103643	+** A different comparison function for SegReader structures. In this
	103644	+** version, it is assumed that each SegReader points to an entry in
	103645	+** a doclist for identical terms. Comparison is made as follows:
	103646	+**
	103647	+** 1) EOF (end of doclist in this case) is greater than not EOF.
	103648	+**
	103649	+** 2) By current docid.
	103650	+**
	103651	+** 3) By segment age. An older segment is considered larger.
	103652	+*/
	103653	+static int fts3SegReaderDoclistCmp(Fts3SegReader pLhs, Fts3SegReader pRhs){
	103654	+ int rc = (pLhs->pOffsetList==0)-(pRhs->pOffsetList==0);
	103655	+ if( rc==0 ){
	103656	+ if( pLhs->iDocid==pRhs->iDocid ){
	103657	+ rc = pRhs->iIdx - pLhs->iIdx;
	103658	+ }else{
	103659	+ rc = (pLhs->iDocid > pRhs->iDocid) ? 1 : -1;
	103660	+ }
	103661	+ }
	103662	+ assert( pLhs->aNode && pRhs->aNode );
	103663	+ return rc;
	103664	+}
	103665	+
	103666	+/*
	103667	+** Compare the term that the Fts3SegReader object passed as the first argument
	103668	+** points to with the term specified by arguments zTerm and nTerm.
	103669	+**
	103670	+** If the pSeg iterator is already at EOF, return 0. Otherwise, return
	103671	+** -ve if the pSeg term is less than zTerm/nTerm, 0 if the two terms are
	103672	+** equal, or +ve if the pSeg term is greater than zTerm/nTerm.
	103673	+*/
	103674	+static int fts3SegReaderTermCmp(
	103675	+ Fts3SegReader pSeg, / Segment reader object */
	103676	+ const char zTerm, / Term to compare to */
	103677	+ int nTerm /* Size of term zTerm in bytes */
	103678	+){
	103679	+ int res = 0;
	103680	+ if( pSeg->aNode ){
	103681	+ if( pSeg->nTerm>nTerm ){
	103682	+ res = memcmp(pSeg->zTerm, zTerm, nTerm);
	103683	+ }else{
	103684	+ res = memcmp(pSeg->zTerm, zTerm, pSeg->nTerm);
	103685	+ }
	103686	+ if( res==0 ){
	103687	+ res = pSeg->nTerm-nTerm;
	103688	+ }
	103689	+ }
	103690	+ return res;
	103691	+}
	103692	+
	103693	+/*
	103694	+** Argument apSegment is an array of nSegment elements. It is known that
	103695	+** the final (nSegment-nSuspect) members are already in sorted order
	103696	+** (according to the comparison function provided). This function shuffles
	103697	+** the array around until all entries are in sorted order.
	103698	+*/
	103699	+static void fts3SegReaderSort(
	103700	+ Fts3SegReader *apSegment, / Array to sort entries of */
	103701	+ int nSegment, /* Size of apSegment array */
	103702	+ int nSuspect, /* Unsorted entry count */
	103703	+ int (xCmp)(Fts3SegReader , Fts3SegReader ) / Comparison function */
	103704	+){
	103705	+ int i; /* Iterator variable */
	103706	+
	103707	+ assert( nSuspect<=nSegment );
	103708	+
	103709	+ if( nSuspect==nSegment ) nSuspect--;
	103710	+ for(i=nSuspect-1; i>=0; i--){
	103711	+ int j;
	103712	+ for(j=i; j<(nSegment-1); j++){
	103713	+ Fts3SegReader *pTmp;
	103714	+ if( xCmp(apSegment[j], apSegment[j+1])<0 ) break;
	103715	+ pTmp = apSegment[j+1];
	103716	+ apSegment[j+1] = apSegment[j];
	103717	+ apSegment[j] = pTmp;
	103718	+ }
	103719	+ }
	103720	+
	103721	+#ifndef NDEBUG
	103722	+ /* Check that the list really is sorted now. */
	103723	+ for(i=0; i<(nSuspect-1); i++){
	103724	+ assert( xCmp(apSegment[i], apSegment[i+1])<0 );
	103725	+ }
	103726	+#endif
	103727	+}
	103728	+
	103729	+/*
	103730	+** Insert a record into the %_segments table.
	103731	+*/
	103732	+static int fts3WriteSegment(
	103733	+ Fts3Table p, / Virtual table handle */
	103734	+ sqlite3_int64 iBlock, /* Block id for new block */
	103735	+ char z, / Pointer to buffer containing block data */
	103736	+ int n /* Size of buffer z in bytes */
	103737	+){
	103738	+ sqlite3_stmt *pStmt;
	103739	+ int rc = fts3SqlStmt(p, SQL_INSERT_SEGMENTS, &pStmt, 0);
	103740	+ if( rc==SQLITE_OK ){
	103741	+ sqlite3_bind_int64(pStmt, 1, iBlock);
	103742	+ rc = sqlite3_bind_blob(pStmt, 2, z, n, SQLITE_STATIC);
	103743	+ if( rc==SQLITE_OK ){
	103744	+ sqlite3_step(pStmt);
	103745	+ rc = sqlite3_reset(pStmt);
	103746	+ }
	103747	+ }
	103748	+ return rc;
	103749	+}
	103750	+
	103751	+/*
	103752	+** Insert a record into the %_segdir table.
	103753	+*/
	103754	+static int fts3WriteSegdir(
	103755	+ Fts3Table p, / Virtual table handle */
	103756	+ int iLevel, /* Value for "level" field */
	103757	+ int iIdx, /* Value for "idx" field */
	103758	+ sqlite3_int64 iStartBlock, /* Value for "start_block" field */
	103759	+ sqlite3_int64 iLeafEndBlock, /* Value for "leaves_end_block" field */
	103760	+ sqlite3_int64 iEndBlock, /* Value for "end_block" field */
	103761	+ char zRoot, / Blob value for "root" field */
	103762	+ int nRoot /* Number of bytes in buffer zRoot */
	103763	+){
	103764	+ sqlite3_stmt *pStmt;
	103765	+ int rc = fts3SqlStmt(p, SQL_INSERT_SEGDIR, &pStmt, 0);
	103766	+ if( rc==SQLITE_OK ){
	103767	+ sqlite3_bind_int(pStmt, 1, iLevel);
	103768	+ sqlite3_bind_int(pStmt, 2, iIdx);
	103769	+ sqlite3_bind_int64(pStmt, 3, iStartBlock);
	103770	+ sqlite3_bind_int64(pStmt, 4, iLeafEndBlock);
	103771	+ sqlite3_bind_int64(pStmt, 5, iEndBlock);
	103772	+ rc = sqlite3_bind_blob(pStmt, 6, zRoot, nRoot, SQLITE_STATIC);
	103773	+ if( rc==SQLITE_OK ){
	103774	+ sqlite3_step(pStmt);
	103775	+ rc = sqlite3_reset(pStmt);
	103776	+ }
	103777	+ }
	103778	+ return rc;
	103779	+}
	103780	+
	103781	+/*
	103782	+** Return the size of the common prefix (if any) shared by zPrev and
	103783	+** zNext, in bytes. For example,
	103784	+**
	103785	+** fts3PrefixCompress("abc", 3, "abcdef", 6) // returns 3
	103786	+** fts3PrefixCompress("abX", 3, "abcdef", 6) // returns 2
	103787	+** fts3PrefixCompress("abX", 3, "Xbcdef", 6) // returns 0
	103788	+*/
	103789	+static int fts3PrefixCompress(
	103790	+ const char zPrev, / Buffer containing previous term */
	103791	+ int nPrev, /* Size of buffer zPrev in bytes */
	103792	+ const char zNext, / Buffer containing next term */
	103793	+ int nNext /* Size of buffer zNext in bytes */
	103794	+){
	103795	+ int n;
	103796	+ for(n=0; n<nPrev && zPrev[n]==zNext[n]; n++);
	103797	+ return n;
	103798	+}
	103799	+
	103800	+/*
	103801	+** Add term zTerm to the SegmentNode. It is guaranteed that zTerm is larger
	103802	+** (according to memcmp) than the previous term.
	103803	+*/
	103804	+static int fts3NodeAddTerm(
	103805	+ Fts3Table p, / Virtual table handle */
	103806	+ SegmentNode *ppTree, / IN/OUT: SegmentNode handle */
	103807	+ int isCopyTerm, /* True if zTerm/nTerm is transient */
	103808	+ const char zTerm, / Pointer to buffer containing term */
	103809	+ int nTerm /* Size of term in bytes */
	103810	+){
	103811	+ SegmentNode pTree = ppTree;
	103812	+ int rc;
	103813	+ SegmentNode *pNew;
	103814	+
	103815	+ /* First try to append the term to the current node. Return early if
	103816	+ ** this is possible.
	103817	+ */
	103818	+ if( pTree ){
	103819	+ int nData = pTree->nData; /* Current size of node in bytes */
	103820	+ int nReq = nData; /* Required space after adding zTerm */
	103821	+ int nPrefix; /* Number of bytes of prefix compression */
	103822	+ int nSuffix; /* Suffix length */
	103823	+
	103824	+ nPrefix = fts3PrefixCompress(pTree->zTerm, pTree->nTerm, zTerm, nTerm);
	103825	+ nSuffix = nTerm-nPrefix;
	103826	+
	103827	+ nReq += sqlite3Fts3VarintLen(nPrefix)+sqlite3Fts3VarintLen(nSuffix)+nSuffix;
	103828	+ if( nReq<=INTERIOR_MAX \|\| !pTree->zTerm ){
	103829	+
	103830	+ if( nReq>INTERIOR_MAX ){
	103831	+ /* An unusual case: this is the first term to be added to the node
	103832	+ ** and the static node buffer (INTERIOR_MAX bytes) is not large
	103833	+ ** enough. Use a separately malloced buffer instead This wastes
	103834	+ ** INTERIOR_MAX bytes, but since this scenario only comes about when
	103835	+ ** the database contain two terms that share a prefix of almost 2KB,
	103836	+ ** this is not expected to be a serious problem.
	103837	+ */
	103838	+ assert( pTree->aData==(char *)&pTree[1] );
	103839	+ pTree->aData = (char *)sqlite3_malloc(nReq);
	103840	+ if( !pTree->aData ){
	103841	+ return SQLITE_NOMEM;
	103842	+ }
	103843	+ }
	103844	+
	103845	+ if( pTree->zTerm ){
	103846	+ /* There is no prefix-length field for first term in a node */
	103847	+ nData += sqlite3Fts3PutVarint(&pTree->aData[nData], nPrefix);
	103848	+ }
	103849	+
	103850	+ nData += sqlite3Fts3PutVarint(&pTree->aData[nData], nSuffix);
	103851	+ memcpy(&pTree->aData[nData], &zTerm[nPrefix], nSuffix);
	103852	+ pTree->nData = nData + nSuffix;
	103853	+ pTree->nEntry++;
	103854	+
	103855	+ if( isCopyTerm ){
	103856	+ if( pTree->nMalloc<nTerm ){
	103857	+ char zNew = sqlite3_realloc(pTree->zMalloc, nTerm2);
	103858	+ if( !zNew ){
	103859	+ return SQLITE_NOMEM;
	103860	+ }
	103861	+ pTree->nMalloc = nTerm*2;
	103862	+ pTree->zMalloc = zNew;
	103863	+ }
	103864	+ pTree->zTerm = pTree->zMalloc;
	103865	+ memcpy(pTree->zTerm, zTerm, nTerm);
	103866	+ pTree->nTerm = nTerm;
	103867	+ }else{
	103868	+ pTree->zTerm = (char *)zTerm;
	103869	+ pTree->nTerm = nTerm;
	103870	+ }
	103871	+ return SQLITE_OK;
	103872	+ }
	103873	+ }
	103874	+
	103875	+ /* If control flows to here, it was not possible to append zTerm to the
	103876	+ ** current node. Create a new node (a right-sibling of the current node).
	103877	+ ** If this is the first node in the tree, the term is added to it.
	103878	+ **
	103879	+ ** Otherwise, the term is not added to the new node, it is left empty for
	103880	+ ** now. Instead, the term is inserted into the parent of pTree. If pTree
	103881	+ ** has no parent, one is created here.
	103882	+ */
	103883	+ pNew = (SegmentNode *)sqlite3_malloc(sizeof(SegmentNode) + INTERIOR_MAX);
	103884	+ if( !pNew ){
	103885	+ return SQLITE_NOMEM;
	103886	+ }
	103887	+ memset(pNew, 0, sizeof(SegmentNode));
	103888	+ pNew->nData = 1 + FTS3_VARINT_MAX;
	103889	+ pNew->aData = (char *)&pNew[1];
	103890	+
	103891	+ if( pTree ){
	103892	+ SegmentNode *pParent = pTree->pParent;
	103893	+ rc = fts3NodeAddTerm(p, &pParent, isCopyTerm, zTerm, nTerm);
	103894	+ if( pTree->pParent==0 ){
	103895	+ pTree->pParent = pParent;
	103896	+ }
	103897	+ pTree->pRight = pNew;
	103898	+ pNew->pLeftmost = pTree->pLeftmost;
	103899	+ pNew->pParent = pParent;
	103900	+ pNew->zMalloc = pTree->zMalloc;
	103901	+ pNew->nMalloc = pTree->nMalloc;
	103902	+ pTree->zMalloc = 0;
	103903	+ }else{
	103904	+ pNew->pLeftmost = pNew;
	103905	+ rc = fts3NodeAddTerm(p, &pNew, isCopyTerm, zTerm, nTerm);
	103906	+ }
	103907	+
	103908	+ *ppTree = pNew;
	103909	+ return rc;
	103910	+}
	103911	+
	103912	+/*
	103913	+** Helper function for fts3NodeWrite().
	103914	+*/
	103915	+static int fts3TreeFinishNode(
	103916	+ SegmentNode *pTree,
	103917	+ int iHeight,
	103918	+ sqlite3_int64 iLeftChild
	103919	+){
	103920	+ int nStart;
	103921	+ assert( iHeight>=1 && iHeight<128 );
	103922	+ nStart = FTS3_VARINT_MAX - sqlite3Fts3VarintLen(iLeftChild);
	103923	+ pTree->aData[nStart] = (char)iHeight;
	103924	+ sqlite3Fts3PutVarint(&pTree->aData[nStart+1], iLeftChild);
	103925	+ return nStart;
	103926	+}
	103927	+
	103928	+/*
	103929	+** Write the buffer for the segment node pTree and all of its peers to the
	103930	+** database. Then call this function recursively to write the parent of
	103931	+** pTree and its peers to the database.
	103932	+**
	103933	+** Except, if pTree is a root node, do not write it to the database. Instead,
	103934	+** set output variables paRoot and pnRoot to contain the root node.
	103935	+**
	103936	+** If successful, SQLITE_OK is returned and output variable *piLast is
	103937	+** set to the largest blockid written to the database (or zero if no
	103938	+** blocks were written to the db). Otherwise, an SQLite error code is
	103939	+** returned.
	103940	+*/
	103941	+static int fts3NodeWrite(
	103942	+ Fts3Table p, / Virtual table handle */
	103943	+ SegmentNode pTree, / SegmentNode handle */
	103944	+ int iHeight, /* Height of this node in tree */
	103945	+ sqlite3_int64 iLeaf, /* Block id of first leaf node */
	103946	+ sqlite3_int64 iFree, /* Block id of next free slot in %_segments */
	103947	+ sqlite3_int64 piLast, / OUT: Block id of last entry written */
	103948	+ char *paRoot, / OUT: Data for root node */
	103949	+ int pnRoot / OUT: Size of root node in bytes */
	103950	+){
	103951	+ int rc = SQLITE_OK;
	103952	+
	103953	+ if( !pTree->pParent ){
	103954	+ /* Root node of the tree. */
	103955	+ int nStart = fts3TreeFinishNode(pTree, iHeight, iLeaf);
	103956	+ *piLast = iFree-1;
	103957	+ *pnRoot = pTree->nData - nStart;
	103958	+ *paRoot = &pTree->aData[nStart];
	103959	+ }else{
	103960	+ SegmentNode *pIter;
	103961	+ sqlite3_int64 iNextFree = iFree;
	103962	+ sqlite3_int64 iNextLeaf = iLeaf;
	103963	+ for(pIter=pTree->pLeftmost; pIter && rc==SQLITE_OK; pIter=pIter->pRight){
	103964	+ int nStart = fts3TreeFinishNode(pIter, iHeight, iNextLeaf);
	103965	+ int nWrite = pIter->nData - nStart;
	103966	+
	103967	+ rc = fts3WriteSegment(p, iNextFree, &pIter->aData[nStart], nWrite);
	103968	+ iNextFree++;
	103969	+ iNextLeaf += (pIter->nEntry+1);
	103970	+ }
	103971	+ if( rc==SQLITE_OK ){
	103972	+ assert( iNextLeaf==iFree );
	103973	+ rc = fts3NodeWrite(
	103974	+ p, pTree->pParent, iHeight+1, iFree, iNextFree, piLast, paRoot, pnRoot
	103975	+ );
	103976	+ }
	103977	+ }
	103978	+
	103979	+ return rc;
	103980	+}
	103981	+
	103982	+/*
	103983	+** Free all memory allocations associated with the tree pTree.
	103984	+*/
	103985	+static void fts3NodeFree(SegmentNode *pTree){
	103986	+ if( pTree ){
	103987	+ SegmentNode *p = pTree->pLeftmost;
	103988	+ fts3NodeFree(p->pParent);
	103989	+ while( p ){
	103990	+ SegmentNode *pRight = p->pRight;
	103991	+ if( p->aData!=(char *)&p[1] ){
	103992	+ sqlite3_free(p->aData);
	103993	+ }
	103994	+ assert( pRight==0 \|\| p->zMalloc==0 );
	103995	+ sqlite3_free(p->zMalloc);
	103996	+ sqlite3_free(p);
	103997	+ p = pRight;
	103998	+ }
	103999	+ }
	104000	+}
	104001	+
	104002	+/*
	104003	+** Add a term to the segment being constructed by the SegmentWriter object
	104004	+** ppWriter. When adding the first term to a segment, ppWriter should
	104005	+** be passed NULL. This function will allocate a new SegmentWriter object
	104006	+** and return it via the input/output variable *ppWriter in this case.
	104007	+**
	104008	+** If successful, SQLITE_OK is returned. Otherwise, an SQLite error code.
	104009	+*/
	104010	+static int fts3SegWriterAdd(
	104011	+ Fts3Table p, / Virtual table handle */
	104012	+ SegmentWriter *ppWriter, / IN/OUT: SegmentWriter handle */
	104013	+ int isCopyTerm, /* True if buffer zTerm must be copied */
	104014	+ const char zTerm, / Pointer to buffer containing term */
	104015	+ int nTerm, /* Size of term in bytes */
	104016	+ const char aDoclist, / Pointer to buffer containing doclist */
	104017	+ int nDoclist /* Size of doclist in bytes */
	104018	+){
	104019	+ int nPrefix; /* Size of term prefix in bytes */
	104020	+ int nSuffix; /* Size of term suffix in bytes */
	104021	+ int nReq; /* Number of bytes required on leaf page */
	104022	+ int nData;
	104023	+ SegmentWriter pWriter = ppWriter;
	104024	+
	104025	+ if( !pWriter ){
	104026	+ int rc;
	104027	+ sqlite3_stmt *pStmt;
	104028	+
	104029	+ /* Allocate the SegmentWriter structure */
	104030	+ pWriter = (SegmentWriter *)sqlite3_malloc(sizeof(SegmentWriter));
	104031	+ if( !pWriter ) return SQLITE_NOMEM;
	104032	+ memset(pWriter, 0, sizeof(SegmentWriter));
	104033	+ *ppWriter = pWriter;
	104034	+
	104035	+ /* Allocate a buffer in which to accumulate data */
	104036	+ pWriter->aData = (char *)sqlite3_malloc(LEAF_MAX);
	104037	+ if( !pWriter->aData ) return SQLITE_NOMEM;
	104038	+ pWriter->nSize = LEAF_MAX;
	104039	+
	104040	+ /* Find the next free blockid in the %_segments table */
	104041	+ rc = fts3SqlStmt(p, SQL_NEXT_SEGMENTS_ID, &pStmt, 0);
	104042	+ if( rc!=SQLITE_OK ) return rc;
	104043	+ if( SQLITE_ROW==sqlite3_step(pStmt) ){
	104044	+ pWriter->iFree = sqlite3_column_int64(pStmt, 0);
	104045	+ pWriter->iFirst = pWriter->iFree;
	104046	+ }
	104047	+ rc = sqlite3_reset(pStmt);
	104048	+ if( rc!=SQLITE_OK ) return rc;
	104049	+ }
	104050	+ nData = pWriter->nData;
	104051	+
	104052	+ nPrefix = fts3PrefixCompress(pWriter->zTerm, pWriter->nTerm, zTerm, nTerm);
	104053	+ nSuffix = nTerm-nPrefix;
	104054	+
	104055	+ /* Figure out how many bytes are required by this new entry */
	104056	+ nReq = sqlite3Fts3VarintLen(nPrefix) + /* varint containing prefix size */
	104057	+ sqlite3Fts3VarintLen(nSuffix) + /* varint containing suffix size */
	104058	+ nSuffix + /* Term suffix */
	104059	+ sqlite3Fts3VarintLen(nDoclist) + /* Size of doclist */
	104060	+ nDoclist; /* Doclist data */
	104061	+
	104062	+ if( nData>0 && nData+nReq>LEAF_MAX ){
	104063	+ int rc;
	104064	+
	104065	+ /* The current leaf node is full. Write it out to the database. */
	104066	+ rc = fts3WriteSegment(p, pWriter->iFree++, pWriter->aData, nData);
	104067	+ if( rc!=SQLITE_OK ) return rc;
	104068	+
	104069	+ /* Add the current term to the interior node tree. The term added to
	104070	+ ** the interior tree must:
	104071	+ **
	104072	+ ** a) be greater than the largest term on the leaf node just written
	104073	+ ** to the database (still available in pWriter->zTerm), and
	104074	+ **
	104075	+ ** b) be less than or equal to the term about to be added to the new
	104076	+ ** leaf node (zTerm/nTerm).
	104077	+ **
	104078	+ ** In other words, it must be the prefix of zTerm 1 byte longer than
	104079	+ ** the common prefix (if any) of zTerm and pWriter->zTerm.
	104080	+ */
	104081	+ assert( nPrefix<nTerm );
	104082	+ rc = fts3NodeAddTerm(p, &pWriter->pTree, isCopyTerm, zTerm, nPrefix+1);
	104083	+ if( rc!=SQLITE_OK ) return rc;
	104084	+
	104085	+ nData = 0;
	104086	+ pWriter->nTerm = 0;
	104087	+
	104088	+ nPrefix = 0;
	104089	+ nSuffix = nTerm;
	104090	+ nReq = 1 + /* varint containing prefix size */
	104091	+ sqlite3Fts3VarintLen(nTerm) + /* varint containing suffix size */
	104092	+ nTerm + /* Term suffix */
	104093	+ sqlite3Fts3VarintLen(nDoclist) + /* Size of doclist */
	104094	+ nDoclist; /* Doclist data */
	104095	+ }
	104096	+
	104097	+ /* If the buffer currently allocated is too small for this entry, realloc
	104098	+ ** the buffer to make it large enough.
	104099	+ */
	104100	+ if( nReq>pWriter->nSize ){
	104101	+ char *aNew = sqlite3_realloc(pWriter->aData, nReq);
	104102	+ if( !aNew ) return SQLITE_NOMEM;
	104103	+ pWriter->aData = aNew;
	104104	+ pWriter->nSize = nReq;
	104105	+ }
	104106	+ assert( nData+nReq<=pWriter->nSize );
	104107	+
	104108	+ /* Append the prefix-compressed term and doclist to the buffer. */
	104109	+ nData += sqlite3Fts3PutVarint(&pWriter->aData[nData], nPrefix);
	104110	+ nData += sqlite3Fts3PutVarint(&pWriter->aData[nData], nSuffix);
	104111	+ memcpy(&pWriter->aData[nData], &zTerm[nPrefix], nSuffix);
	104112	+ nData += nSuffix;
	104113	+ nData += sqlite3Fts3PutVarint(&pWriter->aData[nData], nDoclist);
	104114	+ memcpy(&pWriter->aData[nData], aDoclist, nDoclist);
	104115	+ pWriter->nData = nData + nDoclist;
	104116	+
	104117	+ /* Save the current term so that it can be used to prefix-compress the next.
	104118	+ ** If the isCopyTerm parameter is true, then the buffer pointed to by
	104119	+ ** zTerm is transient, so take a copy of the term data. Otherwise, just
	104120	+ ** store a copy of the pointer.
	104121	+ */
	104122	+ if( isCopyTerm ){
	104123	+ if( nTerm>pWriter->nMalloc ){
	104124	+ char zNew = sqlite3_realloc(pWriter->zMalloc, nTerm2);
	104125	+ if( !zNew ){
	104126	+ return SQLITE_NOMEM;
	104127	+ }
	104128	+ pWriter->nMalloc = nTerm*2;
	104129	+ pWriter->zMalloc = zNew;
	104130	+ pWriter->zTerm = zNew;
	104131	+ }
	104132	+ assert( pWriter->zTerm==pWriter->zMalloc );
	104133	+ memcpy(pWriter->zTerm, zTerm, nTerm);
	104134	+ }else{
	104135	+ pWriter->zTerm = (char *)zTerm;
	104136	+ }
	104137	+ pWriter->nTerm = nTerm;
	104138	+
	104139	+ return SQLITE_OK;
	104140	+}
	104141	+
	104142	+/*
	104143	+** Flush all data associated with the SegmentWriter object pWriter to the
	104144	+** database. This function must be called after all terms have been added
	104145	+** to the segment using fts3SegWriterAdd(). If successful, SQLITE_OK is
	104146	+** returned. Otherwise, an SQLite error code.
	104147	+*/
	104148	+static int fts3SegWriterFlush(
	104149	+ Fts3Table p, / Virtual table handle */
	104150	+ SegmentWriter pWriter, / SegmentWriter to flush to the db */
	104151	+ int iLevel, /* Value for 'level' column of %_segdir */
	104152	+ int iIdx /* Value for 'idx' column of %_segdir */
	104153	+){
	104154	+ int rc; /* Return code */
	104155	+ if( pWriter->pTree ){
	104156	+ sqlite3_int64 iLast; /* Largest block id written to database */
	104157	+ sqlite3_int64 iLastLeaf; /* Largest leaf block id written to db */
	104158	+ char zRoot; / Pointer to buffer containing root node */
	104159	+ int nRoot; /* Size of buffer zRoot */
	104160	+
	104161	+ iLastLeaf = pWriter->iFree;
	104162	+ rc = fts3WriteSegment(p, pWriter->iFree++, pWriter->aData, pWriter->nData);
	104163	+ if( rc==SQLITE_OK ){
	104164	+ rc = fts3NodeWrite(p, pWriter->pTree, 1,
	104165	+ pWriter->iFirst, pWriter->iFree, &iLast, &zRoot, &nRoot);
	104166	+ }
	104167	+ if( rc==SQLITE_OK ){
	104168	+ rc = fts3WriteSegdir(
	104169	+ p, iLevel, iIdx, pWriter->iFirst, iLastLeaf, iLast, zRoot, nRoot);
	104170	+ }
	104171	+ }else{
	104172	+ /* The entire tree fits on the root node. Write it to the segdir table. */
	104173	+ rc = fts3WriteSegdir(
	104174	+ p, iLevel, iIdx, 0, 0, 0, pWriter->aData, pWriter->nData);
	104175	+ }
	104176	+ return rc;
	104177	+}
	104178	+
	104179	+/*
	104180	+** Release all memory held by the SegmentWriter object passed as the
	104181	+** first argument.
	104182	+*/
	104183	+static void fts3SegWriterFree(SegmentWriter *pWriter){
	104184	+ if( pWriter ){
	104185	+ sqlite3_free(pWriter->aData);
	104186	+ sqlite3_free(pWriter->zMalloc);
	104187	+ fts3NodeFree(pWriter->pTree);
	104188	+ sqlite3_free(pWriter);
	104189	+ }
	104190	+}
	104191	+
	104192	+/*
	104193	+** The first value in the apVal[] array is assumed to contain an integer.
	104194	+** This function tests if there exist any documents with docid values that
	104195	+** are different from that integer. i.e. if deleting the document with docid
	104196	+** apVal[0] would mean the FTS3 table were empty.
	104197	+**
	104198	+** If successful, *pisEmpty is set to true if the table is empty except for
	104199	+** document apVal[0], or false otherwise, and SQLITE_OK is returned. If an
	104200	+** error occurs, an SQLite error code is returned.
	104201	+*/
	104202	+static int fts3IsEmpty(Fts3Table p, sqlite3_value apVal, int pisEmpty){
	104203	+ sqlite3_stmt *pStmt;
	104204	+ int rc;
	104205	+ rc = fts3SqlStmt(p, SQL_IS_EMPTY, &pStmt, apVal);
	104206	+ if( rc==SQLITE_OK ){
	104207	+ if( SQLITE_ROW==sqlite3_step(pStmt) ){
	104208	+ *pisEmpty = sqlite3_column_int(pStmt, 0);
	104209	+ }
	104210	+ rc = sqlite3_reset(pStmt);
	104211	+ }
	104212	+ return rc;
	104213	+}
	104214	+
	104215	+/*
	104216	+** Set *pnSegment to the number of segments of level iLevel in the database.
	104217	+**
	104218	+** Return SQLITE_OK if successful, or an SQLite error code if not.
	104219	+*/
	104220	+static int fts3SegmentCount(Fts3Table p, int iLevel, int pnSegment){
	104221	+ sqlite3_stmt *pStmt;
	104222	+ int rc;
	104223	+
	104224	+ assert( iLevel>=0 );
	104225	+ rc = fts3SqlStmt(p, SQL_SELECT_LEVEL_COUNT, &pStmt, 0);
	104226	+ if( rc!=SQLITE_OK ) return rc;
	104227	+ sqlite3_bind_int(pStmt, 1, iLevel);
	104228	+ if( SQLITE_ROW==sqlite3_step(pStmt) ){
	104229	+ *pnSegment = sqlite3_column_int(pStmt, 0);
	104230	+ }
	104231	+ return sqlite3_reset(pStmt);
	104232	+}
	104233	+
	104234	+/*
	104235	+** Set *pnSegment to the total number of segments in the database. Set
	104236	+** *pnMax to the largest segment level in the database (segment levels
	104237	+** are stored in the 'level' column of the %_segdir table).
	104238	+**
	104239	+** Return SQLITE_OK if successful, or an SQLite error code if not.
	104240	+*/
	104241	+static int fts3SegmentCountMax(Fts3Table p, int pnSegment, int *pnMax){
	104242	+ sqlite3_stmt *pStmt;
	104243	+ int rc;
	104244	+
	104245	+ rc = fts3SqlStmt(p, SQL_SELECT_SEGDIR_COUNT_MAX, &pStmt, 0);
	104246	+ if( rc!=SQLITE_OK ) return rc;
	104247	+ if( SQLITE_ROW==sqlite3_step(pStmt) ){
	104248	+ *pnSegment = sqlite3_column_int(pStmt, 0);
	104249	+ *pnMax = sqlite3_column_int(pStmt, 1);
	104250	+ }
	104251	+ return sqlite3_reset(pStmt);
	104252	+}
	104253	+
	104254	+/*
	104255	+** This function is used after merging multiple segments into a single large
	104256	+** segment to delete the old, now redundant, segment b-trees. Specifically,
	104257	+** it:
	104258	+**
	104259	+** 1) Deletes all %_segments entries for the segments associated with
	104260	+** each of the SegReader objects in the array passed as the third
	104261	+** argument, and
	104262	+**
	104263	+** 2) deletes all %_segdir entries with level iLevel, or all %_segdir
	104264	+** entries regardless of level if (iLevel<0).
	104265	+**
	104266	+** SQLITE_OK is returned if successful, otherwise an SQLite error code.
	104267	+*/
	104268	+static int fts3DeleteSegdir(
	104269	+ Fts3Table p, / Virtual table handle */
	104270	+ int iLevel, /* Level of %_segdir entries to delete */
	104271	+ Fts3SegReader *apSegment, / Array of SegReader objects */
	104272	+ int nReader /* Size of array apSegment */
	104273	+){
	104274	+ int rc; /* Return Code */
	104275	+ int i; /* Iterator variable */
	104276	+ sqlite3_stmt pDelete; / SQL statement to delete rows */
	104277	+
	104278	+ rc = fts3SqlStmt(p, SQL_DELETE_SEGMENTS_RANGE, &pDelete, 0);
	104279	+ for(i=0; rc==SQLITE_OK && i<nReader; i++){
	104280	+ Fts3SegReader *pSegment = apSegment[i];
	104281	+ if( pSegment->iStartBlock ){
	104282	+ sqlite3_bind_int64(pDelete, 1, pSegment->iStartBlock);
	104283	+ sqlite3_bind_int64(pDelete, 2, pSegment->iEndBlock);
	104284	+ sqlite3_step(pDelete);
	104285	+ rc = sqlite3_reset(pDelete);
	104286	+ }
	104287	+ }
	104288	+ if( rc!=SQLITE_OK ){
	104289	+ return rc;
	104290	+ }
	104291	+
	104292	+ if( iLevel>=0 ){
	104293	+ rc = fts3SqlStmt(p, SQL_DELETE_SEGDIR_BY_LEVEL, &pDelete, 0);
	104294	+ if( rc==SQLITE_OK ){
	104295	+ sqlite3_bind_int(pDelete, 1, iLevel);
	104296	+ sqlite3_step(pDelete);
	104297	+ rc = sqlite3_reset(pDelete);
	104298	+ }
	104299	+ }else{
	104300	+ rc = fts3SqlExec(p, SQL_DELETE_ALL_SEGDIR, 0);
	104301	+ }
	104302	+
	104303	+ return rc;
	104304	+}
	104305	+
	104306	+/*
	104307	+** When this function is called, buffer ppList (size pnList bytes) contains
	104308	+** a position list that may (or may not) feature multiple columns. This
	104309	+** function adjusts the pointer ppList and the length pnList so that they
	104310	+** identify the subset of the position list that corresponds to column iCol.
	104311	+**
	104312	+** If there are no entries in the input position list for column iCol, then
	104313	+** *pnList is set to zero before returning.
	104314	+*/
	104315	+static void fts3ColumnFilter(
	104316	+ int iCol, /* Column to filter on */
	104317	+ char *ppList, / IN/OUT: Pointer to position list */
	104318	+ int pnList / IN/OUT: Size of buffer ppList in bytes /
	104319	+){
	104320	+ char pList = ppList;
	104321	+ int nList = *pnList;
	104322	+ char *pEnd = &pList[nList];
	104323	+ int iCurrent = 0;
	104324	+ char *p = pList;
	104325	+
	104326	+ assert( iCol>=0 );
	104327	+ while( 1 ){
	104328	+ char c = 0;
	104329	+ while( p<pEnd && (c \| p)&0xFE ) c = p++ & 0x80;
	104330	+
	104331	+ if( iCol==iCurrent ){
	104332	+ nList = (p - pList);
	104333	+ break;
	104334	+ }
	104335	+
	104336	+ nList -= (p - pList);
	104337	+ pList = p;
	104338	+ if( nList==0 ){
	104339	+ break;
	104340	+ }
	104341	+ p = &pList[1];
	104342	+ p += sqlite3Fts3GetVarint32(p, &iCurrent);
	104343	+ }
	104344	+
	104345	+ *ppList = pList;
	104346	+ *pnList = nList;
	104347	+}
	104348	+
	104349	+/*
	104350	+** sqlite3Fts3SegReaderIterate() callback used when merging multiple
	104351	+** segments to create a single, larger segment.
	104352	+*/
	104353	+static int fts3MergeCallback(
	104354	+ Fts3Table *p,
	104355	+ void *pContext,
	104356	+ char *zTerm,
	104357	+ int nTerm,
	104358	+ char *aDoclist,
	104359	+ int nDoclist
	104360	+){
	104361	+ SegmentWriter ppW = (SegmentWriter )pContext;
	104362	+ return fts3SegWriterAdd(p, ppW, 1, zTerm, nTerm, aDoclist, nDoclist);
	104363	+}
	104364	+
	104365	+/*
	104366	+** This function is used to iterate through a contiguous set of terms
	104367	+** stored in the full-text index. It merges data contained in one or
	104368	+** more segments to support this.
	104369	+**
	104370	+** The second argument is passed an array of pointers to SegReader objects
	104371	+** allocated with sqlite3Fts3SegReaderNew(). This function merges the range
	104372	+** of terms selected by each SegReader. If a single term is present in
	104373	+** more than one segment, the associated doclists are merged. For each
	104374	+** term and (possibly merged) doclist in the merged range, the callback
	104375	+** function xFunc is invoked with its arguments set as follows.
	104376	+**
	104377	+** arg 0: Copy of 'p' parameter passed to this function
	104378	+** arg 1: Copy of 'pContext' parameter passed to this function
	104379	+** arg 2: Pointer to buffer containing term
	104380	+** arg 3: Size of arg 2 buffer in bytes
	104381	+** arg 4: Pointer to buffer containing doclist
	104382	+** arg 5: Size of arg 2 buffer in bytes
	104383	+**
	104384	+** The 4th argument to this function is a pointer to a structure of type
	104385	+** Fts3SegFilter, defined in fts3Int.h. The contents of this structure
	104386	+** further restrict the range of terms that callbacks are made for and
	104387	+** modify the behaviour of this function. See comments above structure
	104388	+** definition for details.
	104389	+*/
	104390	+SQLITE_PRIVATE int sqlite3Fts3SegReaderIterate(
	104391	+ Fts3Table p, / Virtual table handle */
	104392	+ Fts3SegReader *apSegment, / Array of Fts3SegReader objects */
	104393	+ int nSegment, /* Size of apSegment array */
	104394	+ Fts3SegFilter pFilter, / Restrictions on range of iteration */
	104395	+ int (xFunc)(Fts3Table , void , char , int, char , int), / Callback */
	104396	+ void pContext / Callback context (2nd argument) */
	104397	+){
	104398	+ int i; /* Iterator variable */
	104399	+ char aBuffer = 0; / Buffer to merge doclists in */
	104400	+ int nAlloc = 0; /* Allocated size of aBuffer buffer */
	104401	+ int rc = SQLITE_OK; /* Return code */
	104402	+
	104403	+ int isIgnoreEmpty = (pFilter->flags & FTS3_SEGMENT_IGNORE_EMPTY);
	104404	+ int isRequirePos = (pFilter->flags & FTS3_SEGMENT_REQUIRE_POS);
	104405	+ int isColFilter = (pFilter->flags & FTS3_SEGMENT_COLUMN_FILTER);
	104406	+ int isPrefix = (pFilter->flags & FTS3_SEGMENT_PREFIX);
	104407	+
	104408	+ /* If the Fts3SegFilter defines a specific term (or term prefix) to search
	104409	+ ** for, then advance each segment iterator until it points to a term of
	104410	+ ** equal or greater value than the specified term. This prevents many
	104411	+ ** unnecessary merge/sort operations for the case where single segment
	104412	+ ** b-tree leaf nodes contain more than one term.
	104413	+ */
	104414	+ if( pFilter->zTerm ){
	104415	+ int nTerm = pFilter->nTerm;
	104416	+ const char *zTerm = pFilter->zTerm;
	104417	+ for(i=0; i<nSegment; i++){
	104418	+ Fts3SegReader *pSeg = apSegment[i];
	104419	+ while( fts3SegReaderTermCmp(pSeg, zTerm, nTerm)<0 ){
	104420	+ rc = fts3SegReaderNext(pSeg);
	104421	+ if( rc!=SQLITE_OK ) goto finished;
	104422	+ }
	104423	+ }
	104424	+ }
	104425	+
	104426	+ fts3SegReaderSort(apSegment, nSegment, nSegment, fts3SegReaderCmp);
	104427	+ while( apSegment[0]->aNode ){
	104428	+ int nTerm = apSegment[0]->nTerm;
	104429	+ char *zTerm = apSegment[0]->zTerm;
	104430	+ int nMerge = 1;
	104431	+
	104432	+ /* If this is a prefix-search, and if the term that apSegment[0] points
	104433	+ ** to does not share a suffix with pFilter->zTerm/nTerm, then all
	104434	+ ** required callbacks have been made. In this case exit early.
	104435	+ **
	104436	+ ** Similarly, if this is a search for an exact match, and the first term
	104437	+ ** of segment apSegment[0] is not a match, exit early.
	104438	+ */
	104439	+ if( pFilter->zTerm ){
	104440	+ if( nTerm<pFilter->nTerm
	104441	+ \|\| (!isPrefix && nTerm>pFilter->nTerm)
	104442	+ \|\| memcmp(zTerm, pFilter->zTerm, pFilter->nTerm)
	104443	+ ){
	104444	+ goto finished;
	104445	+ }
	104446	+ }
	104447	+
	104448	+ while( nMerge<nSegment
	104449	+ && apSegment[nMerge]->aNode
	104450	+ && apSegment[nMerge]->nTerm==nTerm
	104451	+ && 0==memcmp(zTerm, apSegment[nMerge]->zTerm, nTerm)
	104452	+ ){
	104453	+ nMerge++;
	104454	+ }
	104455	+
	104456	+ if( nMerge==1 && !isIgnoreEmpty && !isColFilter && isRequirePos ){
	104457	+ Fts3SegReader *p0 = apSegment[0];
	104458	+ rc = xFunc(p, pContext, zTerm, nTerm, p0->aDoclist, p0->nDoclist);
	104459	+ if( rc!=SQLITE_OK ) goto finished;
	104460	+ }else{
	104461	+ int nDoclist = 0; /* Size of doclist */
	104462	+ sqlite3_int64 iPrev = 0; /* Previous docid stored in doclist */
	104463	+
	104464	+ /* The current term of the first nMerge entries in the array
	104465	+ ** of Fts3SegReader objects is the same. The doclists must be merged
	104466	+ ** and a single term added to the new segment.
	104467	+ */
	104468	+ for(i=0; i<nMerge; i++){
	104469	+ fts3SegReaderFirstDocid(apSegment[i]);
	104470	+ }
	104471	+ fts3SegReaderSort(apSegment, nMerge, nMerge, fts3SegReaderDoclistCmp);
	104472	+ while( apSegment[0]->pOffsetList ){
	104473	+ int j; /* Number of segments that share a docid */
	104474	+ char *pList;
	104475	+ int nList;
	104476	+ int nByte;
	104477	+ sqlite3_int64 iDocid = apSegment[0]->iDocid;
	104478	+ fts3SegReaderNextDocid(apSegment[0], &pList, &nList);
	104479	+ j = 1;
	104480	+ while( j<nMerge
	104481	+ && apSegment[j]->pOffsetList
	104482	+ && apSegment[j]->iDocid==iDocid
	104483	+ ){
	104484	+ fts3SegReaderNextDocid(apSegment[j], 0, 0);
	104485	+ j++;
	104486	+ }
	104487	+
	104488	+ if( isColFilter ){
	104489	+ fts3ColumnFilter(pFilter->iCol, &pList, &nList);
	104490	+ }
	104491	+
	104492	+ if( !isIgnoreEmpty \|\| nList>0 ){
	104493	+ nByte = sqlite3Fts3VarintLen(iDocid-iPrev) + (isRequirePos?nList+1:0);
	104494	+ if( nDoclist+nByte>nAlloc ){
	104495	+ char *aNew;
	104496	+ nAlloc = nDoclist+nByte*2;
	104497	+ aNew = sqlite3_realloc(aBuffer, nAlloc);
	104498	+ if( !aNew ){
	104499	+ rc = SQLITE_NOMEM;
	104500	+ goto finished;
	104501	+ }
	104502	+ aBuffer = aNew;
	104503	+ }
	104504	+ nDoclist += sqlite3Fts3PutVarint(&aBuffer[nDoclist], iDocid-iPrev);
	104505	+ iPrev = iDocid;
	104506	+ if( isRequirePos ){
	104507	+ memcpy(&aBuffer[nDoclist], pList, nList);
	104508	+ nDoclist += nList;
	104509	+ aBuffer[nDoclist++] = '\0';
	104510	+ }
	104511	+ }
	104512	+
	104513	+ fts3SegReaderSort(apSegment, nMerge, j, fts3SegReaderDoclistCmp);
	104514	+ }
	104515	+
	104516	+ if( nDoclist>0 ){
	104517	+ rc = xFunc(p, pContext, zTerm, nTerm, aBuffer, nDoclist);
	104518	+ if( rc!=SQLITE_OK ) goto finished;
	104519	+ }
	104520	+ }
	104521	+
	104522	+ /* If there is a term specified to filter on, and this is not a prefix
	104523	+ ** search, return now. The callback that corresponds to the required
	104524	+ ** term (if such a term exists in the index) has already been made.
	104525	+ */
	104526	+ if( pFilter->zTerm && !isPrefix ){
	104527	+ goto finished;
	104528	+ }
	104529	+
	104530	+ for(i=0; i<nMerge; i++){
	104531	+ rc = fts3SegReaderNext(apSegment[i]);
	104532	+ if( rc!=SQLITE_OK ) goto finished;
	104533	+ }
	104534	+ fts3SegReaderSort(apSegment, nSegment, nMerge, fts3SegReaderCmp);
	104535	+ }
	104536	+
	104537	+ finished:
	104538	+ sqlite3_free(aBuffer);
	104539	+ return rc;
	104540	+}
	104541	+
	104542	+/*
	104543	+** Merge all level iLevel segments in the database into a single
	104544	+** iLevel+1 segment. Or, if iLevel<0, merge all segments into a
	104545	+** single segment with a level equal to the numerically largest level
	104546	+** currently present in the database.
	104547	+**
	104548	+** If this function is called with iLevel<0, but there is only one
	104549	+** segment in the database, SQLITE_DONE is returned immediately.
	104550	+** Otherwise, if successful, SQLITE_OK is returned. If an error occurs,
	104551	+** an SQLite error code is returned.
	104552	+*/
	104553	+static int fts3SegmentMerge(Fts3Table *p, int iLevel){
	104554	+ int i; /* Iterator variable */
	104555	+ int rc; /* Return code */
	104556	+ int iIdx; /* Index of new segment */
	104557	+ int iNewLevel; /* Level to create new segment at */
	104558	+ sqlite3_stmt *pStmt;
	104559	+ SegmentWriter *pWriter = 0;
	104560	+ int nSegment = 0; /* Number of segments being merged */
	104561	+ Fts3SegReader *apSegment = 0; / Array of Segment iterators */
	104562	+ Fts3SegFilter filter; /* Segment term filter condition */
	104563	+
	104564	+ if( iLevel<0 ){
	104565	+ /* This call is to merge all segments in the database to a single
	104566	+ ** segment. The level of the new segment is equal to the the numerically
	104567	+ ** greatest segment level currently present in the database. The index
	104568	+ ** of the new segment is always 0.
	104569	+ */
	104570	+ iIdx = 0;
	104571	+ rc = fts3SegmentCountMax(p, &nSegment, &iNewLevel);
	104572	+ if( nSegment==1 ){
	104573	+ return SQLITE_DONE;
	104574	+ }
	104575	+ }else{
	104576	+ /* This call is to merge all segments at level iLevel. Find the next
	104577	+ ** available segment index at level iLevel+1. The call to
	104578	+ ** fts3AllocateSegdirIdx() will merge the segments at level iLevel+1 to
	104579	+ ** a single iLevel+2 segment if necessary.
	104580	+ */
	104581	+ iNewLevel = iLevel+1;
	104582	+ rc = fts3AllocateSegdirIdx(p, iNewLevel, &iIdx);
	104583	+ if( rc!=SQLITE_OK ) return rc;
	104584	+ rc = fts3SegmentCount(p, iLevel, &nSegment);
	104585	+ }
	104586	+ if( rc!=SQLITE_OK ) return rc;
	104587	+ assert( nSegment>0 );
	104588	+ assert( iNewLevel>=0 );
	104589	+
	104590	+ /* Allocate space for an array of pointers to segment iterators. */
	104591	+ apSegment = (Fts3SegReader*)sqlite3_malloc(sizeof(Fts3SegReader )*nSegment);
	104592	+ if( !apSegment ){
	104593	+ return SQLITE_NOMEM;
	104594	+ }
	104595	+ memset(apSegment, 0, sizeof(Fts3SegReader )nSegment);
	104596	+
	104597	+ /* Allocate a Fts3SegReader structure for each segment being merged. A
	104598	+ ** Fts3SegReader stores the state data required to iterate through all
	104599	+ ** entries on all leaves of a single segment.
	104600	+ */
	104601	+ assert( SQL_SELECT_LEVEL+1==SQL_SELECT_ALL_LEVEL);
	104602	+ rc = fts3SqlStmt(p, SQL_SELECT_LEVEL+(iLevel<0), &pStmt, 0);
	104603	+ if( rc!=SQLITE_OK ) goto finished;
	104604	+ sqlite3_bind_int(pStmt, 1, iLevel);
	104605	+ for(i=0; SQLITE_ROW==(sqlite3_step(pStmt)); i++){
	104606	+ rc = fts3SegReaderNew(p, pStmt, i, &apSegment[i]);
	104607	+ if( rc!=SQLITE_OK ){
	104608	+ goto finished;
	104609	+ }
	104610	+ }
	104611	+ rc = sqlite3_reset(pStmt);
	104612	+ pStmt = 0;
	104613	+ if( rc!=SQLITE_OK ) goto finished;
	104614	+
	104615	+ memset(&filter, 0, sizeof(Fts3SegFilter));
	104616	+ filter.flags = FTS3_SEGMENT_REQUIRE_POS;
	104617	+ filter.flags \|= (iLevel<0 ? FTS3_SEGMENT_IGNORE_EMPTY : 0);
	104618	+ rc = sqlite3Fts3SegReaderIterate(p, apSegment, nSegment,
	104619	+ &filter, fts3MergeCallback, (void *)&pWriter
	104620	+ );
	104621	+ if( rc!=SQLITE_OK ) goto finished;
	104622	+
	104623	+ rc = fts3DeleteSegdir(p, iLevel, apSegment, nSegment);
	104624	+ if( rc==SQLITE_OK ){
	104625	+ rc = fts3SegWriterFlush(p, pWriter, iNewLevel, iIdx);
	104626	+ }
	104627	+
	104628	+ finished:
	104629	+ fts3SegWriterFree(pWriter);
	104630	+ if( apSegment ){
	104631	+ for(i=0; i<nSegment; i++){
	104632	+ sqlite3Fts3SegReaderFree(p, apSegment[i]);
	104633	+ }
	104634	+ sqlite3_free(apSegment);
	104635	+ }
	104636	+ sqlite3_reset(pStmt);
	104637	+ return rc;
	104638	+}
	104639	+
	104640	+/*
	104641	+** This is a comparison function used as a qsort() callback when sorting
	104642	+** an array of pending terms by term. This occurs as part of flushing
	104643	+** the contents of the pending-terms hash table to the database.
	104644	+*/
	104645	+static int qsortCompare(const void lhs, const void rhs){
	104646	+ char z1 = fts3HashKey((Fts3HashElem **)lhs);
	104647	+ char z2 = fts3HashKey((Fts3HashElem **)rhs);
	104648	+ int n1 = fts3HashKeysize((Fts3HashElem *)lhs);
	104649	+ int n2 = fts3HashKeysize((Fts3HashElem *)rhs);
	104650	+
	104651	+ int n = (n1<n2 ? n1 : n2);
	104652	+ int c = memcmp(z1, z2, n);
	104653	+ if( c==0 ){
	104654	+ c = n1 - n2;
	104655	+ }
	104656	+ return c;
	104657	+}
	104658	+
	104659	+
	104660	+/*
	104661	+** Flush the contents of pendingTerms to a level 0 segment.
	104662	+*/
	104663	+SQLITE_PRIVATE int sqlite3Fts3PendingTermsFlush(Fts3Table *p){
	104664	+ Fts3HashElem *pElem;
	104665	+ int idx, rc, i;
	104666	+ Fts3HashElem *apElem; / Array of pointers to hash elements */
	104667	+ int nElem; /* Number of terms in new segment */
	104668	+ SegmentWriter pWriter = 0; / Used to write the segment */
	104669	+
	104670	+ /* Find the number of terms that will make up the new segment. If there
	104671	+ ** are no terms, return early (do not bother to write an empty segment).
	104672	+ */
	104673	+ nElem = fts3HashCount(&p->pendingTerms);
	104674	+ if( nElem==0 ){
	104675	+ assert( p->nPendingData==0 );
	104676	+ return SQLITE_OK;
	104677	+ }
	104678	+
	104679	+ /* Determine the next index at level 0, merging as necessary. */
	104680	+ rc = fts3AllocateSegdirIdx(p, 0, &idx);
	104681	+ if( rc!=SQLITE_OK ){
	104682	+ return rc;
	104683	+ }
	104684	+
	104685	+ apElem = sqlite3_malloc(nElemsizeof(Fts3HashElem ));
	104686	+ if( !apElem ){
	104687	+ return SQLITE_NOMEM;
	104688	+ }
	104689	+
	104690	+ i = 0;
	104691	+ for(pElem=fts3HashFirst(&p->pendingTerms); pElem; pElem=fts3HashNext(pElem)){
	104692	+ apElem[i++] = pElem;
	104693	+ }
	104694	+ assert( i==nElem );
	104695	+
	104696	+ /* TODO(shess) Should we allow user-defined collation sequences,
	104697	+ ** here? I think we only need that once we support prefix searches.
	104698	+ ** Also, should we be using qsort()?
	104699	+ */
	104700	+ if( nElem>1 ){
	104701	+ qsort(apElem, nElem, sizeof(Fts3HashElem *), qsortCompare);
	104702	+ }
	104703	+
	104704	+
	104705	+ /* Write the segment tree into the database. */
	104706	+ for(i=0; rc==SQLITE_OK && i<nElem; i++){
	104707	+ const char *z = fts3HashKey(apElem[i]);
	104708	+ int n = fts3HashKeysize(apElem[i]);
	104709	+ PendingList *pList = fts3HashData(apElem[i]);
	104710	+ rc = fts3SegWriterAdd(p, &pWriter, 0, z, n, pList->aData, pList->nData+1);
	104711	+ }
	104712	+ if( rc==SQLITE_OK ){
	104713	+ rc = fts3SegWriterFlush(p, pWriter, 0, idx);
	104714	+ }
	104715	+
	104716	+ /* Free all allocated resources before returning */
	104717	+ fts3SegWriterFree(pWriter);
	104718	+ sqlite3_free(apElem);
	104719	+ sqlite3Fts3PendingTermsClear(p);
	104720	+ return rc;
	104721	+}
	104722	+
	104723	+/*
	104724	+** This function does the work for the xUpdate method of FTS3 virtual
	104725	+** tables.
	104726	+*/
	104727	+SQLITE_PRIVATE int sqlite3Fts3UpdateMethod(
	104728	+ sqlite3_vtab pVtab, / FTS3 vtab object */
	104729	+ int nArg, /* Size of argument array */
	104730	+ sqlite3_value *apVal, / Array of arguments */
	104731	+ sqlite_int64 pRowid / OUT: The affected (or effected) rowid */
	104732	+){
	104733	+ Fts3Table p = (Fts3Table )pVtab;
	104734	+ int rc = SQLITE_OK; /* Return Code */
	104735	+ int isRemove = 0; /* True for an UPDATE or DELETE */
	104736	+ sqlite3_int64 iRemove; /* Rowid removed by UPDATE or DELETE */
	104737	+
	104738	+ /* If this is a DELETE or UPDATE operation, remove the old record. */
	104739	+ if( sqlite3_value_type(apVal[0])!=SQLITE_NULL ){
	104740	+ int isEmpty;
	104741	+ rc = fts3IsEmpty(p, apVal, &isEmpty);
	104742	+ if( rc==SQLITE_OK ){
	104743	+ if( isEmpty ){
	104744	+ /* Deleting this row means the whole table is empty. In this case
	104745	+ ** delete the contents of all three tables and throw away any
	104746	+ ** data in the pendingTerms hash table.
	104747	+ */
	104748	+ rc = fts3DeleteAll(p);
	104749	+ }else{
	104750	+ isRemove = 1;
	104751	+ iRemove = sqlite3_value_int64(apVal[0]);
	104752	+ rc = fts3PendingTermsDocid(p, iRemove);
	104753	+ if( rc==SQLITE_OK ){
	104754	+ rc = fts3DeleteTerms(p, apVal);
	104755	+ if( rc==SQLITE_OK ){
	104756	+ rc = fts3SqlExec(p, SQL_DELETE_CONTENT, apVal);
	104757	+ }
	104758	+ }
	104759	+ }
	104760	+ }
	104761	+ }
	104762	+
	104763	+ /* If this is an INSERT or UPDATE operation, insert the new record. */
	104764	+ if( nArg>1 && rc==SQLITE_OK ){
	104765	+ rc = fts3InsertData(p, apVal, pRowid);
	104766	+ if( rc==SQLITE_OK && (!isRemove \|\| *pRowid!=iRemove) ){
	104767	+ rc = fts3PendingTermsDocid(p, *pRowid);
	104768	+ }
	104769	+ if( rc==SQLITE_OK ){
	104770	+ rc = fts3InsertTerms(p, apVal);
	104771	+ }
	104772	+ }
	104773	+
	104774	+ return rc;
	104775	+}
	104776	+
	104777	+/*
	104778	+** Flush any data in the pending-terms hash table to disk. If successful,
	104779	+** merge all segments in the database (including the new segment, if
	104780	+** there was any data to flush) into a single segment.
	104781	+*/
	104782	+SQLITE_PRIVATE int sqlite3Fts3Optimize(Fts3Table *p){
	104783	+ int rc = sqlite3Fts3PendingTermsFlush(p);
	104784	+ if( rc==SQLITE_OK ){
	104785	+ rc = fts3SegmentMerge(p, -1);
	104786	+ }
	104787	+ return rc;
	104788	+}
	104789	+
	104790	+#endif
	104791	+
	104792	+/************ End of fts3_write.c ****************************************/
	104793	+/************ Begin file fts3_snippet.c **********************************/
	104794	+/*
	104795	+** 2009 Oct 23
	104796	+**
	104797	+** The author disclaims copyright to this source code. In place of
	104798	+** a legal notice, here is a blessing:
	104799	+**
	104800	+** May you do good and not evil.
	104801	+** May you find forgiveness for yourself and forgive others.
	104802	+** May you share freely, never taking more than you give.
	104803	+**
	104804	+******************************************************************************
	104805	+*/
	104806	+
	104807	+#if !defined(SQLITE_CORE) \|\| defined(SQLITE_ENABLE_FTS3)
	104808	+
	104809	+
	104810	+typedef struct Snippet Snippet;
	104811	+
	104812	+/*
	104813	+** An instance of the following structure keeps track of generated
	104814	+** matching-word offset information and snippets.
	104815	+*/
	104816	+struct Snippet {
	104817	+ int nMatch; /* Total number of matches */
	104818	+ int nAlloc; /* Space allocated for aMatch[] */
	104819	+ struct snippetMatch { /* One entry for each matching term */
	104820	+ char snStatus; /* Status flag for use while constructing snippets */
	104821	+ short int iCol; /* The column that contains the match */
	104822	+ short int iTerm; /* The index in Query.pTerms[] of the matching term */
	104823	+ int iToken; /* The index of the matching document token */
	104824	+ short int nByte; /* Number of bytes in the term */
	104825	+ int iStart; /* The offset to the first character of the term */
	104826	+ } aMatch; / Points to space obtained from malloc */
	104827	+ char zOffset; / Text rendering of aMatch[] */
	104828	+ int nOffset; /* strlen(zOffset) */
	104829	+ char zSnippet; / Snippet text */
	104830	+ int nSnippet; /* strlen(zSnippet) */
	104831	+};
	104832	+
	104833	+
	104834	+/* It is not safe to call isspace(), tolower(), or isalnum() on
	104835	+** hi-bit-set characters. This is the same solution used in the
	104836	+** tokenizer.
	104837	+*/
	104838	+/* TODO(shess) The snippet-generation code should be using the
	104839	+** tokenizer-generated tokens rather than doing its own local
	104840	+** tokenization.
	104841	+*/
	104842	+/* TODO(shess) Is __isascii() a portable version of (c&0x80)==0? */
	104843	+static int safe_isspace(char c){
	104844	+ return (c&0x80)==0 ? isspace(c) : 0;
	104845	+}
	104846	+static int safe_isalnum(char c){
	104847	+ return (c&0x80)==0 ? isalnum(c) : 0;
	104848	+}
	104849	+
	104850	+/*******************************************************************/
	104851	+/* DataBuffer is used to collect data into a buffer in piecemeal
	104852	+** fashion. It implements the usual distinction between amount of
	104853	+** data currently stored (nData) and buffer capacity (nCapacity).
	104854	+**
	104855	+** dataBufferInit - create a buffer with given initial capacity.
	104856	+** dataBufferReset - forget buffer's data, retaining capacity.
	104857	+** dataBufferSwap - swap contents of two buffers.
	104858	+** dataBufferExpand - expand capacity without adding data.
	104859	+** dataBufferAppend - append data.
	104860	+** dataBufferAppend2 - append two pieces of data at once.
	104861	+** dataBufferReplace - replace buffer's data.
	104862	+*/
	104863	+typedef struct DataBuffer {
	104864	+ char pData; / Pointer to malloc'ed buffer. */
	104865	+ int nCapacity; /* Size of pData buffer. */
	104866	+ int nData; /* End of data loaded into pData. */
	104867	+} DataBuffer;
	104868	+
	104869	+static void dataBufferInit(DataBuffer *pBuffer, int nCapacity){
	104870	+ assert( nCapacity>=0 );
	104871	+ pBuffer->nData = 0;
	104872	+ pBuffer->nCapacity = nCapacity;
	104873	+ pBuffer->pData = nCapacity==0 ? NULL : sqlite3_malloc(nCapacity);
	104874	+}
	104875	+static void dataBufferReset(DataBuffer *pBuffer){
	104876	+ pBuffer->nData = 0;
	104877	+}
	104878	+static void dataBufferExpand(DataBuffer *pBuffer, int nAddCapacity){
	104879	+ assert( nAddCapacity>0 );
	104880	+ /* TODO(shess) Consider expanding more aggressively. Note that the
	104881	+ ** underlying malloc implementation may take care of such things for
	104882	+ ** us already.
	104883	+ */
	104884	+ if( pBuffer->nData+nAddCapacity>pBuffer->nCapacity ){
	104885	+ pBuffer->nCapacity = pBuffer->nData+nAddCapacity;
	104886	+ pBuffer->pData = sqlite3_realloc(pBuffer->pData, pBuffer->nCapacity);
	104887	+ }
	104888	+}
	104889	+static void dataBufferAppend(DataBuffer *pBuffer,
	104890	+ const char *pSource, int nSource){
	104891	+ assert( nSource>0 && pSource!=NULL );
	104892	+ dataBufferExpand(pBuffer, nSource);
	104893	+ memcpy(pBuffer->pData+pBuffer->nData, pSource, nSource);
	104894	+ pBuffer->nData += nSource;
	104895	+}
	104896	+static void dataBufferAppend2(DataBuffer *pBuffer,
	104897	+ const char *pSource1, int nSource1,
	104898	+ const char *pSource2, int nSource2){
	104899	+ assert( nSource1>0 && pSource1!=NULL );
	104900	+ assert( nSource2>0 && pSource2!=NULL );
	104901	+ dataBufferExpand(pBuffer, nSource1+nSource2);
	104902	+ memcpy(pBuffer->pData+pBuffer->nData, pSource1, nSource1);
	104903	+ memcpy(pBuffer->pData+pBuffer->nData+nSource1, pSource2, nSource2);
	104904	+ pBuffer->nData += nSource1+nSource2;
	104905	+}
	104906	+static void dataBufferReplace(DataBuffer *pBuffer,
	104907	+ const char *pSource, int nSource){
	104908	+ dataBufferReset(pBuffer);
	104909	+ dataBufferAppend(pBuffer, pSource, nSource);
	104910	+}
	104911	+
	104912	+
	104913	+/* StringBuffer is a null-terminated version of DataBuffer. */
	104914	+typedef struct StringBuffer {
	104915	+ DataBuffer b; /* Includes null terminator. */
	104916	+} StringBuffer;
	104917	+
	104918	+static void initStringBuffer(StringBuffer *sb){
	104919	+ dataBufferInit(&sb->b, 100);
	104920	+ dataBufferReplace(&sb->b, "", 1);
	104921	+}
	104922	+static int stringBufferLength(StringBuffer *sb){
	104923	+ return sb->b.nData-1;
	104924	+}
	104925	+static char stringBufferData(StringBuffer sb){
	104926	+ return sb->b.pData;
	104927	+}
	104928	+
	104929	+static void nappend(StringBuffer sb, const char zFrom, int nFrom){
	104930	+ assert( sb->b.nData>0 );
	104931	+ if( nFrom>0 ){
	104932	+ sb->b.nData--;
	104933	+ dataBufferAppend2(&sb->b, zFrom, nFrom, "", 1);
	104934	+ }
	104935	+}
	104936	+static void append(StringBuffer sb, const char zFrom){
	104937	+ nappend(sb, zFrom, strlen(zFrom));
	104938	+}
	104939	+
	104940	+static int endsInWhiteSpace(StringBuffer *p){
	104941	+ return stringBufferLength(p)>0 &&
	104942	+ safe_isspace(stringBufferData(p)[stringBufferLength(p)-1]);
	104943	+}
	104944	+
	104945	+/* If the StringBuffer ends in something other than white space, add a
	104946	+** single space character to the end.
	104947	+*/
	104948	+static void appendWhiteSpace(StringBuffer *p){
	104949	+ if( stringBufferLength(p)==0 ) return;
	104950	+ if( !endsInWhiteSpace(p) ) append(p, " ");
	104951	+}
	104952	+
	104953	+/* Remove white space from the end of the StringBuffer */
	104954	+static void trimWhiteSpace(StringBuffer *p){
	104955	+ while( endsInWhiteSpace(p) ){
	104956	+ p->b.pData[--p->b.nData-1] = '\0';
	104957	+ }
	104958	+}
	104959	+
	104960	+
	104961	+/*
	104962	+** Release all memory associated with the Snippet structure passed as
	104963	+** an argument.
	104964	+*/
	104965	+static void fts3SnippetFree(Snippet *p){
	104966	+ sqlite3_free(p->aMatch);
	104967	+ sqlite3_free(p->zOffset);
	104968	+ sqlite3_free(p->zSnippet);
	104969	+ sqlite3_free(p);
	104970	+}
	104971	+
	104972	+/*
	104973	+** Append a single entry to the p->aMatch[] log.
	104974	+*/
	104975	+static void snippetAppendMatch(
	104976	+ Snippet p, / Append the entry to this snippet */
	104977	+ int iCol, int iTerm, /* The column and query term */
	104978	+ int iToken, /* Matching token in document */
	104979	+ int iStart, int nByte /* Offset and size of the match */
	104980	+){
	104981	+ int i;
	104982	+ struct snippetMatch *pMatch;
	104983	+ if( p->nMatch+1>=p->nAlloc ){
	104984	+ p->nAlloc = p->nAlloc*2 + 10;
	104985	+ p->aMatch = sqlite3_realloc(p->aMatch, p->nAlloc*sizeof(p->aMatch[0]) );
	104986	+ if( p->aMatch==0 ){
	104987	+ p->nMatch = 0;
	104988	+ p->nAlloc = 0;
	104989	+ return;
	104990	+ }
	104991	+ }
	104992	+ i = p->nMatch++;
	104993	+ pMatch = &p->aMatch[i];
	104994	+ pMatch->iCol = iCol;
	104995	+ pMatch->iTerm = iTerm;
	104996	+ pMatch->iToken = iToken;
	104997	+ pMatch->iStart = iStart;
	104998	+ pMatch->nByte = nByte;
	104999	+}
	105000	+
	105001	+/*
	105002	+** Sizing information for the circular buffer used in snippetOffsetsOfColumn()
	105003	+*/
	105004	+#define FTS3_ROTOR_SZ (32)
	105005	+#define FTS3_ROTOR_MASK (FTS3_ROTOR_SZ-1)
	105006	+
	105007	+/*
	105008	+** Function to iterate through the tokens of a compiled expression.
	105009	+**
	105010	+** Except, skip all tokens on the right-hand side of a NOT operator.
	105011	+** This function is used to find tokens as part of snippet and offset
	105012	+** generation and we do nt want snippets and offsets to report matches
	105013	+** for tokens on the RHS of a NOT.
	105014	+*/
	105015	+static int fts3NextExprToken(Fts3Expr *ppExpr, int piToken){
	105016	+ Fts3Expr p = ppExpr;
	105017	+ int iToken = *piToken;
	105018	+ if( iToken<0 ){
	105019	+ /* In this case the expression p is the root of an expression tree.
	105020	+ ** Move to the first token in the expression tree.
	105021	+ */
	105022	+ while( p->pLeft ){
	105023	+ p = p->pLeft;
	105024	+ }
	105025	+ iToken = 0;
	105026	+ }else{
	105027	+ assert(p && p->eType==FTSQUERY_PHRASE );
	105028	+ if( iToken<(p->pPhrase->nToken-1) ){
	105029	+ iToken++;
	105030	+ }else{
	105031	+ iToken = 0;
	105032	+ while( p->pParent && p->pParent->pLeft!=p ){
	105033	+ assert( p->pParent->pRight==p );
	105034	+ p = p->pParent;
	105035	+ }
	105036	+ p = p->pParent;
	105037	+ if( p ){
	105038	+ assert( p->pRight!=0 );
	105039	+ p = p->pRight;
	105040	+ while( p->pLeft ){
	105041	+ p = p->pLeft;
	105042	+ }
	105043	+ }
	105044	+ }
	105045	+ }
	105046	+
	105047	+ *ppExpr = p;
	105048	+ *piToken = iToken;
	105049	+ return p?1:0;
	105050	+}
	105051	+
	105052	+/*
	105053	+** Return TRUE if the expression node pExpr is located beneath the
	105054	+** RHS of a NOT operator.
	105055	+*/
	105056	+static int fts3ExprBeneathNot(Fts3Expr *p){
	105057	+ Fts3Expr *pParent;
	105058	+ while( p ){
	105059	+ pParent = p->pParent;
	105060	+ if( pParent && pParent->eType==FTSQUERY_NOT && pParent->pRight==p ){
	105061	+ return 1;
	105062	+ }
	105063	+ p = pParent;
	105064	+ }
	105065	+ return 0;
	105066	+}
	105067	+
	105068	+/*
	105069	+** Add entries to pSnippet->aMatch[] for every match that occurs against
	105070	+** document zDoc[0..nDoc-1] which is stored in column iColumn.
	105071	+*/
	105072	+static void snippetOffsetsOfColumn(
	105073	+ Fts3Cursor pCur, / The fulltest search cursor */
	105074	+ Snippet pSnippet, / The Snippet object to be filled in */
	105075	+ int iColumn, /* Index of fulltext table column */
	105076	+ const char zDoc, / Text of the fulltext table column */
	105077	+ int nDoc /* Length of zDoc in bytes */
	105078	+){
	105079	+ const sqlite3_tokenizer_module pTModule; / The tokenizer module */
	105080	+ sqlite3_tokenizer pTokenizer; / The specific tokenizer */
	105081	+ sqlite3_tokenizer_cursor pTCursor; / Tokenizer cursor */
	105082	+ Fts3Table pVtab; / The full text index */
	105083	+ int nColumn; /* Number of columns in the index */
	105084	+ int i, j; /* Loop counters */
	105085	+ int rc; /* Return code */
	105086	+ unsigned int match, prevMatch; /* Phrase search bitmasks */
	105087	+ const char zToken; / Next token from the tokenizer */
	105088	+ int nToken; /* Size of zToken */
	105089	+ int iBegin, iEnd, iPos; /* Offsets of beginning and end */
	105090	+
	105091	+ /* The following variables keep a circular buffer of the last
	105092	+ ** few tokens */
	105093	+ unsigned int iRotor = 0; /* Index of current token */
	105094	+ int iRotorBegin[FTS3_ROTOR_SZ]; /* Beginning offset of token */
	105095	+ int iRotorLen[FTS3_ROTOR_SZ]; /* Length of token */
	105096	+
	105097	+ pVtab = (Fts3Table *)pCur->base.pVtab;
	105098	+ nColumn = pVtab->nColumn;
	105099	+ pTokenizer = pVtab->pTokenizer;
	105100	+ pTModule = pTokenizer->pModule;
	105101	+ rc = pTModule->xOpen(pTokenizer, zDoc, nDoc, &pTCursor);
	105102	+ if( rc ) return;
	105103	+ pTCursor->pTokenizer = pTokenizer;
	105104	+
	105105	+ prevMatch = 0;
	105106	+ while( !pTModule->xNext(pTCursor, &zToken, &nToken, &iBegin, &iEnd, &iPos) ){
	105107	+ Fts3Expr *pIter = pCur->pExpr;
	105108	+ int iIter = -1;
	105109	+ iRotorBegin[iRotor&FTS3_ROTOR_MASK] = iBegin;
	105110	+ iRotorLen[iRotor&FTS3_ROTOR_MASK] = iEnd-iBegin;
	105111	+ match = 0;
	105112	+ for(i=0; i<(FTS3_ROTOR_SZ-1) && fts3NextExprToken(&pIter, &iIter); i++){
	105113	+ int nPhrase; /* Number of tokens in current phrase */
	105114	+ struct PhraseToken pToken; / Current token */
	105115	+ int iCol; /* Column index */
	105116	+
	105117	+ if( fts3ExprBeneathNot(pIter) ) continue;
	105118	+ nPhrase = pIter->pPhrase->nToken;
	105119	+ pToken = &pIter->pPhrase->aToken[iIter];
	105120	+ iCol = pIter->pPhrase->iColumn;
	105121	+ if( iCol>=0 && iCol<nColumn && iCol!=iColumn ) continue;
	105122	+ if( pToken->n>nToken ) continue;
	105123	+ if( !pToken->isPrefix && pToken->n<nToken ) continue;
	105124	+ assert( pToken->n<=nToken );
	105125	+ if( memcmp(pToken->z, zToken, pToken->n) ) continue;
	105126	+ if( iIter>0 && (prevMatch & (1<<i))==0 ) continue;
	105127	+ match \|= 1<<i;
	105128	+ if( i==(FTS3_ROTOR_SZ-2) \|\| nPhrase==iIter+1 ){
	105129	+ for(j=nPhrase-1; j>=0; j--){
	105130	+ int k = (iRotor-j) & FTS3_ROTOR_MASK;
	105131	+ snippetAppendMatch(pSnippet, iColumn, i-j, iPos-j,
	105132	+ iRotorBegin[k], iRotorLen[k]);
	105133	+ }
	105134	+ }
	105135	+ }
	105136	+ prevMatch = match<<1;
	105137	+ iRotor++;
	105138	+ }
	105139	+ pTModule->xClose(pTCursor);
	105140	+}
	105141	+
	105142	+/*
	105143	+** Remove entries from the pSnippet structure to account for the NEAR
	105144	+** operator. When this is called, pSnippet contains the list of token
	105145	+** offsets produced by treating all NEAR operators as AND operators.
	105146	+** This function removes any entries that should not be present after
	105147	+** accounting for the NEAR restriction. For example, if the queried
	105148	+** document is:
	105149	+**
	105150	+** "A B C D E A"
	105151	+**
	105152	+** and the query is:
	105153	+**
	105154	+** A NEAR/0 E
	105155	+**
	105156	+** then when this function is called the Snippet contains token offsets
	105157	+** 0, 4 and 5. This function removes the "0" entry (because the first A
	105158	+** is not near enough to an E).
	105159	+**
	105160	+** When this function is called, the value pointed to by parameter piLeft is
	105161	+** the integer id of the left-most token in the expression tree headed by
	105162	+** pExpr. This function increments *piLeft by the total number of tokens
	105163	+** in the expression tree headed by pExpr.
	105164	+**
	105165	+** Return 1 if any trimming occurs. Return 0 if no trimming is required.
	105166	+*/
	105167	+static int trimSnippetOffsets(
	105168	+ Fts3Expr pExpr, / The search expression */
	105169	+ Snippet pSnippet, / The set of snippet offsets to be trimmed */
	105170	+ int piLeft / Index of left-most token in pExpr */
	105171	+){
	105172	+ if( pExpr ){
	105173	+ if( trimSnippetOffsets(pExpr->pLeft, pSnippet, piLeft) ){
	105174	+ return 1;
	105175	+ }
	105176	+
	105177	+ switch( pExpr->eType ){
	105178	+ case FTSQUERY_PHRASE:
	105179	+ *piLeft += pExpr->pPhrase->nToken;
	105180	+ break;
	105181	+ case FTSQUERY_NEAR: {
	105182	+ /* The right-hand-side of a NEAR operator is always a phrase. The
	105183	+ ** left-hand-side is either a phrase or an expression tree that is
	105184	+ ** itself headed by a NEAR operator. The following initializations
	105185	+ ** set local variable iLeft to the token number of the left-most
	105186	+ ** token in the right-hand phrase, and iRight to the right most
	105187	+ ** token in the same phrase. For example, if we had:
	105188	+ **
	105189	+ ** <col> MATCH '"abc def" NEAR/2 "ghi jkl"'
	105190	+ **
	105191	+ ** then iLeft will be set to 2 (token number of ghi) and nToken will
	105192	+ ** be set to 4.
	105193	+ */
	105194	+ Fts3Expr *pLeft = pExpr->pLeft;
	105195	+ Fts3Expr *pRight = pExpr->pRight;
	105196	+ int iLeft = *piLeft;
	105197	+ int nNear = pExpr->nNear;
	105198	+ int nToken = pRight->pPhrase->nToken;
	105199	+ int jj, ii;
	105200	+ if( pLeft->eType==FTSQUERY_NEAR ){
	105201	+ pLeft = pLeft->pRight;
	105202	+ }
	105203	+ assert( pRight->eType==FTSQUERY_PHRASE );
	105204	+ assert( pLeft->eType==FTSQUERY_PHRASE );
	105205	+ nToken += pLeft->pPhrase->nToken;
	105206	+
	105207	+ for(ii=0; ii<pSnippet->nMatch; ii++){
	105208	+ struct snippetMatch *p = &pSnippet->aMatch[ii];
	105209	+ if( p->iTerm==iLeft ){
	105210	+ int isOk = 0;
	105211	+ /* Snippet ii is an occurence of query term iLeft in the document.
	105212	+ ** It occurs at position (p->iToken) of the document. We now
	105213	+ ** search for an instance of token (iLeft-1) somewhere in the
	105214	+ ** range (p->iToken - nNear)...(p->iToken + nNear + nToken) within
	105215	+ ** the set of snippetMatch structures. If one is found, proceed.
	105216	+ ** If one cannot be found, then remove snippets ii..(ii+N-1)
	105217	+ ** from the matching snippets, where N is the number of tokens
	105218	+ ** in phrase pRight->pPhrase.
	105219	+ */
	105220	+ for(jj=0; isOk==0 && jj<pSnippet->nMatch; jj++){
	105221	+ struct snippetMatch *p2 = &pSnippet->aMatch[jj];
	105222	+ if( p2->iTerm==(iLeft-1) ){
	105223	+ if( p2->iToken>=(p->iToken-nNear-1)
	105224	+ && p2->iToken<(p->iToken+nNear+nToken)
	105225	+ ){
	105226	+ isOk = 1;
	105227	+ }
	105228	+ }
	105229	+ }
	105230	+ if( !isOk ){
	105231	+ int kk;
	105232	+ for(kk=0; kk<pRight->pPhrase->nToken; kk++){
	105233	+ pSnippet->aMatch[kk+ii].iTerm = -2;
	105234	+ }
	105235	+ return 1;
	105236	+ }
	105237	+ }
	105238	+ if( p->iTerm==(iLeft-1) ){
	105239	+ int isOk = 0;
	105240	+ for(jj=0; isOk==0 && jj<pSnippet->nMatch; jj++){
	105241	+ struct snippetMatch *p2 = &pSnippet->aMatch[jj];
	105242	+ if( p2->iTerm==iLeft ){
	105243	+ if( p2->iToken<=(p->iToken+nNear+1)
	105244	+ && p2->iToken>(p->iToken-nNear-nToken)
	105245	+ ){
	105246	+ isOk = 1;
	105247	+ }
	105248	+ }
	105249	+ }
	105250	+ if( !isOk ){
	105251	+ int kk;
	105252	+ for(kk=0; kk<pLeft->pPhrase->nToken; kk++){
	105253	+ pSnippet->aMatch[ii-kk].iTerm = -2;
	105254	+ }
	105255	+ return 1;
	105256	+ }
	105257	+ }
	105258	+ }
	105259	+ break;
	105260	+ }
	105261	+ }
	105262	+
	105263	+ if( trimSnippetOffsets(pExpr->pRight, pSnippet, piLeft) ){
	105264	+ return 1;
	105265	+ }
	105266	+ }
	105267	+ return 0;
	105268	+}
	105269	+
	105270	+/*
	105271	+** Compute all offsets for the current row of the query.
	105272	+** If the offsets have already been computed, this routine is a no-op.
	105273	+*/
	105274	+static int snippetAllOffsets(Fts3Cursor pCsr, Snippet *ppSnippet){
	105275	+ Fts3Table p = (Fts3Table )pCsr->base.pVtab;
	105276	+ int nColumn;
	105277	+ int iColumn, i;
	105278	+ int iFirst, iLast;
	105279	+ int iTerm = 0;
	105280	+ Snippet *pSnippet;
	105281	+
	105282	+ if( pCsr->pExpr==0 ){
	105283	+ return SQLITE_OK;
	105284	+ }
	105285	+
	105286	+ pSnippet = (Snippet *)sqlite3_malloc(sizeof(Snippet));
	105287	+ *ppSnippet = pSnippet;
	105288	+ if( !pSnippet ){
	105289	+ return SQLITE_NOMEM;
	105290	+ }
	105291	+ memset(pSnippet, 0, sizeof(Snippet));
	105292	+
	105293	+ nColumn = p->nColumn;
	105294	+ iColumn = (pCsr->eType - 2);
	105295	+ if( iColumn<0 \|\| iColumn>=nColumn ){
	105296	+ /* Look for matches over all columns of the full-text index */
	105297	+ iFirst = 0;
	105298	+ iLast = nColumn-1;
	105299	+ }else{
	105300	+ /* Look for matches in the iColumn-th column of the index only */
	105301	+ iFirst = iColumn;
	105302	+ iLast = iColumn;
	105303	+ }
	105304	+ for(i=iFirst; i<=iLast; i++){
	105305	+ const char *zDoc;
	105306	+ int nDoc;
	105307	+ zDoc = (const char*)sqlite3_column_text(pCsr->pStmt, i+1);
	105308	+ nDoc = sqlite3_column_bytes(pCsr->pStmt, i+1);
	105309	+ snippetOffsetsOfColumn(pCsr, pSnippet, i, zDoc, nDoc);
	105310	+ }
	105311	+
	105312	+ while( trimSnippetOffsets(pCsr->pExpr, pSnippet, &iTerm) ){
	105313	+ iTerm = 0;
	105314	+ }
	105315	+
	105316	+ return SQLITE_OK;
	105317	+}
	105318	+
	105319	+/*
	105320	+** Convert the information in the aMatch[] array of the snippet
	105321	+** into the string zOffset[0..nOffset-1]. This string is used as
	105322	+** the return of the SQL offsets() function.
	105323	+*/
	105324	+static void snippetOffsetText(Snippet *p){
	105325	+ int i;
	105326	+ int cnt = 0;
	105327	+ StringBuffer sb;
	105328	+ char zBuf[200];
	105329	+ if( p->zOffset ) return;
	105330	+ initStringBuffer(&sb);
	105331	+ for(i=0; i<p->nMatch; i++){
	105332	+ struct snippetMatch *pMatch = &p->aMatch[i];
	105333	+ if( pMatch->iTerm>=0 ){
	105334	+ /* If snippetMatch.iTerm is less than 0, then the match was
	105335	+ ** discarded as part of processing the NEAR operator (see the
	105336	+ ** trimSnippetOffsetsForNear() function for details). Ignore
	105337	+ ** it in this case
	105338	+ */
	105339	+ zBuf[0] = ' ';
	105340	+ sqlite3_snprintf(sizeof(zBuf)-1, &zBuf[cnt>0], "%d %d %d %d",
	105341	+ pMatch->iCol, pMatch->iTerm, pMatch->iStart, pMatch->nByte);
	105342	+ append(&sb, zBuf);
	105343	+ cnt++;
	105344	+ }
	105345	+ }
	105346	+ p->zOffset = stringBufferData(&sb);
	105347	+ p->nOffset = stringBufferLength(&sb);
	105348	+}
	105349	+
	105350	+/*
	105351	+** zDoc[0..nDoc-1] is phrase of text. aMatch[0..nMatch-1] are a set
	105352	+** of matching words some of which might be in zDoc. zDoc is column
	105353	+** number iCol.
	105354	+**
	105355	+** iBreak is suggested spot in zDoc where we could begin or end an
	105356	+** excerpt. Return a value similar to iBreak but possibly adjusted
	105357	+** to be a little left or right so that the break point is better.
	105358	+*/
	105359	+static int wordBoundary(
	105360	+ int iBreak, /* The suggested break point */
	105361	+ const char zDoc, / Document text */
	105362	+ int nDoc, /* Number of bytes in zDoc[] */
	105363	+ struct snippetMatch aMatch, / Matching words */
	105364	+ int nMatch, /* Number of entries in aMatch[] */
	105365	+ int iCol /* The column number for zDoc[] */
	105366	+){
	105367	+ int i;
	105368	+ if( iBreak<=10 ){
	105369	+ return 0;
	105370	+ }
	105371	+ if( iBreak>=nDoc-10 ){
	105372	+ return nDoc;
	105373	+ }
	105374	+ for(i=0; i<nMatch && aMatch[i].iCol<iCol; i++){}
	105375	+ while( i<nMatch && aMatch[i].iStart+aMatch[i].nByte<iBreak ){ i++; }
	105376	+ if( i<nMatch ){
	105377	+ if( aMatch[i].iStart<iBreak+10 ){
	105378	+ return aMatch[i].iStart;
	105379	+ }
	105380	+ if( i>0 && aMatch[i-1].iStart+aMatch[i-1].nByte>=iBreak ){
	105381	+ return aMatch[i-1].iStart;
	105382	+ }
	105383	+ }
	105384	+ for(i=1; i<=10; i++){
	105385	+ if( safe_isspace(zDoc[iBreak-i]) ){
	105386	+ return iBreak - i + 1;
	105387	+ }
	105388	+ if( safe_isspace(zDoc[iBreak+i]) ){
	105389	+ return iBreak + i + 1;
	105390	+ }
	105391	+ }
	105392	+ return iBreak;
	105393	+}
	105394	+
	105395	+
	105396	+
	105397	+/*
	105398	+** Allowed values for Snippet.aMatch[].snStatus
	105399	+*/
	105400	+#define SNIPPET_IGNORE 0 /* It is ok to omit this match from the snippet */
	105401	+#define SNIPPET_DESIRED 1 /* We want to include this match in the snippet */
	105402	+
	105403	+/*
	105404	+** Generate the text of a snippet.
	105405	+*/
	105406	+static void snippetText(
	105407	+ Fts3Cursor pCursor, / The cursor we need the snippet for */
	105408	+ Snippet *pSnippet,
	105409	+ const char zStartMark, / Markup to appear before each match */
	105410	+ const char zEndMark, / Markup to appear after each match */
	105411	+ const char zEllipsis / Ellipsis mark */
	105412	+){
	105413	+ int i, j;
	105414	+ struct snippetMatch *aMatch;
	105415	+ int nMatch;
	105416	+ int nDesired;
	105417	+ StringBuffer sb;
	105418	+ int tailCol;
	105419	+ int tailOffset;
	105420	+ int iCol;
	105421	+ int nDoc;
	105422	+ const char *zDoc;
	105423	+ int iStart, iEnd;
	105424	+ int tailEllipsis = 0;
	105425	+ int iMatch;
	105426	+
	105427	+
	105428	+ sqlite3_free(pSnippet->zSnippet);
	105429	+ pSnippet->zSnippet = 0;
	105430	+ aMatch = pSnippet->aMatch;
	105431	+ nMatch = pSnippet->nMatch;
	105432	+ initStringBuffer(&sb);
	105433	+
	105434	+ for(i=0; i<nMatch; i++){
	105435	+ aMatch[i].snStatus = SNIPPET_IGNORE;
	105436	+ }
	105437	+ nDesired = 0;
	105438	+ for(i=0; i<FTS3_ROTOR_SZ; i++){
	105439	+ for(j=0; j<nMatch; j++){
	105440	+ if( aMatch[j].iTerm==i ){
	105441	+ aMatch[j].snStatus = SNIPPET_DESIRED;
	105442	+ nDesired++;
	105443	+ break;
	105444	+ }
	105445	+ }
	105446	+ }
	105447	+
	105448	+ iMatch = 0;
	105449	+ tailCol = -1;
	105450	+ tailOffset = 0;
	105451	+ for(i=0; i<nMatch && nDesired>0; i++){
	105452	+ if( aMatch[i].snStatus!=SNIPPET_DESIRED ) continue;
	105453	+ nDesired--;
	105454	+ iCol = aMatch[i].iCol;
	105455	+ zDoc = (const char*)sqlite3_column_text(pCursor->pStmt, iCol+1);
	105456	+ nDoc = sqlite3_column_bytes(pCursor->pStmt, iCol+1);
	105457	+ iStart = aMatch[i].iStart - 40;
	105458	+ iStart = wordBoundary(iStart, zDoc, nDoc, aMatch, nMatch, iCol);
	105459	+ if( iStart<=10 ){
	105460	+ iStart = 0;
	105461	+ }
	105462	+ if( iCol==tailCol && iStart<=tailOffset+20 ){
	105463	+ iStart = tailOffset;
	105464	+ }
	105465	+ if( (iCol!=tailCol && tailCol>=0) \|\| iStart!=tailOffset ){
	105466	+ trimWhiteSpace(&sb);
	105467	+ appendWhiteSpace(&sb);
	105468	+ append(&sb, zEllipsis);
	105469	+ appendWhiteSpace(&sb);
	105470	+ }
	105471	+ iEnd = aMatch[i].iStart + aMatch[i].nByte + 40;
	105472	+ iEnd = wordBoundary(iEnd, zDoc, nDoc, aMatch, nMatch, iCol);
	105473	+ if( iEnd>=nDoc-10 ){
	105474	+ iEnd = nDoc;
	105475	+ tailEllipsis = 0;
	105476	+ }else{
	105477	+ tailEllipsis = 1;
	105478	+ }
	105479	+ while( iMatch<nMatch && aMatch[iMatch].iCol<iCol ){ iMatch++; }
	105480	+ while( iStart<iEnd ){
	105481	+ while( iMatch<nMatch && aMatch[iMatch].iStart<iStart
	105482	+ && aMatch[iMatch].iCol<=iCol ){
	105483	+ iMatch++;
	105484	+ }
	105485	+ if( iMatch<nMatch && aMatch[iMatch].iStart<iEnd
	105486	+ && aMatch[iMatch].iCol==iCol ){
	105487	+ nappend(&sb, &zDoc[iStart], aMatch[iMatch].iStart - iStart);
	105488	+ iStart = aMatch[iMatch].iStart;
	105489	+ append(&sb, zStartMark);
	105490	+ nappend(&sb, &zDoc[iStart], aMatch[iMatch].nByte);
	105491	+ append(&sb, zEndMark);
	105492	+ iStart += aMatch[iMatch].nByte;
	105493	+ for(j=iMatch+1; j<nMatch; j++){
	105494	+ if( aMatch[j].iTerm==aMatch[iMatch].iTerm
	105495	+ && aMatch[j].snStatus==SNIPPET_DESIRED ){
	105496	+ nDesired--;
	105497	+ aMatch[j].snStatus = SNIPPET_IGNORE;
	105498	+ }
	105499	+ }
	105500	+ }else{
	105501	+ nappend(&sb, &zDoc[iStart], iEnd - iStart);
	105502	+ iStart = iEnd;
	105503	+ }
	105504	+ }
	105505	+ tailCol = iCol;
	105506	+ tailOffset = iEnd;
	105507	+ }
	105508	+ trimWhiteSpace(&sb);
	105509	+ if( tailEllipsis ){
	105510	+ appendWhiteSpace(&sb);
	105511	+ append(&sb, zEllipsis);
	105512	+ }
	105513	+ pSnippet->zSnippet = stringBufferData(&sb);
	105514	+ pSnippet->nSnippet = stringBufferLength(&sb);
	105515	+}
	105516	+
	105517	+SQLITE_PRIVATE void sqlite3Fts3Offsets(
	105518	+ sqlite3_context pCtx, / SQLite function call context */
	105519	+ Fts3Cursor pCsr / Cursor object */
	105520	+){
	105521	+ Snippet p; / Snippet structure */
	105522	+ int rc = snippetAllOffsets(pCsr, &p);
	105523	+ snippetOffsetText(p);
	105524	+ sqlite3_result_text(pCtx, p->zOffset, p->nOffset, SQLITE_TRANSIENT);
	105525	+ fts3SnippetFree(p);
	105526	+}
	105527	+
	105528	+SQLITE_PRIVATE void sqlite3Fts3Snippet(
	105529	+ sqlite3_context pCtx, / SQLite function call context */
	105530	+ Fts3Cursor pCsr, / Cursor object */
	105531	+ const char zStart, / Snippet start text - "<b>" */
	105532	+ const char zEnd, / Snippet end text - "</b>" */
	105533	+ const char zEllipsis / Snippet ellipsis text - "<b>...</b>" */
	105534	+){
	105535	+ Snippet p; / Snippet structure */
	105536	+ int rc = snippetAllOffsets(pCsr, &p);
	105537	+ snippetText(pCsr, p, zStart, zEnd, zEllipsis);
	105538	+ sqlite3_result_text(pCtx, p->zSnippet, p->nSnippet, SQLITE_TRANSIENT);
	105539	+ fts3SnippetFree(p);
	105540	+}
	105541	+
	105542	+#endif
	105543	+
	105544	+/************ End of fts3_snippet.c **************************************/
107224	105545	/************ Begin file rtree.c *****************************************/
107225	105546	/*
107226	105547	** 2001 September 15
107227	105548	**
107228	105549	** The author disclaims copyright to this source code. In place of
		@@ -107233,12 +105554,10 @@
107233	105554	** May you share freely, never taking more than you give.
107234	105555	**
107235	105556	*************************************************************************
107236	105557	** This file contains code for implementations of the r-tree and r*-tree
107237	105558	** algorithms packaged as an SQLite virtual table module.
107238		-**
107239		-** $Id: rtree.c,v 1.14 2009/08/06 18:36:47 danielk1977 Exp $
107240	105559	*/
107241	105560
107242	105561	#if !defined(SQLITE_CORE) \|\| defined(SQLITE_ENABLE_RTREE)
107243	105562
107244	105563	/*
107245	105564

	--- src/sqlite3.c
	+++ src/sqlite3.c
	@@ -1,8 +1,8 @@
1	/******************************************************************************
2	** This file is an amalgamation of many separate C source files from SQLite
3	** version 3.6.20. By combining all the individual C code files into this
4	** single large file, the entire code can be compiled as a one translation
5	** unit. This allows many compilers to do optimizations that would not be
6	** possible if the files were compiled separately. Performance improvements
7	** of 5% are more are commonly seen when SQLite is compiled as a single
8	** translation unit.
	@@ -91,12 +91,10 @@
91	** May you share freely, never taking more than you give.
92	**
93	*************************************************************************
94	**
95	** This file defines various limits of what SQLite can process.
96	**
97	** @(#) $Id: sqliteLimit.h,v 1.10 2009/01/10 16:15:09 danielk1977 Exp $
98	*/
99
100	/*
101	** The maximum length of a TEXT or BLOB in bytes. This also
102	** limits the size of a row in a table or index.
	@@ -277,15 +275,11 @@
277	** A value of 1 means that a trigger program will not be able to itself
278	** fire any triggers. A value of 0 means that no trigger programs at all
279	** may be executed.
280	*/
281	#ifndef SQLITE_MAX_TRIGGER_DEPTH
282	#if defined(SQLITE_SMALL_STACK)
283	# define SQLITE_MAX_TRIGGER_DEPTH 10
284	#else
285	# define SQLITE_MAX_TRIGGER_DEPTH 1000
286	#endif
287	#endif
288
289	/************ End of sqliteLimit.h ***************************************/
290	/************ Continuing where we left off in sqliteInt.h ****************/
291
	@@ -647,13 +641,13 @@
647	** [sqlite3_libversion_number()], [sqlite3_sourceid()],
648	** [sqlite_version()] and [sqlite_source_id()].
649	**
650	** Requirements: [H10011] [H10014]
651	*/
652	#define SQLITE_VERSION "3.6.20"
653	#define SQLITE_VERSION_NUMBER 3006020
654	#define SQLITE_SOURCE_ID "2009-11-04 23:02:52 f1c09acaca3e205acf5b077c9b2d0fe35f035c1e"
655
656	/*
657	** CAPI3REF: Run-Time Library Version Numbers {H10020} <S60100>
658	** KEYWORDS: sqlite3_version
659	**
	@@ -6316,12 +6310,10 @@
6316	** May you share freely, never taking more than you give.
6317	**
6318	*************************************************************************
6319	** This is the header file for the generic hash-table implemenation
6320	** used in SQLite.
6321	**
6322	** $Id: hash.h,v 1.15 2009/05/02 13:29:38 drh Exp $
6323	*/
6324	#ifndef _SQLITE_HASH_H_
6325	#define _SQLITE_HASH_H_
6326
6327	/* Forward declarations of structures. */
	@@ -6920,12 +6912,10 @@
6920	**
6921	*************************************************************************
6922	** This header file defines the interface that the sqlite B-Tree file
6923	** subsystem. See comments in the source code for a detailed description
6924	** of what each interface routine does.
6925	**
6926	** @(#) $Id: btree.h,v 1.120 2009/07/22 00:35:24 drh Exp $
6927	*/
6928	#ifndef _BTREE_H_
6929	#define _BTREE_H_
6930
6931	/* TODO: This definition is just included so other modules compile. It
	@@ -7058,10 +7048,11 @@
7058	int wrFlag, /* 1 for writing. 0 for read-only */
7059	struct KeyInfo, / First argument to compare function */
7060	BtCursor pCursor / Space to write cursor structure */
7061	);
7062	SQLITE_PRIVATE int sqlite3BtreeCursorSize(void);

7063
7064	SQLITE_PRIVATE int sqlite3BtreeCloseCursor(BtCursor*);
7065	SQLITE_PRIVATE int sqlite3BtreeMovetoUnpacked(
7066	BtCursor*,
7067	UnpackedRecord *pUnKey,
	@@ -7169,12 +7160,10 @@
7169	** Header file for the Virtual DataBase Engine (VDBE)
7170	**
7171	** This header defines the interface to the virtual database engine
7172	** or VDBE. The VDBE implements an abstract machine that runs a
7173	** simple program to access and modify the underlying database.
7174	**
7175	** $Id: vdbe.h,v 1.142 2009/07/24 17:58:53 danielk1977 Exp $
7176	*/
7177	#ifndef _SQLITE_VDBE_H_
7178	#define _SQLITE_VDBE_H_
7179
7180	/*
	@@ -7198,11 +7187,11 @@
7198	** as an instance of the following structure:
7199	*/
7200	struct VdbeOp {
7201	u8 opcode; /* What operation to perform */
7202	signed char p4type; /* One of the P4_xxx constants for p4 */
7203	u8 opflags; /* Not currently used */
7204	u8 p5; /* Fifth parameter is an unsigned character */
7205	int p1; /* First operand */
7206	int p2; /* Second parameter (often the jump destination) */
7207	int p3; /* The third parameter */
7208	union { /* fourth parameter */
	@@ -7476,26 +7465,27 @@
7476	#define OPFLG_JUMP 0x0001 /* jump: P2 holds jmp target */
7477	#define OPFLG_OUT2_PRERELEASE 0x0002 /* out2-prerelease: */
7478	#define OPFLG_IN1 0x0004 /* in1: P1 is an input */
7479	#define OPFLG_IN2 0x0008 /* in2: P2 is an input */
7480	#define OPFLG_IN3 0x0010 /* in3: P3 is an input */
7481	#define OPFLG_OUT3 0x0020 /* out3: P3 is an output */

7482	#define OPFLG_INITIALIZER {\
7483	/* 0 */ 0x00, 0x01, 0x01, 0x04, 0x04, 0x10, 0x00, 0x02,\
7484	/* 8 */ 0x02, 0x02, 0x02, 0x02, 0x00, 0x00, 0x04, 0x04,\
7485	/* 16 */ 0x00, 0x00, 0x00, 0x04, 0x04, 0x05, 0x04, 0x00,\
7486	/* 24 */ 0x00, 0x01, 0x05, 0x05, 0x00, 0x00, 0x00, 0x02,\
7487	/* 32 */ 0x00, 0x00, 0x00, 0x02, 0x10, 0x00, 0x00, 0x00,\
7488	/* 40 */ 0x00, 0x00, 0x00, 0x11, 0x11, 0x11, 0x11, 0x08,\
7489	/* 48 */ 0x11, 0x11, 0x11, 0x11, 0x02, 0x02, 0x00, 0x00,\
7490	/* 56 */ 0x00, 0x00, 0x00, 0x00, 0x02, 0x00, 0x01, 0x01,\
7491	/* 64 */ 0x01, 0x01, 0x01, 0x08, 0x2c, 0x2c, 0x00, 0x02,\
7492	/* 72 */ 0x11, 0x05, 0x05, 0x15, 0x15, 0x15, 0x15, 0x15,\
7493	/* 80 */ 0x15, 0x11, 0x2c, 0x2c, 0x2c, 0x2c, 0x2c, 0x2c,\
7494	/* 88 */ 0x2c, 0x2c, 0x2c, 0x2c, 0x02, 0x04, 0x02, 0x00,\
7495	/* 96 */ 0x02, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,\
7496	/* 104 */ 0x08, 0x21, 0x15, 0x01, 0x02, 0x00, 0x01, 0x08,\
7497	/* 112 */ 0x05, 0x05, 0x05, 0x00, 0x00, 0x00, 0x01, 0x00,\
7498	/* 120 */ 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x01,\
7499	/* 128 */ 0x00, 0x00, 0x02, 0x02, 0x00, 0x00, 0x00, 0x00,\
7500	/* 136 */ 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, 0x04, 0x04,\
7501	/* 144 */ 0x04, 0x04,}
	@@ -7511,10 +7501,11 @@
7511	SQLITE_PRIVATE int sqlite3VdbeAddOp0(Vdbe*,int);
7512	SQLITE_PRIVATE int sqlite3VdbeAddOp1(Vdbe*,int,int);
7513	SQLITE_PRIVATE int sqlite3VdbeAddOp2(Vdbe*,int,int,int);
7514	SQLITE_PRIVATE int sqlite3VdbeAddOp3(Vdbe*,int,int,int,int);
7515	SQLITE_PRIVATE int sqlite3VdbeAddOp4(Vdbe,int,int,int,int,const char zP4,int);

7516	SQLITE_PRIVATE int sqlite3VdbeAddOpList(Vdbe, int nOp, VdbeOpList const aOp);
7517	SQLITE_PRIVATE void sqlite3VdbeChangeP1(Vdbe*, int addr, int P1);
7518	SQLITE_PRIVATE void sqlite3VdbeChangeP2(Vdbe*, int addr, int P2);
7519	SQLITE_PRIVATE void sqlite3VdbeChangeP3(Vdbe*, int addr, int P3);
7520	SQLITE_PRIVATE void sqlite3VdbeChangeP5(Vdbe*, u8 P5);
	@@ -7579,12 +7570,10 @@
7579	**
7580	*************************************************************************
7581	** This header file defines the interface that the sqlite page cache
7582	** subsystem. The page cache subsystem reads and writes a file a page
7583	** at a time and provides a journal for rollback.
7584	**
7585	** @(#) $Id: pager.h,v 1.104 2009/07/24 19:01:19 drh Exp $
7586	*/
7587
7588	#ifndef _PAGER_H_
7589	#define _PAGER_H_
7590
	@@ -7750,12 +7739,10 @@
7750	** May you share freely, never taking more than you give.
7751	**
7752	*************************************************************************
7753	** This header file defines the interface that the sqlite page cache
7754	** subsystem.
7755	**
7756	** @(#) $Id: pcache.h,v 1.20 2009/07/25 11:46:49 danielk1977 Exp $
7757	*/
7758
7759	#ifndef _PCACHE_H_
7760
7761	typedef struct PgHdr PgHdr;
	@@ -7918,12 +7905,10 @@
7918	** "os.c") attempt to abstract the underlying operating system so that
7919	** the SQLite library will work on both POSIX and windows systems.
7920	**
7921	** This header file is #include-ed by sqliteInt.h and thus ends up
7922	** being included by every source file.
7923	**
7924	** $Id: os.h,v 1.108 2009/02/05 16:31:46 drh Exp $
7925	*/
7926	#ifndef _SQLITE_OS_H_
7927	#define _SQLITE_OS_H_
7928
7929	/*
	@@ -8198,12 +8183,10 @@
8198	** better organized.
8199	**
8200	** NOTE: source files should not #include this header file directly.
8201	** Source files should #include the sqliteInt.h file and let that file
8202	** include this one indirectly.
8203	**
8204	** $Id: mutex.h,v 1.9 2008/10/07 15:25:48 drh Exp $
8205	*/
8206
8207
8208	/*
8209	** Figure out what version of the code to use. The choices are
	@@ -8576,10 +8559,11 @@
8576	#define SQLITE_FUNC_CASE 0x02 /* Case-sensitive LIKE-type function */
8577	#define SQLITE_FUNC_EPHEM 0x04 /* Ephemeral. Delete with VDBE */
8578	#define SQLITE_FUNC_NEEDCOLL 0x08 /* sqlite3GetFuncCollSeq() might be called */
8579	#define SQLITE_FUNC_PRIVATE 0x10 /* Allowed for internal use only */
8580	#define SQLITE_FUNC_COUNT 0x20 /* Built-in count() aggregate /

8581
8582	/*
8583	** The following three macros, FUNCTION(), LIKEFUNC() and AGGREGATE() are
8584	** used to create the initializers for the FuncDef structures.
8585	**
	@@ -10290,10 +10274,13 @@
10290	SQLITE_PRIVATE void sqlite3CloseSavepoints(sqlite3 *);
10291	SQLITE_PRIVATE int sqlite3ExprIsConstant(Expr*);
10292	SQLITE_PRIVATE int sqlite3ExprIsConstantNotJoin(Expr*);
10293	SQLITE_PRIVATE int sqlite3ExprIsConstantOrFunction(Expr*);
10294	SQLITE_PRIVATE int sqlite3ExprIsInteger(Expr, int);



10295	SQLITE_PRIVATE int sqlite3IsRowid(const char*);
10296	SQLITE_PRIVATE void sqlite3GenerateRowDelete(Parse, Table, int, int, int, Trigger *, int);
10297	SQLITE_PRIVATE void sqlite3GenerateRowIndexDelete(Parse, Table, int, int*);
10298	SQLITE_PRIVATE int sqlite3GenerateIndexKey(Parse, Index, int, int, int);
10299	SQLITE_PRIVATE void sqlite3GenerateConstraintChecks(Parse,Table,int,int,
	@@ -10461,10 +10448,11 @@
10461	SQLITE_PRIVATE char sqlite3Utf8to16(sqlite3 , u8, char , int, int );
10462	#endif
10463	SQLITE_PRIVATE int sqlite3ValueFromExpr(sqlite3 , Expr , u8, u8, sqlite3_value **);
10464	SQLITE_PRIVATE void sqlite3ValueApplyAffinity(sqlite3_value *, u8, u8);
10465	#ifndef SQLITE_AMALGAMATION

10466	SQLITE_PRIVATE const unsigned char sqlite3UpperToLower[];
10467	SQLITE_PRIVATE const unsigned char sqlite3CtypeMap[];
10468	SQLITE_PRIVATE SQLITE_WSD struct Sqlite3Config sqlite3Config;
10469	SQLITE_PRIVATE SQLITE_WSD FuncDefHash sqlite3GlobalFunctions;
10470	SQLITE_PRIVATE int sqlite3PendingByte;
	@@ -10702,11 +10690,10 @@
10702	*************************************************************************
10703	**
10704	** This file contains definitions of global variables and contants.
10705	*/
10706
10707
10708	/* An array to map all upper-case characters into their corresponding
10709	** lower-case character.
10710	**
10711	** SQLite only considers US-ASCII (or EBCDIC) characters. We do not
10712	** handle case conversions for the UTF character set since the tables
	@@ -10758,10 +10745,11 @@
10758	** isalpha() 0x02
10759	** isdigit() 0x04
10760	** isalnum() 0x06
10761	** isxdigit() 0x08
10762	** toupper() 0x20

10763	**
10764	** Bit 0x20 is set if the mapped character requires translation to upper
10765	** case. i.e. if the character is a lower-case ASCII character.
10766	** If x is a lower-case ASCII character, then its upper-case equivalent
10767	** is (x - 0x20). Therefore toupper() can be implemented as:
	@@ -10768,10 +10756,15 @@
10768	**
10769	** (x & ~(map[x]&0x20))
10770	**
10771	** Standard function tolower() is implemented using the sqlite3UpperToLower[]
10772	** array. tolower() is used more often than toupper() by SQLite.





10773	**
10774	** SQLite's versions are identical to the standard versions assuming a
10775	** locale of "C". They are implemented as macros in sqliteInt.h.
10776	*/
10777	#ifdef SQLITE_ASCII
	@@ -10778,41 +10771,41 @@
10778	SQLITE_PRIVATE const unsigned char sqlite3CtypeMap[256] = {
10779	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 00..07 ........ */
10780	0x00, 0x01, 0x01, 0x01, 0x01, 0x01, 0x00, 0x00, /* 08..0f ........ */
10781	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 10..17 ........ */
10782	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 18..1f ........ */
10783	0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 20..27 !"#$%&' */
10784	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 28..2f ()+,-./ /
10785	0x0c, 0x0c, 0x0c, 0x0c, 0x0c, 0x0c, 0x0c, 0x0c, /* 30..37 01234567 */
10786	0x0c, 0x0c, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 38..3f 89:;<=>? */
10787
10788	0x00, 0x0a, 0x0a, 0x0a, 0x0a, 0x0a, 0x0a, 0x02, /* 40..47 @ABCDEFG */
10789	0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, /* 48..4f HIJKLMNO */
10790	0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, /* 50..57 PQRSTUVW */
10791	0x02, 0x02, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, /* 58..5f XYZ[\]^_ */
10792	0x00, 0x2a, 0x2a, 0x2a, 0x2a, 0x2a, 0x2a, 0x22, /* 60..67 `abcdefg */
10793	0x22, 0x22, 0x22, 0x22, 0x22, 0x22, 0x22, 0x22, /* 68..6f hijklmno */
10794	0x22, 0x22, 0x22, 0x22, 0x22, 0x22, 0x22, 0x22, /* 70..77 pqrstuvw */
10795	0x22, 0x22, 0x22, 0x00, 0x00, 0x00, 0x00, 0x00, /* 78..7f xyz{\|}~. */
10796
10797	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 80..87 ........ */
10798	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 88..8f ........ */
10799	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 90..97 ........ */
10800	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 98..9f ........ */
10801	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* a0..a7 ........ */
10802	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* a8..af ........ */
10803	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* b0..b7 ........ */
10804	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* b8..bf ........ */
10805
10806	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* c0..c7 ........ */
10807	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* c8..cf ........ */
10808	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* d0..d7 ........ */
10809	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* d8..df ........ */
10810	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* e0..e7 ........ */
10811	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* e8..ef ........ */
10812	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* f0..f7 ........ */
10813	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 /* f8..ff ........ */
10814	};
10815	#endif
10816
10817
10818
	@@ -10877,10 +10870,18 @@
10877	** Changing the pending byte during operating results in undefined
10878	** and dileterious behavior.
10879	*/
10880	SQLITE_PRIVATE int sqlite3PendingByte = 0x40000000;
10881








10882	/************ End of global.c ********************************************/
10883	/************ Begin file status.c ****************************************/
10884	/*
10885	** 2008 June 18
10886	**
	@@ -10893,12 +10894,10 @@
10893	**
10894	*************************************************************************
10895	**
10896	** This module implements the sqlite3_status() interface and related
10897	** functionality.
10898	**
10899	** $Id: status.c,v 1.9 2008/09/02 00:52:52 drh Exp $
10900	*/
10901
10902	/*
10903	** Variables in which to record status information.
10904	*/
	@@ -11021,12 +11020,10 @@
11021	**
11022	** There is only one exported symbol in this file - the function
11023	** sqlite3RegisterDateTimeFunctions() found at the bottom of the file.
11024	** All other code has file scope.
11025	**
11026	** $Id: date.c,v 1.107 2009/05/03 20:23:53 drh Exp $
11027	**
11028	** SQLite processes all times and dates as Julian Day numbers. The
11029	** dates and times are stored as the number of days since noon
11030	** in Greenwich on November 24, 4714 B.C. according to the Gregorian
11031	** calendar system.
11032	**
	@@ -12125,12 +12122,10 @@
12125	**
12126	******************************************************************************
12127	**
12128	** This file contains OS interface code that is common to all
12129	** architectures.
12130	**
12131	** $Id: os.c,v 1.127 2009/07/27 11:41:21 danielk1977 Exp $
12132	*/
12133	#define _SQLITE_OS_C_ 1
12134	#undef _SQLITE_OS_C_
12135
12136	/*
	@@ -12252,10 +12247,11 @@
12252	sqlite3_vfs *pVfs,
12253	const char *zPath,
12254	int nPathOut,
12255	char *zPathOut
12256	){

12257	return pVfs->xFullPathname(pVfs, zPath, nPathOut, zPathOut);
12258	}
12259	#ifndef SQLITE_OMIT_LOAD_EXTENSION
12260	SQLITE_PRIVATE void sqlite3OsDlOpen(sqlite3_vfs pVfs, const char *zPath){
12261	return pVfs->xDlOpen(pVfs, zPath);
	@@ -12423,14 +12419,10 @@
12423	** May you find forgiveness for yourself and forgive others.
12424	** May you share freely, never taking more than you give.
12425	**
12426	*************************************************************************
12427	**
12428	** $Id: fault.c,v 1.11 2008/09/02 00:52:52 drh Exp $
12429	*/
12430
12431	/*
12432	** This file contains code to support the concept of "benign"
12433	** malloc failures (when the xMalloc() or xRealloc() method of the
12434	** sqlite3_mem_methods structure fails to allocate a block of memory
12435	** and returns 0).
12436	**
	@@ -12521,12 +12513,10 @@
12521	** This file contains a no-op memory allocation drivers for use when
12522	** SQLITE_ZERO_MALLOC is defined. The allocation drivers implemented
12523	** here always fail. SQLite will not operate with these drivers. These
12524	** are merely placeholders. Real drivers must be substituted using
12525	** sqlite3_config() before SQLite will operate.
12526	**
12527	** $Id: mem0.c,v 1.1 2008/10/28 18:58:20 drh Exp $
12528	*/
12529
12530	/*
12531	** This version of the memory allocator is the default. It is
12532	** used when no other memory allocator is specified using compile-time
	@@ -12585,12 +12575,10 @@
12585	** SQLite will use the standard C-library malloc/realloc/free interface
12586	** to obtain the memory it needs.
12587	**
12588	** This file contains implementations of the low-level memory allocation
12589	** routines specified in the sqlite3_mem_methods object.
12590	**
12591	** $Id: mem1.c,v 1.30 2009/03/23 04:33:33 danielk1977 Exp $
12592	*/
12593
12594	/*
12595	** This version of the memory allocator is the default. It is
12596	** used when no other memory allocator is specified using compile-time
	@@ -12734,12 +12722,10 @@
12734	** information to each allocation in order to help detect and fix memory
12735	** leaks and memory usage errors.
12736	**
12737	** This file contains implementations of the low-level memory allocation
12738	** routines specified in the sqlite3_mem_methods object.
12739	**
12740	** $Id: mem2.c,v 1.45 2009/03/23 04:33:33 danielk1977 Exp $
12741	*/
12742
12743	/*
12744	** This version of the memory allocator is used only if the
12745	** SQLITE_MEMDEBUG macro is defined
	@@ -13183,12 +13169,10 @@
13183	** the amount of memory available to SQLite is fixed and cannot
13184	** be changed.
13185	**
13186	** This version of the memory allocation subsystem is included
13187	** in the build only if SQLITE_ENABLE_MEMSYS3 is defined.
13188	**
13189	** $Id: mem3.c,v 1.25 2008/11/19 16:52:44 danielk1977 Exp $
13190	*/
13191
13192	/*
13193	** This version of the memory allocator is only built into the library
13194	** SQLITE_ENABLE_MEMSYS3 is defined. Defining this symbol does not
	@@ -14442,13 +14426,10 @@
14442	**
14443	*************************************************************************
14444	** This file contains the C functions that implement mutexes.
14445	**
14446	** This file contains code that is common across all mutex implementations.
14447
14448	**
14449	** $Id: mutex.c,v 1.31 2009/07/16 18:21:18 drh Exp $
14450	*/
14451
14452	#if defined(SQLITE_DEBUG) && !defined(SQLITE_MUTEX_OMIT)
14453	/*
14454	** For debugging purposes, record when the mutex subsystem is initialized
	@@ -14609,12 +14590,10 @@
14609	** interface.
14610	**
14611	** If compiled with SQLITE_DEBUG, then additional logic is inserted
14612	** that does error checking on mutexes to make sure they are being
14613	** called correctly.
14614	**
14615	** $Id: mutex_noop.c,v 1.3 2008/12/05 17:17:08 drh Exp $
14616	*/
14617
14618
14619	#if defined(SQLITE_MUTEX_NOOP) && !defined(SQLITE_DEBUG)
14620	/*
	@@ -14783,12 +14762,10 @@
14783	** May you find forgiveness for yourself and forgive others.
14784	** May you share freely, never taking more than you give.
14785	**
14786	*************************************************************************
14787	** This file contains the C functions that implement mutexes for OS/2
14788	**
14789	** $Id: mutex_os2.c,v 1.11 2008/11/22 19:50:54 pweilbacher Exp $
14790	*/
14791
14792	/*
14793	** The code in this file is only used if SQLITE_MUTEX_OS2 is defined.
14794	** See the mutex.h file for details.
	@@ -15058,12 +15035,10 @@
15058	** May you find forgiveness for yourself and forgive others.
15059	** May you share freely, never taking more than you give.
15060	**
15061	*************************************************************************
15062	** This file contains the C functions that implement mutexes for pthreads
15063	**
15064	** $Id: mutex_unix.c,v 1.16 2008/12/08 18:19:18 drh Exp $
15065	*/
15066
15067	/*
15068	** The code in this file is only used if we are compiling threadsafe
15069	** under unix with pthreads.
	@@ -15389,12 +15364,10 @@
15389	** May you find forgiveness for yourself and forgive others.
15390	** May you share freely, never taking more than you give.
15391	**
15392	*************************************************************************
15393	** This file contains the C functions that implement mutexes for win32
15394	**
15395	** $Id: mutex_w32.c,v 1.18 2009/08/10 03:23:21 shane Exp $
15396	*/
15397
15398	/*
15399	** The code in this file is only used if we are compiling multithreaded
15400	** on a win32 system.
	@@ -15676,12 +15649,10 @@
15676	** May you share freely, never taking more than you give.
15677	**
15678	*************************************************************************
15679	**
15680	** Memory allocation functions used throughout sqlite.
15681	**
15682	** $Id: malloc.c,v 1.66 2009/07/17 11:44:07 drh Exp $
15683	*/
15684
15685	/*
15686	** This routine runs when the memory allocator sees that the
15687	** total memory allocation is about to exceed the soft heap
	@@ -16383,12 +16354,10 @@
16383	** the public domain. The original comments are included here for
16384	** completeness. They are very out-of-date but might be useful as
16385	** an historical reference. Most of the "enhancements" have been backed
16386	** out so that the functionality is now the same as standard printf().
16387	**
16388	** $Id: printf.c,v 1.104 2009/06/03 01:24:54 drh Exp $
16389	**
16390	**************************************************************************
16391	**
16392	** The following modules is an enhanced replacement for the "printf" subroutines
16393	** found in the standard C library. The following enhancements are
16394	** supported:
	@@ -17352,12 +17321,10 @@
17352	** This file contains code to implement a pseudo-random number
17353	** generator (PRNG) for SQLite.
17354	**
17355	** Random numbers are used by some of the database backends in order
17356	** to generate random integer keys for tables or random filenames.
17357	**
17358	** $Id: random.c,v 1.29 2008/12/10 19:26:24 drh Exp $
17359	*/
17360
17361
17362	/* All threads share a single random number generator.
17363	** This structure is the current state of the generator.
	@@ -17499,12 +17466,10 @@
17499	**
17500	*************************************************************************
17501	** This file contains routines used to translate between UTF-8,
17502	** UTF-16, UTF-16BE, and UTF-16LE.
17503	**
17504	** $Id: utf.c,v 1.73 2009/04/01 18:40:32 drh Exp $
17505	**
17506	** Notes on UTF-8:
17507	**
17508	** Byte-0 Byte-1 Byte-2 Byte-3 Value
17509	** 0xxxxxxx 00000000 00000000 0xxxxxxx
17510	** 110yyyyy 10xxxxxx 00000000 00000yyy yyxxxxxx
	@@ -17540,12 +17505,10 @@
17540	** This is the header file for information that is private to the
17541	** VDBE. This information used to all be at the top of the single
17542	** source code file "vdbe.c". When that file became too big (over
17543	** 6000 lines long) it was split up into several smaller files and
17544	** this header information was factored out.
17545	**
17546	** $Id: vdbeInt.h,v 1.174 2009/06/23 14:15:04 drh Exp $
17547	*/
17548	#ifndef _VDBEINT_H_
17549	#define _VDBEINT_H_
17550
17551	/*
	@@ -17907,11 +17870,10 @@
17907	SQLITE_PRIVATE int sqlite3VdbeMemFromBtree(BtCursor,int,int,int,Mem);
17908	SQLITE_PRIVATE void sqlite3VdbeMemRelease(Mem *p);
17909	SQLITE_PRIVATE void sqlite3VdbeMemReleaseExternal(Mem *p);
17910	SQLITE_PRIVATE int sqlite3VdbeMemFinalize(Mem, FuncDef);
17911	SQLITE_PRIVATE const char *sqlite3OpcodeName(int);
17912	SQLITE_PRIVATE int sqlite3VdbeOpcodeHasProperty(int, int);
17913	SQLITE_PRIVATE int sqlite3VdbeMemGrow(Mem *pMem, int n, int preserve);
17914	SQLITE_PRIVATE int sqlite3VdbeCloseStatement(Vdbe *, int);
17915	SQLITE_PRIVATE void sqlite3VdbeFrameDelete(VdbeFrame*);
17916	SQLITE_PRIVATE int sqlite3VdbeFrameRestore(VdbeFrame *);
17917	SQLITE_PRIVATE void sqlite3VdbeMemStoreType(Mem *pMem);
	@@ -19574,12 +19536,10 @@
19574	** May you share freely, never taking more than you give.
19575	**
19576	*************************************************************************
19577	** This is the implementation of generic hash-tables
19578	** used in SQLite.
19579	**
19580	** $Id: hash.c,v 1.38 2009/05/09 23:29:12 drh Exp $
19581	*/
19582
19583	/* Turn bulk memory into a hash table object by initializing the
19584	** fields of the Hash structure.
19585	**
	@@ -20011,12 +19971,10 @@
20011	** May you share freely, never taking more than you give.
20012	**
20013	******************************************************************************
20014	**
20015	** This file contains code that is specific to OS/2.
20016	**
20017	** $Id: os_os2.c,v 1.63 2008/12/10 19:26:24 drh Exp $
20018	*/
20019
20020
20021	#if SQLITE_OS_OS2
20022
	@@ -20074,12 +20032,10 @@
20074	** all of the platform-specific files (os_*.c) and is #included into those
20075	** files.
20076	**
20077	** This file should be #included by the os_*.c files only. It is not a
20078	** general purpose header file.
20079	**
20080	** $Id: os_common.h,v 1.38 2009/02/24 18:40:50 danielk1977 Exp $
20081	*/
20082	#ifndef _OS_COMMON_H_
20083	#define _OS_COMMON_H_
20084
20085	/*
	@@ -20136,12 +20092,10 @@
20136	**
20137	******************************************************************************
20138	**
20139	** This file contains inline asm code for retrieving "high-performance"
20140	** counters for x86 class CPUs.
20141	**
20142	** $Id: hwtime.h,v 1.3 2008/08/01 14:33:15 shane Exp $
20143	*/
20144	#ifndef _HWTIME_H_
20145	#define _HWTIME_H_
20146
20147	/*
	@@ -21603,12 +21557,10 @@
21603	** all of the platform-specific files (os_*.c) and is #included into those
21604	** files.
21605	**
21606	** This file should be #included by the os_*.c files only. It is not a
21607	** general purpose header file.
21608	**
21609	** $Id: os_common.h,v 1.38 2009/02/24 18:40:50 danielk1977 Exp $
21610	*/
21611	#ifndef _OS_COMMON_H_
21612	#define _OS_COMMON_H_
21613
21614	/*
	@@ -21665,12 +21617,10 @@
21665	**
21666	******************************************************************************
21667	**
21668	** This file contains inline asm code for retrieving "high-performance"
21669	** counters for x86 class CPUs.
21670	**
21671	** $Id: hwtime.h,v 1.3 2008/08/01 14:33:15 shane Exp $
21672	*/
21673	#ifndef _HWTIME_H_
21674	#define _HWTIME_H_
21675
21676	/*
	@@ -24685,10 +24635,23 @@
24685	rc = ftruncate(((unixFile*)id)->h, (off_t)nByte);
24686	if( rc ){
24687	((unixFile*)id)->lastErrno = errno;
24688	return SQLITE_IOERR_TRUNCATE;
24689	}else{













24690	return SQLITE_OK;
24691	}
24692	}
24693
24694	/*
	@@ -27042,12 +27005,10 @@
27042	** all of the platform-specific files (os_*.c) and is #included into those
27043	** files.
27044	**
27045	** This file should be #included by the os_*.c files only. It is not a
27046	** general purpose header file.
27047	**
27048	** $Id: os_common.h,v 1.38 2009/02/24 18:40:50 danielk1977 Exp $
27049	*/
27050	#ifndef _OS_COMMON_H_
27051	#define _OS_COMMON_H_
27052
27053	/*
	@@ -27104,12 +27065,10 @@
27104	**
27105	******************************************************************************
27106	**
27107	** This file contains inline asm code for retrieving "high-performance"
27108	** counters for x86 class CPUs.
27109	**
27110	** $Id: hwtime.h,v 1.3 2008/08/01 14:33:15 shane Exp $
27111	*/
27112	#ifndef _HWTIME_H_
27113	#define _HWTIME_H_
27114
27115	/*
	@@ -29147,12 +29106,10 @@
29147	** 5 and 500 set operations per Bitvec object, though the number of sets can
29148	** sometimes grow into tens of thousands or larger. The size of the
29149	** Bitvec object is the number of pages in the database file at the
29150	** start of a transaction, and is thus usually less than a few thousand,
29151	** but can be as large as 2 billion for a really big database.
29152	**
29153	** @(#) $Id: bitvec.c,v 1.17 2009/07/25 17:33:26 drh Exp $
29154	*/
29155
29156	/* Size of the Bitvec structure in bytes. */
29157	#define BITVEC_SZ (sizeof(void)128) /* 512 on 32bit. 1024 on 64bit */
29158
	@@ -29536,12 +29493,10 @@
29536	** May you find forgiveness for yourself and forgive others.
29537	** May you share freely, never taking more than you give.
29538	**
29539	*************************************************************************
29540	** This file implements that page cache.
29541	**
29542	** @(#) $Id: pcache.c,v 1.47 2009/07/25 11:46:49 danielk1977 Exp $
29543	*/
29544
29545	/*
29546	** A complete page cache is an instance of this structure.
29547	*/
	@@ -30119,12 +30074,10 @@
30119	** This file implements the default page cache implementation (the
30120	** sqlite3_pcache interface). It also contains part of the implementation
30121	** of the SQLITE_CONFIG_PAGECACHE and sqlite3_release_memory() features.
30122	** If the default page cache implementation is overriden, then neither of
30123	** these two features are available.
30124	**
30125	** @(#) $Id: pcache1.c,v 1.19 2009/07/17 11:44:07 drh Exp $
30126	*/
30127
30128
30129	typedef struct PCache1 PCache1;
30130	typedef struct PgHdr1 PgHdr1;
	@@ -30770,19 +30723,11 @@
30770
30771	h = iNew%pCache->nHash;
30772	pPage->iKey = iNew;
30773	pPage->pNext = pCache->apHash[h];
30774	pCache->apHash[h] = pPage;
30775
30776	/* The xRekey() interface is only used to move pages earlier in the
30777	** database file (in order to move all free pages to the end of the
30778	** file where they can be truncated off.) Hence, it is not possible
30779	** for the new page number to be greater than the largest previously
30780	** fetched page. But we retain the following test in case xRekey()
30781	** begins to be used in different ways in the future.
30782	*/
30783	if( NEVER(iNew>pCache->iMaxKey) ){
30784	pCache->iMaxKey = iNew;
30785	}
30786
30787	pcache1LeaveMutex();
30788	}
	@@ -30954,12 +30899,10 @@
30954	** of the first SMALLEST is O(NlogN). Second and subsequent SMALLEST
30955	** primitives are constant time. The cost of DESTROY is O(N).
30956	**
30957	** There is an added cost of O(N) when switching between TEST and
30958	** SMALLEST primitives.
30959	**
30960	** $Id: rowset.c,v 1.7 2009/05/22 01:00:13 drh Exp $
30961	*/
30962
30963
30964	/*
30965	** Target size for allocation chunks.
	@@ -31338,12 +31281,10 @@
31338	** atomic commit and rollback through the use of a journal file that
31339	** is separate from the database file. The pager also implements file
31340	** locking to prevent two processes from writing the same database
31341	** file simultaneously, or one process from reading the database while
31342	** another is writing.
31343	**
31344	** @(#) $Id: pager.c,v 1.629 2009/08/10 17:48:57 drh Exp $
31345	*/
31346	#ifndef SQLITE_OMIT_DISKIO
31347
31348	/*
31349	** Macros for troubleshooting. Normally turned off
	@@ -32233,14 +32174,14 @@
32233	return rc;
32234	}
32235
32236	/* Check that the values read from the page-size and sector-size fields
32237	** are within range. To be 'in range', both values need to be a power
32238	** of two greater than or equal to 512, and not greater than their
32239	** respective compile time maximum limits.
32240	*/
32241	if( iPageSize<512 \|\| iSectorSize<512
32242	\|\| iPageSize>SQLITE_MAX_PAGE_SIZE \|\| iSectorSize>MAX_SECTOR_SIZE
32243	\|\| ((iPageSize-1)&iPageSize)!=0 \|\| ((iSectorSize-1)&iSectorSize)!=0
32244	){
32245	/* If the either the page-size or sector-size in the journal-header is
32246	** invalid, then the process that wrote the journal-header must have
	@@ -32743,28 +32684,28 @@
32743	){
32744	int rc;
32745	PgHdr pPg; / An existing page in the cache */
32746	Pgno pgno; /* The page number of a page in journal */
32747	u32 cksum; /* Checksum used for sanity checking */
32748	u8 aData; / Temporary storage for the page */
32749	sqlite3_file jfd; / The file descriptor for the journal file */
32750
32751	assert( (isMainJrnl&~1)==0 ); /* isMainJrnl is 0 or 1 */
32752	assert( (isSavepnt&~1)==0 ); /* isSavepnt is 0 or 1 */
32753	assert( isMainJrnl \|\| pDone ); /* pDone always used on sub-journals */
32754	assert( isSavepnt \|\| pDone==0 ); /* pDone never used on non-savepoint */
32755
32756	aData = (u8*)pPager->pTmpSpace;
32757	assert( aData ); /* Temp storage must have already been allocated */
32758
32759	/* Read the page number and page data from the journal or sub-journal
32760	** file. Return an error code to the caller if an IO error occurs.
32761	*/
32762	jfd = isMainJrnl ? pPager->jfd : pPager->sjfd;
32763	rc = read32bits(jfd, *pOffset, &pgno);
32764	if( rc!=SQLITE_OK ) return rc;
32765	rc = sqlite3OsRead(jfd, aData, pPager->pageSize, (*pOffset)+4);
32766	if( rc!=SQLITE_OK ) return rc;
32767	pOffset += pPager->pageSize + 4 + isMainJrnl4;
32768
32769	/* Sanity checking on the page. This is more important that I originally
32770	** thought. If a power failure occurs while the journal is being written,
	@@ -32779,11 +32720,11 @@
32779	return SQLITE_OK;
32780	}
32781	if( isMainJrnl ){
32782	rc = read32bits(jfd, (*pOffset)-4, &cksum);
32783	if( rc ) return rc;
32784	if( !isSavepnt && pager_cksum(pPager, aData)!=cksum ){
32785	return SQLITE_DONE;
32786	}
32787	}
32788
32789	if( pDone && (rc = sqlite3BitvecSet(pDone, pgno))!=SQLITE_OK ){
	@@ -32825,27 +32766,27 @@
32825	** Do not attempt to write if database file has never been opened.
32826	*/
32827	pPg = pager_lookup(pPager, pgno);
32828	assert( pPg \|\| !MEMDB );
32829	PAGERTRACE(("PLAYBACK %d page %d hash(%08x) %s\n",
32830	PAGERID(pPager), pgno, pager_datahash(pPager->pageSize, aData),
32831	(isMainJrnl?"main-journal":"sub-journal")
32832	));
32833	if( (pPager->state>=PAGER_EXCLUSIVE)
32834	&& (pPg==0 \|\| 0==(pPg->flags&PGHDR_NEED_SYNC))
32835	&& isOpen(pPager->fd)
32836	&& !isUnsync
32837	){
32838	i64 ofst = (pgno-1)*(i64)pPager->pageSize;
32839	rc = sqlite3OsWrite(pPager->fd, aData, pPager->pageSize, ofst);
32840	if( pgno>pPager->dbFileSize ){
32841	pPager->dbFileSize = pgno;
32842	}
32843	if( pPager->pBackup ){
32844	CODEC1(pPager, aData, pgno, 3, rc=SQLITE_NOMEM);
32845	sqlite3BackupUpdate(pPager->pBackup, pgno, aData);
32846	CODEC1(pPager, aData, pgno, 0, rc=SQLITE_NOMEM);
32847	}
32848	}else if( !isMainJrnl && pPg==0 ){
32849	/* If this is a rollback of a savepoint and data was not written to
32850	** the database and the page is not in-memory, there is a potential
32851	** problem. When the page is next fetched by the b-tree layer, it
	@@ -32876,11 +32817,11 @@
32876	** of an internal error resulting in an automatic call to
32877	** sqlite3PagerRollback().
32878	*/
32879	void *pData;
32880	pData = pPg->pData;
32881	memcpy(pData, aData, pPager->pageSize);
32882	pPager->xReiniter(pPg);
32883	if( isMainJrnl && (!isSavepnt \|\| *pOffset<=pPager->journalHdr) ){
32884	/* If the contents of this page were just restored from the main
32885	** journal file, then its content must be as they were when the
32886	** transaction was first opened. In this case we can mark the page
	@@ -33101,12 +33042,12 @@
33101	** master journal pointers within created journal files.
33102	**
33103	** For temporary files the effective sector size is always 512 bytes.
33104	**
33105	** Otherwise, for non-temporary files, the effective sector size is
33106	** the value returned by the xSectorSize() method rounded up to 512 if
33107	** it is less than 512, or rounded down to MAX_SECTOR_SIZE if it
33108	** is greater than MAX_SECTOR_SIZE.
33109	*/
33110	static void setSectorSize(Pager *pPager){
33111	assert( isOpen(pPager->fd) \|\| pPager->tempFile );
33112
	@@ -33115,11 +33056,11 @@
33115	** may not have been opened yet, in which case the OsSectorSize()
33116	** call will segfault.
33117	*/
33118	pPager->sectorSize = sqlite3OsSectorSize(pPager->fd);
33119	}
33120	if( pPager->sectorSize<512 ){
33121	pPager->sectorSize = 512;
33122	}
33123	if( pPager->sectorSize>MAX_SECTOR_SIZE ){
33124	assert( MAX_SECTOR_SIZE>=512 );
33125	pPager->sectorSize = MAX_SECTOR_SIZE;
	@@ -34193,11 +34134,13 @@
34193	** than Pager.dbSize, this means sqlite3PagerTruncateImage() was called to
34194	** make the file smaller (presumably by auto-vacuum code). Do not write
34195	** any such pages to the file.
34196	**
34197	** Also, do not write out any page that has the PGHDR_DONT_WRITE flag
34198	** set (set by sqlite3PagerDontWrite()).


34199	*/
34200	if( pgno<=pPager->dbSize && 0==(pList->flags&PGHDR_DONT_WRITE) ){
34201	i64 offset = (pgno-1)(i64)pPager->pageSize; / Offset to write */
34202	char pData; / Data to write */
34203
	@@ -35702,10 +35645,11 @@
35702	SQLITE_PRIVATE int sqlite3PagerIswriteable(DbPage *pPg){
35703	return pPg->flags&PGHDR_DIRTY;
35704	}
35705	#endif
35706

35707	/*
35708	** A call to this routine tells the pager that it is not necessary to
35709	** write the information on page pPg back to the disk, even though
35710	** that page might be marked as dirty. This happens, for example, when
35711	** the page has been added as a leaf of the freelist and so its
	@@ -35727,10 +35671,11 @@
35727	#ifdef SQLITE_CHECK_PAGES
35728	pPg->pageHash = pager_pagehash(pPg);
35729	#endif
35730	}
35731	}

35732
35733	/*
35734	** This routine is called to increment the value of the database file
35735	** change-counter, stored as a 4-byte big-endian integer starting at
35736	** byte offset 24 of the pager file.
	@@ -35766,11 +35711,11 @@
35766	#else
35767	# define DIRECT_MODE isDirectMode
35768	#endif
35769
35770	assert( pPager->state>=PAGER_RESERVED );
35771	if( !pPager->changeCountDone && ALWAYS(pPager->dbSize>0) ){
35772	PgHdr pPgHdr; / Reference to page 1 */
35773	u32 change_counter; /* Initial value of change-counter field */
35774
35775	assert( !pPager->tempFile && isOpen(pPager->fd) );
35776
	@@ -36418,10 +36363,18 @@
36418	Pgno needSyncPgno = 0; /* Old value of pPg->pgno, if sync is required */
36419	int rc; /* Return code */
36420	Pgno origPgno; /* The original page number */
36421
36422	assert( pPg->nRef>0 );








36423
36424	/* If the page being moved is dirty and has not been saved by the latest
36425	** savepoint, then save the current contents of the page into the
36426	** sub-journal now. This is required to handle the following scenario:
36427	**
	@@ -36437,11 +36390,11 @@
36437	**
36438	** subjournalPage() may need to allocate space to store pPg->pgno into
36439	** one or more savepoint bitvecs. This is the reason this function
36440	** may return SQLITE_NOMEM.
36441	*/
36442	if( pPg->flags&PGHDR_DIRTY
36443	&& subjRequiresPage(pPg)
36444	&& SQLITE_OK!=(rc = subjournalPage(pPg))
36445	){
36446	return rc;
36447	}
	@@ -36472,11 +36425,18 @@
36472	pPg->flags &= ~PGHDR_NEED_SYNC;
36473	pPgOld = pager_lookup(pPager, pgno);
36474	assert( !pPgOld \|\| pPgOld->nRef==1 );
36475	if( pPgOld ){
36476	pPg->flags \|= (pPgOld->flags&PGHDR_NEED_SYNC);
36477	sqlite3PcacheDrop(pPgOld);







36478	}
36479
36480	origPgno = pPg->pgno;
36481	sqlite3PcacheMove(pPg, pgno);
36482	sqlite3PcacheMakeDirty(pPg);
	@@ -36517,22 +36477,16 @@
36517	sqlite3PagerUnref(pPgHdr);
36518	}
36519
36520	/*
36521	** For an in-memory database, make sure the original page continues
36522	** to exist, in case the transaction needs to roll back. We allocate
36523	** the page now, instead of at rollback, because we can better deal
36524	** with an out-of-memory error now. Ticket #3761.
36525	*/
36526	if( MEMDB ){
36527	DbPage *pNew;
36528	rc = sqlite3PagerAcquire(pPager, origPgno, &pNew, 1);
36529	if( rc!=SQLITE_OK ){
36530	sqlite3PcacheMove(pPg, origPgno);
36531	return rc;
36532	}
36533	sqlite3PagerUnref(pNew);
36534	}
36535
36536	return SQLITE_OK;
36537	}
36538	#endif
	@@ -36655,12 +36609,10 @@
36655	** May you find forgiveness for yourself and forgive others.
36656	** May you share freely, never taking more than you give.
36657	**
36658	*************************************************************************
36659	**
36660	** $Id: btmutex.c,v 1.17 2009/07/20 12:33:33 drh Exp $
36661	**
36662	** This file contains code used to implement mutexes on Btree objects.
36663	** This code really belongs in btree.c. But btree.c is getting too
36664	** big and we want to break it down some. This packaged seemed like
36665	** a good breakout.
36666	*/
	@@ -36675,12 +36627,10 @@
36675	** May you do good and not evil.
36676	** May you find forgiveness for yourself and forgive others.
36677	** May you share freely, never taking more than you give.
36678	**
36679	*************************************************************************
36680	** $Id: btreeInt.h,v 1.52 2009/07/15 17:25:46 drh Exp $
36681	**
36682	** This file implements a external (disk-based) database using BTrees.
36683	** For a detailed discussion of BTrees, refer to
36684	**
36685	** Donald E. Knuth, THE ART OF COMPUTER PROGRAMMING, Volume 3:
36686	** "Sorting And Searching", pages 473-480. Addison-Wesley
	@@ -37660,12 +37610,10 @@
37660	** May you do good and not evil.
37661	** May you find forgiveness for yourself and forgive others.
37662	** May you share freely, never taking more than you give.
37663	**
37664	*************************************************************************
37665	** $Id: btree.c,v 1.705 2009/08/10 03:57:58 shane Exp $
37666	**
37667	** This file implements a external (disk-based) database using BTrees.
37668	** See the header comment on "btreeInt.h" for additional information.
37669	** Including a description of file format and an overview of operation.
37670	*/
37671
	@@ -40930,12 +40878,12 @@
40930	**
40931	** No checking is done to make sure that page iTable really is the
40932	** root page of a b-tree. If it is not, then the cursor acquired
40933	** will not work correctly.
40934	**
40935	** It is assumed that the sqlite3BtreeCursorSize() bytes of memory
40936	** pointed to by pCur have been zeroed by the caller.
40937	*/
40938	static int btreeCursor(
40939	Btree p, / The btree */
40940	int iTable, /* Root page of table to open */
40941	int wrFlag, /* 1 to write. 0 read-only */
	@@ -41004,11 +40952,23 @@
41004	** sufficient storage to hold a cursor. The BtCursor object is opaque
41005	** to users so they cannot do the sizeof() themselves - they must call
41006	** this routine.
41007	*/
41008	SQLITE_PRIVATE int sqlite3BtreeCursorSize(void){
41009	return sizeof(BtCursor);












41010	}
41011
41012	/*
41013	** Set the cached rowid value of every cursor in the same database file
41014	** as pCur and having the same root page number as pCur. The value is
	@@ -45489,12 +45449,10 @@
45489	** May you share freely, never taking more than you give.
45490	**
45491	*************************************************************************
45492	** This file contains the implementation of the sqlite3_backup_XXX()
45493	** API functions and the related features.
45494	**
45495	** $Id: backup.c,v 1.19 2009/07/06 19:03:13 drh Exp $
45496	*/
45497
45498	/* Macro to find the minimum of two numeric values.
45499	*/
45500	#ifndef MIN
	@@ -46123,12 +46081,10 @@
46123	**
46124	** This file contains code use to manipulate "Mem" structure. A "Mem"
46125	** stores a single value in the VDBE. Mem is an opaque structure visible
46126	** only within the VDBE. Interface routines refer to a Mem using the
46127	** name sqlite_value
46128	**
46129	** $Id: vdbemem.c,v 1.152 2009/07/22 18:07:41 drh Exp $
46130	*/
46131
46132	/*
46133	** Call sqlite3VdbeMemExpandBlob() on the supplied value (type Mem*)
46134	** P if required.
	@@ -46863,13 +46819,10 @@
46863	SQLITE_PRIVATE int sqlite3MemCompare(const Mem pMem1, const Mem pMem2, const CollSeq *pColl){
46864	int rc;
46865	int f1, f2;
46866	int combined_flags;
46867
46868	/* Interchange pMem1 and pMem2 if the collating sequence specifies
46869	** DESC order.
46870	*/
46871	f1 = pMem1->flags;
46872	f2 = pMem2->flags;
46873	combined_flags = f1\|f2;
46874	assert( (combined_flags & MEM_RowSet)==0 );
46875
	@@ -47116,11 +47069,11 @@
47116	*ppVal = 0;
47117	return SQLITE_OK;
47118	}
47119	op = pExpr->op;
47120	if( op==TK_REGISTER ){
47121	op = pExpr->op2;
47122	}
47123
47124	if( op==TK_STRING \|\| op==TK_FLOAT \|\| op==TK_INTEGER ){
47125	pVal = sqlite3ValueNew(db);
47126	if( pVal==0 ) goto no_mem;
	@@ -47229,12 +47182,10 @@
47229	*************************************************************************
47230	** This file contains code used for creating, destroying, and populating
47231	** a VDBE (or an "sqlite3_stmt" as it is known to the outside world.) Prior
47232	** to version 2.8.7, all this code was combined into the vdbe.c source file.
47233	** But that file was getting too big so this subroutines were split out.
47234	**
47235	** $Id: vdbeaux.c,v 1.480 2009/08/08 18:01:08 drh Exp $
47236	*/
47237
47238
47239
47240	/*
	@@ -47410,10 +47361,26 @@
47410	){
47411	int addr = sqlite3VdbeAddOp3(p, op, p1, p2, p3);
47412	sqlite3VdbeChangeP4(p, addr, zP4, p4type);
47413	return addr;
47414	}
















47415
47416	/*
47417	** Create a new symbolic label for an instruction that has yet to be
47418	** coded. The symbolic label is really just a negative number. The
47419	** label can be used as the P2 value of an operation. Later, when
	@@ -47586,10 +47553,12 @@
47586	** This routine is called once after all opcodes have been inserted.
47587	**
47588	** Variable *pMaxFuncArgs is set to the maximum value of any P2 argument
47589	** to an OP_Function, OP_AggStep or OP_VFilter opcode. This is used by
47590	** sqlite3VdbeMakeReady() to size the Vdbe.apArg[] array.


47591	*/
47592	static void resolveP2Values(Vdbe p, int pMaxFuncArgs){
47593	int i;
47594	int nMaxArgs = *pMaxFuncArgs;
47595	Op *pOp;
	@@ -47596,29 +47565,28 @@
47596	int *aLabel = p->aLabel;
47597	p->readOnly = 1;
47598	for(pOp=p->aOp, i=p->nOp-1; i>=0; i--, pOp++){
47599	u8 opcode = pOp->opcode;
47600

47601	if( opcode==OP_Function \|\| opcode==OP_AggStep ){
47602	if( pOp->p5>nMaxArgs ) nMaxArgs = pOp->p5;


47603	#ifndef SQLITE_OMIT_VIRTUALTABLE
47604	}else if( opcode==OP_VUpdate ){
47605	if( pOp->p2>nMaxArgs ) nMaxArgs = pOp->p2;
47606	#endif
47607	}else if( opcode==OP_Transaction && pOp->p2!=0 ){
47608	p->readOnly = 0;
47609	#ifndef SQLITE_OMIT_VIRTUALTABLE
47610	}else if( opcode==OP_VFilter ){
47611	int n;
47612	assert( p->nOp - i >= 3 );
47613	assert( pOp[-1].opcode==OP_Integer );
47614	n = pOp[-1].p1;
47615	if( n>nMaxArgs ) nMaxArgs = n;
47616	#endif
47617	}
47618
47619	if( sqlite3VdbeOpcodeHasProperty(opcode, OPFLG_JUMP) && pOp->p2<0 ){
47620	assert( -1-pOp->p2<p->nLabel );
47621	pOp->p2 = aLabel[-1-pOp->p2];
47622	}
47623	}
47624	sqlite3DbFree(p->db, p->aLabel);
	@@ -47676,11 +47644,11 @@
47676	for(i=0; i<nOp; i++, pIn++){
47677	int p2 = pIn->p2;
47678	VdbeOp *pOut = &p->aOp[i+addr];
47679	pOut->opcode = pIn->opcode;
47680	pOut->p1 = pIn->p1;
47681	if( p2<0 && sqlite3VdbeOpcodeHasProperty(pOut->opcode, OPFLG_JUMP) ){
47682	pOut->p2 = addr + ADDR(p2);
47683	}else{
47684	pOut->p2 = p2;
47685	}
47686	pOut->p3 = pIn->p3;
	@@ -49986,13 +49954,21 @@
49986	Mem mem1;
49987
49988	pKeyInfo = pPKey2->pKeyInfo;
49989	mem1.enc = pKeyInfo->enc;
49990	mem1.db = pKeyInfo->db;
49991	mem1.flags = 0;
49992	mem1.u.i = 0; /* not needed, here to silence compiler warning */
49993	mem1.zMalloc = 0;








49994
49995	idx1 = getVarint32(aKey1, szHdr1);
49996	d1 = szHdr1;
49997	if( pPKey2->flags & UNPACKED_IGNORE_ROWID ){
49998	szHdr1--;
	@@ -50012,51 +49988,56 @@
50012	/* Do the comparison
50013	*/
50014	rc = sqlite3MemCompare(&mem1, &pPKey2->aMem[i],
50015	i<nField ? pKeyInfo->aColl[i] : 0);
50016	if( rc!=0 ){
50017	break;



















50018	}
50019	i++;
50020	}
50021
50022	/* No memory allocation is ever used on mem1. */
50023	if( NEVER(mem1.zMalloc) ) sqlite3VdbeMemRelease(&mem1);
50024
50025	/* If the PREFIX_SEARCH flag is set and all fields except the final
50026	** rowid field were equal, then clear the PREFIX_SEARCH flag and set
50027	** pPKey2->rowid to the value of the rowid field in (pKey1, nKey1).
50028	** This is used by the OP_IsUnique opcode.
50029	*/
50030	if( (pPKey2->flags & UNPACKED_PREFIX_SEARCH) && i==(pPKey2->nField-1) ){
50031	assert( idx1==szHdr1 && rc );
50032	assert( mem1.flags & MEM_Int );
50033	pPKey2->flags &= ~UNPACKED_PREFIX_SEARCH;
50034	pPKey2->rowid = mem1.u.i;
50035	}
50036
50037	if( rc==0 ){
50038	/* rc==0 here means that one of the keys ran out of fields and
50039	** all the fields up to that point were equal. If the UNPACKED_INCRKEY
50040	** flag is set, then break the tie by treating key2 as larger.
50041	** If the UPACKED_PREFIX_MATCH flag is set, then keys with common prefixes
50042	** are considered to be equal. Otherwise, the longer key is the
50043	** larger. As it happens, the pPKey2 will always be the longer
50044	** if there is a difference.
50045	*/
50046	if( pPKey2->flags & UNPACKED_INCRKEY ){
50047	rc = -1;
50048	}else if( pPKey2->flags & UNPACKED_PREFIX_MATCH ){
50049	/* Leave rc==0 */
50050	}else if( idx1<szHdr1 ){
50051	rc = 1;
50052	}
50053	}else if( pKeyInfo->aSortOrder && i<pKeyInfo->nField
50054	&& pKeyInfo->aSortOrder[i] ){
50055	rc = -rc;
50056	}
50057
50058	return rc;
50059	}
50060
50061
50062	/*
	@@ -50272,12 +50253,10 @@
50272	**
50273	*************************************************************************
50274	**
50275	** This file contains code use to implement APIs that are part of the
50276	** VDBE.
50277	**
50278	** $Id: vdbeapi.c,v 1.167 2009/06/25 01:47:12 drh Exp $
50279	*/
50280
50281	#ifndef SQLITE_OMIT_DEPRECATED
50282	/*
50283	** Return TRUE (non-zero) of the statement supplied as an argument needs
	@@ -51528,12 +51507,10 @@
51528	** Various scripts scan this source file in order to generate HTML
51529	** documentation, headers files, or other derived files. The formatting
51530	** of the code in this file is, therefore, important. See other comments
51531	** in this file for details. If in doubt, do not deviate from existing
51532	** commenting and indentation practices when changing or adding code.
51533	**
51534	** $Id: vdbe.c,v 1.874 2009/07/24 17:58:53 danielk1977 Exp $
51535	*/
51536
51537	/*
51538	** The following global variable is incremented every time a cursor
51539	** moves, either by the OP_SeekXX, OP_Next, or OP_Prev opcodes. The test
	@@ -51656,27 +51633,10 @@
51656	}else{
51657	pMem->type = SQLITE_BLOB;
51658	}
51659	}
51660
51661	/*
51662	** Properties of opcodes. The OPFLG_INITIALIZER macro is
51663	** created by mkopcodeh.awk during compilation. Data is obtained
51664	** from the comments following the "case OP_xxxx:" statements in
51665	** this file.
51666	*/
51667	static const unsigned char opcodeProperty[] = OPFLG_INITIALIZER;
51668
51669	/*
51670	** Return true if an opcode has any of the OPFLG_xxx properties
51671	** specified by mask.
51672	*/
51673	SQLITE_PRIVATE int sqlite3VdbeOpcodeHasProperty(int opcode, int mask){
51674	assert( opcode>0 && opcode<(int)sizeof(opcodeProperty) );
51675	return (opcodeProperty[opcode]&mask)!=0;
51676	}
51677
51678	/*
51679	** Allocate VdbeCursor number iCur. Return a pointer to it. Return NULL
51680	** if we run out of memory.
51681	*/
51682	static VdbeCursor *allocateCursor(
	@@ -51707,11 +51667,11 @@
51707	Mem *pMem = &p->aMem[p->nMem-iCur];
51708
51709	int nByte;
51710	VdbeCursor *pCx = 0;
51711	nByte =
51712	sizeof(VdbeCursor) +
51713	(isBtreeCursor?sqlite3BtreeCursorSize():0) +
51714	2nFieldsizeof(u32);
51715
51716	assert( iCur<p->nCursor );
51717	if( p->apCsr[iCur] ){
	@@ -51718,19 +51678,20 @@
51718	sqlite3VdbeFreeCursor(p, p->apCsr[iCur]);
51719	p->apCsr[iCur] = 0;
51720	}
51721	if( SQLITE_OK==sqlite3VdbeMemGrow(pMem, nByte, 0) ){
51722	p->apCsr[iCur] = pCx = (VdbeCursor*)pMem->z;
51723	memset(pMem->z, 0, nByte);
51724	pCx->iDb = iDb;
51725	pCx->nField = nField;
51726	if( nField ){
51727	pCx->aType = (u32 *)&pMem->z[sizeof(VdbeCursor)];
51728	}
51729	if( isBtreeCursor ){
51730	pCx->pCursor = (BtCursor*)
51731	&pMem->z[sizeof(VdbeCursor)+2nFieldsizeof(u32)];

51732	}
51733	}
51734	return pCx;
51735	}
51736
	@@ -51968,12 +51929,10 @@
51968	**
51969	******************************************************************************
51970	**
51971	** This file contains inline asm code for retrieving "high-performance"
51972	** counters for x86 class CPUs.
51973	**
51974	** $Id: hwtime.h,v 1.3 2008/08/01 14:33:15 shane Exp $
51975	*/
51976	#ifndef _HWTIME_H_
51977	#define _HWTIME_H_
51978
51979	/*
	@@ -52132,28 +52091,31 @@
52132	*/
52133	SQLITE_PRIVATE int sqlite3VdbeExec(
52134	Vdbe p / The VDBE */
52135	){
52136	int pc; /* The program counter */

52137	Op pOp; / Current operation */
52138	int rc = SQLITE_OK; /* Value to return */
52139	sqlite3 db = p->db; / The database */

52140	u8 encoding = ENC(db); /* The database encoding */





52141	Mem pIn1 = 0; / 1st input operand */
52142	Mem pIn2 = 0; / 2nd input operand */
52143	Mem pIn3 = 0; / 3rd input operand */
52144	Mem pOut = 0; / Output operand */
52145	u8 opProperty;
52146	int iCompare = 0; /* Result of last OP_Compare operation */
52147	int aPermute = 0; / Permutation of columns for OP_Compare */
52148	#ifdef VDBE_PROFILE
52149	u64 start; /* CPU clock count at start of opcode */
52150	int origPc; /* Program counter at start of opcode */
52151	#endif
52152	#ifndef SQLITE_OMIT_PROGRESS_CALLBACK
52153	int nProgressOps = 0; /* Opcodes executed since progress callback. */
52154	#endif
52155	/********************************************************************
52156	** Automatically generated code
52157	**
52158	** The following union is automatically generated by the
52159	** vdbe-compress.tcl script. The purpose of this union is to
	@@ -52245,14 +52207,12 @@
52245	int szHdr; /* Size of the header size field at start of record */
52246	int avail; /* Number of bytes of available data */
52247	Mem pReg; / PseudoTable input register */
52248	} am;
52249	struct OP_Affinity_stack_vars {
52250	char zAffinity; / The affinity to be applied */
52251	Mem pData0; / First register to which to apply affinity */
52252	Mem pLast; / Last register to which to apply affinity */
52253	Mem pRec; / Current register */
52254	} an;
52255	struct OP_MakeRecord_stack_vars {
52256	u8 zNewRecord; / A buffer to hold the data for the new record */
52257	Mem pRec; / The new record */
52258	u64 nData; /* Number of bytes of data space */
	@@ -52333,18 +52293,19 @@
52333	struct OP_Found_stack_vars {
52334	int alreadyExists;
52335	VdbeCursor *pC;
52336	int res;
52337	UnpackedRecord *pIdxKey;

52338	char aTempRec[ROUND8(sizeof(UnpackedRecord)) + sizeof(Mem)*3 + 7];
52339	} bb;
52340	struct OP_IsUnique_stack_vars {
52341	u16 ii;
52342	VdbeCursor *pCx;
52343	BtCursor *pCrsr;
52344	u16 nField;
52345	Mem *aMem;
52346	UnpackedRecord r; /* B-Tree index search key */
52347	i64 R; /* Rowid stored in register P3 */
52348	} bc;
52349	struct OP_NotExists_stack_vars {
52350	VdbeCursor *pC;
	@@ -52453,63 +52414,58 @@
52453	int j; /* Loop counter */
52454	int nErr; /* Number of errors reported */
52455	char z; / Text of the error report */
52456	Mem pnErr; / Register keeping track of errors remaining */
52457	} bv;
52458	struct OP_RowSetAdd_stack_vars {
52459	Mem *pIdx;
52460	Mem *pVal;
52461	} bw;
52462	struct OP_RowSetRead_stack_vars {
52463	Mem *pIdx;
52464	i64 val;
52465	} bx;
52466	struct OP_RowSetTest_stack_vars {
52467	int iSet;
52468	int exists;
52469	} by;
52470	struct OP_Program_stack_vars {
52471	int nMem; /* Number of memory registers for sub-program */
52472	int nByte; /* Bytes of runtime space required for sub-program */
52473	Mem pRt; / Register to allocate runtime space */
52474	Mem pMem; / Used to iterate through memory cells */
52475	Mem pEnd; / Last memory cell in new array */
52476	VdbeFrame pFrame; / New vdbe frame to execute in */
52477	SubProgram pProgram; / Sub-program to execute */
52478	void t; / Token identifying trigger */
52479	} bz;
52480	struct OP_Param_stack_vars {
52481	VdbeFrame *pFrame;
52482	Mem *pIn;
52483	} ca;
52484	struct OP_MemMax_stack_vars {
52485	Mem *pIn1;
52486	VdbeFrame *pFrame;
52487	} cb;
52488	struct OP_AggStep_stack_vars {
52489	int n;
52490	int i;
52491	Mem *pMem;
52492	Mem *pRec;
52493	sqlite3_context ctx;
52494	sqlite3_value **apVal;
52495	} cc;
52496	struct OP_AggFinal_stack_vars {
52497	Mem *pMem;
52498	} cd;
52499	struct OP_IncrVacuum_stack_vars {
52500	Btree *pBt;
52501	} ce;
52502	struct OP_VBegin_stack_vars {
52503	VTable *pVTab;
52504	} cf;
52505	struct OP_VOpen_stack_vars {
52506	VdbeCursor *pCur;
52507	sqlite3_vtab_cursor *pVtabCursor;
52508	sqlite3_vtab *pVtab;
52509	sqlite3_module *pModule;
52510	} cg;
52511	struct OP_VFilter_stack_vars {
52512	int nArg;
52513	int iQuery;
52514	const sqlite3_module *pModule;
52515	Mem *pQuery;
	@@ -52518,44 +52474,44 @@
52518	sqlite3_vtab *pVtab;
52519	VdbeCursor *pCur;
52520	int res;
52521	int i;
52522	Mem **apArg;
52523	} ch;
52524	struct OP_VColumn_stack_vars {
52525	sqlite3_vtab *pVtab;
52526	const sqlite3_module *pModule;
52527	Mem *pDest;
52528	sqlite3_context sContext;
52529	} ci;
52530	struct OP_VNext_stack_vars {
52531	sqlite3_vtab *pVtab;
52532	const sqlite3_module *pModule;
52533	int res;
52534	VdbeCursor *pCur;
52535	} cj;
52536	struct OP_VRename_stack_vars {
52537	sqlite3_vtab *pVtab;
52538	Mem *pName;
52539	} ck;
52540	struct OP_VUpdate_stack_vars {
52541	sqlite3_vtab *pVtab;
52542	sqlite3_module *pModule;
52543	int nArg;
52544	int i;
52545	sqlite_int64 rowid;
52546	Mem **apArg;
52547	Mem *pX;
52548	} cl;
52549	struct OP_Pagecount_stack_vars {
52550	int p1;
52551	int nPage;
52552	Pager *pPager;
52553	} cm;
52554	struct OP_Trace_stack_vars {
52555	char *zTrace;
52556	} cn;
52557	} u;
52558	/* End automatically generated code
52559	********************************************************************/
52560
52561	assert( p->magic==VDBE_MAGIC_RUN ); /* sqlite3_step() verifies this */
	@@ -52571,20 +52527,23 @@
52571	assert( p->explain==0 );
52572	p->pResultSet = 0;
52573	db->busyHandler.nBusy = 0;
52574	CHECK_FOR_INTERRUPT;
52575	sqlite3VdbeIOTraceSql(p);



52576	#ifdef SQLITE_DEBUG
52577	sqlite3BeginBenignMalloc();
52578	if( p->pc==0
52579	&& ((p->db->flags & SQLITE_VdbeListing) \|\| fileExists(db, "vdbe_explain"))
52580	){
52581	int i;
52582	printf("VDBE Program Listing:\n");
52583	sqlite3VdbePrintSql(p);
52584	for(i=0; i<p->nOp; i++){
52585	sqlite3VdbePrintOp(stdout, i, &p->aOp[i]);
52586	}
52587	}
52588	if( fileExists(db, "vdbe_trace") ){
52589	p->trace = stdout;
52590	}
	@@ -52595,11 +52554,11 @@
52595	if( db->mallocFailed ) goto no_mem;
52596	#ifdef VDBE_PROFILE
52597	origPc = pc;
52598	start = sqlite3Hwtime();
52599	#endif
52600	pOp = &p->aOp[pc];
52601
52602	/* Only allow tracing if SQLITE_DEBUG is defined.
52603	*/
52604	#ifdef SQLITE_DEBUG
52605	if( p->trace ){
	@@ -52636,11 +52595,11 @@
52636	** of VDBE ops have been executed (either since this invocation of
52637	** sqlite3VdbeExec() or since last time the progress callback was called).
52638	** If the progress callback returns non-zero, exit the virtual machine with
52639	** a return code SQLITE_ABORT.
52640	*/
52641	if( db->xProgress ){
52642	if( db->nProgressOps==nProgressOps ){
52643	int prc;
52644	if( sqlite3SafetyOff(db) ) goto abort_due_to_misuse;
52645	prc =db->xProgress(db->pProgressArg);
52646	if( sqlite3SafetyOn(db) ) goto abort_due_to_misuse;
	@@ -52652,70 +52611,51 @@
52652	}
52653	nProgressOps++;
52654	}
52655	#endif
52656
52657	/* Do common setup processing for any opcode that is marked
52658	** with the "out2-prerelease" tag. Such opcodes have a single
52659	** output which is specified by the P2 parameter. The P2 register
52660	** is initialized to a NULL.
52661	*/
52662	opProperty = opcodeProperty[pOp->opcode];
52663	if( (opProperty & OPFLG_OUT2_PRERELEASE)!=0 ){
52664	assert( pOp->p2>0 );
52665	assert( pOp->p2<=p->nMem );
52666	pOut = &p->aMem[pOp->p2];
52667	sqlite3VdbeMemReleaseExternal(pOut);
52668	pOut->flags = MEM_Null;
52669	pOut->n = 0;
52670	}else
52671
52672	/* Do common setup for opcodes marked with one of the following
52673	** combinations of properties.
52674	**
52675	** in1
52676	** in1 in2
52677	** in1 in2 out3
52678	** in1 in3
52679	**
52680	** Variables pIn1, pIn2, and pIn3 are made to point to appropriate
52681	** registers for inputs. Variable pOut points to the output register.
52682	*/
52683	if( (opProperty & OPFLG_IN1)!=0 ){
52684	assert( pOp->p1>0 );
52685	assert( pOp->p1<=p->nMem );
52686	pIn1 = &p->aMem[pOp->p1];
52687	REGISTER_TRACE(pOp->p1, pIn1);
52688	if( (opProperty & OPFLG_IN2)!=0 ){
52689	assert( pOp->p2>0 );
52690	assert( pOp->p2<=p->nMem );
52691	pIn2 = &p->aMem[pOp->p2];
52692	REGISTER_TRACE(pOp->p2, pIn2);
52693	/* As currently implemented, in2 implies out3. There is no reason
52694	** why this has to be, it just worked out that way. */
52695	assert( (opProperty & OPFLG_OUT3)!=0 );
52696	assert( pOp->p3>0 );
52697	assert( pOp->p3<=p->nMem );
52698	pOut = &p->aMem[pOp->p3];
52699	}else if( (opProperty & OPFLG_IN3)!=0 ){
52700	assert( pOp->p3>0 );
52701	assert( pOp->p3<=p->nMem );
52702	pIn3 = &p->aMem[pOp->p3];
52703	REGISTER_TRACE(pOp->p3, pIn3);
52704	}
52705	}else if( (opProperty & OPFLG_IN2)!=0 ){
52706	assert( pOp->p2>0 );
52707	assert( pOp->p2<=p->nMem );
52708	pIn2 = &p->aMem[pOp->p2];
52709	REGISTER_TRACE(pOp->p2, pIn2);
52710	}else if( (opProperty & OPFLG_IN3)!=0 ){
52711	assert( pOp->p3>0 );
52712	assert( pOp->p3<=p->nMem );
52713	pIn3 = &p->aMem[pOp->p3];
52714	REGISTER_TRACE(pOp->p3, pIn3);
52715	}
52716
52717	switch( pOp->opcode ){
52718
52719	/*****************************************************************************
52720	** What follows is a massive switch statement where each case implements a
52721	** separate instruction in the virtual machine. If we follow the usual
	@@ -52767,14 +52707,12 @@
52767	/* Opcode: Gosub P1 P2 * * *
52768	**
52769	** Write the current address onto register P1
52770	** and then jump to address P2.
52771	*/
52772	case OP_Gosub: { /* jump */
52773	assert( pOp->p1>0 );
52774	assert( pOp->p1<=p->nMem );
52775	pIn1 = &p->aMem[pOp->p1];
52776	assert( (pIn1->flags & MEM_Dyn)==0 );
52777	pIn1->flags = MEM_Int;
52778	pIn1->u.i = pc;
52779	REGISTER_TRACE(pOp->p1, pIn1);
52780	pc = pOp->p2 - 1;
	@@ -52784,10 +52722,11 @@
52784	/* Opcode: Return P1 * * * *
52785	**
52786	** Jump to the next instruction after the address in register P1.
52787	*/
52788	case OP_Return: { /* in1 */

52789	assert( pIn1->flags & MEM_Int );
52790	pc = (int)pIn1->u.i;
52791	break;
52792	}
52793
	@@ -52797,10 +52736,11 @@
52797	*/
52798	case OP_Yield: { /* in1 */
52799	#if 0 /* local variables moved into u.aa */
52800	int pcDest;
52801	#endif /* local variables moved into u.aa */

52802	assert( (pIn1->flags & MEM_Dyn)==0 );
52803	pIn1->flags = MEM_Int;
52804	u.aa.pcDest = (int)pIn1->u.i;
52805	pIn1->u.i = pc;
52806	REGISTER_TRACE(pOp->p1, pIn1);
	@@ -52813,10 +52753,11 @@
52813	** Check the value in register P3. If is is NULL then Halt using
52814	** parameter P1, P2, and P4 as if this were a Halt instruction. If the
52815	** value in register P3 is not NULL, then this routine is a no-op.
52816	*/
52817	case OP_HaltIfNull: { /* in3 */

52818	if( (pIn3->flags & MEM_Null)==0 ) break;
52819	/* Fall through into OP_Halt */
52820	}
52821
52822	/* Opcode: Halt P1 P2 * P4 *
	@@ -52852,10 +52793,12 @@
52852	** instruction is set to OE_Ignore, then the sub-program is throwing
52853	** an IGNORE exception. In this case jump to the address specified
52854	** as the p2 of the calling OP_Program. */
52855	pc = p->aOp[pc].p2-1;
52856	}


52857	break;
52858	}
52859
52860	p->rc = pOp->p1;
52861	p->errorAction = (u8)pOp->p2;
	@@ -52878,11 +52821,10 @@
52878	/* Opcode: Integer P1 P2 * * *
52879	**
52880	** The 32-bit integer value P1 is written into register P2.
52881	*/
52882	case OP_Integer: { /* out2-prerelease */
52883	pOut->flags = MEM_Int;
52884	pOut->u.i = pOp->p1;
52885	break;
52886	}
52887
52888	/* Opcode: Int64 * P2 * P4 *
	@@ -52890,11 +52832,10 @@
52890	** P4 is a pointer to a 64-bit integer value.
52891	** Write that value into register P2.
52892	*/
52893	case OP_Int64: { /* out2-prerelease */
52894	assert( pOp->p4.pI64!=0 );
52895	pOut->flags = MEM_Int;
52896	pOut->u.i = *pOp->p4.pI64;
52897	break;
52898	}
52899
52900	/* Opcode: Real * P2 * P4 *
	@@ -52960,10 +52901,11 @@
52960	/* Opcode: Null * P2 * * *
52961	**
52962	** Write a NULL into register P2.
52963	*/
52964	case OP_Null: { /* out2-prerelease */

52965	break;
52966	}
52967
52968
52969	/* Opcode: Blob P1 P2 * P4
	@@ -53009,11 +52951,11 @@
53009	while( u.ab.n-- > 0 ){
53010	u.ab.pVar = &p->aVar[u.ab.p1++];
53011	if( sqlite3VdbeMemTooBig(u.ab.pVar) ){
53012	goto too_big;
53013	}
53014	pOut = &p->aMem[u.ab.p2++];
53015	sqlite3VdbeMemReleaseExternal(pOut);
53016	pOut->flags = MEM_Null;
53017	sqlite3VdbeMemShallowCopy(pOut, u.ab.pVar, MEM_Static);
53018	UPDATE_MAX_BLOBSIZE(pOut);
53019	}
	@@ -53039,15 +52981,15 @@
53039	u.ac.p1 = pOp->p1;
53040	u.ac.p2 = pOp->p2;
53041	assert( u.ac.n>0 && u.ac.p1>0 && u.ac.p2>0 );
53042	assert( u.ac.p1+u.ac.n<=u.ac.p2 \|\| u.ac.p2+u.ac.n<=u.ac.p1 );
53043
53044	pIn1 = &p->aMem[u.ac.p1];
53045	pOut = &p->aMem[u.ac.p2];
53046	while( u.ac.n-- ){
53047	assert( pOut<=&p->aMem[p->nMem] );
53048	assert( pIn1<=&p->aMem[p->nMem] );
53049	u.ac.zMalloc = pOut->zMalloc;
53050	pOut->zMalloc = 0;
53051	sqlite3VdbeMemMove(pOut, pIn1);
53052	pIn1->zMalloc = u.ac.zMalloc;
53053	REGISTER_TRACE(u.ac.p2++, pOut);
	@@ -53062,14 +53004,13 @@
53062	** Make a copy of register P1 into register P2.
53063	**
53064	** This instruction makes a deep copy of the value. A duplicate
53065	** is made of any string or blob constant. See also OP_SCopy.
53066	*/
53067	case OP_Copy: { /* in1 */
53068	assert( pOp->p2>0 );
53069	assert( pOp->p2<=p->nMem );
53070	pOut = &p->aMem[pOp->p2];
53071	assert( pOut!=pIn1 );
53072	sqlite3VdbeMemShallowCopy(pOut, pIn1, MEM_Ephem);
53073	Deephemeralize(pOut);
53074	REGISTER_TRACE(pOp->p2, pOut);
53075	break;
	@@ -53085,15 +53026,13 @@
53085	** Worse, if the original is deallocated, the copy becomes invalid.
53086	** Thus the program must guarantee that the original will not change
53087	** during the lifetime of the copy. Use OP_Copy to make a complete
53088	** copy.
53089	*/
53090	case OP_SCopy: { /* in1 */
53091	REGISTER_TRACE(pOp->p1, pIn1);
53092	assert( pOp->p2>0 );
53093	assert( pOp->p2<=p->nMem );
53094	pOut = &p->aMem[pOp->p2];
53095	assert( pOut!=pIn1 );
53096	sqlite3VdbeMemShallowCopy(pOut, pIn1, MEM_Ephem);
53097	REGISTER_TRACE(pOp->p2, pOut);
53098	break;
53099	}
	@@ -53150,11 +53089,11 @@
53150
53151	/* Make sure the results of the current row are \000 terminated
53152	** and have an assigned type. The results are de-ephemeralized as
53153	** as side effect.
53154	*/
53155	u.ad.pMem = p->pResultSet = &p->aMem[pOp->p1];
53156	for(u.ad.i=0; u.ad.i<pOp->p2; u.ad.i++){
53157	sqlite3VdbeMemNulTerminate(&u.ad.pMem[u.ad.i]);
53158	sqlite3VdbeMemStoreType(&u.ad.pMem[u.ad.i]);
53159	REGISTER_TRACE(pOp->p1+u.ad.i, &u.ad.pMem[u.ad.i]);
53160	}
	@@ -53182,10 +53121,13 @@
53182	case OP_Concat: { /* same as TK_CONCAT, in1, in2, out3 */
53183	#if 0 /* local variables moved into u.ae */
53184	i64 nByte;
53185	#endif /* local variables moved into u.ae */
53186



53187	assert( pIn1!=pOut );
53188	if( (pIn1->flags \| pIn2->flags) & MEM_Null ){
53189	sqlite3VdbeMemSetNull(pOut);
53190	break;
53191	}
	@@ -53257,12 +53199,15 @@
53257	i64 iB; /* Integer value of right operand */
53258	double rA; /* Real value of left operand */
53259	double rB; /* Real value of right operand */
53260	#endif /* local variables moved into u.af */
53261

53262	applyNumericAffinity(pIn1);

53263	applyNumericAffinity(pIn2);

53264	u.af.flags = pIn1->flags \| pIn2->flags;
53265	if( (u.af.flags & MEM_Null)!=0 ) goto arithmetic_result_is_null;
53266	if( (pIn1->flags & pIn2->flags & MEM_Int)==MEM_Int ){
53267	u.af.iA = pIn1->u.i;
53268	u.af.iB = pIn2->u.i;
	@@ -53375,11 +53320,11 @@
53375	u.ag.apVal = p->apArg;
53376	assert( u.ag.apVal \|\| u.ag.n==0 );
53377
53378	assert( u.ag.n==0 \|\| (pOp->p2>0 && pOp->p2+u.ag.n<=p->nMem+1) );
53379	assert( pOp->p3<pOp->p2 \|\| pOp->p3>=pOp->p2+u.ag.n );
53380	u.ag.pArg = &p->aMem[pOp->p2];
53381	for(u.ag.i=0; u.ag.i<u.ag.n; u.ag.i++, u.ag.pArg++){
53382	u.ag.apVal[u.ag.i] = u.ag.pArg;
53383	sqlite3VdbeMemStoreType(u.ag.pArg);
53384	REGISTER_TRACE(pOp->p2, u.ag.pArg);
53385	}
	@@ -53392,11 +53337,11 @@
53392	u.ag.ctx.pVdbeFunc = (VdbeFunc*)pOp->p4.pVdbeFunc;
53393	u.ag.ctx.pFunc = u.ag.ctx.pVdbeFunc->pFunc;
53394	}
53395
53396	assert( pOp->p3>0 && pOp->p3<=p->nMem );
53397	pOut = &p->aMem[pOp->p3];
53398	u.ag.ctx.s.flags = MEM_Null;
53399	u.ag.ctx.s.db = db;
53400	u.ag.ctx.s.xDel = 0;
53401	u.ag.ctx.s.zMalloc = 0;
53402
	@@ -53407,11 +53352,11 @@
53407	sqlite3VdbeMemMove(&u.ag.ctx.s, pOut);
53408	MemSetTypeFlag(&u.ag.ctx.s, MEM_Null);
53409
53410	u.ag.ctx.isError = 0;
53411	if( u.ag.ctx.pFunc->flags & SQLITE_FUNC_NEEDCOLL ){
53412	assert( pOp>p->aOp );
53413	assert( pOp[-1].p4type==P4_COLLSEQ );
53414	assert( pOp[-1].opcode==OP_CollSeq );
53415	u.ag.ctx.pColl = pOp[-1].p4.pColl;
53416	}
53417	if( sqlite3SafetyOff(db) ) goto abort_due_to_misuse;
	@@ -53493,10 +53438,13 @@
53493	#if 0 /* local variables moved into u.ah */
53494	i64 a;
53495	i64 b;
53496	#endif /* local variables moved into u.ah */
53497



53498	if( (pIn1->flags \| pIn2->flags) & MEM_Null ){
53499	sqlite3VdbeMemSetNull(pOut);
53500	break;
53501	}
53502	u.ah.a = sqlite3VdbeIntValue(pIn2);
	@@ -53519,10 +53467,11 @@
53519	** The result is always an integer.
53520	**
53521	** To force any register to be an integer, just add 0.
53522	*/
53523	case OP_AddImm: { /* in1 */

53524	sqlite3VdbeMemIntegerify(pIn1);
53525	pIn1->u.i += pOp->p2;
53526	break;
53527	}
53528
	@@ -53532,10 +53481,11 @@
53532	** in P1 is not an integer and cannot be converted into an integer
53533	** without data loss, then jump immediately to P2, or if P2==0
53534	** raise an SQLITE_MISMATCH exception.
53535	*/
53536	case OP_MustBeInt: { /* jump, in1 */

53537	applyAffinity(pIn1, SQLITE_AFF_NUMERIC, encoding);
53538	if( (pIn1->flags & MEM_Int)==0 ){
53539	if( pOp->p2==0 ){
53540	rc = SQLITE_MISMATCH;
53541	goto abort_due_to_error;
	@@ -53556,10 +53506,11 @@
53556	** has REAL affinity. Such column values may still be stored as
53557	** integers, for space efficiency, but after extraction we want them
53558	** to have only a real value.
53559	*/
53560	case OP_RealAffinity: { /* in1 */

53561	if( pIn1->flags & MEM_Int ){
53562	sqlite3VdbeMemRealify(pIn1);
53563	}
53564	break;
53565	}
	@@ -53573,10 +53524,11 @@
53573	** are afterwards simply interpreted as text.
53574	**
53575	** A NULL value is not changed by this routine. It remains NULL.
53576	*/
53577	case OP_ToText: { /* same as TK_TO_TEXT, in1 */

53578	if( pIn1->flags & MEM_Null ) break;
53579	assert( MEM_Str==(MEM_Blob>>3) );
53580	pIn1->flags \|= (pIn1->flags&MEM_Blob)>>3;
53581	applyAffinity(pIn1, SQLITE_AFF_TEXT, encoding);
53582	rc = ExpandBlob(pIn1);
	@@ -53594,10 +53546,11 @@
53594	** to the underlying data.
53595	**
53596	** A NULL value is not changed by this routine. It remains NULL.
53597	*/
53598	case OP_ToBlob: { /* same as TK_TO_BLOB, in1 */

53599	if( pIn1->flags & MEM_Null ) break;
53600	if( (pIn1->flags & MEM_Blob)==0 ){
53601	applyAffinity(pIn1, SQLITE_AFF_TEXT, encoding);
53602	assert( pIn1->flags & MEM_Str \|\| db->mallocFailed );
53603	MemSetTypeFlag(pIn1, MEM_Blob);
	@@ -53617,10 +53570,11 @@
53617	** is possible.
53618	**
53619	** A NULL value is not changed by this routine. It remains NULL.
53620	*/
53621	case OP_ToNumeric: { /* same as TK_TO_NUMERIC, in1 */

53622	if( (pIn1->flags & (MEM_Null\|MEM_Int\|MEM_Real))==0 ){
53623	sqlite3VdbeMemNumerify(pIn1);
53624	}
53625	break;
53626	}
	@@ -53634,10 +53588,11 @@
53634	** equivalent of atoi() and store 0 if no such conversion is possible.
53635	**
53636	** A NULL value is not changed by this routine. It remains NULL.
53637	*/
53638	case OP_ToInt: { /* same as TK_TO_INT, in1 */

53639	if( (pIn1->flags & MEM_Null)==0 ){
53640	sqlite3VdbeMemIntegerify(pIn1);
53641	}
53642	break;
53643	}
	@@ -53651,10 +53606,11 @@
53651	** equivalent of atoi() and store 0.0 if no such conversion is possible.
53652	**
53653	** A NULL value is not changed by this routine. It remains NULL.
53654	*/
53655	case OP_ToReal: { /* same as TK_TO_REAL, in1 */

53656	if( (pIn1->flags & MEM_Null)==0 ){
53657	sqlite3VdbeMemRealify(pIn1);
53658	}
53659	break;
53660	}
	@@ -53741,10 +53697,12 @@
53741	#if 0 /* local variables moved into u.ai */
53742	int res; /* Result of the comparison of pIn1 against pIn3 */
53743	char affinity; /* Affinity to use for comparison */
53744	#endif /* local variables moved into u.ai */
53745


53746	if( (pIn1->flags \| pIn3->flags)&MEM_Null ){
53747	/* One or both operands are NULL */
53748	if( pOp->p5 & SQLITE_NULLEQ ){
53749	/* If SQLITE_NULLEQ is set (which will only happen if the operator is
53750	** OP_Eq or OP_Ne) then take the jump or not depending on whether
	@@ -53756,11 +53714,11 @@
53756	/* SQLITE_NULLEQ is clear and at least one operand is NULL,
53757	** then the result is always NULL.
53758	** The jump is taken if the SQLITE_JUMPIFNULL bit is set.
53759	*/
53760	if( pOp->p5 & SQLITE_STOREP2 ){
53761	pOut = &p->aMem[pOp->p2];
53762	MemSetTypeFlag(pOut, MEM_Null);
53763	REGISTER_TRACE(pOp->p2, pOut);
53764	}else if( pOp->p5 & SQLITE_JUMPIFNULL ){
53765	pc = pOp->p2-1;
53766	}
	@@ -53788,11 +53746,11 @@
53788	case OP_Gt: u.ai.res = u.ai.res>0; break;
53789	default: u.ai.res = u.ai.res>=0; break;
53790	}
53791
53792	if( pOp->p5 & SQLITE_STOREP2 ){
53793	pOut = &p->aMem[pOp->p2];
53794	MemSetTypeFlag(pOut, MEM_Int);
53795	pOut->u.i = u.ai.res;
53796	REGISTER_TRACE(pOp->p2, pOut);
53797	}else if( u.ai.res ){
53798	pc = pOp->p2-1;
	@@ -53859,16 +53817,16 @@
53859	assert( u.aj.p2>0 && u.aj.p2+u.aj.n<=p->nMem+1 );
53860	}
53861	#endif /* SQLITE_DEBUG */
53862	for(u.aj.i=0; u.aj.i<u.aj.n; u.aj.i++){
53863	u.aj.idx = aPermute ? aPermute[u.aj.i] : u.aj.i;
53864	REGISTER_TRACE(u.aj.p1+u.aj.idx, &p->aMem[u.aj.p1+u.aj.idx]);
53865	REGISTER_TRACE(u.aj.p2+u.aj.idx, &p->aMem[u.aj.p2+u.aj.idx]);
53866	assert( u.aj.i<u.aj.pKeyInfo->nField );
53867	u.aj.pColl = u.aj.pKeyInfo->aColl[u.aj.i];
53868	u.aj.bRev = u.aj.pKeyInfo->aSortOrder[u.aj.i];
53869	iCompare = sqlite3MemCompare(&p->aMem[u.aj.p1+u.aj.idx], &p->aMem[u.aj.p2+u.aj.idx], u.aj.pColl);
53870	if( iCompare ){
53871	if( u.aj.bRev ) iCompare = -iCompare;
53872	break;
53873	}
53874	}
	@@ -53916,15 +53874,17 @@
53916	#if 0 /* local variables moved into u.ak */
53917	int v1; /* Left operand: 0==FALSE, 1==TRUE, 2==UNKNOWN or NULL */
53918	int v2; /* Right operand: 0==FALSE, 1==TRUE, 2==UNKNOWN or NULL */
53919	#endif /* local variables moved into u.ak */
53920

53921	if( pIn1->flags & MEM_Null ){
53922	u.ak.v1 = 2;
53923	}else{
53924	u.ak.v1 = sqlite3VdbeIntValue(pIn1)!=0;
53925	}

53926	if( pIn2->flags & MEM_Null ){
53927	u.ak.v2 = 2;
53928	}else{
53929	u.ak.v2 = sqlite3VdbeIntValue(pIn2)!=0;
53930	}
	@@ -53933,10 +53893,11 @@
53933	u.ak.v1 = and_logic[u.ak.v1*3+u.ak.v2];
53934	}else{
53935	static const unsigned char or_logic[] = { 0, 1, 2, 1, 1, 1, 2, 1, 2 };
53936	u.ak.v1 = or_logic[u.ak.v1*3+u.ak.v2];
53937	}

53938	if( u.ak.v1==2 ){
53939	MemSetTypeFlag(pOut, MEM_Null);
53940	}else{
53941	pOut->u.i = u.ak.v1;
53942	MemSetTypeFlag(pOut, MEM_Int);
	@@ -53948,12 +53909,13 @@
53948	**
53949	** Interpret the value in register P1 as a boolean value. Store the
53950	** boolean complement in register P2. If the value in register P1 is
53951	** NULL, then a NULL is stored in P2.
53952	*/
53953	case OP_Not: { /* same as TK_NOT, in1 */
53954	pOut = &p->aMem[pOp->p2];

53955	if( pIn1->flags & MEM_Null ){
53956	sqlite3VdbeMemSetNull(pOut);
53957	}else{
53958	sqlite3VdbeMemSetInt64(pOut, !sqlite3VdbeIntValue(pIn1));
53959	}
	@@ -53964,12 +53926,13 @@
53964	**
53965	** Interpret the content of register P1 as an integer. Store the
53966	** ones-complement of the P1 value into register P2. If P1 holds
53967	** a NULL then store a NULL in P2.
53968	*/
53969	case OP_BitNot: { /* same as TK_BITNOT, in1 */
53970	pOut = &p->aMem[pOp->p2];

53971	if( pIn1->flags & MEM_Null ){
53972	sqlite3VdbeMemSetNull(pOut);
53973	}else{
53974	sqlite3VdbeMemSetInt64(pOut, ~sqlite3VdbeIntValue(pIn1));
53975	}
	@@ -53991,10 +53954,11 @@
53991	case OP_If: /* jump, in1 */
53992	case OP_IfNot: { /* jump, in1 */
53993	#if 0 /* local variables moved into u.al */
53994	int c;
53995	#endif /* local variables moved into u.al */

53996	if( pIn1->flags & MEM_Null ){
53997	u.al.c = pOp->p3;
53998	}else{
53999	#ifdef SQLITE_OMIT_FLOATING_POINT
54000	u.al.c = sqlite3VdbeIntValue(pIn1)!=0;
	@@ -54012,10 +53976,11 @@
54012	/* Opcode: IsNull P1 P2 * * *
54013	**
54014	** Jump to P2 if the value in register P1 is NULL.
54015	*/
54016	case OP_IsNull: { /* same as TK_ISNULL, jump, in1 */

54017	if( (pIn1->flags & MEM_Null)!=0 ){
54018	pc = pOp->p2 - 1;
54019	}
54020	break;
54021	}
	@@ -54023,10 +53988,11 @@
54023	/* Opcode: NotNull P1 P2 * * *
54024	**
54025	** Jump to P2 if the value in register P1 is not NULL.
54026	*/
54027	case OP_NotNull: { /* same as TK_NOTNULL, jump, in1 */

54028	if( (pIn1->flags & MEM_Null)==0 ){
54029	pc = pOp->p2 - 1;
54030	}
54031	break;
54032	}
	@@ -54081,11 +54047,11 @@
54081	u.am.p2 = pOp->p2;
54082	u.am.pC = 0;
54083	memset(&u.am.sMem, 0, sizeof(u.am.sMem));
54084	assert( u.am.p1<p->nCursor );
54085	assert( pOp->p3>0 && pOp->p3<=p->nMem );
54086	u.am.pDest = &p->aMem[pOp->p3];
54087	MemSetTypeFlag(u.am.pDest, MEM_Null);
54088	u.am.zRec = 0;
54089
54090	/* This block sets the variable u.am.payloadSize to be the total number of
54091	** bytes in the record.
	@@ -54127,11 +54093,11 @@
54127	assert( sqlite3BtreeCursorIsValid(u.am.pCrsr) );
54128	rc = sqlite3BtreeDataSize(u.am.pCrsr, &u.am.payloadSize);
54129	assert( rc==SQLITE_OK ); /* DataSize() cannot fail */
54130	}
54131	}else if( u.am.pC->pseudoTableReg>0 ){
54132	u.am.pReg = &p->aMem[u.am.pC->pseudoTableReg];
54133	assert( u.am.pReg->flags & MEM_Blob );
54134	u.am.payloadSize = u.am.pReg->n;
54135	u.am.zRec = u.am.pReg->z;
54136	u.am.pC->cacheStatus = (pOp->p5&OPFLAG_CLEARCACHE) ? CACHE_STALE : p->cacheCtr;
54137	assert( u.am.payloadSize==0 \|\| u.am.zRec!=0 );
	@@ -54339,22 +54305,23 @@
54339	** string indicates the column affinity that should be used for the nth
54340	** memory cell in the range.
54341	*/
54342	case OP_Affinity: {
54343	#if 0 /* local variables moved into u.an */
54344	char zAffinity; / The affinity to be applied */
54345	Mem pData0; / First register to which to apply affinity */
54346	Mem pLast; / Last register to which to apply affinity */
54347	Mem pRec; / Current register */
54348	#endif /* local variables moved into u.an */
54349
54350	u.an.zAffinity = pOp->p4.z;
54351	u.an.pData0 = &p->aMem[pOp->p1];
54352	u.an.pLast = &u.an.pData0[pOp->p2-1];
54353	for(u.an.pRec=u.an.pData0; u.an.pRec<=u.an.pLast; u.an.pRec++){
54354	ExpandBlob(u.an.pRec);
54355	applyAffinity(u.an.pRec, u.an.zAffinity[u.an.pRec-u.an.pData0], encoding);



54356	}
54357	break;
54358	}
54359
54360	/* Opcode: MakeRecord P1 P2 P3 P4 *
	@@ -54414,11 +54381,11 @@
54414	u.ao.nByte = 0; /* Data space required for this record */
54415	u.ao.nZero = 0; /* Number of zero bytes at the end of the record */
54416	u.ao.nField = pOp->p1;
54417	u.ao.zAffinity = pOp->p4.z;
54418	assert( u.ao.nField>0 && pOp->p2>0 && pOp->p2+u.ao.nField<=p->nMem+1 );
54419	u.ao.pData0 = &p->aMem[u.ao.nField];
54420	u.ao.nField = pOp->p2;
54421	u.ao.pLast = &u.ao.pData0[u.ao.nField-1];
54422	u.ao.file_format = p->minWriteFileFormat;
54423
54424	/* Loop through the elements that will make up the record to figure
	@@ -54458,11 +54425,11 @@
54458	** the new record. The output register (pOp->p3) is not allowed to
54459	** be one of the input registers (because the following call to
54460	** sqlite3VdbeMemGrow() could clobber the value before it is used).
54461	*/
54462	assert( pOp->p3<pOp->p1 \|\| pOp->p3>=pOp->p1+pOp->p2 );
54463	pOut = &p->aMem[pOp->p3];
54464	if( sqlite3VdbeMemGrow(pOut, (int)u.ao.nByte, 0) ){
54465	goto no_mem;
54466	}
54467	u.ao.zNewRecord = (u8 *)pOut->z;
54468
	@@ -54507,11 +54474,10 @@
54507	if( u.ap.pCrsr ){
54508	rc = sqlite3BtreeCount(u.ap.pCrsr, &u.ap.nEntry);
54509	}else{
54510	u.ap.nEntry = 0;
54511	}
54512	pOut->flags = MEM_Int;
54513	pOut->u.i = u.ap.nEntry;
54514	break;
54515	}
54516	#endif
54517
	@@ -54836,11 +54802,10 @@
54836	assert( db->aDb[u.at.iDb].pBt!=0 );
54837	assert( (p->btreeMask & (1<<u.at.iDb))!=0 );
54838
54839	sqlite3BtreeGetMeta(db->aDb[u.at.iDb].pBt, u.at.iCookie, (u32 *)&u.at.iMeta);
54840	pOut->u.i = u.at.iMeta;
54841	MemSetTypeFlag(pOut, MEM_Int);
54842	break;
54843	}
54844
54845	/* Opcode: SetCookie P1 P2 P3 * *
54846	**
	@@ -54859,10 +54824,11 @@
54859	assert( pOp->p2<SQLITE_N_BTREE_META );
54860	assert( pOp->p1>=0 && pOp->p1<db->nDb );
54861	assert( (p->btreeMask & (1<<pOp->p1))!=0 );
54862	u.au.pDb = &db->aDb[pOp->p1];
54863	assert( u.au.pDb->pBt!=0 );

54864	sqlite3VdbeMemIntegerify(pIn3);
54865	/* See note about index shifting on OP_ReadCookie */
54866	rc = sqlite3BtreeUpdateMeta(u.au.pDb->pBt, pOp->p2, (int)pIn3->u.i);
54867	if( pOp->p2==BTREE_SCHEMA_VERSION ){
54868	/* When the schema cookie changes, record the new cookie internally */
	@@ -55021,11 +54987,11 @@
55021	u.aw.wrFlag = 0;
55022	}
55023	if( pOp->p5 ){
55024	assert( u.aw.p2>0 );
55025	assert( u.aw.p2<=p->nMem );
55026	pIn2 = &p->aMem[u.aw.p2];
55027	sqlite3VdbeMemIntegerify(pIn2);
55028	u.aw.p2 = (int)pIn2->u.i;
55029	/* The u.aw.p2 value always comes from a prior OP_CreateTable opcode and
55030	** that opcode will always set the u.aw.p2 value to 2 or more or else fail.
55031	** If there were a failure, the prepared statement would have halted
	@@ -55243,17 +55209,21 @@
55243	assert( pOp->p1>=0 && pOp->p1<p->nCursor );
55244	assert( pOp->p2!=0 );
55245	u.az.pC = p->apCsr[pOp->p1];
55246	assert( u.az.pC!=0 );
55247	assert( u.az.pC->pseudoTableReg==0 );



55248	if( u.az.pC->pCursor!=0 ){
55249	u.az.oc = pOp->opcode;
55250	u.az.pC->nullRow = 0;
55251	if( u.az.pC->isTable ){
55252	/* The input value in P3 might be of any type: integer, real, string,
55253	** blob, or NULL. But it needs to be an integer before we can do
55254	** the seek, so covert it. */

55255	applyNumericAffinity(pIn3);
55256	u.az.iKey = sqlite3VdbeIntValue(pIn3);
55257	u.az.pC->rowidIsValid = 0;
55258
55259	/* If the P3 value could not be converted into an integer without
	@@ -55272,16 +55242,16 @@
55272	if( u.az.iKey==SMALLEST_INT64 && (pIn3->r<(double)u.az.iKey \|\| pIn3->r>0) ){
55273	/* The P3 value is too large in magnitude to be expressed as an
55274	** integer. */
55275	u.az.res = 1;
55276	if( pIn3->r<0 ){
55277	if( u.az.oc==OP_SeekGt \|\| u.az.oc==OP_SeekGe ){
55278	rc = sqlite3BtreeFirst(u.az.pC->pCursor, &u.az.res);
55279	if( rc!=SQLITE_OK ) goto abort_due_to_error;
55280	}
55281	}else{
55282	if( u.az.oc==OP_SeekLt \|\| u.az.oc==OP_SeekLe ){
55283	rc = sqlite3BtreeLast(u.az.pC->pCursor, &u.az.res);
55284	if( rc!=SQLITE_OK ) goto abort_due_to_error;
55285	}
55286	}
55287	if( u.az.res ){
	@@ -55309,16 +55279,26 @@
55309	u.az.nField = pOp->p4.i;
55310	assert( pOp->p4type==P4_INT32 );
55311	assert( u.az.nField>0 );
55312	u.az.r.pKeyInfo = u.az.pC->pKeyInfo;
55313	u.az.r.nField = (u16)u.az.nField;
55314	if( u.az.oc==OP_SeekGt \|\| u.az.oc==OP_SeekLe ){
55315	u.az.r.flags = UNPACKED_INCRKEY;
55316	}else{
55317	u.az.r.flags = 0;
55318	}
55319	u.az.r.aMem = &p->aMem[pOp->p3];










55320	rc = sqlite3BtreeMovetoUnpacked(u.az.pC->pCursor, &u.az.r, 0, 0, &u.az.res);
55321	if( rc!=SQLITE_OK ){
55322	goto abort_due_to_error;
55323	}
55324	u.az.pC->rowidIsValid = 0;
	@@ -55326,11 +55306,11 @@
55326	u.az.pC->deferredMoveto = 0;
55327	u.az.pC->cacheStatus = CACHE_STALE;
55328	#ifdef SQLITE_TEST
55329	sqlite3_search_count++;
55330	#endif
55331	if( u.az.oc==OP_SeekGe \|\| u.az.oc==OP_SeekGt ){
55332	if( u.az.res<0 \|\| (u.az.res==0 && u.az.oc==OP_SeekGt) ){
55333	rc = sqlite3BtreeNext(u.az.pC->pCursor, &u.az.res);
55334	if( rc!=SQLITE_OK ) goto abort_due_to_error;
55335	u.az.pC->rowidIsValid = 0;
55336	}else{
	@@ -55381,45 +55361,40 @@
55381	u.ba.pC = p->apCsr[pOp->p1];
55382	assert( u.ba.pC!=0 );
55383	if( ALWAYS(u.ba.pC->pCursor!=0) ){
55384	assert( u.ba.pC->isTable );
55385	u.ba.pC->nullRow = 0;

55386	u.ba.pC->movetoTarget = sqlite3VdbeIntValue(pIn2);
55387	u.ba.pC->rowidIsValid = 0;
55388	u.ba.pC->deferredMoveto = 1;
55389	}
55390	break;
55391	}
55392
55393
55394	/* Opcode: Found P1 P2 P3 * *
55395	**
55396	** Register P3 holds a blob constructed by MakeRecord. P1 is an index.
55397	** If an entry that matches the value in register p3 exists in P1 then
55398	** jump to P2. If the P3 value does not match any entry in P1
55399	** then fall thru. The P1 cursor is left pointing at the matching entry
55400	** if it exists.
55401	**
55402	** This instruction is used to implement the IN operator where the
55403	** left-hand side is a SELECT statement. P1 may be a true index, or it
55404	** may be a temporary index that holds the results of the SELECT
55405	** statement. This instruction is also used to implement the
55406	** DISTINCT keyword in SELECT statements.
55407	**
55408	** This instruction checks if index P1 contains a record for which
55409	** the first N serialized values exactly match the N serialized values
55410	** in the record in register P3, where N is the total number of values in
55411	** the P3 record (the P3 record is a prefix of the P1 record).
55412	**
55413	** See also: NotFound, IsUnique, NotExists
55414	*/
55415	/* Opcode: NotFound P1 P2 P3 * *
55416	**
55417	** Register P3 holds a blob constructed by MakeRecord. P1 is
55418	** an index. If no entry exists in P1 that matches the blob then jump
55419	** to P2. If an entry does existing, fall through. The cursor is left
55420	** pointing to the entry that matches.
55421	**
55422	** See also: Found, NotExists, IsUnique
55423	*/
55424	case OP_NotFound: /* jump, in3 */
55425	case OP_Found: { /* jump, in3 */
	@@ -55426,36 +55401,47 @@
55426	#if 0 /* local variables moved into u.bb */
55427	int alreadyExists;
55428	VdbeCursor *pC;
55429	int res;
55430	UnpackedRecord *pIdxKey;

55431	char aTempRec[ROUND8(sizeof(UnpackedRecord)) + sizeof(Mem)*3 + 7];
55432	#endif /* local variables moved into u.bb */
55433
55434	#ifdef SQLITE_TEST
55435	sqlite3_found_count++;
55436	#endif
55437
55438	u.bb.alreadyExists = 0;
55439	assert( pOp->p1>=0 && pOp->p1<p->nCursor );

55440	u.bb.pC = p->apCsr[pOp->p1];
55441	assert( u.bb.pC!=0 );

55442	if( ALWAYS(u.bb.pC->pCursor!=0) ){
55443
55444	assert( u.bb.pC->isTable==0 );
55445	assert( pIn3->flags & MEM_Blob );
55446	ExpandBlob(pIn3);
55447	u.bb.pIdxKey = sqlite3VdbeRecordUnpack(u.bb.pC->pKeyInfo, pIn3->n, pIn3->z,
55448	u.bb.aTempRec, sizeof(u.bb.aTempRec));
55449	if( u.bb.pIdxKey==0 ){
55450	goto no_mem;
55451	}
55452	if( pOp->opcode==OP_Found ){






55453	u.bb.pIdxKey->flags \|= UNPACKED_PREFIX_MATCH;
55454	}
55455	rc = sqlite3BtreeMovetoUnpacked(u.bb.pC->pCursor, u.bb.pIdxKey, 0, 0, &u.bb.res);
55456	sqlite3VdbeDeleteUnpackedRecord(u.bb.pIdxKey);


55457	if( rc!=SQLITE_OK ){
55458	break;
55459	}
55460	u.bb.alreadyExists = (u.bb.res==0);
55461	u.bb.pC->deferredMoveto = 0;
	@@ -55469,13 +55455,14 @@
55469	break;
55470	}
55471
55472	/* Opcode: IsUnique P1 P2 P3 P4 *
55473	**
55474	** Cursor P1 is open on an index. So it has no data and its key consists
55475	** of a record generated by OP_MakeRecord where the last field is the
55476	** rowid of the entry that the index refers to.

55477	**
55478	** The P3 register contains an integer record number. Call this record
55479	** number R. Register P4 is the first in a set of N contiguous registers
55480	** that make up an unpacked index key that can be used with cursor P1.
55481	** The value of N can be inferred from the cursor. N includes the rowid
	@@ -55498,16 +55485,17 @@
55498	#if 0 /* local variables moved into u.bc */
55499	u16 ii;
55500	VdbeCursor *pCx;
55501	BtCursor *pCrsr;
55502	u16 nField;
55503	Mem *aMem;
55504	UnpackedRecord r; /* B-Tree index search key */
55505	i64 R; /* Rowid stored in register P3 */
55506	#endif /* local variables moved into u.bc */
55507
55508	u.bc.aMem = &p->aMem[pOp->p4.i];

55509	/* Assert that the values of parameters P1 and P4 are in range. */
55510	assert( pOp->p4type==P4_INT32 );
55511	assert( pOp->p4.i>0 && pOp->p4.i<=p->nMem );
55512	assert( pOp->p1>=0 && pOp->p1<p->nCursor );
55513
	@@ -55519,24 +55507,24 @@
55519	u.bc.pCrsr = u.bc.pCx->pCursor;
55520
55521	/* If any of the values are NULL, take the jump. */
55522	u.bc.nField = u.bc.pCx->pKeyInfo->nField;
55523	for(u.bc.ii=0; u.bc.ii<u.bc.nField; u.bc.ii++){
55524	if( u.bc.aMem[u.bc.ii].flags & MEM_Null ){
55525	pc = pOp->p2 - 1;
55526	u.bc.pCrsr = 0;
55527	break;
55528	}
55529	}
55530	assert( (u.bc.aMem[u.bc.nField].flags & MEM_Null)==0 );
55531
55532	if( u.bc.pCrsr!=0 ){
55533	/* Populate the index search key. */
55534	u.bc.r.pKeyInfo = u.bc.pCx->pKeyInfo;
55535	u.bc.r.nField = u.bc.nField + 1;
55536	u.bc.r.flags = UNPACKED_PREFIX_SEARCH;
55537	u.bc.r.aMem = u.bc.aMem;
55538
55539	/* Extract the value of u.bc.R from register P3. */
55540	sqlite3VdbeMemIntegerify(pIn3);
55541	u.bc.R = pIn3->u.i;
55542
	@@ -55573,10 +55561,11 @@
55573	BtCursor *pCrsr;
55574	int res;
55575	u64 iKey;
55576	#endif /* local variables moved into u.bd */
55577

55578	assert( pIn3->flags & MEM_Int );
55579	assert( pOp->p1>=0 && pOp->p1<p->nCursor );
55580	u.bd.pC = p->apCsr[pOp->p1];
55581	assert( u.bd.pC!=0 );
55582	assert( u.bd.pC->isTable );
	@@ -55616,11 +55605,10 @@
55616	*/
55617	case OP_Sequence: { /* out2-prerelease */
55618	assert( pOp->p1>=0 && pOp->p1<p->nCursor );
55619	assert( p->apCsr[pOp->p1]!=0 );
55620	pOut->u.i = p->apCsr[pOp->p1]->seqCount++;
55621	MemSetTypeFlag(pOut, MEM_Int);
55622	break;
55623	}
55624
55625
55626	/* Opcode: NewRowid P1 P2 P3 * *
	@@ -55712,11 +55700,11 @@
55712	assert( pOp->p3<=u.be.pFrame->nMem );
55713	u.be.pMem = &u.be.pFrame->aMem[pOp->p3];
55714	}else{
55715	/* Assert that P3 is a valid memory cell. */
55716	assert( pOp->p3<=p->nMem );
55717	u.be.pMem = &p->aMem[pOp->p3];
55718	}
55719
55720	REGISTER_TRACE(pOp->p3, u.be.pMem);
55721	sqlite3VdbeMemIntegerify(u.be.pMem);
55722	assert( (u.be.pMem->flags & MEM_Int)!=0 ); /* mem(P3) holds an integer */
	@@ -55755,11 +55743,10 @@
55755	}
55756	u.be.pC->rowidIsValid = 0;
55757	u.be.pC->deferredMoveto = 0;
55758	u.be.pC->cacheStatus = CACHE_STALE;
55759	}
55760	MemSetTypeFlag(pOut, MEM_Int);
55761	pOut->u.i = u.be.v;
55762	break;
55763	}
55764
55765	/* Opcode: Insert P1 P2 P3 P4 P5
	@@ -55818,21 +55805,21 @@
55818	const char zDb; / database name - used by the update hook */
55819	const char zTbl; / Table name - used by the opdate hook */
55820	int op; /* Opcode for update hook: SQLITE_UPDATE or SQLITE_INSERT */
55821	#endif /* local variables moved into u.bf */
55822
55823	u.bf.pData = &p->aMem[pOp->p2];
55824	assert( pOp->p1>=0 && pOp->p1<p->nCursor );
55825	u.bf.pC = p->apCsr[pOp->p1];
55826	assert( u.bf.pC!=0 );
55827	assert( u.bf.pC->pCursor!=0 );
55828	assert( u.bf.pC->pseudoTableReg==0 );
55829	assert( u.bf.pC->isTable );
55830	REGISTER_TRACE(pOp->p2, u.bf.pData);
55831
55832	if( pOp->opcode==OP_Insert ){
55833	u.bf.pKey = &p->aMem[pOp->p3];
55834	assert( u.bf.pKey->flags & MEM_Int );
55835	REGISTER_TRACE(pOp->p3, u.bf.pKey);
55836	u.bf.iKey = u.bf.pKey->u.i;
55837	}else{
55838	assert( pOp->opcode==OP_InsertInt );
	@@ -55980,11 +55967,11 @@
55980	BtCursor *pCrsr;
55981	u32 n;
55982	i64 n64;
55983	#endif /* local variables moved into u.bh */
55984
55985	pOut = &p->aMem[pOp->p2];
55986
55987	/* Note that RowKey and RowData are really exactly the same instruction */
55988	assert( pOp->p1>=0 && pOp->p1<p->nCursor );
55989	u.bh.pC = p->apCsr[pOp->p1];
55990	assert( u.bh.pC->isTable \|\| pOp->opcode==OP_RowKey );
	@@ -56055,11 +56042,11 @@
56055	assert( pOp->p1>=0 && pOp->p1<p->nCursor );
56056	u.bi.pC = p->apCsr[pOp->p1];
56057	assert( u.bi.pC!=0 );
56058	assert( u.bi.pC->pseudoTableReg==0 );
56059	if( u.bi.pC->nullRow ){
56060	/* Do nothing so that reg[P2] remains NULL */
56061	break;
56062	}else if( u.bi.pC->deferredMoveto ){
56063	u.bi.v = u.bi.pC->movetoTarget;
56064	#ifndef SQLITE_OMIT_VIRTUALTABLE
56065	}else if( u.bi.pC->pVtabCursor ){
	@@ -56083,11 +56070,10 @@
56083	rc = sqlite3BtreeKeySize(u.bi.pC->pCursor, &u.bi.v);
56084	assert( rc==SQLITE_OK ); /* Always so because of CursorMoveto() above */
56085	}
56086	}
56087	pOut->u.i = u.bi.v;
56088	MemSetTypeFlag(pOut, MEM_Int);
56089	break;
56090	}
56091
56092	/* Opcode: NullRow P1 * * * *
56093	**
	@@ -56278,10 +56264,11 @@
56278	#endif /* local variables moved into u.bn */
56279
56280	assert( pOp->p1>=0 && pOp->p1<p->nCursor );
56281	u.bn.pC = p->apCsr[pOp->p1];
56282	assert( u.bn.pC!=0 );

56283	assert( pIn2->flags & MEM_Blob );
56284	u.bn.pCrsr = u.bn.pC->pCursor;
56285	if( ALWAYS(u.bn.pCrsr!=0) ){
56286	assert( u.bn.pC->isTable==0 );
56287	rc = ExpandBlob(pIn2);
	@@ -56320,11 +56307,11 @@
56320	u.bo.pCrsr = u.bo.pC->pCursor;
56321	if( ALWAYS(u.bo.pCrsr!=0) ){
56322	u.bo.r.pKeyInfo = u.bo.pC->pKeyInfo;
56323	u.bo.r.nField = (u16)pOp->p3;
56324	u.bo.r.flags = 0;
56325	u.bo.r.aMem = &p->aMem[pOp->p2];
56326	rc = sqlite3BtreeMovetoUnpacked(u.bo.pCrsr, &u.bo.r, 0, 0, &u.bo.res);
56327	if( rc==SQLITE_OK && u.bo.res==0 ){
56328	rc = sqlite3BtreeDelete(u.bo.pCrsr);
56329	}
56330	assert( u.bo.pC->deferredMoveto==0 );
	@@ -56350,10 +56337,11 @@
56350
56351	assert( pOp->p1>=0 && pOp->p1<p->nCursor );
56352	u.bp.pC = p->apCsr[pOp->p1];
56353	assert( u.bp.pC!=0 );
56354	u.bp.pCrsr = u.bp.pC->pCursor;

56355	if( ALWAYS(u.bp.pCrsr!=0) ){
56356	rc = sqlite3VdbeCursorMoveto(u.bp.pC);
56357	if( NEVER(rc) ) goto abort_due_to_error;
56358	assert( u.bp.pC->deferredMoveto==0 );
56359	assert( u.bp.pC->isTable==0 );
	@@ -56360,12 +56348,12 @@
56360	if( !u.bp.pC->nullRow ){
56361	rc = sqlite3VdbeIdxRowid(db, u.bp.pCrsr, &u.bp.rowid);
56362	if( rc!=SQLITE_OK ){
56363	goto abort_due_to_error;
56364	}
56365	MemSetTypeFlag(pOut, MEM_Int);
56366	pOut->u.i = u.bp.rowid;

56367	}
56368	}
56369	break;
56370	}
56371
	@@ -56393,12 +56381,12 @@
56393	** Otherwise fall through to the next instruction.
56394	**
56395	** If P5 is non-zero then the key value is increased by an epsilon prior
56396	** to the comparison. This makes the opcode work like IdxLE.
56397	*/
56398	case OP_IdxLT: /* jump, in3 */
56399	case OP_IdxGE: { /* jump, in3 */
56400	#if 0 /* local variables moved into u.bq */
56401	VdbeCursor *pC;
56402	int res;
56403	UnpackedRecord r;
56404	#endif /* local variables moved into u.bq */
	@@ -56415,11 +56403,11 @@
56415	if( pOp->p5 ){
56416	u.bq.r.flags = UNPACKED_INCRKEY \| UNPACKED_IGNORE_ROWID;
56417	}else{
56418	u.bq.r.flags = UNPACKED_IGNORE_ROWID;
56419	}
56420	u.bq.r.aMem = &p->aMem[pOp->p3];
56421	rc = sqlite3VdbeIdxKeyCompare(u.bq.pC, &u.bq.r, &u.bq.res);
56422	if( pOp->opcode==OP_IdxLT ){
56423	u.bq.res = -u.bq.res;
56424	}else{
56425	assert( pOp->opcode==OP_IdxGE );
	@@ -56467,23 +56455,25 @@
56467	}
56468	}
56469	#else
56470	u.br.iCnt = db->activeVdbeCnt;
56471	#endif

56472	if( u.br.iCnt>1 ){
56473	rc = SQLITE_LOCKED;
56474	p->errorAction = OE_Abort;
56475	}else{
56476	u.br.iDb = pOp->p3;
56477	assert( u.br.iCnt==1 );
56478	assert( (p->btreeMask & (1<<u.br.iDb))!=0 );
56479	rc = sqlite3BtreeDropTable(db->aDb[u.br.iDb].pBt, pOp->p1, &u.br.iMoved);
56480	MemSetTypeFlag(pOut, MEM_Int);
56481	pOut->u.i = u.br.iMoved;
56482	#ifndef SQLITE_OMIT_AUTOVACUUM
56483	if( rc==SQLITE_OK && u.br.iMoved!=0 ){
56484	sqlite3RootPageMoved(&db->aDb[u.br.iDb], u.br.iMoved, pOp->p1);

56485	}
56486	#endif
56487	}
56488	break;
56489	}
	@@ -56517,11 +56507,11 @@
56517	db->aDb[pOp->p2].pBt, pOp->p1, (pOp->p3 ? &u.bs.nChange : 0)
56518	);
56519	if( pOp->p3 ){
56520	p->nChange += u.bs.nChange;
56521	if( pOp->p3>0 ){
56522	p->aMem[pOp->p3].u.i += u.bs.nChange;
56523	}
56524	}
56525	break;
56526	}
56527
	@@ -56566,11 +56556,10 @@
56566	}else{
56567	u.bt.flags = BTREE_ZERODATA;
56568	}
56569	rc = sqlite3BtreeCreateTable(u.bt.pDb->pBt, &u.bt.pgno, u.bt.flags);
56570	pOut->u.i = u.bt.pgno;
56571	MemSetTypeFlag(pOut, MEM_Int);
56572	break;
56573	}
56574
56575	/* Opcode: ParseSchema P1 P2 * P4 *
56576	**
	@@ -56733,14 +56722,14 @@
56733	u.bv.nRoot = pOp->p2;
56734	assert( u.bv.nRoot>0 );
56735	u.bv.aRoot = sqlite3DbMallocRaw(db, sizeof(int)*(u.bv.nRoot+1) );
56736	if( u.bv.aRoot==0 ) goto no_mem;
56737	assert( pOp->p3>0 && pOp->p3<=p->nMem );
56738	u.bv.pnErr = &p->aMem[pOp->p3];
56739	assert( (u.bv.pnErr->flags & MEM_Int)!=0 );
56740	assert( (u.bv.pnErr->flags & (MEM_Str\|MEM_Blob))==0 );
56741	pIn1 = &p->aMem[pOp->p1];
56742	for(u.bv.j=0; u.bv.j<u.bv.nRoot; u.bv.j++){
56743	u.bv.aRoot[u.bv.j] = (int)sqlite3VdbeIntValue(&pIn1[u.bv.j]);
56744	}
56745	u.bv.aRoot[u.bv.j] = 0;
56746	assert( pOp->p5<db->nDb );
	@@ -56768,53 +56757,43 @@
56768	** Insert the integer value held by register P2 into a boolean index
56769	** held in register P1.
56770	**
56771	** An assertion fails if P2 is not an integer.
56772	*/
56773	case OP_RowSetAdd: { /* in2 */
56774	#if 0 /* local variables moved into u.bw */
56775	Mem *pIdx;
56776	Mem *pVal;
56777	#endif /* local variables moved into u.bw */
56778	assert( pOp->p1>0 && pOp->p1<=p->nMem );
56779	u.bw.pIdx = &p->aMem[pOp->p1];
56780	assert( pOp->p2>0 && pOp->p2<=p->nMem );
56781	u.bw.pVal = &p->aMem[pOp->p2];
56782	assert( (u.bw.pVal->flags & MEM_Int)!=0 );
56783	if( (u.bw.pIdx->flags & MEM_RowSet)==0 ){
56784	sqlite3VdbeMemSetRowSet(u.bw.pIdx);
56785	if( (u.bw.pIdx->flags & MEM_RowSet)==0 ) goto no_mem;
56786	}
56787	sqlite3RowSetInsert(u.bw.pIdx->u.pRowSet, u.bw.pVal->u.i);
56788	break;
56789	}
56790
56791	/* Opcode: RowSetRead P1 P2 P3 * *
56792	**
56793	** Extract the smallest value from boolean index P1 and put that value into
56794	** register P3. Or, if boolean index P1 is initially empty, leave P3
56795	** unchanged and jump to instruction P2.
56796	*/
56797	case OP_RowSetRead: { /* jump, out3 */
56798	#if 0 /* local variables moved into u.bx */
56799	Mem *pIdx;
56800	i64 val;
56801	#endif /* local variables moved into u.bx */
56802	assert( pOp->p1>0 && pOp->p1<=p->nMem );
56803	CHECK_FOR_INTERRUPT;
56804	u.bx.pIdx = &p->aMem[pOp->p1];
56805	pOut = &p->aMem[pOp->p3];
56806	if( (u.bx.pIdx->flags & MEM_RowSet)==0
56807	\|\| sqlite3RowSetNext(u.bx.pIdx->u.pRowSet, &u.bx.val)==0
56808	){
56809	/* The boolean index is empty */
56810	sqlite3VdbeMemSetNull(u.bx.pIdx);
56811	pc = pOp->p2 - 1;
56812	}else{
56813	/* A value was pulled from the index */
56814	assert( pOp->p3>0 && pOp->p3<=p->nMem );
56815	sqlite3VdbeMemSetInt64(pOut, u.bx.val);
56816	}
56817	break;
56818	}
56819
56820	/* Opcode: RowSetTest P1 P2 P3 P4
	@@ -56839,16 +56818,18 @@
56839	** inserted, there is no need to search to see if the same value was
56840	** previously inserted as part of set X (only if it was previously
56841	** inserted as part of some other set).
56842	*/
56843	case OP_RowSetTest: { /* jump, in1, in3 */
56844	#if 0 /* local variables moved into u.by */
56845	int iSet;
56846	int exists;
56847	#endif /* local variables moved into u.by */
56848
56849	u.by.iSet = pOp->p4.i;


56850	assert( pIn3->flags&MEM_Int );
56851
56852	/* If there is anything other than a rowset object in memory cell P1,
56853	** delete it now and initialize P1 with an empty rowset
56854	*/
	@@ -56856,21 +56837,21 @@
56856	sqlite3VdbeMemSetRowSet(pIn1);
56857	if( (pIn1->flags & MEM_RowSet)==0 ) goto no_mem;
56858	}
56859
56860	assert( pOp->p4type==P4_INT32 );
56861	assert( u.by.iSet==-1 \|\| u.by.iSet>=0 );
56862	if( u.by.iSet ){
56863	u.by.exists = sqlite3RowSetTest(pIn1->u.pRowSet,
56864	(u8)(u.by.iSet>=0 ? u.by.iSet & 0xf : 0xff),
56865	pIn3->u.i);
56866	if( u.by.exists ){
56867	pc = pOp->p2 - 1;
56868	break;
56869	}
56870	}
56871	if( u.by.iSet>=0 ){
56872	sqlite3RowSetInsert(pIn1->u.pRowSet, pIn3->u.i);
56873	}
56874	break;
56875	}
56876
	@@ -56889,24 +56870,24 @@
56889	** memory required by the sub-vdbe at runtime.
56890	**
56891	** P4 is a pointer to the VM containing the trigger program.
56892	*/
56893	case OP_Program: { /* jump */
56894	#if 0 /* local variables moved into u.bz */
56895	int nMem; /* Number of memory registers for sub-program */
56896	int nByte; /* Bytes of runtime space required for sub-program */
56897	Mem pRt; / Register to allocate runtime space */
56898	Mem pMem; / Used to iterate through memory cells */
56899	Mem pEnd; / Last memory cell in new array */
56900	VdbeFrame pFrame; / New vdbe frame to execute in */
56901	SubProgram pProgram; / Sub-program to execute */
56902	void t; / Token identifying trigger */
56903	#endif /* local variables moved into u.bz */
56904
56905	u.bz.pProgram = pOp->p4.pProgram;
56906	u.bz.pRt = &p->aMem[pOp->p3];
56907	assert( u.bz.pProgram->nOp>0 );
56908
56909	/* If the p5 flag is clear, then recursive invocation of triggers is
56910	** disabled for backwards compatibility (p5 is set if this sub-program
56911	** is really a trigger, not a foreign key action, and the flag set
56912	** and cleared by the "PRAGMA recursive_triggers" command is clear).
	@@ -56916,79 +56897,79 @@
56916	** SubProgram (if the trigger may be executed with more than one different
56917	** ON CONFLICT algorithm). SubProgram structures associated with a
56918	** single trigger all have the same value for the SubProgram.token
56919	** variable. */
56920	if( pOp->p5 ){
56921	u.bz.t = u.bz.pProgram->token;
56922	for(u.bz.pFrame=p->pFrame; u.bz.pFrame && u.bz.pFrame->token!=u.bz.t; u.bz.pFrame=u.bz.pFrame->pParent);
56923	if( u.bz.pFrame ) break;
56924	}
56925
56926	if( p->nFrame>=db->aLimit[SQLITE_LIMIT_TRIGGER_DEPTH] ){
56927	rc = SQLITE_ERROR;
56928	sqlite3SetString(&p->zErrMsg, db, "too many levels of trigger recursion");
56929	break;
56930	}
56931
56932	/* Register u.bz.pRt is used to store the memory required to save the state
56933	** of the current program, and the memory required at runtime to execute
56934	** the trigger program. If this trigger has been fired before, then u.bz.pRt
56935	** is already allocated. Otherwise, it must be initialized. */
56936	if( (u.bz.pRt->flags&MEM_Frame)==0 ){
56937	/* SubProgram.nMem is set to the number of memory cells used by the
56938	** program stored in SubProgram.aOp. As well as these, one memory
56939	** cell is required for each cursor used by the program. Set local
56940	** variable u.bz.nMem (and later, VdbeFrame.nChildMem) to this value.
56941	*/
56942	u.bz.nMem = u.bz.pProgram->nMem + u.bz.pProgram->nCsr;
56943	u.bz.nByte = ROUND8(sizeof(VdbeFrame))
56944	+ u.bz.nMem * sizeof(Mem)
56945	+ u.bz.pProgram->nCsr * sizeof(VdbeCursor *);
56946	u.bz.pFrame = sqlite3DbMallocZero(db, u.bz.nByte);
56947	if( !u.bz.pFrame ){
56948	goto no_mem;
56949	}
56950	sqlite3VdbeMemRelease(u.bz.pRt);
56951	u.bz.pRt->flags = MEM_Frame;
56952	u.bz.pRt->u.pFrame = u.bz.pFrame;
56953
56954	u.bz.pFrame->v = p;
56955	u.bz.pFrame->nChildMem = u.bz.nMem;
56956	u.bz.pFrame->nChildCsr = u.bz.pProgram->nCsr;
56957	u.bz.pFrame->pc = pc;
56958	u.bz.pFrame->aMem = p->aMem;
56959	u.bz.pFrame->nMem = p->nMem;
56960	u.bz.pFrame->apCsr = p->apCsr;
56961	u.bz.pFrame->nCursor = p->nCursor;
56962	u.bz.pFrame->aOp = p->aOp;
56963	u.bz.pFrame->nOp = p->nOp;
56964	u.bz.pFrame->token = u.bz.pProgram->token;
56965
56966	u.bz.pEnd = &VdbeFrameMem(u.bz.pFrame)[u.bz.pFrame->nChildMem];
56967	for(u.bz.pMem=VdbeFrameMem(u.bz.pFrame); u.bz.pMem!=u.bz.pEnd; u.bz.pMem++){
56968	u.bz.pMem->flags = MEM_Null;
56969	u.bz.pMem->db = db;
56970	}
56971	}else{
56972	u.bz.pFrame = u.bz.pRt->u.pFrame;
56973	assert( u.bz.pProgram->nMem+u.bz.pProgram->nCsr==u.bz.pFrame->nChildMem );
56974	assert( u.bz.pProgram->nCsr==u.bz.pFrame->nChildCsr );
56975	assert( pc==u.bz.pFrame->pc );
56976	}
56977
56978	p->nFrame++;
56979	u.bz.pFrame->pParent = p->pFrame;
56980	u.bz.pFrame->lastRowid = db->lastRowid;
56981	u.bz.pFrame->nChange = p->nChange;
56982	p->nChange = 0;
56983	p->pFrame = u.bz.pFrame;
56984	p->aMem = &VdbeFrameMem(u.bz.pFrame)[-1];
56985	p->nMem = u.bz.pFrame->nChildMem;
56986	p->nCursor = (u16)u.bz.pFrame->nChildCsr;
56987	p->apCsr = (VdbeCursor **)&p->aMem[p->nMem+1];
56988	p->aOp = u.bz.pProgram->aOp;
56989	p->nOp = u.bz.pProgram->nOp;
56990	pc = -1;
56991
56992	break;
56993	}
56994
	@@ -57003,17 +56984,17 @@
57003	** The address of the cell in the parent frame is determined by adding
57004	** the value of the P1 argument to the value of the P1 argument to the
57005	** calling OP_Program instruction.
57006	*/
57007	case OP_Param: { /* out2-prerelease */
57008	#if 0 /* local variables moved into u.ca */
57009	VdbeFrame *pFrame;
57010	Mem *pIn;
57011	#endif /* local variables moved into u.ca */
57012	u.ca.pFrame = p->pFrame;
57013	u.ca.pIn = &u.ca.pFrame->aMem[pOp->p1 + u.ca.pFrame->aOp[u.ca.pFrame->pc].p1];
57014	sqlite3VdbeMemShallowCopy(pOut, u.ca.pIn, MEM_Ephem);
57015	break;
57016	}
57017
57018	#endif /* #ifndef SQLITE_OMIT_TRIGGER */
57019
	@@ -57065,24 +57046,25 @@
57065	**
57066	** This instruction throws an error if the memory cell is not initially
57067	** an integer.
57068	*/
57069	case OP_MemMax: { /* in2 */
57070	#if 0 /* local variables moved into u.cb */
57071	Mem *pIn1;
57072	VdbeFrame *pFrame;
57073	#endif /* local variables moved into u.cb */
57074	if( p->pFrame ){
57075	for(u.cb.pFrame=p->pFrame; u.cb.pFrame->pParent; u.cb.pFrame=u.cb.pFrame->pParent);
57076	u.cb.pIn1 = &u.cb.pFrame->aMem[pOp->p1];
57077	}else{
57078	u.cb.pIn1 = &p->aMem[pOp->p1];
57079	}
57080	sqlite3VdbeMemIntegerify(u.cb.pIn1);

57081	sqlite3VdbeMemIntegerify(pIn2);
57082	if( u.cb.pIn1->u.i<pIn2->u.i){
57083	u.cb.pIn1->u.i = pIn2->u.i;
57084	}
57085	break;
57086	}
57087	#endif /* SQLITE_OMIT_AUTOINCREMENT */
57088
	@@ -57092,10 +57074,11 @@
57092	**
57093	** It is illegal to use this instruction on a register that does
57094	** not contain an integer. An assertion fault will result if you try.
57095	*/
57096	case OP_IfPos: { /* jump, in1 */

57097	assert( pIn1->flags&MEM_Int );
57098	if( pIn1->u.i>0 ){
57099	pc = pOp->p2 - 1;
57100	}
57101	break;
	@@ -57107,26 +57090,30 @@
57107	**
57108	** It is illegal to use this instruction on a register that does
57109	** not contain an integer. An assertion fault will result if you try.
57110	*/
57111	case OP_IfNeg: { /* jump, in1 */

57112	assert( pIn1->flags&MEM_Int );
57113	if( pIn1->u.i<0 ){
57114	pc = pOp->p2 - 1;
57115	}
57116	break;
57117	}
57118
57119	/* Opcode: IfZero P1 P2 * * *
57120	**
57121	** If the value of register P1 is exactly 0, jump to P2.

57122	**
57123	** It is illegal to use this instruction on a register that does
57124	** not contain an integer. An assertion fault will result if you try.
57125	*/
57126	case OP_IfZero: { /* jump, in1 */

57127	assert( pIn1->flags&MEM_Int );

57128	if( pIn1->u.i==0 ){
57129	pc = pOp->p2 - 1;
57130	}
57131	break;
57132	}
	@@ -57140,51 +57127,51 @@
57140	**
57141	** The P5 arguments are taken from register P2 and its
57142	** successors.
57143	*/
57144	case OP_AggStep: {
57145	#if 0 /* local variables moved into u.cc */
57146	int n;
57147	int i;
57148	Mem *pMem;
57149	Mem *pRec;
57150	sqlite3_context ctx;
57151	sqlite3_value **apVal;
57152	#endif /* local variables moved into u.cc */
57153
57154	u.cc.n = pOp->p5;
57155	assert( u.cc.n>=0 );
57156	u.cc.pRec = &p->aMem[pOp->p2];
57157	u.cc.apVal = p->apArg;
57158	assert( u.cc.apVal \|\| u.cc.n==0 );
57159	for(u.cc.i=0; u.cc.i<u.cc.n; u.cc.i++, u.cc.pRec++){
57160	u.cc.apVal[u.cc.i] = u.cc.pRec;
57161	sqlite3VdbeMemStoreType(u.cc.pRec);
57162	}
57163	u.cc.ctx.pFunc = pOp->p4.pFunc;
57164	assert( pOp->p3>0 && pOp->p3<=p->nMem );
57165	u.cc.ctx.pMem = u.cc.pMem = &p->aMem[pOp->p3];
57166	u.cc.pMem->n++;
57167	u.cc.ctx.s.flags = MEM_Null;
57168	u.cc.ctx.s.z = 0;
57169	u.cc.ctx.s.zMalloc = 0;
57170	u.cc.ctx.s.xDel = 0;
57171	u.cc.ctx.s.db = db;
57172	u.cc.ctx.isError = 0;
57173	u.cc.ctx.pColl = 0;
57174	if( u.cc.ctx.pFunc->flags & SQLITE_FUNC_NEEDCOLL ){
57175	assert( pOp>p->aOp );
57176	assert( pOp[-1].p4type==P4_COLLSEQ );
57177	assert( pOp[-1].opcode==OP_CollSeq );
57178	u.cc.ctx.pColl = pOp[-1].p4.pColl;
57179	}
57180	(u.cc.ctx.pFunc->xStep)(&u.cc.ctx, u.cc.n, u.cc.apVal);
57181	if( u.cc.ctx.isError ){
57182	sqlite3SetString(&p->zErrMsg, db, "%s", sqlite3_value_text(&u.cc.ctx.s));
57183	rc = u.cc.ctx.isError;
57184	}
57185	sqlite3VdbeMemRelease(&u.cc.ctx.s);
57186	break;
57187	}
57188
57189	/* Opcode: AggFinal P1 P2 * P4 *
57190	**
	@@ -57197,23 +57184,23 @@
57197	** functions that can take varying numbers of arguments. The
57198	** P4 argument is only needed for the degenerate case where
57199	** the step function was not previously called.
57200	*/
57201	case OP_AggFinal: {
57202	#if 0 /* local variables moved into u.cd */
57203	Mem *pMem;
57204	#endif /* local variables moved into u.cd */
57205	assert( pOp->p1>0 && pOp->p1<=p->nMem );
57206	u.cd.pMem = &p->aMem[pOp->p1];
57207	assert( (u.cd.pMem->flags & ~(MEM_Null\|MEM_Agg))==0 );
57208	rc = sqlite3VdbeMemFinalize(u.cd.pMem, pOp->p4.pFunc);
57209	if( rc ){
57210	sqlite3SetString(&p->zErrMsg, db, "%s", sqlite3_value_text(u.cd.pMem));
57211	}
57212	sqlite3VdbeChangeEncoding(u.cd.pMem, encoding);
57213	UPDATE_MAX_BLOBSIZE(u.cd.pMem);
57214	if( sqlite3VdbeMemTooBig(u.cd.pMem) ){
57215	goto too_big;
57216	}
57217	break;
57218	}
57219
	@@ -57239,18 +57226,18 @@
57239	** Perform a single step of the incremental vacuum procedure on
57240	** the P1 database. If the vacuum has finished, jump to instruction
57241	** P2. Otherwise, fall through to the next instruction.
57242	*/
57243	case OP_IncrVacuum: { /* jump */
57244	#if 0 /* local variables moved into u.ce */
57245	Btree *pBt;
57246	#endif /* local variables moved into u.ce */
57247
57248	assert( pOp->p1>=0 && pOp->p1<db->nDb );
57249	assert( (p->btreeMask & (1<<pOp->p1))!=0 );
57250	u.ce.pBt = db->aDb[pOp->p1].pBt;
57251	rc = sqlite3BtreeIncrVacuum(u.ce.pBt);
57252	if( rc==SQLITE_DONE ){
57253	pc = pOp->p2 - 1;
57254	rc = SQLITE_OK;
57255	}
57256	break;
	@@ -57316,19 +57303,19 @@
57316	** Also, whether or not P4 is set, check that this is not being called from
57317	** within a callback to a virtual table xSync() method. If it is, the error
57318	** code will be set to SQLITE_LOCKED.
57319	*/
57320	case OP_VBegin: {
57321	#if 0 /* local variables moved into u.cf */
57322	VTable *pVTab;
57323	#endif /* local variables moved into u.cf */
57324	u.cf.pVTab = pOp->p4.pVtab;
57325	rc = sqlite3VtabBegin(db, u.cf.pVTab);
57326	if( u.cf.pVTab ){
57327	sqlite3DbFree(db, p->zErrMsg);
57328	p->zErrMsg = u.cf.pVTab->pVtab->zErrMsg;
57329	u.cf.pVTab->pVtab->zErrMsg = 0;
57330	}
57331	break;
57332	}
57333	#endif /* SQLITE_OMIT_VIRTUALTABLE */
57334
	@@ -57364,40 +57351,40 @@
57364	** P4 is a pointer to a virtual table object, an sqlite3_vtab structure.
57365	** P1 is a cursor number. This opcode opens a cursor to the virtual
57366	** table and stores that cursor in P1.
57367	*/
57368	case OP_VOpen: {
57369	#if 0 /* local variables moved into u.cg */
57370	VdbeCursor *pCur;
57371	sqlite3_vtab_cursor *pVtabCursor;
57372	sqlite3_vtab *pVtab;
57373	sqlite3_module *pModule;
57374	#endif /* local variables moved into u.cg */
57375
57376	u.cg.pCur = 0;
57377	u.cg.pVtabCursor = 0;
57378	u.cg.pVtab = pOp->p4.pVtab->pVtab;
57379	u.cg.pModule = (sqlite3_module *)u.cg.pVtab->pModule;
57380	assert(u.cg.pVtab && u.cg.pModule);
57381	if( sqlite3SafetyOff(db) ) goto abort_due_to_misuse;
57382	rc = u.cg.pModule->xOpen(u.cg.pVtab, &u.cg.pVtabCursor);
57383	sqlite3DbFree(db, p->zErrMsg);
57384	p->zErrMsg = u.cg.pVtab->zErrMsg;
57385	u.cg.pVtab->zErrMsg = 0;
57386	if( sqlite3SafetyOn(db) ) goto abort_due_to_misuse;
57387	if( SQLITE_OK==rc ){
57388	/* Initialize sqlite3_vtab_cursor base class */
57389	u.cg.pVtabCursor->pVtab = u.cg.pVtab;
57390
57391	/* Initialise vdbe cursor object */
57392	u.cg.pCur = allocateCursor(p, pOp->p1, 0, -1, 0);
57393	if( u.cg.pCur ){
57394	u.cg.pCur->pVtabCursor = u.cg.pVtabCursor;
57395	u.cg.pCur->pModule = u.cg.pVtabCursor->pVtab->pModule;
57396	}else{
57397	db->mallocFailed = 1;
57398	u.cg.pModule->xClose(u.cg.pVtabCursor);
57399	}
57400	}
57401	break;
57402	}
57403	#endif /* SQLITE_OMIT_VIRTUALTABLE */
	@@ -57420,11 +57407,11 @@
57420	** xFilter as argv. Register P3+2 becomes argv[0] when passed to xFilter.
57421	**
57422	** A jump is made to P2 if the result set after filtering would be empty.
57423	*/
57424	case OP_VFilter: { /* jump */
57425	#if 0 /* local variables moved into u.ch */
57426	int nArg;
57427	int iQuery;
57428	const sqlite3_module *pModule;
57429	Mem *pQuery;
57430	Mem *pArgc;
	@@ -57432,52 +57419,52 @@
57432	sqlite3_vtab *pVtab;
57433	VdbeCursor *pCur;
57434	int res;
57435	int i;
57436	Mem **apArg;
57437	#endif /* local variables moved into u.ch */
57438
57439	u.ch.pQuery = &p->aMem[pOp->p3];
57440	u.ch.pArgc = &u.ch.pQuery[1];
57441	u.ch.pCur = p->apCsr[pOp->p1];
57442	REGISTER_TRACE(pOp->p3, u.ch.pQuery);
57443	assert( u.ch.pCur->pVtabCursor );
57444	u.ch.pVtabCursor = u.ch.pCur->pVtabCursor;
57445	u.ch.pVtab = u.ch.pVtabCursor->pVtab;
57446	u.ch.pModule = u.ch.pVtab->pModule;
57447
57448	/* Grab the index number and argc parameters */
57449	assert( (u.ch.pQuery->flags&MEM_Int)!=0 && u.ch.pArgc->flags==MEM_Int );
57450	u.ch.nArg = (int)u.ch.pArgc->u.i;
57451	u.ch.iQuery = (int)u.ch.pQuery->u.i;
57452
57453	/* Invoke the xFilter method */
57454	{
57455	u.ch.res = 0;
57456	u.ch.apArg = p->apArg;
57457	for(u.ch.i = 0; u.ch.i<u.ch.nArg; u.ch.i++){
57458	u.ch.apArg[u.ch.i] = &u.ch.pArgc[u.ch.i+1];
57459	sqlite3VdbeMemStoreType(u.ch.apArg[u.ch.i]);
57460	}
57461
57462	if( sqlite3SafetyOff(db) ) goto abort_due_to_misuse;
57463	p->inVtabMethod = 1;
57464	rc = u.ch.pModule->xFilter(u.ch.pVtabCursor, u.ch.iQuery, pOp->p4.z, u.ch.nArg, u.ch.apArg);
57465	p->inVtabMethod = 0;
57466	sqlite3DbFree(db, p->zErrMsg);
57467	p->zErrMsg = u.ch.pVtab->zErrMsg;
57468	u.ch.pVtab->zErrMsg = 0;
57469	if( rc==SQLITE_OK ){
57470	u.ch.res = u.ch.pModule->xEof(u.ch.pVtabCursor);
57471	}
57472	if( sqlite3SafetyOn(db) ) goto abort_due_to_misuse;
57473
57474	if( u.ch.res ){
57475	pc = pOp->p2 - 1;
57476	}
57477	}
57478	u.ch.pCur->nullRow = 0;
57479
57480	break;
57481	}
57482	#endif /* SQLITE_OMIT_VIRTUALTABLE */
57483
	@@ -57487,60 +57474,60 @@
57487	** Store the value of the P2-th column of
57488	** the row of the virtual-table that the
57489	** P1 cursor is pointing to into register P3.
57490	*/
57491	case OP_VColumn: {
57492	#if 0 /* local variables moved into u.ci */
57493	sqlite3_vtab *pVtab;
57494	const sqlite3_module *pModule;
57495	Mem *pDest;
57496	sqlite3_context sContext;
57497	#endif /* local variables moved into u.ci */
57498
57499	VdbeCursor *pCur = p->apCsr[pOp->p1];
57500	assert( pCur->pVtabCursor );
57501	assert( pOp->p3>0 && pOp->p3<=p->nMem );
57502	u.ci.pDest = &p->aMem[pOp->p3];
57503	if( pCur->nullRow ){
57504	sqlite3VdbeMemSetNull(u.ci.pDest);
57505	break;
57506	}
57507	u.ci.pVtab = pCur->pVtabCursor->pVtab;
57508	u.ci.pModule = u.ci.pVtab->pModule;
57509	assert( u.ci.pModule->xColumn );
57510	memset(&u.ci.sContext, 0, sizeof(u.ci.sContext));
57511
57512	/* The output cell may already have a buffer allocated. Move
57513	** the current contents to u.ci.sContext.s so in case the user-function
57514	** can use the already allocated buffer instead of allocating a
57515	** new one.
57516	*/
57517	sqlite3VdbeMemMove(&u.ci.sContext.s, u.ci.pDest);
57518	MemSetTypeFlag(&u.ci.sContext.s, MEM_Null);
57519
57520	if( sqlite3SafetyOff(db) ) goto abort_due_to_misuse;
57521	rc = u.ci.pModule->xColumn(pCur->pVtabCursor, &u.ci.sContext, pOp->p2);
57522	sqlite3DbFree(db, p->zErrMsg);
57523	p->zErrMsg = u.ci.pVtab->zErrMsg;
57524	u.ci.pVtab->zErrMsg = 0;
57525	if( u.ci.sContext.isError ){
57526	rc = u.ci.sContext.isError;
57527	}
57528
57529	/* Copy the result of the function to the P3 register. We
57530	** do this regardless of whether or not an error occurred to ensure any
57531	** dynamic allocation in u.ci.sContext.s (a Mem struct) is released.
57532	*/
57533	sqlite3VdbeChangeEncoding(&u.ci.sContext.s, encoding);
57534	REGISTER_TRACE(pOp->p3, u.ci.pDest);
57535	sqlite3VdbeMemMove(u.ci.pDest, &u.ci.sContext.s);
57536	UPDATE_MAX_BLOBSIZE(u.ci.pDest);
57537
57538	if( sqlite3SafetyOn(db) ){
57539	goto abort_due_to_misuse;
57540	}
57541	if( sqlite3VdbeMemTooBig(u.ci.pDest) ){
57542	goto too_big;
57543	}
57544	break;
57545	}
57546	#endif /* SQLITE_OMIT_VIRTUALTABLE */
	@@ -57551,46 +57538,46 @@
57551	** Advance virtual table P1 to the next row in its result set and
57552	** jump to instruction P2. Or, if the virtual table has reached
57553	** the end of its result set, then fall through to the next instruction.
57554	*/
57555	case OP_VNext: { /* jump */
57556	#if 0 /* local variables moved into u.cj */
57557	sqlite3_vtab *pVtab;
57558	const sqlite3_module *pModule;
57559	int res;
57560	VdbeCursor *pCur;
57561	#endif /* local variables moved into u.cj */
57562
57563	u.cj.res = 0;
57564	u.cj.pCur = p->apCsr[pOp->p1];
57565	assert( u.cj.pCur->pVtabCursor );
57566	if( u.cj.pCur->nullRow ){
57567	break;
57568	}
57569	u.cj.pVtab = u.cj.pCur->pVtabCursor->pVtab;
57570	u.cj.pModule = u.cj.pVtab->pModule;
57571	assert( u.cj.pModule->xNext );
57572
57573	/* Invoke the xNext() method of the module. There is no way for the
57574	** underlying implementation to return an error if one occurs during
57575	** xNext(). Instead, if an error occurs, true is returned (indicating that
57576	** data is available) and the error code returned when xColumn or
57577	** some other method is next invoked on the save virtual table cursor.
57578	*/
57579	if( sqlite3SafetyOff(db) ) goto abort_due_to_misuse;
57580	p->inVtabMethod = 1;
57581	rc = u.cj.pModule->xNext(u.cj.pCur->pVtabCursor);
57582	p->inVtabMethod = 0;
57583	sqlite3DbFree(db, p->zErrMsg);
57584	p->zErrMsg = u.cj.pVtab->zErrMsg;
57585	u.cj.pVtab->zErrMsg = 0;
57586	if( rc==SQLITE_OK ){
57587	u.cj.res = u.cj.pModule->xEof(u.cj.pCur->pVtabCursor);
57588	}
57589	if( sqlite3SafetyOn(db) ) goto abort_due_to_misuse;
57590
57591	if( !u.cj.res ){
57592	/* If there is data, jump to P2 */
57593	pc = pOp->p2 - 1;
57594	}
57595	break;
57596	}
	@@ -57602,25 +57589,25 @@
57602	** P4 is a pointer to a virtual table object, an sqlite3_vtab structure.
57603	** This opcode invokes the corresponding xRename method. The value
57604	** in register P1 is passed as the zName argument to the xRename method.
57605	*/
57606	case OP_VRename: {
57607	#if 0 /* local variables moved into u.ck */
57608	sqlite3_vtab *pVtab;
57609	Mem *pName;
57610	#endif /* local variables moved into u.ck */
57611
57612	u.ck.pVtab = pOp->p4.pVtab->pVtab;
57613	u.ck.pName = &p->aMem[pOp->p1];
57614	assert( u.ck.pVtab->pModule->xRename );
57615	REGISTER_TRACE(pOp->p1, u.ck.pName);
57616	assert( u.ck.pName->flags & MEM_Str );
57617	if( sqlite3SafetyOff(db) ) goto abort_due_to_misuse;
57618	rc = u.ck.pVtab->pModule->xRename(u.ck.pVtab, u.ck.pName->z);
57619	sqlite3DbFree(db, p->zErrMsg);
57620	p->zErrMsg = u.ck.pVtab->zErrMsg;
57621	u.ck.pVtab->zErrMsg = 0;
57622	if( sqlite3SafetyOn(db) ) goto abort_due_to_misuse;
57623
57624	break;
57625	}
57626	#endif
	@@ -57648,41 +57635,41 @@
57648	** P1 is a boolean flag. If it is set to true and the xUpdate call
57649	** is successful, then the value returned by sqlite3_last_insert_rowid()
57650	** is set to the value of the rowid for the row just inserted.
57651	*/
57652	case OP_VUpdate: {
57653	#if 0 /* local variables moved into u.cl */
57654	sqlite3_vtab *pVtab;
57655	sqlite3_module *pModule;
57656	int nArg;
57657	int i;
57658	sqlite_int64 rowid;
57659	Mem **apArg;
57660	Mem *pX;
57661	#endif /* local variables moved into u.cl */
57662
57663	u.cl.pVtab = pOp->p4.pVtab->pVtab;
57664	u.cl.pModule = (sqlite3_module *)u.cl.pVtab->pModule;
57665	u.cl.nArg = pOp->p2;
57666	assert( pOp->p4type==P4_VTAB );
57667	if( ALWAYS(u.cl.pModule->xUpdate) ){
57668	u.cl.apArg = p->apArg;
57669	u.cl.pX = &p->aMem[pOp->p3];
57670	for(u.cl.i=0; u.cl.i<u.cl.nArg; u.cl.i++){
57671	sqlite3VdbeMemStoreType(u.cl.pX);
57672	u.cl.apArg[u.cl.i] = u.cl.pX;
57673	u.cl.pX++;
57674	}
57675	if( sqlite3SafetyOff(db) ) goto abort_due_to_misuse;
57676	rc = u.cl.pModule->xUpdate(u.cl.pVtab, u.cl.nArg, u.cl.apArg, &u.cl.rowid);
57677	sqlite3DbFree(db, p->zErrMsg);
57678	p->zErrMsg = u.cl.pVtab->zErrMsg;
57679	u.cl.pVtab->zErrMsg = 0;
57680	if( sqlite3SafetyOn(db) ) goto abort_due_to_misuse;
57681	if( rc==SQLITE_OK && pOp->p1 ){
57682	assert( u.cl.nArg>1 && u.cl.apArg[0] && (u.cl.apArg[0]->flags&MEM_Null) );
57683	db->lastRowid = u.cl.rowid;
57684	}
57685	p->nChange++;
57686	}
57687	break;
57688	}
	@@ -57692,25 +57679,24 @@
57692	/* Opcode: Pagecount P1 P2 * * *
57693	**
57694	** Write the current number of pages in database P1 to memory cell P2.
57695	*/
57696	case OP_Pagecount: { /* out2-prerelease */
57697	#if 0 /* local variables moved into u.cm */
57698	int p1;
57699	int nPage;
57700	Pager *pPager;
57701	#endif /* local variables moved into u.cm */
57702
57703	u.cm.p1 = pOp->p1;
57704	u.cm.pPager = sqlite3BtreePager(db->aDb[u.cm.p1].pBt);
57705	rc = sqlite3PagerPagecount(u.cm.pPager, &u.cm.nPage);
57706	/* OP_Pagecount is always called from within a read transaction. The
57707	** page count has already been successfully read and cached. So the
57708	** sqlite3PagerPagecount() call above cannot fail. */
57709	if( ALWAYS(rc==SQLITE_OK) ){
57710	pOut->flags = MEM_Int;
57711	pOut->u.i = u.cm.nPage;
57712	}
57713	break;
57714	}
57715	#endif
57716
	@@ -57719,22 +57705,22 @@
57719	**
57720	** If tracing is enabled (by the sqlite3_trace()) interface, then
57721	** the UTF-8 string contained in P4 is emitted on the trace callback.
57722	*/
57723	case OP_Trace: {
57724	#if 0 /* local variables moved into u.cn */
57725	char *zTrace;
57726	#endif /* local variables moved into u.cn */
57727
57728	u.cn.zTrace = (pOp->p4.z ? pOp->p4.z : p->zSql);
57729	if( u.cn.zTrace ){
57730	if( db->xTrace ){
57731	db->xTrace(db->pTraceArg, u.cn.zTrace);
57732	}
57733	#ifdef SQLITE_DEBUG
57734	if( (db->flags & SQLITE_SqlTrace)!=0 ){
57735	sqlite3DebugPrintf("SQL-trace: %s\n", u.cn.zTrace);
57736	}
57737	#endif /* SQLITE_DEBUG */
57738	}
57739	break;
57740	}
	@@ -57769,11 +57755,11 @@
57769	u64 elapsed = sqlite3Hwtime() - start;
57770	pOp->cycles += elapsed;
57771	pOp->cnt++;
57772	#if 0
57773	fprintf(stdout, "%10llu ", elapsed);
57774	sqlite3VdbePrintOp(stdout, origPc, &p->aOp[origPc]);
57775	#endif
57776	}
57777	#endif
57778
57779	/* The following code adds nothing to the actual functionality
	@@ -57785,15 +57771,15 @@
57785	assert( pc>=-1 && pc<p->nOp );
57786
57787	#ifdef SQLITE_DEBUG
57788	if( p->trace ){
57789	if( rc!=0 ) fprintf(p->trace,"rc=%d\n",rc);
57790	if( opProperty & OPFLG_OUT2_PRERELEASE ){
57791	registerTrace(p->trace, pOp->p2, pOut);
57792	}
57793	if( opProperty & OPFLG_OUT3 ){
57794	registerTrace(p->trace, pOp->p3, pOut);
57795	}
57796	}
57797	#endif /* SQLITE_DEBUG */
57798	#endif /* NDEBUG */
57799	} /* The end of the for(;;) loop the loops through opcodes */
	@@ -57805,10 +57791,11 @@
57805	assert( rc );
57806	p->rc = rc;
57807	sqlite3VdbeHalt(p);
57808	if( rc==SQLITE_IOERR_NOMEM ) db->mallocFailed = 1;
57809	rc = SQLITE_ERROR;

57810
57811	/* This is the only way out of this procedure. We have to
57812	** release the mutexes on btrees that were acquired at the
57813	** top. */
57814	vdbe_return:
	@@ -57872,12 +57859,10 @@
57872	** May you share freely, never taking more than you give.
57873	**
57874	*************************************************************************
57875	**
57876	** This file contains code used to implement incremental BLOB I/O.
57877	**
57878	** $Id: vdbeblob.c,v 1.35 2009/07/02 07:47:33 danielk1977 Exp $
57879	*/
57880
57881
57882	#ifndef SQLITE_OMIT_INCRBLOB
57883
	@@ -58266,16 +58251,10 @@
58266	** May you find forgiveness for yourself and forgive others.
58267	** May you share freely, never taking more than you give.
58268	**
58269	*************************************************************************
58270	**
58271	** @(#) $Id: journal.c,v 1.9 2009/01/20 17:06:27 danielk1977 Exp $
58272	*/
58273
58274	#ifdef SQLITE_ENABLE_ATOMIC_WRITE
58275
58276	/*
58277	** This file implements a special kind of sqlite3_file object used
58278	** by SQLite to create journal files if the atomic-write optimization
58279	** is enabled.
58280	**
58281	** The distinctive characteristic of this sqlite3_file is that the
	@@ -58286,11 +58265,11 @@
58286	**
58287	** 1) The in-memory representation grows too large for the allocated
58288	** buffer, or
58289	** 2) The sqlite3JournalCreate() function is called.
58290	*/
58291
58292
58293
58294	/*
58295	** A JournalFile object is a subclass of sqlite3_file used by
58296	** as an open file handle for journal files.
	@@ -58511,12 +58490,10 @@
58511	*************************************************************************
58512	**
58513	** This file contains code use to implement an in-memory rollback journal.
58514	** The in-memory rollback journal is used to journal transactions for
58515	** ":memory:" databases and when the journal_mode=MEMORY pragma is used.
58516	**
58517	** @(#) $Id: memjournal.c,v 1.12 2009/05/04 11:42:30 danielk1977 Exp $
58518	*/
58519
58520	/* Forward references to internal structures */
58521	typedef struct MemJournal MemJournal;
58522	typedef struct FilePoint FilePoint;
	@@ -58770,12 +58747,10 @@
58770	** May you share freely, never taking more than you give.
58771	**
58772	*************************************************************************
58773	** This file contains routines used for walking the parser tree for
58774	** an SQL statement.
58775	**
58776	** $Id: walker.c,v 1.7 2009/06/15 23:15:59 drh Exp $
58777	*/
58778
58779
58780	/*
58781	** Walk an expression tree. Invoke the callback once for each node
	@@ -58910,12 +58885,10 @@
58910	*************************************************************************
58911	**
58912	** This file contains routines used for walking the parser tree and
58913	** resolve all identifiers by associating them with a particular
58914	** table and column.
58915	**
58916	** $Id: resolve.c,v 1.30 2009/06/15 23:15:59 drh Exp $
58917	*/
58918
58919	/*
58920	** Turn the pExpr expression into an alias for the iCol-th column of the
58921	** result set in pEList.
	@@ -61342,10 +61315,98 @@
61342	p->flags \|= EP_IntValue;
61343	p->u.iValue = *pValue;
61344	}
61345	return rc;
61346	}
























































































61347
61348	/*
61349	** Return TRUE if the given string is a row-id column name.
61350	*/
61351	SQLITE_PRIVATE int sqlite3IsRowid(const char *z){
	@@ -61430,20 +61491,20 @@
61430	** be found with <column> as its left-most column.
61431	**
61432	** When the b-tree is being used for membership tests, the calling function
61433	** needs to know whether or not the structure contains an SQL NULL
61434	** value in order to correctly evaluate expressions like "X IN (Y, Z)".
61435	** If there is a chance that the b-tree might contain a NULL value at
61436	** runtime, then a register is allocated and the register number written
61437	** to *prNotFound. If there is no chance that the b-tree contains a
61438	** NULL value, then *prNotFound is left unchanged.
61439	**
61440	** If a register is allocated and its location stored in *prNotFound, then
61441	** its initial value is NULL. If the b-tree does not remain constant
61442	** for the duration of the query (i.e. the SELECT that generates the b-tree
61443	** is a correlated subquery) then the value of the allocated register is
61444	** reset to NULL each time the b-tree is repopulated. This allows the
61445	** caller to use vdbe code equivalent to the following:
61446	**
61447	** if( register==NULL ){
61448	** has_null = <test if data structure contains null>
61449	** register = 1
	@@ -61631,11 +61692,11 @@
61631	}
61632
61633	affinity = sqlite3ExprAffinity(pLeft);
61634
61635	/* Whether this is an 'x IN(SELECT...)' or an 'x IN(<exprlist>)'
61636	** expression it is handled the same way. A virtual table is
61637	** filled with single-field index keys representing the results
61638	** from the SELECT or the <exprlist>.
61639	**
61640	** If the 'x' expression is a column value, or the SELECT...
61641	** statement returns a column value, then the affinity of that
	@@ -61779,10 +61840,132 @@
61779
61780	return rReg;
61781	}
61782	#endif /* SQLITE_OMIT_SUBQUERY */
61783


























































































































61784	/*
61785	** Duplicate an 8-byte value
61786	*/
61787	static char dup8bytes(Vdbe v, const char *in){
61788	char *out = sqlite3DbMallocRaw(sqlite3VdbeDb(v), 8);
	@@ -62479,10 +62662,31 @@
62479	pDef = sqlite3FindFunction(db, zId, nId, nFarg, enc, 0);
62480	if( pDef==0 ){
62481	sqlite3ErrorMsg(pParse, "unknown function: %.*s()", nId, zId);
62482	break;
62483	}





















62484	if( pFarg ){
62485	r1 = sqlite3GetTempRange(pParse, nFarg);
62486	sqlite3ExprCachePush(pParse); /* Ticket 2ea2425d34be */
62487	sqlite3ExprCodeExprList(pParse, pFarg, r1, 1);
62488	sqlite3ExprCachePop(pParse, 1); /* Ticket 2ea2425d34be */
	@@ -62536,99 +62740,23 @@
62536	testcase( op==TK_SELECT );
62537	inReg = sqlite3CodeSubselect(pParse, pExpr, 0, 0);
62538	break;
62539	}
62540	case TK_IN: {
62541	int rNotFound = 0;
62542	int rMayHaveNull = 0;
62543	int j2, j3, j4, j5;
62544	char affinity;
62545	int eType;
62546
62547	VdbeNoopComment((v, "begin IN expr r%d", target));
62548	eType = sqlite3FindInIndex(pParse, pExpr, &rMayHaveNull);
62549	if( rMayHaveNull ){
62550	rNotFound = ++pParse->nMem;
62551	}
62552
62553	/* Figure out the affinity to use to create a key from the results
62554	** of the expression. affinityStr stores a static string suitable for
62555	** P4 of OP_MakeRecord.
62556	*/
62557	affinity = comparisonAffinity(pExpr);
62558
62559
62560	/* Code the <expr> from "<expr> IN (...)". The temporary table
62561	** pExpr->iTable contains the values that make up the (...) set.
62562	*/
62563	sqlite3ExprCachePush(pParse);
62564	sqlite3ExprCode(pParse, pExpr->pLeft, target);
62565	j2 = sqlite3VdbeAddOp1(v, OP_IsNull, target);
62566	if( eType==IN_INDEX_ROWID ){
62567	j3 = sqlite3VdbeAddOp1(v, OP_MustBeInt, target);
62568	j4 = sqlite3VdbeAddOp3(v, OP_NotExists, pExpr->iTable, 0, target);
62569	sqlite3VdbeAddOp2(v, OP_Integer, 1, target);
62570	j5 = sqlite3VdbeAddOp0(v, OP_Goto);
62571	sqlite3VdbeJumpHere(v, j3);
62572	sqlite3VdbeJumpHere(v, j4);
62573	sqlite3VdbeAddOp2(v, OP_Integer, 0, target);
62574	}else{
62575	r2 = regFree2 = sqlite3GetTempReg(pParse);
62576
62577	/* Create a record and test for set membership. If the set contains
62578	** the value, then jump to the end of the test code. The target
62579	** register still contains the true (1) value written to it earlier.
62580	*/
62581	sqlite3VdbeAddOp4(v, OP_MakeRecord, target, 1, r2, &affinity, 1);
62582	sqlite3VdbeAddOp2(v, OP_Integer, 1, target);
62583	j5 = sqlite3VdbeAddOp3(v, OP_Found, pExpr->iTable, 0, r2);
62584
62585	/* If the set membership test fails, then the result of the
62586	** "x IN (...)" expression must be either 0 or NULL. If the set
62587	** contains no NULL values, then the result is 0. If the set
62588	** contains one or more NULL values, then the result of the
62589	** expression is also NULL.
62590	*/
62591	if( rNotFound==0 ){
62592	/* This branch runs if it is known at compile time (now) that
62593	** the set contains no NULL values. This happens as the result
62594	** of a "NOT NULL" constraint in the database schema. No need
62595	** to test the data structure at runtime in this case.
62596	*/
62597	sqlite3VdbeAddOp2(v, OP_Integer, 0, target);
62598	}else{
62599	/* This block populates the rNotFound register with either NULL
62600	** or 0 (an integer value). If the data structure contains one
62601	** or more NULLs, then set rNotFound to NULL. Otherwise, set it
62602	** to 0. If register rMayHaveNull is already set to some value
62603	** other than NULL, then the test has already been run and
62604	** rNotFound is already populated.
62605	*/
62606	static const char nullRecord[] = { 0x02, 0x00 };
62607	j3 = sqlite3VdbeAddOp1(v, OP_NotNull, rMayHaveNull);
62608	sqlite3VdbeAddOp2(v, OP_Null, 0, rNotFound);
62609	sqlite3VdbeAddOp4(v, OP_Blob, 2, rMayHaveNull, 0,
62610	nullRecord, P4_STATIC);
62611	j4 = sqlite3VdbeAddOp3(v, OP_Found, pExpr->iTable, 0, rMayHaveNull);
62612	sqlite3VdbeAddOp2(v, OP_Integer, 0, rNotFound);
62613	sqlite3VdbeJumpHere(v, j4);
62614	sqlite3VdbeJumpHere(v, j3);
62615
62616	/* Copy the value of register rNotFound (which is either NULL or 0)
62617	** into the target register. This will be the result of the
62618	** expression.
62619	*/
62620	sqlite3VdbeAddOp2(v, OP_Copy, rNotFound, target);
62621	}
62622	}
62623	sqlite3VdbeJumpHere(v, j2);
62624	sqlite3VdbeJumpHere(v, j5);
62625	sqlite3ExprCachePop(pParse, 1);
62626	VdbeComment((v, "end IN expr r%d", target));
62627	break;
62628	}
62629	#endif
62630	/*
62631	** x BETWEEN y AND z
62632	**
62633	** This is equivalent to
62634	**
	@@ -63054,10 +63182,66 @@
63054	sqlite3ExprHardCopy(pParse, target, n);
63055	}
63056	}
63057	return n;
63058	}
























































63059
63060	/*
63061	** Generate code for a boolean expression such that a jump is made
63062	** to the label "dest" if the expression is true but execution
63063	** continues straight thru if the expression is false.
	@@ -63154,40 +63338,20 @@
63154	sqlite3VdbeAddOp2(v, op, r1, dest);
63155	testcase( regFree1==0 );
63156	break;
63157	}
63158	case TK_BETWEEN: {
63159	/* x BETWEEN y AND z
63160	**
63161	** Is equivalent to
63162	**
63163	** x>=y AND x<=z
63164	**
63165	** Code it as such, taking care to do the common subexpression
63166	** elementation of x.
63167	*/
63168	Expr exprAnd;
63169	Expr compLeft;
63170	Expr compRight;
63171	Expr exprX;
63172
63173	assert( !ExprHasProperty(pExpr, EP_xIsSelect) );
63174	exprX = *pExpr->pLeft;
63175	exprAnd.op = TK_AND;
63176	exprAnd.pLeft = &compLeft;
63177	exprAnd.pRight = &compRight;
63178	compLeft.op = TK_GE;
63179	compLeft.pLeft = &exprX;
63180	compLeft.pRight = pExpr->x.pList->a[0].pExpr;
63181	compRight.op = TK_LE;
63182	compRight.pLeft = &exprX;
63183	compRight.pRight = pExpr->x.pList->a[1].pExpr;
63184	exprX.iTable = sqlite3ExprCodeTemp(pParse, &exprX, &regFree1);
63185	testcase( regFree1==0 );
63186	exprX.op = TK_REGISTER;
63187	testcase( jumpIfNull==0 );
63188	sqlite3ExprIfTrue(pParse, &exprAnd, dest, jumpIfNull);








63189	break;
63190	}
63191	default: {
63192	r1 = sqlite3ExprCodeTemp(pParse, pExpr, &regFree1);
63193	sqlite3VdbeAddOp3(v, OP_If, r1, dest, jumpIfNull!=0);
	@@ -63267,10 +63431,11 @@
63267	sqlite3VdbeResolveLabel(v, d2);
63268	sqlite3ExprCachePop(pParse, 1);
63269	break;
63270	}
63271	case TK_NOT: {

63272	sqlite3ExprIfTrue(pParse, pExpr->pLeft, dest, jumpIfNull);
63273	break;
63274	}
63275	case TK_LT:
63276	case TK_LE:
	@@ -63314,40 +63479,22 @@
63314	sqlite3VdbeAddOp2(v, op, r1, dest);
63315	testcase( regFree1==0 );
63316	break;
63317	}
63318	case TK_BETWEEN: {
63319	/* x BETWEEN y AND z
63320	**
63321	** Is equivalent to
63322	**
63323	** x>=y AND x<=z
63324	**
63325	** Code it as such, taking care to do the common subexpression
63326	** elementation of x.
63327	*/
63328	Expr exprAnd;
63329	Expr compLeft;
63330	Expr compRight;
63331	Expr exprX;
63332
63333	assert( !ExprHasProperty(pExpr, EP_xIsSelect) );
63334	exprX = *pExpr->pLeft;
63335	exprAnd.op = TK_AND;
63336	exprAnd.pLeft = &compLeft;
63337	exprAnd.pRight = &compRight;
63338	compLeft.op = TK_GE;
63339	compLeft.pLeft = &exprX;
63340	compLeft.pRight = pExpr->x.pList->a[0].pExpr;
63341	compRight.op = TK_LE;
63342	compRight.pLeft = &exprX;
63343	compRight.pRight = pExpr->x.pList->a[1].pExpr;
63344	exprX.iTable = sqlite3ExprCodeTemp(pParse, &exprX, &regFree1);
63345	testcase( regFree1==0 );
63346	exprX.op = TK_REGISTER;
63347	testcase( jumpIfNull==0 );
63348	sqlite3ExprIfFalse(pParse, &exprAnd, dest, jumpIfNull);










63349	break;
63350	}
63351	default: {
63352	r1 = sqlite3ExprCodeTemp(pParse, pExpr, &regFree1);
63353	sqlite3VdbeAddOp3(v, OP_IfNot, r1, dest, jumpIfNull!=0);
	@@ -63692,12 +63839,10 @@
63692	** May you share freely, never taking more than you give.
63693	**
63694	*************************************************************************
63695	** This file contains C code routines that used to generate VDBE code
63696	** that implements the ALTER TABLE command.
63697	**
63698	** $Id: alter.c,v 1.62 2009/07/24 17:58:53 danielk1977 Exp $
63699	*/
63700
63701	/*
63702	** The code in this file only exists if we are not omitting the
63703	** ALTER TABLE logic from the build.
	@@ -64488,12 +64633,10 @@
64488	** May you find forgiveness for yourself and forgive others.
64489	** May you share freely, never taking more than you give.
64490	**
64491	*************************************************************************
64492	** This file contains code associated with the ANALYZE command.
64493	**
64494	** @(#) $Id: analyze.c,v 1.52 2009/04/16 17:45:48 drh Exp $
64495	*/
64496	#ifndef SQLITE_OMIT_ANALYZE
64497
64498	/*
64499	** This routine generates code that opens the sqlite_stat1 table for
	@@ -65143,12 +65286,10 @@
65143	** May you find forgiveness for yourself and forgive others.
65144	** May you share freely, never taking more than you give.
65145	**
65146	*************************************************************************
65147	** This file contains code used to implement the ATTACH and DETACH commands.
65148	**
65149	** $Id: attach.c,v 1.93 2009/05/31 21:21:41 drh Exp $
65150	*/
65151
65152	#ifndef SQLITE_OMIT_ATTACH
65153	/*
65154	** Resolve an expression that was part of an ATTACH or DETACH statement. This
	@@ -65685,12 +65826,10 @@
65685	*************************************************************************
65686	** This file contains code used to implement the sqlite3_set_authorizer()
65687	** API. This facility is an optional feature of the library. Embedded
65688	** systems that do not need this facility may omit it by recompiling
65689	** the library with -DSQLITE_OMIT_AUTHORIZATION=1
65690	**
65691	** $Id: auth.c,v 1.32 2009/07/02 18:40:35 danielk1977 Exp $
65692	*/
65693
65694	/*
65695	** All of the code in this file may be omitted by defining a single
65696	** macro.
	@@ -65946,12 +66085,10 @@
65946	** DROP INDEX
65947	** creating ID lists
65948	** BEGIN TRANSACTION
65949	** COMMIT
65950	** ROLLBACK
65951	**
65952	** $Id: build.c,v 1.557 2009/07/24 17:58:53 danielk1977 Exp $
65953	*/
65954
65955	/*
65956	** This routine is called when a new SQL statement is beginning to
65957	** be parsed. Initialize the pParse structure as needed.
	@@ -69645,12 +69782,10 @@
69645	**
69646	*************************************************************************
69647	**
69648	** This file contains functions used to access the internal hash tables
69649	** of user defined functions and collation sequences.
69650	**
69651	** $Id: callback.c,v 1.42 2009/06/17 00:35:31 drh Exp $
69652	*/
69653
69654
69655	/*
69656	** Invoke the 'collation needed' callback to request a collation sequence
	@@ -70098,12 +70233,10 @@
70098	** May you share freely, never taking more than you give.
70099	**
70100	*************************************************************************
70101	** This file contains C code routines that are called by the parser
70102	** in order to generate code for DELETE FROM statements.
70103	**
70104	** $Id: delete.c,v 1.207 2009/08/08 18:01:08 drh Exp $
70105	*/
70106
70107	/*
70108	** Look up every table that is named in pSrc. If any table is not found,
70109	** add an error message to pParse->zErrMsg and return NULL. If all tables
	@@ -70722,11 +70855,10 @@
70722	}
70723	sqlite3ReleaseTempRange(pParse, regBase, nCol+1);
70724	return regBase;
70725	}
70726
70727
70728	/************ End of delete.c ********************************************/
70729	/************ Begin file func.c ******************************************/
70730	/*
70731	** 2002 February 23
70732	**
	@@ -70880,10 +71012,12 @@
70880	** p1 is 1-indexed. So substr(x,1,1) returns the first character
70881	** of x. If x is text, then we actually count UTF-8 characters.
70882	** If x is a blob, then we count bytes.
70883	**
70884	** If p1 is negative, then we begin abs(p1) from the end of x[].


70885	*/
70886	static void substrFunc(
70887	sqlite3_context *context,
70888	int argc,
70889	sqlite3_value **argv
	@@ -70900,24 +71034,26 @@
70900	\|\| (argc==3 && sqlite3_value_type(argv[2])==SQLITE_NULL)
70901	){
70902	return;
70903	}
70904	p0type = sqlite3_value_type(argv[0]);

70905	if( p0type==SQLITE_BLOB ){
70906	len = sqlite3_value_bytes(argv[0]);
70907	z = sqlite3_value_blob(argv[0]);
70908	if( z==0 ) return;
70909	assert( len==sqlite3_value_bytes(argv[0]) );
70910	}else{
70911	z = sqlite3_value_text(argv[0]);
70912	if( z==0 ) return;
70913	len = 0;
70914	for(z2=z; *z2; len++){
70915	SQLITE_SKIP_UTF8(z2);


70916	}
70917	}
70918	p1 = sqlite3_value_int(argv[1]);
70919	if( argc==3 ){
70920	p2 = sqlite3_value_int(argv[2]);
70921	if( p2<0 ){
70922	p2 = -p2;
70923	negP2 = 1;
	@@ -70943,14 +71079,10 @@
70943	p2 += p1;
70944	p1 = 0;
70945	}
70946	}
70947	assert( p1>=0 && p2>=0 );
70948	if( p1+p2>len ){
70949	p2 = len-p1;
70950	if( p2<0 ) p2 = 0;
70951	}
70952	if( p0type!=SQLITE_BLOB ){
70953	while( *z && p1 ){
70954	SQLITE_SKIP_UTF8(z);
70955	p1--;
70956	}
	@@ -70957,10 +71089,14 @@
70957	for(z2=z; *z2 && p2; p2--){
70958	SQLITE_SKIP_UTF8(z2);
70959	}
70960	sqlite3_result_text(context, (char*)z, (int)(z2-z), SQLITE_TRANSIENT);
70961	}else{




70962	sqlite3_result_blob(context, (char*)&z[p1], (int)p2, SQLITE_TRANSIENT);
70963	}
70964	}
70965
70966	/*
	@@ -71058,10 +71194,18 @@
71058	sqlite3_result_text(context, (char *)z1, -1, sqlite3_free);
71059	}
71060	}
71061	}
71062








71063	/*
71064	** Implementation of the IFNULL(), NVL(), and COALESCE() functions.
71065	** All three do the same thing. They return the first non-NULL
71066	** argument.
71067	*/
	@@ -71076,10 +71220,12 @@
71076	sqlite3_result_value(context, argv[i]);
71077	break;
71078	}
71079	}
71080	}


71081
71082	/*
71083	** Implementation of random(). Return a random integer.
71084	*/
71085	static void randomFunc(
	@@ -72160,14 +72306,16 @@
72160	FUNCTION(round, 2, 0, 0, roundFunc ),
72161	#endif
72162	FUNCTION(upper, 1, 0, 0, upperFunc ),
72163	FUNCTION(lower, 1, 0, 0, lowerFunc ),
72164	FUNCTION(coalesce, 1, 0, 0, 0 ),
72165	FUNCTION(coalesce, -1, 0, 0, ifnullFunc ),
72166	FUNCTION(coalesce, 0, 0, 0, 0 ),


72167	FUNCTION(hex, 1, 0, 0, hexFunc ),
72168	FUNCTION(ifnull, 2, 0, 1, ifnullFunc ),

72169	FUNCTION(random, 0, 0, 0, randomFunc ),
72170	FUNCTION(randomblob, 1, 0, 0, randomBlob ),
72171	FUNCTION(nullif, 2, 0, 1, nullifFunc ),
72172	FUNCTION(sqlite_version, 0, 0, 0, versionFunc ),
72173	FUNCTION(sqlite_source_id, 0, 0, 0, sourceidFunc ),
	@@ -72613,11 +72761,11 @@
72613	sqlite3VdbeAddOp2(v, OP_Goto, 0, iOk);
72614	}
72615
72616	sqlite3VdbeAddOp3(v, OP_MakeRecord, regTemp, nCol, regRec);
72617	sqlite3VdbeChangeP4(v, -1, sqlite3IndexAffinityStr(v, pIdx), 0);
72618	sqlite3VdbeAddOp3(v, OP_Found, iCur, iOk, regRec);
72619
72620	sqlite3ReleaseTempReg(pParse, regRec);
72621	sqlite3ReleaseTempRange(pParse, regTemp, nCol);
72622	}
72623	}
	@@ -73419,12 +73567,10 @@
73419	** May you share freely, never taking more than you give.
73420	**
73421	*************************************************************************
73422	** This file contains C code routines that are called by the parser
73423	** to handle INSERT statements in SQLite.
73424	**
73425	** $Id: insert.c,v 1.270 2009/07/24 17:58:53 danielk1977 Exp $
73426	*/
73427
73428	/*
73429	** Generate code that will open a table for reading.
73430	*/
	@@ -75240,12 +75386,10 @@
75240	*************************************************************************
75241	** Main file for the SQLite library. The routines in this file
75242	** implement the programmer interface to the library. Routines in
75243	** other files are for internal use by SQLite and should not be
75244	** accessed by users of the library.
75245	**
75246	** $Id: legacy.c,v 1.35 2009/08/07 16:56:00 danielk1977 Exp $
75247	*/
75248
75249
75250	/*
75251	** Execute SQL code. Return one of the SQLITE_ success/failure
	@@ -75386,12 +75530,10 @@
75386	** May you share freely, never taking more than you give.
75387	**
75388	*************************************************************************
75389	** This file contains code used to dynamically load extensions into
75390	** the SQLite library.
75391	**
75392	** $Id: loadext.c,v 1.60 2009/06/03 01:24:54 drh Exp $
75393	*/
75394
75395	#ifndef SQLITE_CORE
75396	#define SQLITE_CORE 1 /* Disable the API redefinition in sqlite3ext.h */
75397	#endif
	@@ -75411,12 +75553,10 @@
75411	** This header file defines the SQLite interface for use by
75412	** shared libraries that want to be imported as extensions into
75413	** an SQLite instance. Shared libraries that intend to be loaded
75414	** as extensions by SQLite should #include this file instead of
75415	** sqlite3.h.
75416	**
75417	** @(#) $Id: sqlite3ext.h,v 1.25 2008/10/12 00:27:54 shane Exp $
75418	*/
75419	#ifndef _SQLITE3EXT_H_
75420	#define _SQLITE3EXT_H_
75421
75422	typedef struct sqlite3_api_routines sqlite3_api_routines;
	@@ -76380,12 +76520,10 @@
76380	** May you find forgiveness for yourself and forgive others.
76381	** May you share freely, never taking more than you give.
76382	**
76383	*************************************************************************
76384	** This file contains code used to implement the PRAGMA command.
76385	**
76386	** $Id: pragma.c,v 1.214 2009/07/02 07:47:33 danielk1977 Exp $
76387	*/
76388
76389	/* Ignore this whole file if pragmas are disabled
76390	*/
76391	#if !defined(SQLITE_OMIT_PRAGMA)
	@@ -77499,10 +77637,11 @@
77499	sqlite3VdbeAddOp2(v, OP_Integer, 0, 2); /* reg(2) will count entries */
77500	loopTop = sqlite3VdbeAddOp2(v, OP_Rewind, 1, 0);
77501	sqlite3VdbeAddOp2(v, OP_AddImm, 2, 1); /* increment entry count */
77502	for(j=0, pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext, j++){
77503	int jmp2;

77504	static const VdbeOpList idxErr[] = {
77505	{ OP_AddImm, 1, -1, 0},
77506	{ OP_String8, 0, 3, 0}, /* 1 */
77507	{ OP_Rowid, 1, 4, 0},
77508	{ OP_String8, 0, 5, 0}, /* 3 */
	@@ -77512,12 +77651,12 @@
77512	{ OP_Concat, 6, 3, 3},
77513	{ OP_ResultRow, 3, 1, 0},
77514	{ OP_IfPos, 1, 0, 0}, /* 9 */
77515	{ OP_Halt, 0, 0, 0},
77516	};
77517	sqlite3GenerateIndexKey(pParse, pIdx, 1, 3, 1);
77518	jmp2 = sqlite3VdbeAddOp3(v, OP_Found, j+2, 0, 3);
77519	addr = sqlite3VdbeAddOpList(v, ArraySize(idxErr), idxErr);
77520	sqlite3VdbeChangeP4(v, addr+1, "rowid ", P4_STATIC);
77521	sqlite3VdbeChangeP4(v, addr+3, " missing from index ", P4_STATIC);
77522	sqlite3VdbeChangeP4(v, addr+4, pIdx->zName, P4_STATIC);
77523	sqlite3VdbeJumpHere(v, addr+9);
	@@ -77820,12 +77959,10 @@
77820	**
77821	*************************************************************************
77822	** This file contains the implementation of the sqlite3_prepare()
77823	** interface, and routines that contribute to loading the database schema
77824	** from disk.
77825	**
77826	** $Id: prepare.c,v 1.131 2009/08/06 17:43:31 drh Exp $
77827	*/
77828
77829	/*
77830	** Fill the InitData structure with an error message that indicates
77831	** that the database is corrupt.
	@@ -78694,12 +78831,10 @@
78694	** May you share freely, never taking more than you give.
78695	**
78696	*************************************************************************
78697	** This file contains C code routines that are called by the parser
78698	** to handle SELECT statements in SQLite.
78699	**
78700	** $Id: select.c,v 1.526 2009/08/01 15:09:58 drh Exp $
78701	*/
78702
78703
78704	/*
78705	** Delete all the content of a Select structure but do not deallocate
	@@ -79123,12 +79258,12 @@
79123	Vdbe *v;
79124	int r1;
79125
79126	v = pParse->pVdbe;
79127	r1 = sqlite3GetTempReg(pParse);

79128	sqlite3VdbeAddOp3(v, OP_MakeRecord, iMem, N, r1);
79129	sqlite3VdbeAddOp3(v, OP_Found, iTab, addrRepeat, r1);
79130	sqlite3VdbeAddOp2(v, OP_IdxInsert, iTab, r1);
79131	sqlite3ReleaseTempReg(pParse, r1);
79132	}
79133
79134	/*
	@@ -79365,12 +79500,11 @@
79365	/* Jump to the end of the loop if the LIMIT is reached.
79366	*/
79367	if( p->iLimit ){
79368	assert( pOrderBy==0 ); /* If there is an ORDER BY, the call to
79369	** pushOntoSorter() would have cleared p->iLimit */
79370	sqlite3VdbeAddOp2(v, OP_AddImm, p->iLimit, -1);
79371	sqlite3VdbeAddOp2(v, OP_IfZero, p->iLimit, iBreak);
79372	}
79373	}
79374
79375	/*
79376	** Given an expression list, generate a KeyInfo structure that records
	@@ -79992,11 +80126,11 @@
79992	*/
79993	static void computeLimitRegisters(Parse pParse, Select p, int iBreak){
79994	Vdbe *v = 0;
79995	int iLimit = 0;
79996	int iOffset;
79997	int addr1;
79998	if( p->iLimit ) return;
79999
80000	/*
80001	** "LIMIT -1" always shows all rows. There is some
80002	** contraversy about what the correct behavior should be.
	@@ -80007,14 +80141,22 @@
80007	assert( p->pOffset==0 \|\| p->pLimit!=0 );
80008	if( p->pLimit ){
80009	p->iLimit = iLimit = ++pParse->nMem;
80010	v = sqlite3GetVdbe(pParse);
80011	if( NEVER(v==0) ) return; /* VDBE should have already been allocated */
80012	sqlite3ExprCode(pParse, p->pLimit, iLimit);
80013	sqlite3VdbeAddOp1(v, OP_MustBeInt, iLimit);
80014	VdbeComment((v, "LIMIT counter"));
80015	sqlite3VdbeAddOp2(v, OP_IfZero, iLimit, iBreak);








80016	if( p->pOffset ){
80017	p->iOffset = iOffset = ++pParse->nMem;
80018	pParse->nMem++; /* Allocate an extra register for limit+offset */
80019	sqlite3ExprCode(pParse, p->pOffset, iOffset);
80020	sqlite3VdbeAddOp1(v, OP_MustBeInt, iOffset);
	@@ -80346,11 +80488,11 @@
80346	iCont = sqlite3VdbeMakeLabel(v);
80347	computeLimitRegisters(pParse, p, iBreak);
80348	sqlite3VdbeAddOp2(v, OP_Rewind, tab1, iBreak);
80349	r1 = sqlite3GetTempReg(pParse);
80350	iStart = sqlite3VdbeAddOp2(v, OP_RowKey, tab1, r1);
80351	sqlite3VdbeAddOp3(v, OP_NotFound, tab2, iCont, r1);
80352	sqlite3ReleaseTempReg(pParse, r1);
80353	selectInnerLoop(pParse, p, p->pEList, tab1, p->pEList->nExpr,
80354	0, -1, &dest, iCont, iBreak);
80355	sqlite3VdbeResolveLabel(v, iCont);
80356	sqlite3VdbeAddOp2(v, OP_Next, tab1, iStart);
	@@ -80565,12 +80707,11 @@
80565	}
80566
80567	/* Jump to the end of the loop if the LIMIT is reached.
80568	*/
80569	if( p->iLimit ){
80570	sqlite3VdbeAddOp2(v, OP_AddImm, p->iLimit, -1);
80571	sqlite3VdbeAddOp2(v, OP_IfZero, p->iLimit, iBreak);
80572	}
80573
80574	/* Generate the subroutine return
80575	*/
80576	sqlite3VdbeResolveLabel(v, iContinue);
	@@ -82934,12 +83075,10 @@
82934	** interface routines. These are just wrappers around the main
82935	** interface routine of sqlite3_exec().
82936	**
82937	** These routines are in a separate files so that they will not be linked
82938	** if they are not used.
82939	**
82940	** $Id: table.c,v 1.40 2009/04/10 14:28:00 drh Exp $
82941	*/
82942
82943	#ifndef SQLITE_OMIT_GET_TABLE
82944
82945	/*
	@@ -83126,13 +83265,11 @@
83126	** May you do good and not evil.
83127	** May you find forgiveness for yourself and forgive others.
83128	** May you share freely, never taking more than you give.
83129	**
83130	*************************************************************************
83131	**
83132	**
83133	** $Id: trigger.c,v 1.143 2009/08/10 03:57:58 shane Exp $
83134	*/
83135
83136	#ifndef SQLITE_OMIT_TRIGGER
83137	/*
83138	** Delete a linked list of TriggerStep structures.
	@@ -84210,12 +84347,10 @@
84210	** May you share freely, never taking more than you give.
84211	**
84212	*************************************************************************
84213	** This file contains C code routines that are called by the parser
84214	** to handle UPDATE statements.
84215	**
84216	** $Id: update.c,v 1.207 2009/08/08 18:01:08 drh Exp $
84217	*/
84218
84219	#ifndef SQLITE_OMIT_VIRTUALTABLE
84220	/* Forward declaration */
84221	static void updateVirtualTable(
	@@ -84843,12 +84978,10 @@
84843	*************************************************************************
84844	** This file contains code used to implement the VACUUM command.
84845	**
84846	** Most of the code in this file may be omitted by defining the
84847	** SQLITE_OMIT_VACUUM macro.
84848	**
84849	** $Id: vacuum.c,v 1.91 2009/07/02 07:47:33 danielk1977 Exp $
84850	*/
84851
84852	#if !defined(SQLITE_OMIT_VACUUM) && !defined(SQLITE_OMIT_ATTACH)
84853	/*
84854	** Execute zSql on database db. Return an error code.
	@@ -85149,12 +85282,10 @@
85149	** May you find forgiveness for yourself and forgive others.
85150	** May you share freely, never taking more than you give.
85151	**
85152	*************************************************************************
85153	** This file contains code used to help implement virtual tables.
85154	**
85155	** $Id: vtab.c,v 1.94 2009/08/08 18:01:08 drh Exp $
85156	*/
85157	#ifndef SQLITE_OMIT_VIRTUALTABLE
85158
85159	/*
85160	** The actual function that does the work of creating a new module.
	@@ -86118,12 +86249,10 @@
86118	** the WHERE clause of SQL statements. This module is responsible for
86119	** generating the code that loops through a table looking for applicable
86120	** rows. Indices are selected and used to speed the search when doing
86121	** so is applicable. Because this module is responsible for selecting
86122	** indices, you might also think of this module as the "query optimizer".
86123	**
86124	** $Id: where.c,v 1.411 2009/07/31 06:14:52 danielk1977 Exp $
86125	*/
86126
86127	/*
86128	** Trace output macros
86129	*/
	@@ -88698,20 +88827,43 @@
88698
88699	/*
88700	** Code an OP_Affinity opcode to apply the column affinity string zAff
88701	** to the n registers starting at base.
88702	**
88703	** Buffer zAff was allocated using sqlite3DbMalloc(). It is the
88704	** responsibility of this function to arrange for it to be eventually
88705	** freed using sqlite3DbFree().



88706	*/
88707	static void codeApplyAffinity(Parse pParse, int base, int n, char zAff){
88708	Vdbe *v = pParse->pVdbe;




88709	assert( v!=0 );
88710	sqlite3VdbeAddOp2(v, OP_Affinity, base, n);
88711	sqlite3VdbeChangeP4(v, -1, zAff, P4_DYNAMIC);
88712	sqlite3ExprCacheAffinityChange(pParse, base, n);
















88713	}
88714
88715
88716	/*
88717	** Generate code for a single equality term of the WHERE clause. An equality
	@@ -88778,11 +88930,11 @@
88778	return iReg;
88779	}
88780
88781	/*
88782	** Generate code that will evaluate all == and IN constraints for an
88783	** index. The values for all constraints are left on the stack.
88784	**
88785	** For example, consider table t1(a,b,c,d,e,f) with index i1(a,b,c).
88786	** Suppose the WHERE clause is this: a==5 AND b IN (1,2,3) AND c>5 AND c<10
88787	** The index has as many as three equality constraints, but in this
88788	** example, the third "c" value is an inequality. So only two
	@@ -88790,11 +88942,12 @@
88790	** a==5 and b IN (1,2,3). The current values for a and b will be stored
88791	** in consecutive registers and the index of the first register is returned.
88792	**
88793	** In the example above nEq==2. But this subroutine works for any value
88794	** of nEq including 0. If nEq==0, this routine is nearly a no-op.
88795	** The only thing it does is allocate the pLevel->iMem memory cell.

88796	**
88797	** This routine always allocates at least one memory cell and returns
88798	** the index of that memory cell. The code that
88799	** calls this routine will use that memory cell to store the termination
88800	** key value of the loop. If one or more IN operators appear, then
	@@ -88868,15 +89021,19 @@
88868	}
88869	}
88870	testcase( pTerm->eOperator & WO_ISNULL );
88871	testcase( pTerm->eOperator & WO_IN );
88872	if( (pTerm->eOperator & (WO_ISNULL\|WO_IN))==0 ){
88873	sqlite3VdbeAddOp2(v, OP_IsNull, regBase+j, pLevel->addrBrk);
88874	if( zAff
88875	&& sqlite3CompareAffinity(pTerm->pExpr->pRight, zAff[j])==SQLITE_AFF_NONE
88876	){
88877	zAff[j] = SQLITE_AFF_NONE;




88878	}
88879	}
88880	}
88881	*pzAff = zAff;
88882	return regBase;
	@@ -89198,19 +89355,22 @@
89198	/* Seek the index cursor to the start of the range. */
89199	nConstraint = nEq;
89200	if( pRangeStart ){
89201	Expr *pRight = pRangeStart->pExpr->pRight;
89202	sqlite3ExprCode(pParse, pRight, regBase+nEq);
89203	sqlite3VdbeAddOp2(v, OP_IsNull, regBase+nEq, addrNxt);
89204	if( zAff
89205	&& sqlite3CompareAffinity(pRight, zAff[nConstraint])==SQLITE_AFF_NONE
89206	){
89207	/* Since the comparison is to be performed with no conversions applied
89208	** to the operands, set the affinity to apply to pRight to
89209	** SQLITE_AFF_NONE. */
89210	zAff[nConstraint] = SQLITE_AFF_NONE;
89211	}



89212	nConstraint++;
89213	}else if( isMinQuery ){
89214	sqlite3VdbeAddOp2(v, OP_Null, 0, regBase+nEq);
89215	nConstraint++;
89216	startEq = 0;
	@@ -89223,34 +89383,36 @@
89223	testcase( op==OP_Last );
89224	testcase( op==OP_SeekGt );
89225	testcase( op==OP_SeekGe );
89226	testcase( op==OP_SeekLe );
89227	testcase( op==OP_SeekLt );
89228	sqlite3VdbeAddOp4(v, op, iIdxCur, addrNxt, regBase,
89229	SQLITE_INT_TO_PTR(nConstraint), P4_INT32);
89230
89231	/* Load the value for the inequality constraint at the end of the
89232	** range (if any).
89233	*/
89234	nConstraint = nEq;
89235	if( pRangeEnd ){
89236	Expr *pRight = pRangeEnd->pExpr->pRight;
89237	sqlite3ExprCacheRemove(pParse, regBase+nEq);
89238	sqlite3ExprCode(pParse, pRight, regBase+nEq);
89239	sqlite3VdbeAddOp2(v, OP_IsNull, regBase+nEq, addrNxt);
89240	zAff = sqlite3DbStrDup(pParse->db, zAff);
89241	if( zAff
89242	&& sqlite3CompareAffinity(pRight, zAff[nConstraint])==SQLITE_AFF_NONE
89243	){
89244	/* Since the comparison is to be performed with no conversions applied
89245	** to the operands, set the affinity to apply to pRight to
89246	** SQLITE_AFF_NONE. */
89247	zAff[nConstraint] = SQLITE_AFF_NONE;
89248	}


89249	codeApplyAffinity(pParse, regBase, nEq+1, zAff);
89250	nConstraint++;
89251	}

89252
89253	/* Top of the loop body */
89254	pLevel->p2 = sqlite3VdbeCurrentAddr(v);
89255
89256	/* Check if the index cursor is past the end of the range. */
	@@ -89257,12 +89419,11 @@
89257	op = aEndOp[(pRangeEnd \|\| nEq) * (1 + bRev)];
89258	testcase( op==OP_Noop );
89259	testcase( op==OP_IdxGE );
89260	testcase( op==OP_IdxLT );
89261	if( op!=OP_Noop ){
89262	sqlite3VdbeAddOp4(v, op, iIdxCur, addrNxt, regBase,
89263	SQLITE_INT_TO_PTR(nConstraint), P4_INT32);
89264	sqlite3VdbeChangeP5(v, endEq!=bRev ?1:0);
89265	}
89266
89267	/* If there are inequality constraints, check that the value
89268	** of the table column that the inequality contrains is not NULL.
	@@ -89387,13 +89548,12 @@
89387	if( (wctrlFlags & WHERE_DUPLICATES_OK)==0 ){
89388	int iSet = ((ii==pOrWc->nTerm-1)?-1:ii);
89389	int r;
89390	r = sqlite3ExprCodeGetColumn(pParse, pTabItem->pTab, -1, iCur,
89391	regRowid, 0);
89392	sqlite3VdbeAddOp4(v, OP_RowSetTest, regRowset,
89393	sqlite3VdbeCurrentAddr(v)+2,
89394	r, SQLITE_INT_TO_PTR(iSet), P4_INT32);
89395	}
89396	sqlite3VdbeAddOp2(v, OP_Gosub, regReturn, iLoopBody);
89397
89398	/* Finish the loop through table entries that match term pOrTerm. */
89399	sqlite3WhereEnd(pSubWInfo);
	@@ -89920,11 +90080,12 @@
89920	sqlite3OpenTable(pParse, pTabItem->iCursor, iDb, pTab, op);
89921	if( !pWInfo->okOnePass && pTab->nCol<BMS ){
89922	Bitmask b = pTabItem->colUsed;
89923	int n = 0;
89924	for(; b; b=b>>1, n++){}
89925	sqlite3VdbeChangeP4(v, sqlite3VdbeCurrentAddr(v)-1, SQLITE_INT_TO_PTR(n), P4_INT32);

89926	assert( n<=pTab->nCol );
89927	}
89928	}else{
89929	sqlite3TableLock(pParse, iDb, pTab->tnum, 0, pTab->zName);
89930	}
	@@ -90048,11 +90209,15 @@
90048	}
90049	sqlite3VdbeResolveLabel(v, pLevel->addrBrk);
90050	if( pLevel->iLeftJoin ){
90051	int addr;
90052	addr = sqlite3VdbeAddOp1(v, OP_IfPos, pLevel->iLeftJoin);
90053	sqlite3VdbeAddOp1(v, OP_NullRow, pTabList->a[i].iCursor);




90054	if( pLevel->iIdxCur>=0 ){
90055	sqlite3VdbeAddOp1(v, OP_NullRow, pLevel->iIdxCur);
90056	}
90057	if( pLevel->op==OP_Return ){
90058	sqlite3VdbeAddOp2(v, OP_Gosub, pLevel->p1, pLevel->addrFirst);
	@@ -90099,11 +90264,10 @@
90099	*/
90100	if( (pLevel->plan.wsFlags & WHERE_INDEXED)!=0 && !db->mallocFailed){
90101	int k, j, last;
90102	VdbeOp *pOp;
90103	Index *pIdx = pLevel->plan.u.pIdx;
90104	int useIndexOnly = pLevel->plan.wsFlags & WHERE_IDX_ONLY;
90105
90106	assert( pIdx!=0 );
90107	pOp = sqlite3VdbeGetOp(v, pWInfo->iTop);
90108	last = sqlite3VdbeCurrentAddr(v);
90109	for(k=pWInfo->iTop; k<last; k++, pOp++){
	@@ -90114,16 +90278,15 @@
90114	pOp->p2 = j;
90115	pOp->p1 = pLevel->iIdxCur;
90116	break;
90117	}
90118	}
90119	assert(!useIndexOnly \|\| j<pIdx->nColumn);

90120	}else if( pOp->opcode==OP_Rowid ){
90121	pOp->p1 = pLevel->iIdxCur;
90122	pOp->opcode = OP_IdxRowid;
90123	}else if( pOp->opcode==OP_NullRow && useIndexOnly ){
90124	pOp->opcode = OP_Noop;
90125	}
90126	}
90127	}
90128	}
90129
	@@ -90239,10 +90402,21 @@
90239	){
90240	pOut->pExpr = sqlite3PExpr(pParse, op, pOperand->pExpr, 0, 0);
90241	pOut->zStart = pOperand->zStart;
90242	pOut->zEnd = &pPostOp->z[pPostOp->n];
90243	}











90244
90245	/* Construct an expression node for a unary prefix operator
90246	*/
90247	static void spanUnaryPrefix(
90248	ExprSpan pOut, / Write the new expression node here */
	@@ -92857,21 +93031,17 @@
92857	{spanUnaryPostfix(&yygotominor.yy346,pParse,TK_NOTNULL,&yymsp[-2].minor.yy346,&yymsp[0].minor.yy0);}
92858	break;
92859	case 216: /* expr ::= expr IS expr */
92860	{
92861	spanBinaryExpr(&yygotominor.yy346,pParse,TK_IS,&yymsp[-2].minor.yy346,&yymsp[0].minor.yy346);
92862	if( pParse->db->mallocFailed==0 && yymsp[0].minor.yy346.pExpr->op==TK_NULL ){
92863	yygotominor.yy346.pExpr->op = TK_ISNULL;
92864	}
92865	}
92866	break;
92867	case 217: /* expr ::= expr IS NOT expr */
92868	{
92869	spanBinaryExpr(&yygotominor.yy346,pParse,TK_ISNOT,&yymsp[-3].minor.yy346,&yymsp[0].minor.yy346);
92870	if( pParse->db->mallocFailed==0 && yymsp[0].minor.yy346.pExpr->op==TK_NULL ){
92871	yygotominor.yy346.pExpr->op = TK_NOTNULL;
92872	}
92873	}
92874	break;
92875	case 218: /* expr ::= NOT expr */
92876	case 219: /* expr ::= BITNOT expr */ yytestcase(yyruleno==219);
92877	{spanUnaryPrefix(&yygotominor.yy346,pParse,yymsp[-1].major,&yymsp[0].minor.yy346,&yymsp[-1].minor.yy0);}
	@@ -93551,12 +93721,10 @@
93551	** An tokenizer for SQL
93552	**
93553	** This file contains C code that splits an SQL input string up into
93554	** individual tokens and sends those tokens one-by-one over to the
93555	** parser for analysis.
93556	**
93557	** $Id: tokenize.c,v 1.163 2009/07/03 22:54:37 drh Exp $
93558	*/
93559
93560	/*
93561	** The charMap() macro maps alphabetic characters into their
93562	** lower-case ASCII equivalent. On ASCII machines, this is just
	@@ -93892,20 +94060,11 @@
93892	** middle of identfiers. But many SQL implementations do.
93893	** SQLite will allow '$' in identifiers for compatibility.
93894	** But the feature is undocumented.
93895	*/
93896	#ifdef SQLITE_ASCII
93897	SQLITE_PRIVATE const char sqlite3IsAsciiIdChar[] = {
93898	/* x0 x1 x2 x3 x4 x5 x6 x7 x8 x9 xA xB xC xD xE xF */
93899	0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 2x */
93900	1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, /* 3x */
93901	0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* 4x */
93902	1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 1, /* 5x */
93903	0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* 6x */
93904	1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, /* 7x */
93905	};
93906	#define IdChar(C) (((c=C)&0x80)!=0 \|\| (c>0x1f && sqlite3IsAsciiIdChar[c-0x20]))
93907	#endif
93908	#ifdef SQLITE_EBCDIC
93909	SQLITE_PRIVATE const char sqlite3IsEbcdicIdChar[] = {
93910	/* x0 x1 x2 x3 x4 x5 x6 x7 x8 x9 xA xB xC xD xE xF */
93911	0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, /* 4x */
	@@ -94361,22 +94520,19 @@
94361	**
94362	** This file contains C code that implements the sqlite3_complete() API.
94363	** This code used to be part of the tokenizer.c source file. But by
94364	** separating it out, the code will be automatically omitted from
94365	** static links that do not use it.
94366	**
94367	** $Id: complete.c,v 1.8 2009/04/28 04:46:42 drh Exp $
94368	*/
94369	#ifndef SQLITE_OMIT_COMPLETE
94370
94371	/*
94372	** This is defined in tokenize.c. We just have to import the definition.
94373	*/
94374	#ifndef SQLITE_AMALGAMATION
94375	#ifdef SQLITE_ASCII
94376	SQLITE_PRIVATE const char sqlite3IsAsciiIdChar[];
94377	#define IdChar(C) (((c=C)&0x80)!=0 \|\| (c>0x1f && sqlite3IsAsciiIdChar[c-0x20]))
94378	#endif
94379	#ifdef SQLITE_EBCDIC
94380	SQLITE_PRIVATE const char sqlite3IsEbcdicIdChar[];
94381	#define IdChar(C) (((c=C)>=0x42 && sqlite3IsEbcdicIdChar[c-0x40]))
94382	#endif
	@@ -94529,11 +94685,13 @@
94529	if( *zSql==0 ) return 0;
94530	token = tkOTHER;
94531	break;
94532	}
94533	default: {
94534	int c;


94535	if( IdChar((u8)*zSql) ){
94536	/* Keywords and unquoted identifiers */
94537	int nId;
94538	for(nId=1; IdChar(zSql[nId]); nId++){}
94539	#ifdef SQLITE_OMIT_TRIGGER
	@@ -97015,12 +97173,10 @@
97015	**
97016	*************************************************************************
97017	**
97018	** This file contains the implementation of the sqlite3_unlock_notify()
97019	** API method and its associated functionality.
97020	**
97021	** $Id: notify.c,v 1.4 2009/04/07 22:06:57 drh Exp $
97022	*/
97023
97024	/* Omit this entire file if SQLITE_ENABLE_UNLOCK_NOTIFY is not defined. */
97025	#ifdef SQLITE_ENABLE_UNLOCK_NOTIFY
97026
	@@ -97615,380 +97771,17 @@
97615
97616	#if defined(SQLITE_ENABLE_FTS3) && !defined(SQLITE_CORE)
97617	# define SQLITE_CORE 1
97618	#endif
97619
97620
97621	/************ Include fts3_expr.h in the middle of fts3.c ****************/
97622	/************ Begin file fts3_expr.h *************************************/
97623	/*
97624	** 2008 Nov 28
97625	**
97626	** The author disclaims copyright to this source code. In place of
97627	** a legal notice, here is a blessing:
97628	**
97629	** May you do good and not evil.
97630	** May you find forgiveness for yourself and forgive others.
97631	** May you share freely, never taking more than you give.
97632	**
97633	******************************************************************************
97634	**
97635	*/
97636
97637	/************ Include fts3_tokenizer.h in the middle of fts3_expr.h ******/
97638	/************ Begin file fts3_tokenizer.h ********************************/
97639	/*
97640	** 2006 July 10
97641	**
97642	** The author disclaims copyright to this source code.
97643	**
97644	*************************************************************************
97645	** Defines the interface to tokenizers used by fulltext-search. There
97646	** are three basic components:
97647	**
97648	** sqlite3_tokenizer_module is a singleton defining the tokenizer
97649	** interface functions. This is essentially the class structure for
97650	** tokenizers.
97651	**
97652	** sqlite3_tokenizer is used to define a particular tokenizer, perhaps
97653	** including customization information defined at creation time.
97654	**
97655	** sqlite3_tokenizer_cursor is generated by a tokenizer to generate
97656	** tokens from a particular input.
97657	*/
97658	#ifndef _FTS3_TOKENIZER_H_
97659	#define _FTS3_TOKENIZER_H_
97660
97661	/* TODO(shess) Only used for SQLITE_OK and SQLITE_DONE at this time.
97662	** If tokenizers are to be allowed to call sqlite3_*() functions, then
97663	** we will need a way to register the API consistently.
97664	*/
97665
97666	/*
97667	** Structures used by the tokenizer interface. When a new tokenizer
97668	** implementation is registered, the caller provides a pointer to
97669	** an sqlite3_tokenizer_module containing pointers to the callback
97670	** functions that make up an implementation.
97671	**
97672	** When an fts3 table is created, it passes any arguments passed to
97673	** the tokenizer clause of the CREATE VIRTUAL TABLE statement to the
97674	** sqlite3_tokenizer_module.xCreate() function of the requested tokenizer
97675	** implementation. The xCreate() function in turn returns an
97676	** sqlite3_tokenizer structure representing the specific tokenizer to
97677	** be used for the fts3 table (customized by the tokenizer clause arguments).
97678	**
97679	** To tokenize an input buffer, the sqlite3_tokenizer_module.xOpen()
97680	** method is called. It returns an sqlite3_tokenizer_cursor object
97681	** that may be used to tokenize a specific input buffer based on
97682	** the tokenization rules supplied by a specific sqlite3_tokenizer
97683	** object.
97684	*/
97685	typedef struct sqlite3_tokenizer_module sqlite3_tokenizer_module;
97686	typedef struct sqlite3_tokenizer sqlite3_tokenizer;
97687	typedef struct sqlite3_tokenizer_cursor sqlite3_tokenizer_cursor;
97688
97689	struct sqlite3_tokenizer_module {
97690
97691	/*
97692	** Structure version. Should always be set to 0.
97693	*/
97694	int iVersion;
97695
97696	/*
97697	** Create a new tokenizer. The values in the argv[] array are the
97698	** arguments passed to the "tokenizer" clause of the CREATE VIRTUAL
97699	** TABLE statement that created the fts3 table. For example, if
97700	** the following SQL is executed:
97701	**
97702	** CREATE .. USING fts3( ... , tokenizer <tokenizer-name> arg1 arg2)
97703	**
97704	** then argc is set to 2, and the argv[] array contains pointers
97705	** to the strings "arg1" and "arg2".
97706	**
97707	** This method should return either SQLITE_OK (0), or an SQLite error
97708	** code. If SQLITE_OK is returned, then *ppTokenizer should be set
97709	** to point at the newly created tokenizer structure. The generic
97710	** sqlite3_tokenizer.pModule variable should not be initialised by
97711	** this callback. The caller will do so.
97712	*/
97713	int (*xCreate)(
97714	int argc, /* Size of argv array */
97715	const char constargv, /* Tokenizer argument strings */
97716	sqlite3_tokenizer *ppTokenizer / OUT: Created tokenizer */
97717	);
97718
97719	/*
97720	** Destroy an existing tokenizer. The fts3 module calls this method
97721	** exactly once for each successful call to xCreate().
97722	*/
97723	int (xDestroy)(sqlite3_tokenizer pTokenizer);
97724
97725	/*
97726	** Create a tokenizer cursor to tokenize an input buffer. The caller
97727	** is responsible for ensuring that the input buffer remains valid
97728	** until the cursor is closed (using the xClose() method).
97729	*/
97730	int (*xOpen)(
97731	sqlite3_tokenizer pTokenizer, / Tokenizer object */
97732	const char pInput, int nBytes, / Input buffer */
97733	sqlite3_tokenizer_cursor *ppCursor / OUT: Created tokenizer cursor */
97734	);
97735
97736	/*
97737	** Destroy an existing tokenizer cursor. The fts3 module calls this
97738	** method exactly once for each successful call to xOpen().
97739	*/
97740	int (xClose)(sqlite3_tokenizer_cursor pCursor);
97741
97742	/*
97743	** Retrieve the next token from the tokenizer cursor pCursor. This
97744	** method should either return SQLITE_OK and set the values of the
97745	** "OUT" variables identified below, or SQLITE_DONE to indicate that
97746	** the end of the buffer has been reached, or an SQLite error code.
97747	**
97748	** *ppToken should be set to point at a buffer containing the
97749	** normalized version of the token (i.e. after any case-folding and/or
97750	** stemming has been performed). *pnBytes should be set to the length
97751	** of this buffer in bytes. The input text that generated the token is
97752	** identified by the byte offsets returned in *piStartOffset and
97753	** piEndOffset. piStartOffset should be set to the index of the first
97754	** byte of the token in the input buffer. *piEndOffset should be set
97755	** to the index of the first byte just past the end of the token in
97756	** the input buffer.
97757	**
97758	** The buffer *ppToken is set to point at is managed by the tokenizer
97759	** implementation. It is only required to be valid until the next call
97760	** to xNext() or xClose().
97761	*/
97762	/* TODO(shess) current implementation requires pInput to be
97763	** nul-terminated. This should either be fixed, or pInput/nBytes
97764	** should be converted to zInput.
97765	*/
97766	int (*xNext)(
97767	sqlite3_tokenizer_cursor pCursor, / Tokenizer cursor */
97768	const char *ppToken, int pnBytes, /* OUT: Normalized text for token */
97769	int piStartOffset, / OUT: Byte offset of token in input buffer */
97770	int piEndOffset, / OUT: Byte offset of end of token in input buffer */
97771	int piPosition / OUT: Number of tokens returned before this one */
97772	);
97773	};
97774
97775	struct sqlite3_tokenizer {
97776	const sqlite3_tokenizer_module pModule; / The module for this tokenizer */
97777	/* Tokenizer implementations will typically add additional fields */
97778	};
97779
97780	struct sqlite3_tokenizer_cursor {
97781	sqlite3_tokenizer pTokenizer; / Tokenizer for this cursor. */
97782	/* Tokenizer implementations will typically add additional fields */
97783	};
97784
97785	#endif /* _FTS3_TOKENIZER_H_ */
97786
97787	/************ End of fts3_tokenizer.h ************************************/
97788	/************ Continuing where we left off in fts3_expr.h ****************/
97789
97790	/*
97791	** The following describes the syntax supported by the fts3 MATCH
97792	** operator in a similar format to that used by the lemon parser
97793	** generator. This module does not use actually lemon, it uses a
97794	** custom parser.
97795	**
97796	** query ::= andexpr (OR andexpr)*.
97797	**
97798	** andexpr ::= notexpr (AND? notexpr)*.
97799	**
97800	** notexpr ::= nearexpr (NOT nearexpr\|-TOKEN)*.
97801	** notexpr ::= LP query RP.
97802	**
97803	** nearexpr ::= phrase (NEAR distance_opt nearexpr)*.
97804	**
97805	** distance_opt ::= .
97806	** distance_opt ::= / INTEGER.
97807	**
97808	** phrase ::= TOKEN.
97809	** phrase ::= COLUMN:TOKEN.
97810	** phrase ::= "TOKEN TOKEN TOKEN...".
97811	*/
97812
97813	typedef struct Fts3Expr Fts3Expr;
97814	typedef struct Fts3Phrase Fts3Phrase;
97815
97816	/*
97817	** A "phrase" is a sequence of one or more tokens that must match in
97818	** sequence. A single token is the base case and the most common case.
97819	** For a sequence of tokens contained in "...", nToken will be the number
97820	** of tokens in the string.
97821	*/
97822	struct Fts3Phrase {
97823	int nToken; /* Number of tokens in the phrase */
97824	int iColumn; /* Index of column this phrase must match */
97825	int isNot; /* Phrase prefixed by unary not (-) operator */
97826	struct PhraseToken {
97827	char z; / Text of the token */
97828	int n; /* Number of bytes in buffer pointed to by z */
97829	int isPrefix; /* True if token ends in with a "" character /
97830	} aToken[1]; /* One entry for each token in the phrase */
97831	};
97832
97833	/*
97834	** A tree of these objects forms the RHS of a MATCH operator.
97835	*/
97836	struct Fts3Expr {
97837	int eType; /* One of the FTSQUERY_XXX values defined below */
97838	int nNear; /* Valid if eType==FTSQUERY_NEAR */
97839	Fts3Expr pParent; / pParent->pLeft==this or pParent->pRight==this */
97840	Fts3Expr pLeft; / Left operand */
97841	Fts3Expr pRight; / Right operand */
97842	Fts3Phrase pPhrase; / Valid if eType==FTSQUERY_PHRASE */
97843	};
97844
97845	SQLITE_PRIVATE int sqlite3Fts3ExprParse(sqlite3_tokenizer , char *, int, int,
97846	const char , int, Fts3Expr *);
97847	SQLITE_PRIVATE void sqlite3Fts3ExprFree(Fts3Expr *);
97848
97849	/*
97850	** Candidate values for Fts3Query.eType. Note that the order of the first
97851	** four values is in order of precedence when parsing expressions. For
97852	** example, the following:
97853	**
97854	** "a OR b AND c NOT d NEAR e"
97855	**
97856	** is equivalent to:
97857	**
97858	** "a OR (b AND (c NOT (d NEAR e)))"
97859	*/
97860	#define FTSQUERY_NEAR 1
97861	#define FTSQUERY_NOT 2
97862	#define FTSQUERY_AND 3
97863	#define FTSQUERY_OR 4
97864	#define FTSQUERY_PHRASE 5
97865
97866	#ifdef SQLITE_TEST
97867	SQLITE_PRIVATE void sqlite3Fts3ExprInitTestInterface(sqlite3 *db);
97868	#endif
97869
97870	/************ End of fts3_expr.h *****************************************/
97871	/************ Continuing where we left off in fts3.c *********************/
97872	/************ Include fts3_hash.h in the middle of fts3.c ****************/
97873	/************ Begin file fts3_hash.h *************************************/
97874	/*
97875	** 2001 September 22
97876	**
97877	** The author disclaims copyright to this source code. In place of
97878	** a legal notice, here is a blessing:
97879	**
97880	** May you do good and not evil.
97881	** May you find forgiveness for yourself and forgive others.
97882	** May you share freely, never taking more than you give.
97883	**
97884	*************************************************************************
97885	** This is the header file for the generic hash-table implemenation
97886	** used in SQLite. We've modified it slightly to serve as a standalone
97887	** hash table implementation for the full-text indexing module.
97888	**
97889	*/
97890	#ifndef _FTS3_HASH_H_
97891	#define _FTS3_HASH_H_
97892
97893	/* Forward declarations of structures. */
97894	typedef struct fts3Hash fts3Hash;
97895	typedef struct fts3HashElem fts3HashElem;
97896
97897	/* A complete hash table is an instance of the following structure.
97898	** The internals of this structure are intended to be opaque -- client
97899	** code should not attempt to access or modify the fields of this structure
97900	** directly. Change this structure only by using the routines below.
97901	** However, many of the "procedures" and "functions" for modifying and
97902	** accessing this structure are really macros, so we can't really make
97903	** this structure opaque.
97904	*/
97905	struct fts3Hash {
97906	char keyClass; /* HASH_INT, _POINTER, _STRING, _BINARY */
97907	char copyKey; /* True if copy of key made on insert */
97908	int count; /* Number of entries in this table */
97909	fts3HashElem first; / The first element of the array */
97910	int htsize; /* Number of buckets in the hash table */
97911	struct _fts3ht { /* the hash table */
97912	int count; /* Number of entries with this hash */
97913	fts3HashElem chain; / Pointer to first entry with this hash */
97914	} *ht;
97915	};
97916
97917	/* Each element in the hash table is an instance of the following
97918	** structure. All elements are stored on a single doubly-linked list.
97919	**
97920	** Again, this structure is intended to be opaque, but it can't really
97921	** be opaque because it is used by macros.
97922	*/
97923	struct fts3HashElem {
97924	fts3HashElem next, prev; /* Next and previous elements in the table */
97925	void data; / Data associated with this element */
97926	void pKey; int nKey; / Key associated with this element */
97927	};
97928
97929	/*
97930	** There are 2 different modes of operation for a hash table:
97931	**
97932	** FTS3_HASH_STRING pKey points to a string that is nKey bytes long
97933	** (including the null-terminator, if any). Case
97934	** is respected in comparisons.
97935	**
97936	** FTS3_HASH_BINARY pKey points to binary data nKey bytes long.
97937	** memcmp() is used to compare keys.
97938	**
97939	** A copy of the key is made if the copyKey parameter to fts3HashInit is 1.
97940	*/
97941	#define FTS3_HASH_STRING 1
97942	#define FTS3_HASH_BINARY 2
97943
97944	/*
97945	** Access routines. To delete, insert a NULL pointer.
97946	*/
97947	SQLITE_PRIVATE void sqlite3Fts3HashInit(fts3Hash*, int keytype, int copyKey);
97948	SQLITE_PRIVATE void sqlite3Fts3HashInsert(fts3Hash, const void pKey, int nKey, void pData);
97949	SQLITE_PRIVATE void sqlite3Fts3HashFind(const fts3Hash, const void *pKey, int nKey);
97950	SQLITE_PRIVATE void sqlite3Fts3HashClear(fts3Hash*);
97951
97952	/*
97953	** Shorthand for the functions above
97954	*/
97955	#define fts3HashInit sqlite3Fts3HashInit
97956	#define fts3HashInsert sqlite3Fts3HashInsert
97957	#define fts3HashFind sqlite3Fts3HashFind
97958	#define fts3HashClear sqlite3Fts3HashClear
97959
97960	/*
97961	** Macros for looping over all elements of a hash table. The idiom is
97962	** like this:
97963	**
97964	** fts3Hash h;
97965	** fts3HashElem *p;
97966	** ...
97967	** for(p=fts3HashFirst(&h); p; p=fts3HashNext(p)){
97968	** SomeStructure *pData = fts3HashData(p);
97969	** // do something with pData
97970	** }
97971	*/
97972	#define fts3HashFirst(H) ((H)->first)
97973	#define fts3HashNext(E) ((E)->next)
97974	#define fts3HashData(E) ((E)->data)
97975	#define fts3HashKey(E) ((E)->pKey)
97976	#define fts3HashKeysize(E) ((E)->nKey)
97977
97978	/*
97979	** Number of entries in a hash table
97980	*/
97981	#define fts3HashCount(H) ((H)->count)
97982
97983	#endif /* _FTS3_HASH_H_ */
97984
97985	/************ End of fts3_hash.h *****************************************/
97986	/************ Continuing where we left off in fts3.c *********************/
97987	#ifndef SQLITE_CORE
97988	SQLITE_EXTENSION_INIT1
97989	#endif

97990
97991
97992	/* TODO(shess) MAN, this thing needs some refactoring. At minimum, it
97993	** would be nice to order the file better, perhaps something along the
97994	** lines of:
	@@ -98006,29 +97799,10 @@
98006	# define FTSTRACE(A) printf A; fflush(stdout)
98007	#else
98008	# define FTSTRACE(A)
98009	#endif
98010
98011	/* It is not safe to call isspace(), tolower(), or isalnum() on
98012	** hi-bit-set characters. This is the same solution used in the
98013	** tokenizer.
98014	*/
98015	/* TODO(shess) The snippet-generation code should be using the
98016	** tokenizer-generated tokens rather than doing its own local
98017	** tokenization.
98018	*/
98019	/* TODO(shess) Is __isascii() a portable version of (c&0x80)==0? */
98020	static int safe_isspace(char c){
98021	return (c&0x80)==0 ? isspace(c) : 0;
98022	}
98023	static int safe_tolower(char c){
98024	return (c&0x80)==0 ? tolower(c) : c;
98025	}
98026	static int safe_isalnum(char c){
98027	return (c&0x80)==0 ? isalnum(c) : 0;
98028	}
98029
98030	typedef enum DocListType {
98031	DL_DOCIDS, /* docids only */
98032	DL_POSITIONS, /* docids + positions */
98033	DL_POSITIONS_OFFSETS /* docids + positions + offsets */
98034	} DocListType;
	@@ -98050,15 +97824,10 @@
98050	POS_END = 0, /* end of this position list */
98051	POS_COLUMN, /* followed by new column number */
98052	POS_BASE
98053	};
98054
98055	/* MERGE_COUNT controls how often we merge segments (see comment at
98056	** top of file).
98057	*/
98058	#define MERGE_COUNT 16
98059
98060	/* utility functions */
98061
98062	/* CLEAR() and SCRAMBLE() abstract memset() on a pointer to a single
98063	** record to prevent errors of the form:
98064	**
	@@ -98073,2225 +97842,71 @@
98073	# define SCRAMBLE(b) memset(b, 0x55, sizeof(*(b)))
98074	#else
98075	# define SCRAMBLE(b)
98076	#endif
98077
98078	/* We may need up to VARINT_MAX bytes to store an encoded 64-bit integer. */
98079	#define VARINT_MAX 10
98080
98081	/* Write a 64-bit variable-length integer to memory starting at p[0].
98082	* The length of data written will be between 1 and VARINT_MAX bytes.
98083	* The number of bytes written is returned. */
98084	static int fts3PutVarint(char *p, sqlite_int64 v){
98085	unsigned char q = (unsigned char ) p;
98086	sqlite_uint64 vu = v;
98087	do{
98088	*q++ = (unsigned char) ((vu & 0x7f) \| 0x80);
98089	vu >>= 7;
98090	}while( vu!=0 );
98091	q[-1] &= 0x7f; /* turn off high bit in final byte */
98092	assert( q - (unsigned char *)p <= VARINT_MAX );
98093	return (int) (q - (unsigned char *)p);
98094	}
98095
98096	/* Read a 64-bit variable-length integer from memory starting at p[0].
98097	* Return the number of bytes read, or 0 on error.
98098	* The value is stored in v. /
98099	static int fts3GetVarint(const char p, sqlite_int64 v){


98100	const unsigned char q = (const unsigned char ) p;
98101	sqlite_uint64 x = 0, y = 1;
98102	while( (*q & 0x80) == 0x80 ){
98103	x += y * (*q++ & 0x7f);
98104	y <<= 7;
98105	if( q - (unsigned char )p >= VARINT_MAX ){ / bad data */
98106	assert( 0 );
98107	return 0;
98108	}
98109	}
98110	x += y * (*q++);
98111	*v = (sqlite_int64) x;
98112	return (int) (q - (unsigned char *)p);
98113	}
98114
98115	static int fts3GetVarint32(const char p, int pi){




98116	sqlite_int64 i;
98117	int ret = fts3GetVarint(p, &i);
98118	*pi = (int) i;
98119	assert( *pi==i );
98120	return ret;
98121	}
98122
98123	/*******************************************************************/
98124	/* DataBuffer is used to collect data into a buffer in piecemeal
98125	** fashion. It implements the usual distinction between amount of
98126	** data currently stored (nData) and buffer capacity (nCapacity).
98127	**
98128	** dataBufferInit - create a buffer with given initial capacity.
98129	** dataBufferReset - forget buffer's data, retaining capacity.
98130	** dataBufferDestroy - free buffer's data.
98131	** dataBufferSwap - swap contents of two buffers.
98132	** dataBufferExpand - expand capacity without adding data.
98133	** dataBufferAppend - append data.
98134	** dataBufferAppend2 - append two pieces of data at once.
98135	** dataBufferReplace - replace buffer's data.
98136	*/
98137	typedef struct DataBuffer {
98138	char pData; / Pointer to malloc'ed buffer. */
98139	int nCapacity; /* Size of pData buffer. */
98140	int nData; /* End of data loaded into pData. */
98141	} DataBuffer;
98142
98143	static void dataBufferInit(DataBuffer *pBuffer, int nCapacity){
98144	assert( nCapacity>=0 );
98145	pBuffer->nData = 0;
98146	pBuffer->nCapacity = nCapacity;
98147	pBuffer->pData = nCapacity==0 ? NULL : sqlite3_malloc(nCapacity);
98148	}
98149	static void dataBufferReset(DataBuffer *pBuffer){
98150	pBuffer->nData = 0;
98151	}
98152	static void dataBufferDestroy(DataBuffer *pBuffer){
98153	if( pBuffer->pData!=NULL ) sqlite3_free(pBuffer->pData);
98154	SCRAMBLE(pBuffer);
98155	}
98156	static void dataBufferSwap(DataBuffer pBuffer1, DataBuffer pBuffer2){
98157	DataBuffer tmp = *pBuffer1;
98158	pBuffer1 = pBuffer2;
98159	*pBuffer2 = tmp;
98160	}
98161	static void dataBufferExpand(DataBuffer *pBuffer, int nAddCapacity){
98162	assert( nAddCapacity>0 );
98163	/* TODO(shess) Consider expanding more aggressively. Note that the
98164	** underlying malloc implementation may take care of such things for
98165	** us already.
98166	*/
98167	if( pBuffer->nData+nAddCapacity>pBuffer->nCapacity ){
98168	pBuffer->nCapacity = pBuffer->nData+nAddCapacity;
98169	pBuffer->pData = sqlite3_realloc(pBuffer->pData, pBuffer->nCapacity);
98170	}
98171	}
98172	static void dataBufferAppend(DataBuffer *pBuffer,
98173	const char *pSource, int nSource){
98174	assert( nSource>0 && pSource!=NULL );
98175	dataBufferExpand(pBuffer, nSource);
98176	memcpy(pBuffer->pData+pBuffer->nData, pSource, nSource);
98177	pBuffer->nData += nSource;
98178	}
98179	static void dataBufferAppend2(DataBuffer *pBuffer,
98180	const char *pSource1, int nSource1,
98181	const char *pSource2, int nSource2){
98182	assert( nSource1>0 && pSource1!=NULL );
98183	assert( nSource2>0 && pSource2!=NULL );
98184	dataBufferExpand(pBuffer, nSource1+nSource2);
98185	memcpy(pBuffer->pData+pBuffer->nData, pSource1, nSource1);
98186	memcpy(pBuffer->pData+pBuffer->nData+nSource1, pSource2, nSource2);
98187	pBuffer->nData += nSource1+nSource2;
98188	}
98189	static void dataBufferReplace(DataBuffer *pBuffer,
98190	const char *pSource, int nSource){
98191	dataBufferReset(pBuffer);
98192	dataBufferAppend(pBuffer, pSource, nSource);
98193	}
98194
98195	/* StringBuffer is a null-terminated version of DataBuffer. */
98196	typedef struct StringBuffer {
98197	DataBuffer b; /* Includes null terminator. */
98198	} StringBuffer;
98199
98200	static void initStringBuffer(StringBuffer *sb){
98201	dataBufferInit(&sb->b, 100);
98202	dataBufferReplace(&sb->b, "", 1);
98203	}
98204	static int stringBufferLength(StringBuffer *sb){
98205	return sb->b.nData-1;
98206	}
98207	static char stringBufferData(StringBuffer sb){
98208	return sb->b.pData;
98209	}
98210	static void stringBufferDestroy(StringBuffer *sb){
98211	dataBufferDestroy(&sb->b);
98212	}
98213
98214	static void nappend(StringBuffer sb, const char zFrom, int nFrom){
98215	assert( sb->b.nData>0 );
98216	if( nFrom>0 ){
98217	sb->b.nData--;
98218	dataBufferAppend2(&sb->b, zFrom, nFrom, "", 1);
98219	}
98220	}
98221	static void append(StringBuffer sb, const char zFrom){
98222	nappend(sb, zFrom, strlen(zFrom));
98223	}
98224
98225	/* Append a list of strings separated by commas. */
98226	static void appendList(StringBuffer sb, int nString, char *azString){
98227	int i;
98228	for(i=0; i<nString; ++i){
98229	if( i>0 ) append(sb, ", ");
98230	append(sb, azString[i]);
98231	}
98232	}
98233
98234	static int endsInWhiteSpace(StringBuffer *p){
98235	return stringBufferLength(p)>0 &&
98236	safe_isspace(stringBufferData(p)[stringBufferLength(p)-1]);
98237	}
98238
98239	/* If the StringBuffer ends in something other than white space, add a
98240	** single space character to the end.
98241	*/
98242	static void appendWhiteSpace(StringBuffer *p){
98243	if( stringBufferLength(p)==0 ) return;
98244	if( !endsInWhiteSpace(p) ) append(p, " ");
98245	}
98246
98247	/* Remove white space from the end of the StringBuffer */
98248	static void trimWhiteSpace(StringBuffer *p){
98249	while( endsInWhiteSpace(p) ){
98250	p->b.pData[--p->b.nData-1] = '\0';
98251	}
98252	}
98253
98254	/*******************************************************************/
98255	/* DLReader is used to read document elements from a doclist. The
98256	** current docid is cached, so dlrDocid() is fast. DLReader does not
98257	** own the doclist buffer.
98258	**
98259	** dlrAtEnd - true if there's no more data to read.
98260	** dlrDocid - docid of current document.
98261	** dlrDocData - doclist data for current document (including docid).
98262	** dlrDocDataBytes - length of same.
98263	** dlrAllDataBytes - length of all remaining data.
98264	** dlrPosData - position data for current document.
98265	** dlrPosDataLen - length of pos data for current document (incl POS_END).
98266	** dlrStep - step to current document.
98267	** dlrInit - initial for doclist of given type against given data.
98268	** dlrDestroy - clean up.
98269	**
98270	** Expected usage is something like:
98271	**
98272	** DLReader reader;
98273	** dlrInit(&reader, pData, nData);
98274	** while( !dlrAtEnd(&reader) ){
98275	** // calls to dlrDocid() and kin.
98276	** dlrStep(&reader);
98277	** }
98278	** dlrDestroy(&reader);
98279	*/
98280	typedef struct DLReader {
98281	DocListType iType;
98282	const char *pData;
98283	int nData;
98284
98285	sqlite_int64 iDocid;
98286	int nElement;
98287	} DLReader;
98288
98289	static int dlrAtEnd(DLReader *pReader){
98290	assert( pReader->nData>=0 );
98291	return pReader->nData==0;
98292	}
98293	static sqlite_int64 dlrDocid(DLReader *pReader){
98294	assert( !dlrAtEnd(pReader) );
98295	return pReader->iDocid;
98296	}
98297	static const char dlrDocData(DLReader pReader){
98298	assert( !dlrAtEnd(pReader) );
98299	return pReader->pData;
98300	}
98301	static int dlrDocDataBytes(DLReader *pReader){
98302	assert( !dlrAtEnd(pReader) );
98303	return pReader->nElement;
98304	}
98305	static int dlrAllDataBytes(DLReader *pReader){
98306	assert( !dlrAtEnd(pReader) );
98307	return pReader->nData;
98308	}
98309	/* TODO(shess) Consider adding a field to track iDocid varint length
98310	** to make these two functions faster. This might matter (a tiny bit)
98311	** for queries.
98312	*/
98313	static const char dlrPosData(DLReader pReader){
98314	sqlite_int64 iDummy;
98315	int n = fts3GetVarint(pReader->pData, &iDummy);
98316	assert( !dlrAtEnd(pReader) );
98317	return pReader->pData+n;
98318	}
98319	static int dlrPosDataLen(DLReader *pReader){
98320	sqlite_int64 iDummy;
98321	int n = fts3GetVarint(pReader->pData, &iDummy);
98322	assert( !dlrAtEnd(pReader) );
98323	return pReader->nElement-n;
98324	}
98325	static void dlrStep(DLReader *pReader){
98326	assert( !dlrAtEnd(pReader) );
98327
98328	/* Skip past current doclist element. */
98329	assert( pReader->nElement<=pReader->nData );
98330	pReader->pData += pReader->nElement;
98331	pReader->nData -= pReader->nElement;
98332
98333	/* If there is more data, read the next doclist element. */
98334	if( pReader->nData!=0 ){
98335	sqlite_int64 iDocidDelta;
98336	int iDummy, n = fts3GetVarint(pReader->pData, &iDocidDelta);
98337	pReader->iDocid += iDocidDelta;
98338	if( pReader->iType>=DL_POSITIONS ){
98339	assert( n<pReader->nData );
98340	while( 1 ){
98341	n += fts3GetVarint32(pReader->pData+n, &iDummy);
98342	assert( n<=pReader->nData );
98343	if( iDummy==POS_END ) break;
98344	if( iDummy==POS_COLUMN ){
98345	n += fts3GetVarint32(pReader->pData+n, &iDummy);
98346	assert( n<pReader->nData );
98347	}else if( pReader->iType==DL_POSITIONS_OFFSETS ){
98348	n += fts3GetVarint32(pReader->pData+n, &iDummy);
98349	n += fts3GetVarint32(pReader->pData+n, &iDummy);
98350	assert( n<pReader->nData );
98351	}
98352	}
98353	}
98354	pReader->nElement = n;
98355	assert( pReader->nElement<=pReader->nData );
98356	}
98357	}
98358	static void dlrInit(DLReader *pReader, DocListType iType,
98359	const char *pData, int nData){
98360	assert( pData!=NULL && nData!=0 );
98361	pReader->iType = iType;
98362	pReader->pData = pData;
98363	pReader->nData = nData;
98364	pReader->nElement = 0;
98365	pReader->iDocid = 0;
98366
98367	/* Load the first element's data. There must be a first element. */
98368	dlrStep(pReader);
98369	}
98370	static void dlrDestroy(DLReader *pReader){
98371	SCRAMBLE(pReader);
98372	}
98373
98374	#ifndef NDEBUG
98375	/* Verify that the doclist can be validly decoded. Also returns the
98376	** last docid found because it is convenient in other assertions for
98377	** DLWriter.
98378	*/
98379	static void docListValidate(DocListType iType, const char *pData, int nData,
98380	sqlite_int64 *pLastDocid){
98381	sqlite_int64 iPrevDocid = 0;
98382	assert( nData>0 );
98383	assert( pData!=0 );
98384	assert( pData+nData>pData );
98385	while( nData!=0 ){
98386	sqlite_int64 iDocidDelta;
98387	int n = fts3GetVarint(pData, &iDocidDelta);
98388	iPrevDocid += iDocidDelta;
98389	if( iType>DL_DOCIDS ){
98390	int iDummy;
98391	while( 1 ){
98392	n += fts3GetVarint32(pData+n, &iDummy);
98393	if( iDummy==POS_END ) break;
98394	if( iDummy==POS_COLUMN ){
98395	n += fts3GetVarint32(pData+n, &iDummy);
98396	}else if( iType>DL_POSITIONS ){
98397	n += fts3GetVarint32(pData+n, &iDummy);
98398	n += fts3GetVarint32(pData+n, &iDummy);
98399	}
98400	assert( n<=nData );
98401	}
98402	}
98403	assert( n<=nData );
98404	pData += n;
98405	nData -= n;
98406	}
98407	if( pLastDocid ) *pLastDocid = iPrevDocid;
98408	}
98409	#define ASSERT_VALID_DOCLIST(i, p, n, o) docListValidate(i, p, n, o)
98410	#else
98411	#define ASSERT_VALID_DOCLIST(i, p, n, o) assert( 1 )
98412	#endif
98413
98414	/*******************************************************************/
98415	/* DLWriter is used to write doclist data to a DataBuffer. DLWriter
98416	** always appends to the buffer and does not own it.
98417	**
98418	** dlwInit - initialize to write a given type doclistto a buffer.
98419	** dlwDestroy - clear the writer's memory. Does not free buffer.
98420	** dlwAppend - append raw doclist data to buffer.
98421	** dlwCopy - copy next doclist from reader to writer.
98422	** dlwAdd - construct doclist element and append to buffer.
98423	** Only apply dlwAdd() to DL_DOCIDS doclists (else use PLWriter).
98424	*/
98425	typedef struct DLWriter {
98426	DocListType iType;
98427	DataBuffer *b;
98428	sqlite_int64 iPrevDocid;
98429	#ifndef NDEBUG
98430	int has_iPrevDocid;
98431	#endif
98432	} DLWriter;
98433
98434	static void dlwInit(DLWriter pWriter, DocListType iType, DataBuffer b){
98435	pWriter->b = b;
98436	pWriter->iType = iType;
98437	pWriter->iPrevDocid = 0;
98438	#ifndef NDEBUG
98439	pWriter->has_iPrevDocid = 0;
98440	#endif
98441	}
98442	static void dlwDestroy(DLWriter *pWriter){
98443	SCRAMBLE(pWriter);
98444	}
98445	/* iFirstDocid is the first docid in the doclist in pData. It is
98446	** needed because pData may point within a larger doclist, in which
98447	** case the first item would be delta-encoded.
98448	**
98449	** iLastDocid is the final docid in the doclist in pData. It is
98450	** needed to create the new iPrevDocid for future delta-encoding. The
98451	** code could decode the passed doclist to recreate iLastDocid, but
98452	** the only current user (docListMerge) already has decoded this
98453	** information.
98454	*/
98455	/* TODO(shess) This has become just a helper for docListMerge.
98456	** Consider a refactor to make this cleaner.
98457	*/
98458	static void dlwAppend(DLWriter *pWriter,
98459	const char *pData, int nData,
98460	sqlite_int64 iFirstDocid, sqlite_int64 iLastDocid){
98461	sqlite_int64 iDocid = 0;
98462	char c[VARINT_MAX];
98463	int nFirstOld, nFirstNew; /* Old and new varint len of first docid. */
98464	#ifndef NDEBUG
98465	sqlite_int64 iLastDocidDelta;
98466	#endif
98467
98468	/* Recode the initial docid as delta from iPrevDocid. */
98469	nFirstOld = fts3GetVarint(pData, &iDocid);
98470	assert( nFirstOld<nData \|\| (nFirstOld==nData && pWriter->iType==DL_DOCIDS) );
98471	nFirstNew = fts3PutVarint(c, iFirstDocid-pWriter->iPrevDocid);
98472
98473	/* Verify that the incoming doclist is valid AND that it ends with
98474	** the expected docid. This is essential because we'll trust this
98475	** docid in future delta-encoding.
98476	*/
98477	ASSERT_VALID_DOCLIST(pWriter->iType, pData, nData, &iLastDocidDelta);
98478	assert( iLastDocid==iFirstDocid-iDocid+iLastDocidDelta );
98479
98480	/* Append recoded initial docid and everything else. Rest of docids
98481	** should have been delta-encoded from previous initial docid.
98482	*/
98483	if( nFirstOld<nData ){
98484	dataBufferAppend2(pWriter->b, c, nFirstNew,
98485	pData+nFirstOld, nData-nFirstOld);
98486	}else{
98487	dataBufferAppend(pWriter->b, c, nFirstNew);
98488	}
98489	pWriter->iPrevDocid = iLastDocid;
98490	}
98491	static void dlwCopy(DLWriter pWriter, DLReader pReader){
98492	dlwAppend(pWriter, dlrDocData(pReader), dlrDocDataBytes(pReader),
98493	dlrDocid(pReader), dlrDocid(pReader));
98494	}
98495	static void dlwAdd(DLWriter *pWriter, sqlite_int64 iDocid){
98496	char c[VARINT_MAX];
98497	int n = fts3PutVarint(c, iDocid-pWriter->iPrevDocid);
98498
98499	/* Docids must ascend. */
98500	assert( !pWriter->has_iPrevDocid \|\| iDocid>pWriter->iPrevDocid );
98501	assert( pWriter->iType==DL_DOCIDS );
98502
98503	dataBufferAppend(pWriter->b, c, n);
98504	pWriter->iPrevDocid = iDocid;
98505	#ifndef NDEBUG
98506	pWriter->has_iPrevDocid = 1;
98507	#endif
98508	}
98509
98510	/*******************************************************************/
98511	/* PLReader is used to read data from a document's position list. As
98512	** the caller steps through the list, data is cached so that varints
98513	** only need to be decoded once.
98514	**
98515	** plrInit, plrDestroy - create/destroy a reader.
98516	** plrColumn, plrPosition, plrStartOffset, plrEndOffset - accessors
98517	** plrAtEnd - at end of stream, only call plrDestroy once true.
98518	** plrStep - step to the next element.
98519	*/
98520	typedef struct PLReader {
98521	/* These refer to the next position's data. nData will reach 0 when
98522	** reading the last position, so plrStep() signals EOF by setting
98523	** pData to NULL.
98524	*/
98525	const char *pData;
98526	int nData;
98527
98528	DocListType iType;
98529	int iColumn; /* the last column read */
98530	int iPosition; /* the last position read */
98531	int iStartOffset; /* the last start offset read */
98532	int iEndOffset; /* the last end offset read */
98533	} PLReader;
98534
98535	static int plrAtEnd(PLReader *pReader){
98536	return pReader->pData==NULL;
98537	}
98538	static int plrColumn(PLReader *pReader){
98539	assert( !plrAtEnd(pReader) );
98540	return pReader->iColumn;
98541	}
98542	static int plrPosition(PLReader *pReader){
98543	assert( !plrAtEnd(pReader) );
98544	return pReader->iPosition;
98545	}
98546	static int plrStartOffset(PLReader *pReader){
98547	assert( !plrAtEnd(pReader) );
98548	return pReader->iStartOffset;
98549	}
98550	static int plrEndOffset(PLReader *pReader){
98551	assert( !plrAtEnd(pReader) );
98552	return pReader->iEndOffset;
98553	}
98554	static void plrStep(PLReader *pReader){
98555	int i, n;
98556
98557	assert( !plrAtEnd(pReader) );
98558
98559	if( pReader->nData==0 ){
98560	pReader->pData = NULL;
98561	return;
98562	}
98563
98564	n = fts3GetVarint32(pReader->pData, &i);
98565	if( i==POS_COLUMN ){
98566	n += fts3GetVarint32(pReader->pData+n, &pReader->iColumn);
98567	pReader->iPosition = 0;
98568	pReader->iStartOffset = 0;
98569	n += fts3GetVarint32(pReader->pData+n, &i);
98570	}
98571	/* Should never see adjacent column changes. */
98572	assert( i!=POS_COLUMN );
98573
98574	if( i==POS_END ){
98575	pReader->nData = 0;
98576	pReader->pData = NULL;
98577	return;
98578	}
98579
98580	pReader->iPosition += i-POS_BASE;
98581	if( pReader->iType==DL_POSITIONS_OFFSETS ){
98582	n += fts3GetVarint32(pReader->pData+n, &i);
98583	pReader->iStartOffset += i;
98584	n += fts3GetVarint32(pReader->pData+n, &i);
98585	pReader->iEndOffset = pReader->iStartOffset+i;
98586	}
98587	assert( n<=pReader->nData );
98588	pReader->pData += n;
98589	pReader->nData -= n;
98590	}
98591
98592	static void plrInit(PLReader pReader, DLReader pDLReader){
98593	pReader->pData = dlrPosData(pDLReader);
98594	pReader->nData = dlrPosDataLen(pDLReader);
98595	pReader->iType = pDLReader->iType;
98596	pReader->iColumn = 0;
98597	pReader->iPosition = 0;
98598	pReader->iStartOffset = 0;
98599	pReader->iEndOffset = 0;
98600	plrStep(pReader);
98601	}
98602	static void plrDestroy(PLReader *pReader){
98603	SCRAMBLE(pReader);
98604	}
98605
98606	/*******************************************************************/
98607	/* PLWriter is used in constructing a document's position list. As a
98608	** convenience, if iType is DL_DOCIDS, PLWriter becomes a no-op.
98609	** PLWriter writes to the associated DLWriter's buffer.
98610	**
98611	** plwInit - init for writing a document's poslist.
98612	** plwDestroy - clear a writer.
98613	** plwAdd - append position and offset information.
98614	** plwCopy - copy next position's data from reader to writer.
98615	** plwTerminate - add any necessary doclist terminator.
98616	**
98617	** Calling plwAdd() after plwTerminate() may result in a corrupt
98618	** doclist.
98619	*/
98620	/* TODO(shess) Until we've written the second item, we can cache the
98621	** first item's information. Then we'd have three states:
98622	**
98623	** - initialized with docid, no positions.
98624	** - docid and one position.
98625	** - docid and multiple positions.
98626	**
98627	** Only the last state needs to actually write to dlw->b, which would
98628	** be an improvement in the DLCollector case.
98629	*/
98630	typedef struct PLWriter {
98631	DLWriter *dlw;
98632
98633	int iColumn; /* the last column written */
98634	int iPos; /* the last position written */
98635	int iOffset; /* the last start offset written */
98636	} PLWriter;
98637
98638	/* TODO(shess) In the case where the parent is reading these values
98639	** from a PLReader, we could optimize to a copy if that PLReader has
98640	** the same type as pWriter.
98641	*/
98642	static void plwAdd(PLWriter *pWriter, int iColumn, int iPos,
98643	int iStartOffset, int iEndOffset){
98644	/* Worst-case space for POS_COLUMN, iColumn, iPosDelta,
98645	** iStartOffsetDelta, and iEndOffsetDelta.
98646	*/
98647	char c[5*VARINT_MAX];
98648	int n = 0;
98649
98650	/* Ban plwAdd() after plwTerminate(). */
98651	assert( pWriter->iPos!=-1 );
98652
98653	if( pWriter->dlw->iType==DL_DOCIDS ) return;
98654
98655	if( iColumn!=pWriter->iColumn ){
98656	n += fts3PutVarint(c+n, POS_COLUMN);
98657	n += fts3PutVarint(c+n, iColumn);
98658	pWriter->iColumn = iColumn;
98659	pWriter->iPos = 0;
98660	pWriter->iOffset = 0;
98661	}
98662	assert( iPos>=pWriter->iPos );
98663	n += fts3PutVarint(c+n, POS_BASE+(iPos-pWriter->iPos));
98664	pWriter->iPos = iPos;
98665	if( pWriter->dlw->iType==DL_POSITIONS_OFFSETS ){
98666	assert( iStartOffset>=pWriter->iOffset );
98667	n += fts3PutVarint(c+n, iStartOffset-pWriter->iOffset);
98668	pWriter->iOffset = iStartOffset;
98669	assert( iEndOffset>=iStartOffset );
98670	n += fts3PutVarint(c+n, iEndOffset-iStartOffset);
98671	}
98672	dataBufferAppend(pWriter->dlw->b, c, n);
98673	}
98674	static void plwCopy(PLWriter pWriter, PLReader pReader){
98675	plwAdd(pWriter, plrColumn(pReader), plrPosition(pReader),
98676	plrStartOffset(pReader), plrEndOffset(pReader));
98677	}
98678	static void plwInit(PLWriter pWriter, DLWriter dlw, sqlite_int64 iDocid){
98679	char c[VARINT_MAX];
98680	int n;
98681
98682	pWriter->dlw = dlw;
98683
98684	/* Docids must ascend. */
98685	assert( !pWriter->dlw->has_iPrevDocid \|\| iDocid>pWriter->dlw->iPrevDocid );
98686	n = fts3PutVarint(c, iDocid-pWriter->dlw->iPrevDocid);
98687	dataBufferAppend(pWriter->dlw->b, c, n);
98688	pWriter->dlw->iPrevDocid = iDocid;
98689	#ifndef NDEBUG
98690	pWriter->dlw->has_iPrevDocid = 1;
98691	#endif
98692
98693	pWriter->iColumn = 0;
98694	pWriter->iPos = 0;
98695	pWriter->iOffset = 0;
98696	}
98697	/* TODO(shess) Should plwDestroy() also terminate the doclist? But
98698	** then plwDestroy() would no longer be just a destructor, it would
98699	** also be doing work, which isn't consistent with the overall idiom.
98700	** Another option would be for plwAdd() to always append any necessary
98701	** terminator, so that the output is always correct. But that would
98702	** add incremental work to the common case with the only benefit being
98703	** API elegance. Punt for now.
98704	*/
98705	static void plwTerminate(PLWriter *pWriter){
98706	if( pWriter->dlw->iType>DL_DOCIDS ){
98707	char c[VARINT_MAX];
98708	int n = fts3PutVarint(c, POS_END);
98709	dataBufferAppend(pWriter->dlw->b, c, n);
98710	}
98711	#ifndef NDEBUG
98712	/* Mark as terminated for assert in plwAdd(). */
98713	pWriter->iPos = -1;
98714	#endif
98715	}
98716	static void plwDestroy(PLWriter *pWriter){
98717	SCRAMBLE(pWriter);
98718	}
98719
98720	/*******************************************************************/
98721	/* DLCollector wraps PLWriter and DLWriter to provide a
98722	** dynamically-allocated doclist area to use during tokenization.
98723	**
98724	** dlcNew - malloc up and initialize a collector.
98725	** dlcDelete - destroy a collector and all contained items.
98726	** dlcAddPos - append position and offset information.
98727	** dlcAddDoclist - add the collected doclist to the given buffer.
98728	** dlcNext - terminate the current document and open another.
98729	*/
98730	typedef struct DLCollector {
98731	DataBuffer b;
98732	DLWriter dlw;
98733	PLWriter plw;
98734	} DLCollector;
98735
98736	/* TODO(shess) This could also be done by calling plwTerminate() and
98737	** dataBufferAppend(). I tried that, expecting nominal performance
98738	** differences, but it seemed to pretty reliably be worth 1% to code
98739	** it this way. I suspect it is the incremental malloc overhead (some
98740	** percentage of the plwTerminate() calls will cause a realloc), so
98741	** this might be worth revisiting if the DataBuffer implementation
98742	** changes.
98743	*/
98744	static void dlcAddDoclist(DLCollector pCollector, DataBuffer b){
98745	if( pCollector->dlw.iType>DL_DOCIDS ){
98746	char c[VARINT_MAX];
98747	int n = fts3PutVarint(c, POS_END);
98748	dataBufferAppend2(b, pCollector->b.pData, pCollector->b.nData, c, n);
98749	}else{
98750	dataBufferAppend(b, pCollector->b.pData, pCollector->b.nData);
98751	}
98752	}
98753	static void dlcNext(DLCollector *pCollector, sqlite_int64 iDocid){
98754	plwTerminate(&pCollector->plw);
98755	plwDestroy(&pCollector->plw);
98756	plwInit(&pCollector->plw, &pCollector->dlw, iDocid);
98757	}
98758	static void dlcAddPos(DLCollector *pCollector, int iColumn, int iPos,
98759	int iStartOffset, int iEndOffset){
98760	plwAdd(&pCollector->plw, iColumn, iPos, iStartOffset, iEndOffset);
98761	}
98762
98763	static DLCollector *dlcNew(sqlite_int64 iDocid, DocListType iType){
98764	DLCollector *pCollector = sqlite3_malloc(sizeof(DLCollector));
98765	dataBufferInit(&pCollector->b, 0);
98766	dlwInit(&pCollector->dlw, iType, &pCollector->b);
98767	plwInit(&pCollector->plw, &pCollector->dlw, iDocid);
98768	return pCollector;
98769	}
98770	static void dlcDelete(DLCollector *pCollector){
98771	plwDestroy(&pCollector->plw);
98772	dlwDestroy(&pCollector->dlw);
98773	dataBufferDestroy(&pCollector->b);
98774	SCRAMBLE(pCollector);
98775	sqlite3_free(pCollector);
98776	}
98777
98778
98779	/* Copy the doclist data of iType in pData/nData into *out, trimming
98780	** unnecessary data as we go. Only columns matching iColumn are
98781	** copied, all columns copied if iColumn is -1. Elements with no
98782	** matching columns are dropped. The output is an iOutType doclist.
98783	*/
98784	/* NOTE(shess) This code is only valid after all doclists are merged.
98785	** If this is run before merges, then doclist items which represent
98786	** deletion will be trimmed, and will thus not effect a deletion
98787	** during the merge.
98788	*/
98789	static void docListTrim(DocListType iType, const char *pData, int nData,
98790	int iColumn, DocListType iOutType, DataBuffer *out){
98791	DLReader dlReader;
98792	DLWriter dlWriter;
98793
98794	assert( iOutType<=iType );
98795
98796	dlrInit(&dlReader, iType, pData, nData);
98797	dlwInit(&dlWriter, iOutType, out);
98798
98799	while( !dlrAtEnd(&dlReader) ){
98800	PLReader plReader;
98801	PLWriter plWriter;
98802	int match = 0;
98803
98804	plrInit(&plReader, &dlReader);
98805
98806	while( !plrAtEnd(&plReader) ){
98807	if( iColumn==-1 \|\| plrColumn(&plReader)==iColumn ){
98808	if( !match ){
98809	plwInit(&plWriter, &dlWriter, dlrDocid(&dlReader));
98810	match = 1;
98811	}
98812	plwAdd(&plWriter, plrColumn(&plReader), plrPosition(&plReader),
98813	plrStartOffset(&plReader), plrEndOffset(&plReader));
98814	}
98815	plrStep(&plReader);
98816	}
98817	if( match ){
98818	plwTerminate(&plWriter);
98819	plwDestroy(&plWriter);
98820	}
98821
98822	plrDestroy(&plReader);
98823	dlrStep(&dlReader);
98824	}
98825	dlwDestroy(&dlWriter);
98826	dlrDestroy(&dlReader);
98827	}
98828
98829	/* Used by docListMerge() to keep doclists in the ascending order by
98830	** docid, then ascending order by age (so the newest comes first).
98831	*/
98832	typedef struct OrderedDLReader {
98833	DLReader *pReader;
98834
98835	/* TODO(shess) If we assume that docListMerge pReaders is ordered by
98836	** age (which we do), then we could use pReader comparisons to break
98837	** ties.
98838	*/
98839	int idx;
98840	} OrderedDLReader;
98841
98842	/* Order eof to end, then by docid asc, idx desc. */
98843	static int orderedDLReaderCmp(OrderedDLReader r1, OrderedDLReader r2){
98844	if( dlrAtEnd(r1->pReader) ){
98845	if( dlrAtEnd(r2->pReader) ) return 0; /* Both atEnd(). */
98846	return 1; /* Only r1 atEnd(). */
98847	}
98848	if( dlrAtEnd(r2->pReader) ) return -1; /* Only r2 atEnd(). */
98849
98850	if( dlrDocid(r1->pReader)<dlrDocid(r2->pReader) ) return -1;
98851	if( dlrDocid(r1->pReader)>dlrDocid(r2->pReader) ) return 1;
98852
98853	/* Descending on idx. */
98854	return r2->idx-r1->idx;
98855	}
98856
98857	/* Bubble p[0] to appropriate place in p[1..n-1]. Assumes that
98858	** p[1..n-1] is already sorted.
98859	*/
98860	/* TODO(shess) Is this frequent enough to warrant a binary search?
98861	** Before implementing that, instrument the code to check. In most
98862	** current usage, I expect that p[0] will be less than p[1] a very
98863	** high proportion of the time.
98864	*/
98865	static void orderedDLReaderReorder(OrderedDLReader *p, int n){
98866	while( n>1 && orderedDLReaderCmp(p, p+1)>0 ){
98867	OrderedDLReader tmp = p[0];
98868	p[0] = p[1];
98869	p[1] = tmp;
98870	n--;
98871	p++;
98872	}
98873	}
98874
98875	/* Given an array of doclist readers, merge their doclist elements
98876	** into out in sorted order (by docid), dropping elements from older
98877	** readers when there is a duplicate docid. pReaders is assumed to be
98878	** ordered by age, oldest first.
98879	*/
98880	/* TODO(shess) nReaders must be <= MERGE_COUNT. This should probably
98881	** be fixed.
98882	*/
98883	static void docListMerge(DataBuffer *out,
98884	DLReader *pReaders, int nReaders){
98885	OrderedDLReader readers[MERGE_COUNT];
98886	DLWriter writer;
98887	int i, n;
98888	const char *pStart = 0;
98889	int nStart = 0;
98890	sqlite_int64 iFirstDocid = 0, iLastDocid = 0;
98891
98892	assert( nReaders>0 );
98893	if( nReaders==1 ){
98894	dataBufferAppend(out, dlrDocData(pReaders), dlrAllDataBytes(pReaders));
98895	return;
98896	}
98897
98898	assert( nReaders<=MERGE_COUNT );
98899	n = 0;
98900	for(i=0; i<nReaders; i++){
98901	assert( pReaders[i].iType==pReaders[0].iType );
98902	readers[i].pReader = pReaders+i;
98903	readers[i].idx = i;
98904	n += dlrAllDataBytes(&pReaders[i]);
98905	}
98906	/* Conservatively size output to sum of inputs. Output should end
98907	** up strictly smaller than input.
98908	*/
98909	dataBufferExpand(out, n);
98910
98911	/* Get the readers into sorted order. */
98912	while( i-->0 ){
98913	orderedDLReaderReorder(readers+i, nReaders-i);
98914	}
98915
98916	dlwInit(&writer, pReaders[0].iType, out);
98917	while( !dlrAtEnd(readers[0].pReader) ){
98918	sqlite_int64 iDocid = dlrDocid(readers[0].pReader);
98919
98920	/* If this is a continuation of the current buffer to copy, extend
98921	** that buffer. memcpy() seems to be more efficient if it has a
98922	** lots of data to copy.
98923	*/
98924	if( dlrDocData(readers[0].pReader)==pStart+nStart ){
98925	nStart += dlrDocDataBytes(readers[0].pReader);
98926	}else{
98927	if( pStart!=0 ){
98928	dlwAppend(&writer, pStart, nStart, iFirstDocid, iLastDocid);
98929	}
98930	pStart = dlrDocData(readers[0].pReader);
98931	nStart = dlrDocDataBytes(readers[0].pReader);
98932	iFirstDocid = iDocid;
98933	}
98934	iLastDocid = iDocid;
98935	dlrStep(readers[0].pReader);
98936
98937	/* Drop all of the older elements with the same docid. */
98938	for(i=1; i<nReaders &&
98939	!dlrAtEnd(readers[i].pReader) &&
98940	dlrDocid(readers[i].pReader)==iDocid; i++){
98941	dlrStep(readers[i].pReader);
98942	}
98943
98944	/* Get the readers back into order. */
98945	while( i-->0 ){
98946	orderedDLReaderReorder(readers+i, nReaders-i);
98947	}
98948	}
98949
98950	/* Copy over any remaining elements. */
98951	if( nStart>0 ) dlwAppend(&writer, pStart, nStart, iFirstDocid, iLastDocid);
98952	dlwDestroy(&writer);
98953	}
98954
98955	/* Helper function for posListUnion(). Compares the current position
98956	** between left and right, returning as standard C idiom of <0 if
98957	** left<right, >0 if left>right, and 0 if left==right. "End" always
98958	** compares greater.
98959	*/
98960	static int posListCmp(PLReader pLeft, PLReader pRight){
98961	assert( pLeft->iType==pRight->iType );
98962	if( pLeft->iType==DL_DOCIDS ) return 0;
98963
98964	if( plrAtEnd(pLeft) ) return plrAtEnd(pRight) ? 0 : 1;
98965	if( plrAtEnd(pRight) ) return -1;
98966
98967	if( plrColumn(pLeft)<plrColumn(pRight) ) return -1;
98968	if( plrColumn(pLeft)>plrColumn(pRight) ) return 1;
98969
98970	if( plrPosition(pLeft)<plrPosition(pRight) ) return -1;
98971	if( plrPosition(pLeft)>plrPosition(pRight) ) return 1;
98972	if( pLeft->iType==DL_POSITIONS ) return 0;
98973
98974	if( plrStartOffset(pLeft)<plrStartOffset(pRight) ) return -1;
98975	if( plrStartOffset(pLeft)>plrStartOffset(pRight) ) return 1;
98976
98977	if( plrEndOffset(pLeft)<plrEndOffset(pRight) ) return -1;
98978	if( plrEndOffset(pLeft)>plrEndOffset(pRight) ) return 1;
98979
98980	return 0;
98981	}
98982
98983	/* Write the union of position lists in pLeft and pRight to pOut.
98984	** "Union" in this case meaning "All unique position tuples". Should
98985	** work with any doclist type, though both inputs and the output
98986	** should be the same type.
98987	*/
98988	static void posListUnion(DLReader pLeft, DLReader pRight, DLWriter *pOut){
98989	PLReader left, right;
98990	PLWriter writer;
98991
98992	assert( dlrDocid(pLeft)==dlrDocid(pRight) );
98993	assert( pLeft->iType==pRight->iType );
98994	assert( pLeft->iType==pOut->iType );
98995
98996	plrInit(&left, pLeft);
98997	plrInit(&right, pRight);
98998	plwInit(&writer, pOut, dlrDocid(pLeft));
98999
99000	while( !plrAtEnd(&left) \|\| !plrAtEnd(&right) ){
99001	int c = posListCmp(&left, &right);
99002	if( c<0 ){
99003	plwCopy(&writer, &left);
99004	plrStep(&left);
99005	}else if( c>0 ){
99006	plwCopy(&writer, &right);
99007	plrStep(&right);
99008	}else{
99009	plwCopy(&writer, &left);
99010	plrStep(&left);
99011	plrStep(&right);
99012	}
99013	}
99014
99015	plwTerminate(&writer);
99016	plwDestroy(&writer);
99017	plrDestroy(&left);
99018	plrDestroy(&right);
99019	}
99020
99021	/* Write the union of doclists in pLeft and pRight to pOut. For
99022	** docids in common between the inputs, the union of the position
99023	** lists is written. Inputs and outputs are always type DL_DEFAULT.
99024	*/
99025	static void docListUnion(
99026	const char *pLeft, int nLeft,
99027	const char *pRight, int nRight,
99028	DataBuffer pOut / Write the combined doclist here */
99029	){
99030	DLReader left, right;
99031	DLWriter writer;
99032
99033	if( nLeft==0 ){
99034	if( nRight!=0) dataBufferAppend(pOut, pRight, nRight);
99035	return;
99036	}
99037	if( nRight==0 ){
99038	dataBufferAppend(pOut, pLeft, nLeft);
99039	return;
99040	}
99041
99042	dlrInit(&left, DL_DEFAULT, pLeft, nLeft);
99043	dlrInit(&right, DL_DEFAULT, pRight, nRight);
99044	dlwInit(&writer, DL_DEFAULT, pOut);
99045
99046	while( !dlrAtEnd(&left) \|\| !dlrAtEnd(&right) ){
99047	if( dlrAtEnd(&right) ){
99048	dlwCopy(&writer, &left);
99049	dlrStep(&left);
99050	}else if( dlrAtEnd(&left) ){
99051	dlwCopy(&writer, &right);
99052	dlrStep(&right);
99053	}else if( dlrDocid(&left)<dlrDocid(&right) ){
99054	dlwCopy(&writer, &left);
99055	dlrStep(&left);
99056	}else if( dlrDocid(&left)>dlrDocid(&right) ){
99057	dlwCopy(&writer, &right);
99058	dlrStep(&right);
99059	}else{
99060	posListUnion(&left, &right, &writer);
99061	dlrStep(&left);
99062	dlrStep(&right);
99063	}
99064	}
99065
99066	dlrDestroy(&left);
99067	dlrDestroy(&right);
99068	dlwDestroy(&writer);
99069	}
99070
99071	/*
99072	** This function is used as part of the implementation of phrase and
99073	** NEAR matching.
99074	**
99075	** pLeft and pRight are DLReaders positioned to the same docid in
99076	** lists of type DL_POSITION. This function writes an entry to the
99077	** DLWriter pOut for each position in pRight that is less than
99078	** (nNear+1) greater (but not equal to or smaller) than a position
99079	** in pLeft. For example, if nNear is 0, and the positions contained
99080	** by pLeft and pRight are:
99081	**
99082	** pLeft: 5 10 15 20
99083	** pRight: 6 9 17 21
99084	**
99085	** then the docid is added to pOut. If pOut is of type DL_POSITIONS,
99086	** then a positionids "6" and "21" are also added to pOut.
99087	**
99088	** If boolean argument isSaveLeft is true, then positionids are copied
99089	** from pLeft instead of pRight. In the example above, the positions "5"
99090	** and "20" would be added instead of "6" and "21".
99091	*/
99092	static void posListPhraseMerge(
99093	DLReader *pLeft,
99094	DLReader *pRight,
99095	int nNear,
99096	int isSaveLeft,
99097	DLWriter *pOut
99098	){
99099	PLReader left, right;
99100	PLWriter writer;
99101	int match = 0;
99102
99103	assert( dlrDocid(pLeft)==dlrDocid(pRight) );
99104	assert( pOut->iType!=DL_POSITIONS_OFFSETS );
99105
99106	plrInit(&left, pLeft);
99107	plrInit(&right, pRight);
99108
99109	while( !plrAtEnd(&left) && !plrAtEnd(&right) ){
99110	if( plrColumn(&left)<plrColumn(&right) ){
99111	plrStep(&left);
99112	}else if( plrColumn(&left)>plrColumn(&right) ){
99113	plrStep(&right);
99114	}else if( plrPosition(&left)>=plrPosition(&right) ){
99115	plrStep(&right);
99116	}else{
99117	if( (plrPosition(&right)-plrPosition(&left))<=(nNear+1) ){
99118	if( !match ){
99119	plwInit(&writer, pOut, dlrDocid(pLeft));
99120	match = 1;
99121	}
99122	if( !isSaveLeft ){
99123	plwAdd(&writer, plrColumn(&right), plrPosition(&right), 0, 0);
99124	}else{
99125	plwAdd(&writer, plrColumn(&left), plrPosition(&left), 0, 0);
99126	}
99127	plrStep(&right);
99128	}else{
99129	plrStep(&left);
99130	}
99131	}
99132	}
99133
99134	if( match ){
99135	plwTerminate(&writer);
99136	plwDestroy(&writer);
99137	}
99138
99139	plrDestroy(&left);
99140	plrDestroy(&right);
99141	}
99142
99143	/*
99144	** Compare the values pointed to by the PLReaders passed as arguments.
99145	** Return -1 if the value pointed to by pLeft is considered less than
99146	** the value pointed to by pRight, +1 if it is considered greater
99147	** than it, or 0 if it is equal. i.e.
99148	**
99149	** (pLeft - pRight)
99150	**
99151	** A PLReader that is in the EOF condition is considered greater than
99152	** any other. If neither argument is in EOF state, the return value of
99153	** plrColumn() is used. If the plrColumn() values are equal, the
99154	** comparison is on the basis of plrPosition().
99155	*/
99156	static int plrCompare(PLReader pLeft, PLReader pRight){
99157	assert(!plrAtEnd(pLeft) \|\| !plrAtEnd(pRight));
99158
99159	if( plrAtEnd(pRight) \|\| plrAtEnd(pLeft) ){
99160	return (plrAtEnd(pRight) ? -1 : 1);
99161	}
99162	if( plrColumn(pLeft)!=plrColumn(pRight) ){
99163	return ((plrColumn(pLeft)<plrColumn(pRight)) ? -1 : 1);
99164	}
99165	if( plrPosition(pLeft)!=plrPosition(pRight) ){
99166	return ((plrPosition(pLeft)<plrPosition(pRight)) ? -1 : 1);
99167	}
99168	return 0;
99169	}
99170
99171	/* We have two doclists with positions: pLeft and pRight. Depending
99172	** on the value of the nNear parameter, perform either a phrase
99173	** intersection (if nNear==0) or a NEAR intersection (if nNear>0)
99174	** and write the results into pOut.
99175	**
99176	** A phrase intersection means that two documents only match
99177	** if pLeft.iPos+1==pRight.iPos.
99178	**
99179	** A NEAR intersection means that two documents only match if
99180	** (abs(pLeft.iPos-pRight.iPos)<nNear).
99181	**
99182	** If a NEAR intersection is requested, then the nPhrase argument should
99183	** be passed the number of tokens in the two operands to the NEAR operator
99184	** combined. For example:
99185	**
99186	** Query syntax nPhrase
99187	** ------------------------------------
99188	** "A B C" NEAR "D E" 5
99189	** A NEAR B 2
99190	**
99191	** iType controls the type of data written to pOut. If iType is
99192	** DL_POSITIONS, the positions are those from pRight.
99193	*/
99194	static void docListPhraseMerge(
99195	const char *pLeft, int nLeft,
99196	const char *pRight, int nRight,
99197	int nNear, /* 0 for a phrase merge, non-zero for a NEAR merge */
99198	int nPhrase, /* Number of tokens in left+right operands to NEAR */
99199	DocListType iType, /* Type of doclist to write to pOut */
99200	DataBuffer pOut / Write the combined doclist here */
99201	){
99202	DLReader left, right;
99203	DLWriter writer;
99204
99205	if( nLeft==0 \|\| nRight==0 ) return;
99206
99207	assert( iType!=DL_POSITIONS_OFFSETS );
99208
99209	dlrInit(&left, DL_POSITIONS, pLeft, nLeft);
99210	dlrInit(&right, DL_POSITIONS, pRight, nRight);
99211	dlwInit(&writer, iType, pOut);
99212
99213	while( !dlrAtEnd(&left) && !dlrAtEnd(&right) ){
99214	if( dlrDocid(&left)<dlrDocid(&right) ){
99215	dlrStep(&left);
99216	}else if( dlrDocid(&right)<dlrDocid(&left) ){
99217	dlrStep(&right);
99218	}else{
99219	if( nNear==0 ){
99220	posListPhraseMerge(&left, &right, 0, 0, &writer);
99221	}else{
99222	/* This case occurs when two terms (simple terms or phrases) are
99223	* connected by a NEAR operator, span (nNear+1). i.e.
99224	*
99225	* '"terrible company" NEAR widget'
99226	*/
99227	DataBuffer one = {0, 0, 0};
99228	DataBuffer two = {0, 0, 0};
99229
99230	DLWriter dlwriter2;
99231	DLReader dr1 = {0, 0, 0, 0, 0};
99232	DLReader dr2 = {0, 0, 0, 0, 0};
99233
99234	dlwInit(&dlwriter2, iType, &one);
99235	posListPhraseMerge(&right, &left, nNear-3+nPhrase, 1, &dlwriter2);
99236	dlwInit(&dlwriter2, iType, &two);
99237	posListPhraseMerge(&left, &right, nNear-1, 0, &dlwriter2);
99238
99239	if( one.nData) dlrInit(&dr1, iType, one.pData, one.nData);
99240	if( two.nData) dlrInit(&dr2, iType, two.pData, two.nData);
99241
99242	if( !dlrAtEnd(&dr1) \|\| !dlrAtEnd(&dr2) ){
99243	PLReader pr1 = {0};
99244	PLReader pr2 = {0};
99245
99246	PLWriter plwriter;
99247	plwInit(&plwriter, &writer, dlrDocid(dlrAtEnd(&dr1)?&dr2:&dr1));
99248
99249	if( one.nData ) plrInit(&pr1, &dr1);
99250	if( two.nData ) plrInit(&pr2, &dr2);
99251	while( !plrAtEnd(&pr1) \|\| !plrAtEnd(&pr2) ){
99252	int iCompare = plrCompare(&pr1, &pr2);
99253	switch( iCompare ){
99254	case -1:
99255	plwCopy(&plwriter, &pr1);
99256	plrStep(&pr1);
99257	break;
99258	case 1:
99259	plwCopy(&plwriter, &pr2);
99260	plrStep(&pr2);
99261	break;
99262	case 0:
99263	plwCopy(&plwriter, &pr1);
99264	plrStep(&pr1);
99265	plrStep(&pr2);
99266	break;
99267	}
99268	}
99269	plwTerminate(&plwriter);
99270	}
99271	dataBufferDestroy(&one);
99272	dataBufferDestroy(&two);
99273	}
99274	dlrStep(&left);
99275	dlrStep(&right);
99276	}
99277	}
99278
99279	dlrDestroy(&left);
99280	dlrDestroy(&right);
99281	dlwDestroy(&writer);
99282	}
99283
99284	/* We have two DL_DOCIDS doclists: pLeft and pRight.
99285	** Write the intersection of these two doclists into pOut as a
99286	** DL_DOCIDS doclist.
99287	*/
99288	static void docListAndMerge(
99289	const char *pLeft, int nLeft,
99290	const char *pRight, int nRight,
99291	DataBuffer pOut / Write the combined doclist here */
99292	){
99293	DLReader left, right;
99294	DLWriter writer;
99295
99296	if( nLeft==0 \|\| nRight==0 ) return;
99297
99298	dlrInit(&left, DL_DOCIDS, pLeft, nLeft);
99299	dlrInit(&right, DL_DOCIDS, pRight, nRight);
99300	dlwInit(&writer, DL_DOCIDS, pOut);
99301
99302	while( !dlrAtEnd(&left) && !dlrAtEnd(&right) ){
99303	if( dlrDocid(&left)<dlrDocid(&right) ){
99304	dlrStep(&left);
99305	}else if( dlrDocid(&right)<dlrDocid(&left) ){
99306	dlrStep(&right);
99307	}else{
99308	dlwAdd(&writer, dlrDocid(&left));
99309	dlrStep(&left);
99310	dlrStep(&right);
99311	}
99312	}
99313
99314	dlrDestroy(&left);
99315	dlrDestroy(&right);
99316	dlwDestroy(&writer);
99317	}
99318
99319	/* We have two DL_DOCIDS doclists: pLeft and pRight.
99320	** Write the union of these two doclists into pOut as a
99321	** DL_DOCIDS doclist.
99322	*/
99323	static void docListOrMerge(
99324	const char *pLeft, int nLeft,
99325	const char *pRight, int nRight,
99326	DataBuffer pOut / Write the combined doclist here */
99327	){
99328	DLReader left, right;
99329	DLWriter writer;
99330
99331	if( nLeft==0 ){
99332	if( nRight!=0 ) dataBufferAppend(pOut, pRight, nRight);
99333	return;
99334	}
99335	if( nRight==0 ){
99336	dataBufferAppend(pOut, pLeft, nLeft);
99337	return;
99338	}
99339
99340	dlrInit(&left, DL_DOCIDS, pLeft, nLeft);
99341	dlrInit(&right, DL_DOCIDS, pRight, nRight);
99342	dlwInit(&writer, DL_DOCIDS, pOut);
99343
99344	while( !dlrAtEnd(&left) \|\| !dlrAtEnd(&right) ){
99345	if( dlrAtEnd(&right) ){
99346	dlwAdd(&writer, dlrDocid(&left));
99347	dlrStep(&left);
99348	}else if( dlrAtEnd(&left) ){
99349	dlwAdd(&writer, dlrDocid(&right));
99350	dlrStep(&right);
99351	}else if( dlrDocid(&left)<dlrDocid(&right) ){
99352	dlwAdd(&writer, dlrDocid(&left));
99353	dlrStep(&left);
99354	}else if( dlrDocid(&right)<dlrDocid(&left) ){
99355	dlwAdd(&writer, dlrDocid(&right));
99356	dlrStep(&right);
99357	}else{
99358	dlwAdd(&writer, dlrDocid(&left));
99359	dlrStep(&left);
99360	dlrStep(&right);
99361	}
99362	}
99363
99364	dlrDestroy(&left);
99365	dlrDestroy(&right);
99366	dlwDestroy(&writer);
99367	}
99368
99369	/* We have two DL_DOCIDS doclists: pLeft and pRight.
99370	** Write into pOut as DL_DOCIDS doclist containing all documents that
99371	** occur in pLeft but not in pRight.
99372	*/
99373	static void docListExceptMerge(
99374	const char *pLeft, int nLeft,
99375	const char *pRight, int nRight,
99376	DataBuffer pOut / Write the combined doclist here */
99377	){
99378	DLReader left, right;
99379	DLWriter writer;
99380
99381	if( nLeft==0 ) return;
99382	if( nRight==0 ){
99383	dataBufferAppend(pOut, pLeft, nLeft);
99384	return;
99385	}
99386
99387	dlrInit(&left, DL_DOCIDS, pLeft, nLeft);
99388	dlrInit(&right, DL_DOCIDS, pRight, nRight);
99389	dlwInit(&writer, DL_DOCIDS, pOut);
99390
99391	while( !dlrAtEnd(&left) ){
99392	while( !dlrAtEnd(&right) && dlrDocid(&right)<dlrDocid(&left) ){
99393	dlrStep(&right);
99394	}
99395	if( dlrAtEnd(&right) \|\| dlrDocid(&left)<dlrDocid(&right) ){
99396	dlwAdd(&writer, dlrDocid(&left));
99397	}
99398	dlrStep(&left);
99399	}
99400
99401	dlrDestroy(&left);
99402	dlrDestroy(&right);
99403	dlwDestroy(&writer);
99404	}
99405
99406	static char string_dup_n(const char s, int n){
99407	char *str = sqlite3_malloc(n + 1);
99408	memcpy(str, s, n);
99409	str[n] = '\0';
99410	return str;
99411	}
99412
99413	/* Duplicate a string; the caller must free() the returned string.
99414	* (We don't use strdup() since it is not part of the standard C library and
99415	* may not be available everywhere.) */
99416	static char string_dup(const char s){
99417	return string_dup_n(s, strlen(s));
99418	}
99419
99420	/* Format a string, replacing each occurrence of the % character with
99421	* zDb.zName. This may be more convenient than sqlite_mprintf()
99422	* when one string is used repeatedly in a format string.
99423	* The caller must free() the returned string. */
99424	static char string_format(const char zFormat,
99425	const char zDb, const char zName){
99426	const char *p;
99427	size_t len = 0;
99428	size_t nDb = strlen(zDb);
99429	size_t nName = strlen(zName);
99430	size_t nFullTableName = nDb+1+nName;
99431	char *result;
99432	char *r;
99433
99434	/* first compute length needed */
99435	for(p = zFormat ; *p ; ++p){
99436	len += (*p=='%' ? nFullTableName : 1);
99437	}
99438	len += 1; /* for null terminator */
99439
99440	r = result = sqlite3_malloc(len);
99441	for(p = zFormat; *p; ++p){
99442	if( *p=='%' ){
99443	memcpy(r, zDb, nDb);
99444	r += nDb;
99445	*r++ = '.';
99446	memcpy(r, zName, nName);
99447	r += nName;
99448	} else {
99449	r++ = p;
99450	}
99451	}
99452	*r++ = '\0';
99453	assert( r == result + len );
99454	return result;
99455	}
99456
99457	static int sql_exec(sqlite3 db, const char zDb, const char *zName,
99458	const char *zFormat){
99459	char *zCommand = string_format(zFormat, zDb, zName);
99460	int rc;
99461	FTSTRACE(("FTS3 sql: %s\n", zCommand));
99462	rc = sqlite3_exec(db, zCommand, NULL, 0, NULL);
99463	sqlite3_free(zCommand);
99464	return rc;
99465	}
99466
99467	static int sql_prepare(sqlite3 db, const char zDb, const char *zName,
99468	sqlite3_stmt *ppStmt, const char zFormat){
99469	char *zCommand = string_format(zFormat, zDb, zName);
99470	int rc;
99471	FTSTRACE(("FTS3 prepare: %s\n", zCommand));
99472	rc = sqlite3_prepare_v2(db, zCommand, -1, ppStmt, NULL);
99473	sqlite3_free(zCommand);
99474	return rc;
99475	}
99476
99477	/* end utility functions */
99478
99479	/* Forward reference */
99480	typedef struct fulltext_vtab fulltext_vtab;
99481
99482	/*
99483	** An instance of the following structure keeps track of generated
99484	** matching-word offset information and snippets.
99485	*/
99486	typedef struct Snippet {
99487	int nMatch; /* Total number of matches */
99488	int nAlloc; /* Space allocated for aMatch[] */
99489	struct snippetMatch { /* One entry for each matching term */
99490	char snStatus; /* Status flag for use while constructing snippets */
99491	short int iCol; /* The column that contains the match */
99492	short int iTerm; /* The index in Query.pTerms[] of the matching term */
99493	int iToken; /* The index of the matching document token */
99494	short int nByte; /* Number of bytes in the term */
99495	int iStart; /* The offset to the first character of the term */
99496	} aMatch; / Points to space obtained from malloc */
99497	char zOffset; / Text rendering of aMatch[] */
99498	int nOffset; /* strlen(zOffset) */
99499	char zSnippet; / Snippet text */
99500	int nSnippet; /* strlen(zSnippet) */
99501	} Snippet;
99502
99503
99504	typedef enum QueryType {
99505	QUERY_GENERIC, /* table scan */
99506	QUERY_DOCID, /* lookup by docid */
99507	QUERY_FULLTEXT /* QUERY_FULLTEXT + [i] is a full-text search for column i*/
99508	} QueryType;
99509
99510	typedef enum fulltext_statement {
99511	CONTENT_INSERT_STMT,
99512	CONTENT_SELECT_STMT,
99513	CONTENT_UPDATE_STMT,
99514	CONTENT_DELETE_STMT,
99515	CONTENT_EXISTS_STMT,
99516
99517	BLOCK_INSERT_STMT,
99518	BLOCK_SELECT_STMT,
99519	BLOCK_DELETE_STMT,
99520	BLOCK_DELETE_ALL_STMT,
99521
99522	SEGDIR_MAX_INDEX_STMT,
99523	SEGDIR_SET_STMT,
99524	SEGDIR_SELECT_LEVEL_STMT,
99525	SEGDIR_SPAN_STMT,
99526	SEGDIR_DELETE_STMT,
99527	SEGDIR_SELECT_SEGMENT_STMT,
99528	SEGDIR_SELECT_ALL_STMT,
99529	SEGDIR_DELETE_ALL_STMT,
99530	SEGDIR_COUNT_STMT,
99531
99532	MAX_STMT /* Always at end! */
99533	} fulltext_statement;
99534
99535	/* These must exactly match the enum above. */
99536	/* TODO(shess): Is there some risk that a statement will be used in two
99537	** cursors at once, e.g. if a query joins a virtual table to itself?
99538	** If so perhaps we should move some of these to the cursor object.
99539	*/
99540	static const char *const fulltext_zStatement[MAX_STMT] = {
99541	/* CONTENT_INSERT / NULL, / generated in contentInsertStatement() */
99542	/* CONTENT_SELECT / NULL, / generated in contentSelectStatement() */
99543	/* CONTENT_UPDATE / NULL, / generated in contentUpdateStatement() */
99544	/* CONTENT_DELETE */ "delete from %_content where docid = ?",
99545	/* CONTENT_EXISTS */ "select docid from %_content limit 1",
99546
99547	/* BLOCK_INSERT */
99548	"insert into %_segments (blockid, block) values (null, ?)",
99549	/* BLOCK_SELECT */ "select block from %_segments where blockid = ?",
99550	/* BLOCK_DELETE */ "delete from %_segments where blockid between ? and ?",
99551	/* BLOCK_DELETE_ALL */ "delete from %_segments",
99552
99553	/* SEGDIR_MAX_INDEX */ "select max(idx) from %_segdir where level = ?",
99554	/* SEGDIR_SET */ "insert into %_segdir values (?, ?, ?, ?, ?, ?)",
99555	/* SEGDIR_SELECT_LEVEL */
99556	"select start_block, leaves_end_block, root from %_segdir "
99557	" where level = ? order by idx",
99558	/* SEGDIR_SPAN */
99559	"select min(start_block), max(end_block) from %_segdir "
99560	" where level = ? and start_block <> 0",
99561	/* SEGDIR_DELETE */ "delete from %_segdir where level = ?",
99562
99563	/* NOTE(shess): The first three results of the following two
99564	** statements must match.
99565	*/
99566	/* SEGDIR_SELECT_SEGMENT */
99567	"select start_block, leaves_end_block, root from %_segdir "
99568	" where level = ? and idx = ?",
99569	/* SEGDIR_SELECT_ALL */
99570	"select start_block, leaves_end_block, root from %_segdir "
99571	" order by level desc, idx asc",
99572	/* SEGDIR_DELETE_ALL */ "delete from %_segdir",
99573	/* SEGDIR_COUNT / "select count(), ifnull(max(level),0) from %_segdir",
99574	};
99575
99576	/*
99577	** A connection to a fulltext index is an instance of the following
99578	** structure. The xCreate and xConnect methods create an instance
99579	** of this structure and xDestroy and xDisconnect free that instance.
99580	** All other methods receive a pointer to the structure as one of their
99581	** arguments.
99582	*/
99583	struct fulltext_vtab {
99584	sqlite3_vtab base; /* Base class used by SQLite core */
99585	sqlite3 db; / The database connection */
99586	const char zDb; / logical database name */
99587	const char zName; / virtual table name */
99588	int nColumn; /* number of columns in virtual table */
99589	char *azColumn; / column names. malloced */
99590	char *azContentColumn; / column names in content table; malloced */
99591	sqlite3_tokenizer pTokenizer; / tokenizer for inserts and queries */
99592
99593	/* Precompiled statements which we keep as long as the table is
99594	** open.
99595	*/
99596	sqlite3_stmt *pFulltextStatements[MAX_STMT];
99597
99598	/* Precompiled statements used for segment merges. We run a
99599	** separate select across the leaf level of each tree being merged.
99600	*/
99601	sqlite3_stmt *pLeafSelectStmts[MERGE_COUNT];
99602	/* The statement used to prepare pLeafSelectStmts. */
99603	#define LEAF_SELECT \
99604	"select block from %_segments where blockid between ? and ? order by blockid"
99605
99606	/* These buffer pending index updates during transactions.
99607	** nPendingData estimates the memory size of the pending data. It
99608	** doesn't include the hash-bucket overhead, nor any malloc
99609	** overhead. When nPendingData exceeds kPendingThreshold, the
99610	** buffer is flushed even before the transaction closes.
99611	** pendingTerms stores the data, and is only valid when nPendingData
99612	** is >=0 (nPendingData<0 means pendingTerms has not been
99613	** initialized). iPrevDocid is the last docid written, used to make
99614	** certain we're inserting in sorted order.
99615	*/
99616	int nPendingData;
99617	#define kPendingThreshold (110241024)
99618	sqlite_int64 iPrevDocid;
99619	fts3Hash pendingTerms;
99620	};
99621
99622	/*
99623	** When the core wants to do a query, it create a cursor using a
99624	** call to xOpen. This structure is an instance of a cursor. It
99625	** is destroyed by xClose.
99626	*/
99627	typedef struct fulltext_cursor {
99628	sqlite3_vtab_cursor base; /* Base class used by SQLite core */
99629	QueryType iCursorType; /* Copy of sqlite3_index_info.idxNum */
99630	sqlite3_stmt pStmt; / Prepared statement in use by the cursor */
99631	int eof; /* True if at End Of Results */
99632	Fts3Expr pExpr; / Parsed MATCH query string */
99633	Snippet snippet; /* Cached snippet for the current row */
99634	int iColumn; /* Column being searched */
99635	DataBuffer result; /* Doclist results from fulltextQuery */
99636	DLReader reader; /* Result reader if result not empty */
99637	} fulltext_cursor;
99638
99639	static fulltext_vtab cursor_vtab(fulltext_cursor c){
99640	return (fulltext_vtab *) c->base.pVtab;
99641	}
99642
99643	static const sqlite3_module fts3Module; /* forward declaration */
99644
99645	/* Return a dynamically generated statement of the form
99646	* insert into %_content (docid, ...) values (?, ...)
99647	*/
99648	static const char contentInsertStatement(fulltext_vtab v){
99649	StringBuffer sb;
99650	int i;
99651
99652	initStringBuffer(&sb);
99653	append(&sb, "insert into %_content (docid, ");
99654	appendList(&sb, v->nColumn, v->azContentColumn);
99655	append(&sb, ") values (?");
99656	for(i=0; i<v->nColumn; ++i)
99657	append(&sb, ", ?");
99658	append(&sb, ")");
99659	return stringBufferData(&sb);
99660	}
99661
99662	/* Return a dynamically generated statement of the form
99663	* select <content columns> from %_content where docid = ?
99664	*/
99665	static const char contentSelectStatement(fulltext_vtab v){
99666	StringBuffer sb;
99667	initStringBuffer(&sb);
99668	append(&sb, "SELECT ");
99669	appendList(&sb, v->nColumn, v->azContentColumn);
99670	append(&sb, " FROM %_content WHERE docid = ?");
99671	return stringBufferData(&sb);
99672	}
99673
99674	/* Return a dynamically generated statement of the form
99675	* update %_content set [col_0] = ?, [col_1] = ?, ...
99676	* where docid = ?
99677	*/
99678	static const char contentUpdateStatement(fulltext_vtab v){
99679	StringBuffer sb;
99680	int i;
99681
99682	initStringBuffer(&sb);
99683	append(&sb, "update %_content set ");
99684	for(i=0; i<v->nColumn; ++i) {
99685	if( i>0 ){
99686	append(&sb, ", ");
99687	}
99688	append(&sb, v->azContentColumn[i]);
99689	append(&sb, " = ?");
99690	}
99691	append(&sb, " where docid = ?");
99692	return stringBufferData(&sb);
99693	}
99694
99695	/* Puts a freshly-prepared statement determined by iStmt in *ppStmt.
99696	** If the indicated statement has never been prepared, it is prepared
99697	** and cached, otherwise the cached version is reset.
99698	*/
99699	static int sql_get_statement(fulltext_vtab *v, fulltext_statement iStmt,
99700	sqlite3_stmt **ppStmt){
99701	assert( iStmt<MAX_STMT );
99702	if( v->pFulltextStatements[iStmt]==NULL ){
99703	const char *zStmt;
99704	int rc;
99705	switch( iStmt ){
99706	case CONTENT_INSERT_STMT:
99707	zStmt = contentInsertStatement(v); break;
99708	case CONTENT_SELECT_STMT:
99709	zStmt = contentSelectStatement(v); break;
99710	case CONTENT_UPDATE_STMT:
99711	zStmt = contentUpdateStatement(v); break;
99712	default:
99713	zStmt = fulltext_zStatement[iStmt];
99714	}
99715	rc = sql_prepare(v->db, v->zDb, v->zName, &v->pFulltextStatements[iStmt],
99716	zStmt);
99717	if( zStmt != fulltext_zStatement[iStmt]) sqlite3_free((void *) zStmt);
99718	if( rc!=SQLITE_OK ) return rc;
99719	} else {
99720	int rc = sqlite3_reset(v->pFulltextStatements[iStmt]);
99721	if( rc!=SQLITE_OK ) return rc;
99722	}
99723
99724	*ppStmt = v->pFulltextStatements[iStmt];
99725	return SQLITE_OK;
99726	}
99727
99728	/* Like sqlite3_step(), but convert SQLITE_DONE to SQLITE_OK and
99729	** SQLITE_ROW to SQLITE_ERROR. Useful for statements like UPDATE,
99730	** where we expect no results.
99731	*/
99732	static int sql_single_step(sqlite3_stmt *s){
99733	int rc = sqlite3_step(s);
99734	return (rc==SQLITE_DONE) ? SQLITE_OK : rc;
99735	}
99736
99737	/* Like sql_get_statement(), but for special replicated LEAF_SELECT
99738	** statements. idx -1 is a special case for an uncached version of
99739	** the statement (used in the optimize implementation).
99740	*/
99741	/* TODO(shess) Write version for generic statements and then share
99742	** that between the cached-statement functions.
99743	*/
99744	static int sql_get_leaf_statement(fulltext_vtab *v, int idx,
99745	sqlite3_stmt **ppStmt){
99746	assert( idx>=-1 && idx<MERGE_COUNT );
99747	if( idx==-1 ){
99748	return sql_prepare(v->db, v->zDb, v->zName, ppStmt, LEAF_SELECT);
99749	}else if( v->pLeafSelectStmts[idx]==NULL ){
99750	int rc = sql_prepare(v->db, v->zDb, v->zName, &v->pLeafSelectStmts[idx],
99751	LEAF_SELECT);
99752	if( rc!=SQLITE_OK ) return rc;
99753	}else{
99754	int rc = sqlite3_reset(v->pLeafSelectStmts[idx]);
99755	if( rc!=SQLITE_OK ) return rc;
99756	}
99757
99758	*ppStmt = v->pLeafSelectStmts[idx];
99759	return SQLITE_OK;
99760	}
99761
99762	/* insert into %_content (docid, ...) values ([docid], [pValues])
99763	** If the docid contains SQL NULL, then a unique docid will be
99764	** generated.
99765	*/
99766	static int content_insert(fulltext_vtab v, sqlite3_value docid,
99767	sqlite3_value **pValues){
99768	sqlite3_stmt *s;
99769	int i;
99770	int rc = sql_get_statement(v, CONTENT_INSERT_STMT, &s);
99771	if( rc!=SQLITE_OK ) return rc;
99772
99773	rc = sqlite3_bind_value(s, 1, docid);
99774	if( rc!=SQLITE_OK ) return rc;
99775
99776	for(i=0; i<v->nColumn; ++i){
99777	rc = sqlite3_bind_value(s, 2+i, pValues[i]);
99778	if( rc!=SQLITE_OK ) return rc;
99779	}
99780
99781	return sql_single_step(s);
99782	}
99783
99784	/* update %_content set col0 = pValues[0], col1 = pValues[1], ...
99785	* where docid = [iDocid] */
99786	static int content_update(fulltext_vtab v, sqlite3_value *pValues,
99787	sqlite_int64 iDocid){
99788	sqlite3_stmt *s;
99789	int i;
99790	int rc = sql_get_statement(v, CONTENT_UPDATE_STMT, &s);
99791	if( rc!=SQLITE_OK ) return rc;
99792
99793	for(i=0; i<v->nColumn; ++i){
99794	rc = sqlite3_bind_value(s, 1+i, pValues[i]);
99795	if( rc!=SQLITE_OK ) return rc;
99796	}
99797
99798	rc = sqlite3_bind_int64(s, 1+v->nColumn, iDocid);
99799	if( rc!=SQLITE_OK ) return rc;
99800
99801	return sql_single_step(s);
99802	}
99803
99804	static void freeStringArray(int nString, const char **pString){
99805	int i;
99806
99807	for (i=0 ; i < nString ; ++i) {
99808	if( pString[i]!=NULL ) sqlite3_free((void *) pString[i]);
99809	}
99810	sqlite3_free((void *) pString);
99811	}
99812
99813	/* select * from %_content where docid = [iDocid]
99814	* The caller must delete the returned array and all strings in it.
99815	* null fields will be NULL in the returned array.
99816	*
99817	* TODO: Perhaps we should return pointer/length strings here for consistency
99818	* with other code which uses pointer/length. */
99819	static int content_select(fulltext_vtab *v, sqlite_int64 iDocid,
99820	const char ***pValues){
99821	sqlite3_stmt *s;
99822	const char **values;
99823	int i;
99824	int rc;
99825
99826	*pValues = NULL;
99827
99828	rc = sql_get_statement(v, CONTENT_SELECT_STMT, &s);
99829	if( rc!=SQLITE_OK ) return rc;
99830
99831	rc = sqlite3_bind_int64(s, 1, iDocid);
99832	if( rc!=SQLITE_OK ) return rc;
99833
99834	rc = sqlite3_step(s);
99835	if( rc!=SQLITE_ROW ) return rc;
99836
99837	values = (const char *) sqlite3_malloc(v->nColumn sizeof(const char *));
99838	for(i=0; i<v->nColumn; ++i){
99839	if( sqlite3_column_type(s, i)==SQLITE_NULL ){
99840	values[i] = NULL;
99841	}else{
99842	values[i] = string_dup((char*)sqlite3_column_text(s, i));
99843	}
99844	}
99845
99846	/* We expect only one row. We must execute another sqlite3_step()
99847	* to complete the iteration; otherwise the table will remain locked. */
99848	rc = sqlite3_step(s);
99849	if( rc==SQLITE_DONE ){
99850	*pValues = values;
99851	return SQLITE_OK;
99852	}
99853
99854	freeStringArray(v->nColumn, values);
99855	return rc;
99856	}
99857
99858	/* delete from %_content where docid = [iDocid ] */
99859	static int content_delete(fulltext_vtab *v, sqlite_int64 iDocid){
99860	sqlite3_stmt *s;
99861	int rc = sql_get_statement(v, CONTENT_DELETE_STMT, &s);
99862	if( rc!=SQLITE_OK ) return rc;
99863
99864	rc = sqlite3_bind_int64(s, 1, iDocid);
99865	if( rc!=SQLITE_OK ) return rc;
99866
99867	return sql_single_step(s);
99868	}
99869
99870	/* Returns SQLITE_ROW if any rows exist in %_content, SQLITE_DONE if
99871	** no rows exist, and any error in case of failure.
99872	*/
99873	static int content_exists(fulltext_vtab *v){
99874	sqlite3_stmt *s;
99875	int rc = sql_get_statement(v, CONTENT_EXISTS_STMT, &s);
99876	if( rc!=SQLITE_OK ) return rc;
99877
99878	rc = sqlite3_step(s);
99879	if( rc!=SQLITE_ROW ) return rc;
99880
99881	/* We expect only one row. We must execute another sqlite3_step()
99882	* to complete the iteration; otherwise the table will remain locked. */
99883	rc = sqlite3_step(s);
99884	if( rc==SQLITE_DONE ) return SQLITE_ROW;
99885	if( rc==SQLITE_ROW ) return SQLITE_ERROR;
99886	return rc;
99887	}
99888
99889	/* insert into %_segments values ([pData])
99890	** returns assigned blockid in *piBlockid
99891	*/
99892	static int block_insert(fulltext_vtab v, const char pData, int nData,
99893	sqlite_int64 *piBlockid){
99894	sqlite3_stmt *s;
99895	int rc = sql_get_statement(v, BLOCK_INSERT_STMT, &s);
99896	if( rc!=SQLITE_OK ) return rc;
99897
99898	rc = sqlite3_bind_blob(s, 1, pData, nData, SQLITE_STATIC);
99899	if( rc!=SQLITE_OK ) return rc;
99900
99901	rc = sqlite3_step(s);
99902	if( rc==SQLITE_ROW ) return SQLITE_ERROR;
99903	if( rc!=SQLITE_DONE ) return rc;
99904
99905	/* blockid column is an alias for rowid. */
99906	*piBlockid = sqlite3_last_insert_rowid(v->db);
99907	return SQLITE_OK;
99908	}
99909
99910	/* delete from %_segments
99911	** where blockid between [iStartBlockid] and [iEndBlockid]
99912	**
99913	** Deletes the range of blocks, inclusive, used to delete the blocks
99914	** which form a segment.
99915	*/
99916	static int block_delete(fulltext_vtab *v,
99917	sqlite_int64 iStartBlockid, sqlite_int64 iEndBlockid){
99918	sqlite3_stmt *s;
99919	int rc = sql_get_statement(v, BLOCK_DELETE_STMT, &s);
99920	if( rc!=SQLITE_OK ) return rc;
99921
99922	rc = sqlite3_bind_int64(s, 1, iStartBlockid);
99923	if( rc!=SQLITE_OK ) return rc;
99924
99925	rc = sqlite3_bind_int64(s, 2, iEndBlockid);
99926	if( rc!=SQLITE_OK ) return rc;
99927
99928	return sql_single_step(s);
99929	}
99930
99931	/* Returns SQLITE_ROW with *pidx set to the maximum segment idx found
99932	** at iLevel. Returns SQLITE_DONE if there are no segments at
99933	** iLevel. Otherwise returns an error.
99934	*/
99935	static int segdir_max_index(fulltext_vtab v, int iLevel, int pidx){
99936	sqlite3_stmt *s;
99937	int rc = sql_get_statement(v, SEGDIR_MAX_INDEX_STMT, &s);
99938	if( rc!=SQLITE_OK ) return rc;
99939
99940	rc = sqlite3_bind_int(s, 1, iLevel);
99941	if( rc!=SQLITE_OK ) return rc;
99942
99943	rc = sqlite3_step(s);
99944	/* Should always get at least one row due to how max() works. */
99945	if( rc==SQLITE_DONE ) return SQLITE_DONE;
99946	if( rc!=SQLITE_ROW ) return rc;
99947
99948	/* NULL means that there were no inputs to max(). */
99949	if( SQLITE_NULL==sqlite3_column_type(s, 0) ){
99950	rc = sqlite3_step(s);
99951	if( rc==SQLITE_ROW ) return SQLITE_ERROR;
99952	return rc;
99953	}
99954
99955	*pidx = sqlite3_column_int(s, 0);
99956
99957	/* We expect only one row. We must execute another sqlite3_step()
99958	* to complete the iteration; otherwise the table will remain locked. */
99959	rc = sqlite3_step(s);
99960	if( rc==SQLITE_ROW ) return SQLITE_ERROR;
99961	if( rc!=SQLITE_DONE ) return rc;
99962	return SQLITE_ROW;
99963	}
99964
99965	/* insert into %_segdir values (
99966	** [iLevel], [idx],
99967	** [iStartBlockid], [iLeavesEndBlockid], [iEndBlockid],
99968	** [pRootData]
99969	** )
99970	*/
99971	static int segdir_set(fulltext_vtab *v, int iLevel, int idx,
99972	sqlite_int64 iStartBlockid,
99973	sqlite_int64 iLeavesEndBlockid,
99974	sqlite_int64 iEndBlockid,
99975	const char *pRootData, int nRootData){
99976	sqlite3_stmt *s;
99977	int rc = sql_get_statement(v, SEGDIR_SET_STMT, &s);
99978	if( rc!=SQLITE_OK ) return rc;
99979
99980	rc = sqlite3_bind_int(s, 1, iLevel);
99981	if( rc!=SQLITE_OK ) return rc;
99982
99983	rc = sqlite3_bind_int(s, 2, idx);
99984	if( rc!=SQLITE_OK ) return rc;
99985
99986	rc = sqlite3_bind_int64(s, 3, iStartBlockid);
99987	if( rc!=SQLITE_OK ) return rc;
99988
99989	rc = sqlite3_bind_int64(s, 4, iLeavesEndBlockid);
99990	if( rc!=SQLITE_OK ) return rc;
99991
99992	rc = sqlite3_bind_int64(s, 5, iEndBlockid);
99993	if( rc!=SQLITE_OK ) return rc;
99994
99995	rc = sqlite3_bind_blob(s, 6, pRootData, nRootData, SQLITE_STATIC);
99996	if( rc!=SQLITE_OK ) return rc;
99997
99998	return sql_single_step(s);
99999	}
100000
100001	/* Queries %_segdir for the block span of the segments in level
100002	** iLevel. Returns SQLITE_DONE if there are no blocks for iLevel,
100003	** SQLITE_ROW if there are blocks, else an error.
100004	*/
100005	static int segdir_span(fulltext_vtab *v, int iLevel,
100006	sqlite_int64 *piStartBlockid,
100007	sqlite_int64 *piEndBlockid){
100008	sqlite3_stmt *s;
100009	int rc = sql_get_statement(v, SEGDIR_SPAN_STMT, &s);
100010	if( rc!=SQLITE_OK ) return rc;
100011
100012	rc = sqlite3_bind_int(s, 1, iLevel);
100013	if( rc!=SQLITE_OK ) return rc;
100014
100015	rc = sqlite3_step(s);
100016	if( rc==SQLITE_DONE ) return SQLITE_DONE; /* Should never happen */
100017	if( rc!=SQLITE_ROW ) return rc;
100018
100019	/* This happens if all segments at this level are entirely inline. */
100020	if( SQLITE_NULL==sqlite3_column_type(s, 0) ){
100021	/* We expect only one row. We must execute another sqlite3_step()
100022	* to complete the iteration; otherwise the table will remain locked. */
100023	int rc2 = sqlite3_step(s);
100024	if( rc2==SQLITE_ROW ) return SQLITE_ERROR;
100025	return rc2;
100026	}
100027
100028	*piStartBlockid = sqlite3_column_int64(s, 0);
100029	*piEndBlockid = sqlite3_column_int64(s, 1);
100030
100031	/* We expect only one row. We must execute another sqlite3_step()
100032	* to complete the iteration; otherwise the table will remain locked. */
100033	rc = sqlite3_step(s);
100034	if( rc==SQLITE_ROW ) return SQLITE_ERROR;
100035	if( rc!=SQLITE_DONE ) return rc;
100036	return SQLITE_ROW;
100037	}
100038
100039	/* Delete the segment blocks and segment directory records for all
100040	** segments at iLevel.
100041	*/
100042	static int segdir_delete(fulltext_vtab *v, int iLevel){
100043	sqlite3_stmt *s;
100044	sqlite_int64 iStartBlockid, iEndBlockid;
100045	int rc = segdir_span(v, iLevel, &iStartBlockid, &iEndBlockid);
100046	if( rc!=SQLITE_ROW && rc!=SQLITE_DONE ) return rc;
100047
100048	if( rc==SQLITE_ROW ){
100049	rc = block_delete(v, iStartBlockid, iEndBlockid);
100050	if( rc!=SQLITE_OK ) return rc;
100051	}
100052
100053	/* Delete the segment directory itself. */
100054	rc = sql_get_statement(v, SEGDIR_DELETE_STMT, &s);
100055	if( rc!=SQLITE_OK ) return rc;
100056
100057	rc = sqlite3_bind_int64(s, 1, iLevel);
100058	if( rc!=SQLITE_OK ) return rc;
100059
100060	return sql_single_step(s);
100061	}
100062
100063	/* Delete entire fts index, SQLITE_OK on success, relevant error on
100064	** failure.
100065	*/
100066	static int segdir_delete_all(fulltext_vtab *v){
100067	sqlite3_stmt *s;
100068	int rc = sql_get_statement(v, SEGDIR_DELETE_ALL_STMT, &s);
100069	if( rc!=SQLITE_OK ) return rc;
100070
100071	rc = sql_single_step(s);
100072	if( rc!=SQLITE_OK ) return rc;
100073
100074	rc = sql_get_statement(v, BLOCK_DELETE_ALL_STMT, &s);
100075	if( rc!=SQLITE_OK ) return rc;
100076
100077	return sql_single_step(s);
100078	}
100079
100080	/* Returns SQLITE_OK with *pnSegments set to the number of entries in
100081	** %_segdir and *piMaxLevel set to the highest level which has a
100082	** segment. Otherwise returns the SQLite error which caused failure.
100083	*/
100084	static int segdir_count(fulltext_vtab v, int pnSegments, int *piMaxLevel){
100085	sqlite3_stmt *s;
100086	int rc = sql_get_statement(v, SEGDIR_COUNT_STMT, &s);
100087	if( rc!=SQLITE_OK ) return rc;
100088
100089	rc = sqlite3_step(s);
100090	/* TODO(shess): This case should not be possible? Should stronger
100091	** measures be taken if it happens?
100092	*/
100093	if( rc==SQLITE_DONE ){
100094	*pnSegments = 0;
100095	*piMaxLevel = 0;
100096	return SQLITE_OK;
100097	}
100098	if( rc!=SQLITE_ROW ) return rc;
100099
100100	*pnSegments = sqlite3_column_int(s, 0);
100101	*piMaxLevel = sqlite3_column_int(s, 1);
100102
100103	/* We expect only one row. We must execute another sqlite3_step()
100104	* to complete the iteration; otherwise the table will remain locked. */
100105	rc = sqlite3_step(s);
100106	if( rc==SQLITE_DONE ) return SQLITE_OK;
100107	if( rc==SQLITE_ROW ) return SQLITE_ERROR;
100108	return rc;
100109	}
100110
100111	/* TODO(shess) clearPendingTerms() is far down the file because
100112	** writeZeroSegment() is far down the file because LeafWriter is far
100113	** down the file. Consider refactoring the code to move the non-vtab
100114	** code above the vtab code so that we don't need this forward
100115	** reference.
100116	*/
100117	static int clearPendingTerms(fulltext_vtab *v);
100118
100119	/*
100120	** Free the memory used to contain a fulltext_vtab structure.
100121	*/
100122	static void fulltext_vtab_destroy(fulltext_vtab *v){
100123	int iStmt, i;
100124
100125	FTSTRACE(("FTS3 Destroy %p\n", v));
100126	for( iStmt=0; iStmt<MAX_STMT; iStmt++ ){
100127	if( v->pFulltextStatements[iStmt]!=NULL ){
100128	sqlite3_finalize(v->pFulltextStatements[iStmt]);
100129	v->pFulltextStatements[iStmt] = NULL;
100130	}
100131	}
100132
100133	for( i=0; i<MERGE_COUNT; i++ ){
100134	if( v->pLeafSelectStmts[i]!=NULL ){
100135	sqlite3_finalize(v->pLeafSelectStmts[i]);
100136	v->pLeafSelectStmts[i] = NULL;
100137	}
100138	}
100139
100140	if( v->pTokenizer!=NULL ){
100141	v->pTokenizer->pModule->xDestroy(v->pTokenizer);
100142	v->pTokenizer = NULL;
100143	}
100144
100145	clearPendingTerms(v);
100146
100147	sqlite3_free(v->azColumn);
100148	for(i = 0; i < v->nColumn; ++i) {
100149	sqlite3_free(v->azContentColumn[i]);
100150	}
100151	sqlite3_free(v->azContentColumn);
100152	sqlite3_free(v);
100153	}
100154
100155	/*
100156	** Token types for parsing the arguments to xConnect or xCreate.
100157	*/
100158	#define TOKEN_EOF 0 /* End of file */
100159	#define TOKEN_SPACE 1 /* Any kind of whitespace */
100160	#define TOKEN_ID 2 /* An identifier */
100161	#define TOKEN_STRING 3 /* A string literal */
100162	#define TOKEN_PUNCT 4 /* A single punctuation character */
100163
100164	/*
100165	** If X is a character that can be used in an identifier then
100166	** ftsIdChar(X) will be true. Otherwise it is false.
100167	**
100168	** For ASCII, any character with the high-order bit set is
100169	** allowed in an identifier. For 7-bit characters,
100170	** isFtsIdChar[X] must be 1.
100171	**
100172	** Ticket #1066. the SQL standard does not allow '$' in the
100173	** middle of identfiers. But many SQL implementations do.
100174	** SQLite will allow '$' in identifiers for compatibility.
100175	** But the feature is undocumented.
100176	*/
100177	static const char isFtsIdChar[] = {
100178	/* x0 x1 x2 x3 x4 x5 x6 x7 x8 x9 xA xB xC xD xE xF */
100179	0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 2x */
100180	1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, /* 3x */
100181	0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* 4x */
100182	1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 1, /* 5x */
100183	0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* 6x */
100184	1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, /* 7x */
100185	};
100186	#define ftsIdChar(C) (((c=C)&0x80)!=0 \|\| (c>0x1f && isFtsIdChar[c-0x20]))
100187
100188
100189	/*
100190	** Return the length of the token that begins at z[0].
100191	** Store the token type in *tokenType before returning.
100192	*/
100193	static int ftsGetToken(const char z, int tokenType){
100194	int i, c;
100195	switch( *z ){
100196	case 0: {
100197	*tokenType = TOKEN_EOF;
100198	return 0;
100199	}
100200	case ' ': case '\t': case '\n': case '\f': case '\r': {
100201	for(i=1; safe_isspace(z[i]); i++){}
100202	*tokenType = TOKEN_SPACE;
100203	return i;
100204	}
100205	case '`':
100206	case '\'':
100207	case '"': {
100208	int delim = z[0];
100209	for(i=1; (c=z[i])!=0; i++){
100210	if( c==delim ){
100211	if( z[i+1]==delim ){
100212	i++;
100213	}else{
100214	break;
100215	}
100216	}
100217	}
100218	*tokenType = TOKEN_STRING;
100219	return i + (c!=0);
100220	}
100221	case '[': {
100222	for(i=1, c=z[0]; c!=']' && (c=z[i])!=0; i++){}
100223	*tokenType = TOKEN_ID;
100224	return i;
100225	}
100226	default: {
100227	if( !ftsIdChar(*z) ){
100228	break;
100229	}
100230	for(i=1; ftsIdChar(z[i]); i++){}
100231	*tokenType = TOKEN_ID;
100232	return i;
100233	}
100234	}
100235	*tokenType = TOKEN_PUNCT;
100236	return 1;
100237	}
100238
100239	/*
100240	** A token extracted from a string is an instance of the following
100241	** structure.
100242	*/
100243	typedef struct FtsToken {
100244	const char z; / Pointer to token text. Not '\000' terminated */
100245	short int n; /* Length of the token text in bytes. */
100246	} FtsToken;
100247
100248	/*
100249	** Given a input string (which is really one of the argv[] parameters
100250	** passed into xConnect or xCreate) split the string up into tokens.
100251	** Return an array of pointers to '\000' terminated strings, one string
100252	** for each non-whitespace token.
100253	**
100254	** The returned array is terminated by a single NULL pointer.
100255	**
100256	** Space to hold the returned array is obtained from a single
100257	** malloc and should be freed by passing the return value to free().
100258	** The individual strings within the token list are all a part of
100259	** the single memory allocation and will all be freed at once.
100260	*/
100261	static char *tokenizeString(const char z, int *pnToken){
100262	int nToken = 0;
100263	FtsToken aToken = sqlite3_malloc( strlen(z) sizeof(aToken[0]) );
100264	int n = 1;
100265	int e, i;
100266	int totalSize = 0;
100267	char **azToken;
100268	char *zCopy;
100269	while( n>0 ){
100270	n = ftsGetToken(z, &e);
100271	if( e!=TOKEN_SPACE ){
100272	aToken[nToken].z = z;
100273	aToken[nToken].n = n;
100274	nToken++;
100275	totalSize += n+1;
100276	}
100277	z += n;
100278	}
100279	azToken = (char*)sqlite3_malloc( nTokensizeof(char*) + totalSize );
100280	zCopy = (char*)&azToken[nToken];
100281	nToken--;
100282	for(i=0; i<nToken; i++){
100283	azToken[i] = zCopy;
100284	n = aToken[i].n;
100285	memcpy(zCopy, aToken[i].z, n);
100286	zCopy[n] = 0;
100287	zCopy += n+1;
100288	}
100289	azToken[nToken] = 0;
100290	sqlite3_free(aToken);
100291	*pnToken = nToken;
100292	return azToken;
100293	}
100294
100295	/*
100296	** Convert an SQL-style quoted string into a normal string by removing
100297	** the quote characters. The conversion is done in-place. If the
	@@ -100303,14 +97918,14 @@
100303	** "abc" becomes abc
100304	** 'xyz' becomes xyz
100305	** [pqr] becomes pqr
100306	** `mno` becomes mno
100307	*/
100308	static void dequoteString(char *z){
100309	int quote;
100310	int i, j;
100311	if( z==0 ) return;
100312	quote = z[0];
100313	switch( quote ){
100314	case '\'': break;
100315	case '"': break;
100316	case '`': break; /* For MySQL compatibility */
	@@ -100330,1363 +97945,1413 @@
100330	z[j++] = z[i];
100331	}
100332	}
100333	}
100334
100335	/*
100336	** The input azIn is a NULL-terminated list of tokens. Remove the first
100337	** token and all punctuation tokens. Remove the quotes from
100338	** around string literal tokens.
100339	**
100340	** Example:
100341	**
100342	** input: tokenize chinese ( 'simplifed' , 'mixed' )
100343	** output: chinese simplifed mixed
100344	**
100345	** Another example:
100346	**
100347	** input: delimiters ( '[' , ']' , '...' )
100348	** output: [ ] ...
100349	*/
100350	static void tokenListToIdList(char **azIn){
100351	int i, j;
100352	if( azIn ){
100353	for(i=0, j=-1; azIn[i]; i++){
100354	if( safe_isalnum(azIn[i][0]) \|\| azIn[i][1] ){
100355	dequoteString(azIn[i]);
100356	if( j>=0 ){
100357	azIn[j] = azIn[i];
100358	}
100359	j++;
100360	}
100361	}
100362	azIn[j] = 0;
100363	}
100364	}
100365
100366
100367	/*
100368	** Find the first alphanumeric token in the string zIn. Null-terminate
100369	** this token. Remove any quotation marks. And return a pointer to
100370	** the result.
100371	*/
100372	static char firstToken(char zIn, char **pzTail){
100373	int n, ttype;
100374	while(1){
100375	n = ftsGetToken(zIn, &ttype);
100376	if( ttype==TOKEN_SPACE ){
100377	zIn += n;
100378	}else if( ttype==TOKEN_EOF ){
100379	*pzTail = zIn;
100380	return 0;
100381	}else{
100382	zIn[n] = 0;
100383	*pzTail = &zIn[1];
100384	dequoteString(zIn);
100385	return zIn;
100386	}
100387	}
100388	/NOTREACHED/
100389	}
100390
100391	/* Return true if...
100392	**
100393	** * s begins with the string t, ignoring case
100394	** * s is longer than t
100395	** * The first character of s beyond t is not a alphanumeric
100396	**
100397	** Ignore leading space in *s.
100398	**
100399	** To put it another way, return true if the first token of
100400	** s[] is t[].
100401	*/
100402	static int startsWith(const char s, const char t){
100403	while( safe_isspace(*s) ){ s++; }
100404	while( *t ){
100405	if( safe_tolower(s++)!=safe_tolower(t++) ) return 0;
100406	}
100407	return s!='_' && !safe_isalnum(s);
100408	}
100409
100410	/*
100411	** An instance of this structure defines the "spec" of a
100412	** full text index. This structure is populated by parseSpec
100413	** and use by fulltextConnect and fulltextCreate.
100414	*/
100415	typedef struct TableSpec {
100416	const char zDb; / Logical database name */
100417	const char zName; / Name of the full-text index */
100418	int nColumn; /* Number of columns to be indexed */
100419	char *azColumn; / Original names of columns to be indexed */
100420	char *azContentColumn; / Column names for %_content */
100421	char *azTokenizer; / Name of tokenizer and its arguments */
100422	} TableSpec;
100423
100424	/*
100425	** Reclaim all of the memory used by a TableSpec
100426	*/
100427	static void clearTableSpec(TableSpec *p) {
100428	sqlite3_free(p->azColumn);
100429	sqlite3_free(p->azContentColumn);
100430	sqlite3_free(p->azTokenizer);
100431	}
100432
100433	/* Parse a CREATE VIRTUAL TABLE statement, which looks like this:
100434	*
100435	* CREATE VIRTUAL TABLE email
100436	* USING fts3(subject, body, tokenize mytokenizer(myarg))
100437	*
100438	* We return parsed information in a TableSpec structure.
100439	*
100440	*/
100441	static int parseSpec(TableSpec pSpec, int argc, const char const*argv,
100442	char**pzErr){
100443	int i, n;
100444	char z, zDummy;
100445	char **azArg;
100446	const char zTokenizer = 0; / argv[] entry describing the tokenizer */
100447
100448	assert( argc>=3 );
100449	/* Current interface:
100450	** argv[0] - module name
100451	** argv[1] - database name
100452	** argv[2] - table name
100453	** argv[3..] - columns, optionally followed by tokenizer specification
100454	** and snippet delimiters specification.
100455	*/
100456
100457	/* Make a copy of the complete argv[][] array in a single allocation.
100458	** The argv[][] array is read-only and transient. We can write to the
100459	** copy in order to modify things and the copy is persistent.
100460	*/
100461	CLEAR(pSpec);
100462	for(i=n=0; i<argc; i++){
100463	n += strlen(argv[i]) + 1;
100464	}
100465	azArg = sqlite3_malloc( sizeof(char)argc + n );
100466	if( azArg==0 ){
100467	return SQLITE_NOMEM;
100468	}
100469	z = (char*)&azArg[argc];
100470	for(i=0; i<argc; i++){
100471	azArg[i] = z;
100472	strcpy(z, argv[i]);
100473	z += strlen(z)+1;
100474	}
100475
100476	/* Identify the column names and the tokenizer and delimiter arguments
100477	** in the argv[][] array.
100478	*/
100479	pSpec->zDb = azArg[1];
100480	pSpec->zName = azArg[2];
100481	pSpec->nColumn = 0;
100482	pSpec->azColumn = azArg;
100483	zTokenizer = "tokenize simple";
100484	for(i=3; i<argc; ++i){
100485	if( startsWith(azArg[i],"tokenize") ){
100486	zTokenizer = azArg[i];
100487	}else{
100488	z = azArg[pSpec->nColumn] = firstToken(azArg[i], &zDummy);
100489	pSpec->nColumn++;
100490	}
100491	}
100492	if( pSpec->nColumn==0 ){
100493	azArg[0] = "content";
100494	pSpec->nColumn = 1;
100495	}
100496
100497	/*
100498	** Construct the list of content column names.
100499	**
100500	** Each content column name will be of the form cNNAAAA
100501	** where NN is the column number and AAAA is the sanitized
100502	** column name. "sanitized" means that special characters are
100503	** converted to "_". The cNN prefix guarantees that all column
100504	** names are unique.
100505	**
100506	** The AAAA suffix is not strictly necessary. It is included
100507	** for the convenience of people who might examine the generated
100508	** %_content table and wonder what the columns are used for.
100509	*/
100510	pSpec->azContentColumn = sqlite3_malloc( pSpec->nColumn * sizeof(char *) );
100511	if( pSpec->azContentColumn==0 ){
100512	clearTableSpec(pSpec);
100513	return SQLITE_NOMEM;
100514	}
100515	for(i=0; i<pSpec->nColumn; i++){
100516	char *p;
100517	pSpec->azContentColumn[i] = sqlite3_mprintf("c%d%s", i, azArg[i]);
100518	for (p = pSpec->azContentColumn[i]; *p ; ++p) {
100519	if( !safe_isalnum(p) ) p = '_';
100520	}
100521	}
100522
100523	/*
100524	** Parse the tokenizer specification string.
100525	*/
100526	pSpec->azTokenizer = tokenizeString(zTokenizer, &n);
100527	tokenListToIdList(pSpec->azTokenizer);
100528
100529	return SQLITE_OK;
100530	}
100531
100532	/*
100533	** Generate a CREATE TABLE statement that describes the schema of
100534	** the virtual table. Return a pointer to this schema string.
100535	**
100536	** Space is obtained from sqlite3_mprintf() and should be freed
100537	** using sqlite3_free().
100538	*/
100539	static char *fulltextSchema(
100540	int nColumn, /* Number of columns */
100541	const char const azColumn, /* List of columns */
100542	const char zTableName / Name of the table */
100543	){
100544	int i;
100545	char zSchema, zNext;
100546	const char *zSep = "(";
100547	zSchema = sqlite3_mprintf("CREATE TABLE x");
100548	for(i=0; i<nColumn; i++){
100549	zNext = sqlite3_mprintf("%s%s%Q", zSchema, zSep, azColumn[i]);
100550	sqlite3_free(zSchema);
100551	zSchema = zNext;
100552	zSep = ",";
100553	}
100554	zNext = sqlite3_mprintf("%s,%Q HIDDEN", zSchema, zTableName);
100555	sqlite3_free(zSchema);
100556	zSchema = zNext;
100557	zNext = sqlite3_mprintf("%s,docid HIDDEN)", zSchema);
100558	sqlite3_free(zSchema);
100559	return zNext;
100560	}
100561
100562	/*
100563	** Build a new sqlite3_vtab structure that will describe the
100564	** fulltext index defined by spec.
100565	*/
100566	static int constructVtab(
100567	sqlite3 db, / The SQLite database connection */
100568	fts3Hash pHash, / Hash table containing tokenizers */
100569	TableSpec spec, / Parsed spec information from parseSpec() */
100570	sqlite3_vtab *ppVTab, / Write the resulting vtab structure here */
100571	char *pzErr / Write any error message here */
100572	){
100573	int rc;
100574	int n;
100575	fulltext_vtab *v = 0;
100576	const sqlite3_tokenizer_module *m = NULL;
100577	char *schema;
100578
100579	char const zTok; / Name of tokenizer to use for this fts table */
100580	int nTok; /* Length of zTok, including nul terminator */
100581
100582	v = (fulltext_vtab *) sqlite3_malloc(sizeof(fulltext_vtab));
100583	if( v==0 ) return SQLITE_NOMEM;
100584	CLEAR(v);
100585	/* sqlite will initialize v->base */
100586	v->db = db;
100587	v->zDb = spec->zDb; /* Freed when azColumn is freed */
100588	v->zName = spec->zName; /* Freed when azColumn is freed */
100589	v->nColumn = spec->nColumn;
100590	v->azContentColumn = spec->azContentColumn;
100591	spec->azContentColumn = 0;
100592	v->azColumn = spec->azColumn;
100593	spec->azColumn = 0;
100594
100595	if( spec->azTokenizer==0 ){
100596	return SQLITE_NOMEM;
100597	}
100598
100599	zTok = spec->azTokenizer[0];
100600	if( !zTok ){
100601	zTok = "simple";
100602	}
100603	nTok = strlen(zTok)+1;
100604
100605	m = (sqlite3_tokenizer_module *)sqlite3Fts3HashFind(pHash, zTok, nTok);
100606	if( !m ){
100607	*pzErr = sqlite3_mprintf("unknown tokenizer: %s", spec->azTokenizer[0]);
100608	rc = SQLITE_ERROR;
100609	goto err;
100610	}
100611
100612	for(n=0; spec->azTokenizer[n]; n++){}
100613	if( n ){
100614	rc = m->xCreate(n-1, (const charconst)&spec->azTokenizer[1],
100615	&v->pTokenizer);
100616	}else{
100617	rc = m->xCreate(0, 0, &v->pTokenizer);
100618	}
100619	if( rc!=SQLITE_OK ) goto err;
100620	v->pTokenizer->pModule = m;
100621
100622	/* TODO: verify the existence of backing tables foo_content, foo_term */
100623
100624	schema = fulltextSchema(v->nColumn, (const charconst)v->azColumn,
100625	spec->zName);
100626	rc = sqlite3_declare_vtab(db, schema);
100627	sqlite3_free(schema);
100628	if( rc!=SQLITE_OK ) goto err;
100629
100630	memset(v->pFulltextStatements, 0, sizeof(v->pFulltextStatements));
100631
100632	/* Indicate that the buffer is not live. */
100633	v->nPendingData = -1;
100634
100635	*ppVTab = &v->base;
100636	FTSTRACE(("FTS3 Connect %p\n", v));
100637
100638	return rc;
100639
100640	err:
100641	fulltext_vtab_destroy(v);
100642	return rc;
100643	}
100644
100645	static int fulltextConnect(
100646	sqlite3 *db,
100647	void *pAux,
100648	int argc, const char constargv,
100649	sqlite3_vtab **ppVTab,
100650	char **pzErr
100651	){
100652	TableSpec spec;
100653	int rc = parseSpec(&spec, argc, argv, pzErr);
100654	if( rc!=SQLITE_OK ) return rc;
100655
100656	rc = constructVtab(db, (fts3Hash *)pAux, &spec, ppVTab, pzErr);
100657	clearTableSpec(&spec);
100658	return rc;
100659	}
100660
100661	/* The %_content table holds the text of each document, with
100662	** the docid column exposed as the SQLite rowid for the table.
100663	*/
100664	/* TODO(shess) This comment needs elaboration to match the updated
100665	** code. Work it into the top-of-file comment at that time.
100666	*/
100667	static int fulltextCreate(sqlite3 db, void pAux,
100668	int argc, const char * const *argv,
100669	sqlite3_vtab ppVTab, char pzErr){
100670	int rc;
100671	TableSpec spec;
100672	StringBuffer schema;
100673	FTSTRACE(("FTS3 Create\n"));
100674
100675	rc = parseSpec(&spec, argc, argv, pzErr);
100676	if( rc!=SQLITE_OK ) return rc;
100677
100678	initStringBuffer(&schema);
100679	append(&schema, "CREATE TABLE %_content(");
100680	append(&schema, " docid INTEGER PRIMARY KEY,");
100681	appendList(&schema, spec.nColumn, spec.azContentColumn);
100682	append(&schema, ")");
100683	rc = sql_exec(db, spec.zDb, spec.zName, stringBufferData(&schema));
100684	stringBufferDestroy(&schema);
100685	if( rc!=SQLITE_OK ) goto out;
100686
100687	rc = sql_exec(db, spec.zDb, spec.zName,
100688	"create table %_segments("
100689	" blockid INTEGER PRIMARY KEY,"
100690	" block blob"
100691	");"
100692	);
100693	if( rc!=SQLITE_OK ) goto out;
100694
100695	rc = sql_exec(db, spec.zDb, spec.zName,
100696	"create table %_segdir("
100697	" level integer,"
100698	" idx integer,"
100699	" start_block integer,"
100700	" leaves_end_block integer,"
100701	" end_block integer,"
100702	" root blob,"
100703	" primary key(level, idx)"
100704	");");
100705	if( rc!=SQLITE_OK ) goto out;
100706
100707	rc = constructVtab(db, (fts3Hash *)pAux, &spec, ppVTab, pzErr);
100708
100709	out:
100710	clearTableSpec(&spec);
100711	return rc;
100712	}
100713
100714	/* Decide how to handle an SQL query. */
100715	static int fulltextBestIndex(sqlite3_vtab pVTab, sqlite3_index_info pInfo){
100716	fulltext_vtab v = (fulltext_vtab )pVTab;
100717	int i;
100718	FTSTRACE(("FTS3 BestIndex\n"));
100719
100720	for(i=0; i<pInfo->nConstraint; ++i){
100721	const struct sqlite3_index_constraint *pConstraint;
100722	pConstraint = &pInfo->aConstraint[i];
100723	if( pConstraint->usable ) {
100724	if( (pConstraint->iColumn==-1 \|\| pConstraint->iColumn==v->nColumn+1) &&
100725	pConstraint->op==SQLITE_INDEX_CONSTRAINT_EQ ){
100726	pInfo->idxNum = QUERY_DOCID; /* lookup by docid */
100727	FTSTRACE(("FTS3 QUERY_DOCID\n"));
100728	} else if( pConstraint->iColumn>=0 && pConstraint->iColumn<=v->nColumn &&
100729	pConstraint->op==SQLITE_INDEX_CONSTRAINT_MATCH ){
100730	/* full-text search */
100731	pInfo->idxNum = QUERY_FULLTEXT + pConstraint->iColumn;
100732	FTSTRACE(("FTS3 QUERY_FULLTEXT %d\n", pConstraint->iColumn));
100733	} else continue;
100734
100735	pInfo->aConstraintUsage[i].argvIndex = 1;
100736	pInfo->aConstraintUsage[i].omit = 1;
100737
100738	/* An arbitrary value for now.
100739	* TODO: Perhaps docid matches should be considered cheaper than
100740	* full-text searches. */
100741	pInfo->estimatedCost = 1.0;
100742
100743	return SQLITE_OK;
100744	}
100745	}
100746	pInfo->idxNum = QUERY_GENERIC;
100747	return SQLITE_OK;
100748	}
100749
100750	static int fulltextDisconnect(sqlite3_vtab *pVTab){
100751	FTSTRACE(("FTS3 Disconnect %p\n", pVTab));
100752	fulltext_vtab_destroy((fulltext_vtab *)pVTab);
100753	return SQLITE_OK;
100754	}
100755
100756	static int fulltextDestroy(sqlite3_vtab *pVTab){
100757	fulltext_vtab v = (fulltext_vtab )pVTab;
100758	int rc;
100759
100760	FTSTRACE(("FTS3 Destroy %p\n", pVTab));
100761	rc = sql_exec(v->db, v->zDb, v->zName,
100762	"drop table if exists %_content;"
100763	"drop table if exists %_segments;"
100764	"drop table if exists %_segdir;"
100765	);
100766	if( rc!=SQLITE_OK ) return rc;
100767
100768	fulltext_vtab_destroy((fulltext_vtab *)pVTab);
100769	return SQLITE_OK;
100770	}
100771
100772	static int fulltextOpen(sqlite3_vtab pVTab, sqlite3_vtab_cursor *ppCursor){
100773	fulltext_cursor *c;
100774
100775	c = (fulltext_cursor *) sqlite3_malloc(sizeof(fulltext_cursor));
100776	if( c ){
100777	memset(c, 0, sizeof(fulltext_cursor));
100778	/* sqlite will initialize c->base */
100779	*ppCursor = &c->base;
100780	FTSTRACE(("FTS3 Open %p: %p\n", pVTab, c));
100781	return SQLITE_OK;
100782	}else{
100783	return SQLITE_NOMEM;
100784	}
100785	}
100786
100787	/* Free all of the dynamically allocated memory held by the
100788	** Snippet
100789	*/
100790	static void snippetClear(Snippet *p){
100791	sqlite3_free(p->aMatch);
100792	sqlite3_free(p->zOffset);
100793	sqlite3_free(p->zSnippet);
100794	CLEAR(p);
100795	}
100796
100797	/*
100798	** Append a single entry to the p->aMatch[] log.
100799	*/
100800	static void snippetAppendMatch(
100801	Snippet p, / Append the entry to this snippet */
100802	int iCol, int iTerm, /* The column and query term */
100803	int iToken, /* Matching token in document */
100804	int iStart, int nByte /* Offset and size of the match */
100805	){
100806	int i;
100807	struct snippetMatch *pMatch;
100808	if( p->nMatch+1>=p->nAlloc ){
100809	p->nAlloc = p->nAlloc*2 + 10;
100810	p->aMatch = sqlite3_realloc(p->aMatch, p->nAlloc*sizeof(p->aMatch[0]) );
100811	if( p->aMatch==0 ){
100812	p->nMatch = 0;
100813	p->nAlloc = 0;
100814	return;
100815	}
100816	}
100817	i = p->nMatch++;
100818	pMatch = &p->aMatch[i];
100819	pMatch->iCol = iCol;
100820	pMatch->iTerm = iTerm;
100821	pMatch->iToken = iToken;
100822	pMatch->iStart = iStart;
100823	pMatch->nByte = nByte;
100824	}
100825
100826	/*
100827	** Sizing information for the circular buffer used in snippetOffsetsOfColumn()
100828	*/
100829	#define FTS3_ROTOR_SZ (32)
100830	#define FTS3_ROTOR_MASK (FTS3_ROTOR_SZ-1)
100831
100832	/*
100833	** Function to iterate through the tokens of a compiled expression.
100834	**
100835	** Except, skip all tokens on the right-hand side of a NOT operator.
100836	** This function is used to find tokens as part of snippet and offset
100837	** generation and we do nt want snippets and offsets to report matches
100838	** for tokens on the RHS of a NOT.
100839	*/
100840	static int fts3NextExprToken(Fts3Expr *ppExpr, int piToken){
100841	Fts3Expr p = ppExpr;
100842	int iToken = *piToken;
100843	if( iToken<0 ){
100844	/* In this case the expression p is the root of an expression tree.
100845	** Move to the first token in the expression tree.
100846	*/
100847	while( p->pLeft ){
100848	p = p->pLeft;
100849	}
100850	iToken = 0;
100851	}else{
100852	assert(p && p->eType==FTSQUERY_PHRASE );
100853	if( iToken<(p->pPhrase->nToken-1) ){
100854	iToken++;
100855	}else{
100856	iToken = 0;
100857	while( p->pParent && p->pParent->pLeft!=p ){
100858	assert( p->pParent->pRight==p );
100859	p = p->pParent;
100860	}
100861	p = p->pParent;
100862	if( p ){
100863	assert( p->pRight!=0 );
100864	p = p->pRight;
100865	while( p->pLeft ){
100866	p = p->pLeft;
100867	}
100868	}
100869	}
100870	}
100871
100872	*ppExpr = p;
100873	*piToken = iToken;
100874	return p?1:0;
100875	}
100876
100877	/*
100878	** Return TRUE if the expression node pExpr is located beneath the
100879	** RHS of a NOT operator.
100880	*/
100881	static int fts3ExprBeneathNot(Fts3Expr *p){
100882	Fts3Expr *pParent;
100883	while( p ){
100884	pParent = p->pParent;
100885	if( pParent && pParent->eType==FTSQUERY_NOT && pParent->pRight==p ){
100886	return 1;
100887	}
100888	p = pParent;
100889	}
100890	return 0;
100891	}
100892
100893	/*
100894	** Add entries to pSnippet->aMatch[] for every match that occurs against
100895	** document zDoc[0..nDoc-1] which is stored in column iColumn.
100896	*/
100897	static void snippetOffsetsOfColumn(
100898	fulltext_cursor pCur, / The fulltest search cursor */
100899	Snippet pSnippet, / The Snippet object to be filled in */
100900	int iColumn, /* Index of fulltext table column */
100901	const char zDoc, / Text of the fulltext table column */
100902	int nDoc /* Length of zDoc in bytes */
100903	){
100904	const sqlite3_tokenizer_module pTModule; / The tokenizer module */
100905	sqlite3_tokenizer pTokenizer; / The specific tokenizer */
100906	sqlite3_tokenizer_cursor pTCursor; / Tokenizer cursor */
100907	fulltext_vtab pVtab; / The full text index */
100908	int nColumn; /* Number of columns in the index */
100909	int i, j; /* Loop counters */
100910	int rc; /* Return code */
100911	unsigned int match, prevMatch; /* Phrase search bitmasks */
100912	const char zToken; / Next token from the tokenizer */
100913	int nToken; /* Size of zToken */
100914	int iBegin, iEnd, iPos; /* Offsets of beginning and end */
100915
100916	/* The following variables keep a circular buffer of the last
100917	** few tokens */
100918	unsigned int iRotor = 0; /* Index of current token */
100919	int iRotorBegin[FTS3_ROTOR_SZ]; /* Beginning offset of token */
100920	int iRotorLen[FTS3_ROTOR_SZ]; /* Length of token */
100921
100922	pVtab = cursor_vtab(pCur);
100923	nColumn = pVtab->nColumn;
100924	pTokenizer = pVtab->pTokenizer;
100925	pTModule = pTokenizer->pModule;
100926	rc = pTModule->xOpen(pTokenizer, zDoc, nDoc, &pTCursor);
100927	if( rc ) return;
100928	pTCursor->pTokenizer = pTokenizer;
100929
100930	prevMatch = 0;
100931	while( !pTModule->xNext(pTCursor, &zToken, &nToken, &iBegin, &iEnd, &iPos) ){
100932	Fts3Expr *pIter = pCur->pExpr;
100933	int iIter = -1;
100934	iRotorBegin[iRotor&FTS3_ROTOR_MASK] = iBegin;
100935	iRotorLen[iRotor&FTS3_ROTOR_MASK] = iEnd-iBegin;
100936	match = 0;
100937	for(i=0; i<(FTS3_ROTOR_SZ-1) && fts3NextExprToken(&pIter, &iIter); i++){
100938	int nPhrase; /* Number of tokens in current phrase */
100939	struct PhraseToken pToken; / Current token */
100940	int iCol; /* Column index */
100941
100942	if( fts3ExprBeneathNot(pIter) ) continue;
100943	nPhrase = pIter->pPhrase->nToken;
100944	pToken = &pIter->pPhrase->aToken[iIter];
100945	iCol = pIter->pPhrase->iColumn;
100946	if( iCol>=0 && iCol<nColumn && iCol!=iColumn ) continue;
100947	if( pToken->n>nToken ) continue;
100948	if( !pToken->isPrefix && pToken->n<nToken ) continue;
100949	assert( pToken->n<=nToken );
100950	if( memcmp(pToken->z, zToken, pToken->n) ) continue;
100951	if( iIter>0 && (prevMatch & (1<<i))==0 ) continue;
100952	match \|= 1<<i;
100953	if( i==(FTS3_ROTOR_SZ-2) \|\| nPhrase==iIter+1 ){
100954	for(j=nPhrase-1; j>=0; j--){
100955	int k = (iRotor-j) & FTS3_ROTOR_MASK;
100956	snippetAppendMatch(pSnippet, iColumn, i-j, iPos-j,
100957	iRotorBegin[k], iRotorLen[k]);
100958	}
100959	}
100960	}
100961	prevMatch = match<<1;
100962	iRotor++;
100963	}
100964	pTModule->xClose(pTCursor);
100965	}
100966
100967	/*
100968	** Remove entries from the pSnippet structure to account for the NEAR
100969	** operator. When this is called, pSnippet contains the list of token
100970	** offsets produced by treating all NEAR operators as AND operators.
100971	** This function removes any entries that should not be present after
100972	** accounting for the NEAR restriction. For example, if the queried
100973	** document is:
100974	**
100975	** "A B C D E A"
100976	**
100977	** and the query is:
100978	**
100979	** A NEAR/0 E
100980	**
100981	** then when this function is called the Snippet contains token offsets
100982	** 0, 4 and 5. This function removes the "0" entry (because the first A
100983	** is not near enough to an E).
100984	**
100985	** When this function is called, the value pointed to by parameter piLeft is
100986	** the integer id of the left-most token in the expression tree headed by
100987	** pExpr. This function increments *piLeft by the total number of tokens
100988	** in the expression tree headed by pExpr.
100989	**
100990	** Return 1 if any trimming occurs. Return 0 if no trimming is required.
100991	*/
100992	static int trimSnippetOffsets(
100993	Fts3Expr pExpr, / The search expression */
100994	Snippet pSnippet, / The set of snippet offsets to be trimmed */
100995	int piLeft / Index of left-most token in pExpr */
100996	){
100997	if( pExpr ){
100998	if( trimSnippetOffsets(pExpr->pLeft, pSnippet, piLeft) ){
100999	return 1;
101000	}
101001
101002	switch( pExpr->eType ){
101003	case FTSQUERY_PHRASE:
101004	*piLeft += pExpr->pPhrase->nToken;
101005	break;
101006	case FTSQUERY_NEAR: {
101007	/* The right-hand-side of a NEAR operator is always a phrase. The
101008	** left-hand-side is either a phrase or an expression tree that is
101009	** itself headed by a NEAR operator. The following initializations
101010	** set local variable iLeft to the token number of the left-most
101011	** token in the right-hand phrase, and iRight to the right most
101012	** token in the same phrase. For example, if we had:
101013	**
101014	** <col> MATCH '"abc def" NEAR/2 "ghi jkl"'
101015	**
101016	** then iLeft will be set to 2 (token number of ghi) and nToken will
101017	** be set to 4.
101018	*/
101019	Fts3Expr *pLeft = pExpr->pLeft;
101020	Fts3Expr *pRight = pExpr->pRight;
101021	int iLeft = *piLeft;
101022	int nNear = pExpr->nNear;
101023	int nToken = pRight->pPhrase->nToken;
101024	int jj, ii;
101025	if( pLeft->eType==FTSQUERY_NEAR ){
101026	pLeft = pLeft->pRight;
101027	}
101028	assert( pRight->eType==FTSQUERY_PHRASE );
101029	assert( pLeft->eType==FTSQUERY_PHRASE );
101030	nToken += pLeft->pPhrase->nToken;
101031
101032	for(ii=0; ii<pSnippet->nMatch; ii++){
101033	struct snippetMatch *p = &pSnippet->aMatch[ii];
101034	if( p->iTerm==iLeft ){
101035	int isOk = 0;
101036	/* Snippet ii is an occurence of query term iLeft in the document.
101037	** It occurs at position (p->iToken) of the document. We now
101038	** search for an instance of token (iLeft-1) somewhere in the
101039	** range (p->iToken - nNear)...(p->iToken + nNear + nToken) within
101040	** the set of snippetMatch structures. If one is found, proceed.
101041	** If one cannot be found, then remove snippets ii..(ii+N-1)
101042	** from the matching snippets, where N is the number of tokens
101043	** in phrase pRight->pPhrase.
101044	*/
101045	for(jj=0; isOk==0 && jj<pSnippet->nMatch; jj++){
101046	struct snippetMatch *p2 = &pSnippet->aMatch[jj];
101047	if( p2->iTerm==(iLeft-1) ){
101048	if( p2->iToken>=(p->iToken-nNear-1)
101049	&& p2->iToken<(p->iToken+nNear+nToken)
101050	){
101051	isOk = 1;
101052	}
101053	}
101054	}
101055	if( !isOk ){
101056	int kk;
101057	for(kk=0; kk<pRight->pPhrase->nToken; kk++){
101058	pSnippet->aMatch[kk+ii].iTerm = -2;
101059	}
101060	return 1;
101061	}
101062	}
101063	if( p->iTerm==(iLeft-1) ){
101064	int isOk = 0;
101065	for(jj=0; isOk==0 && jj<pSnippet->nMatch; jj++){
101066	struct snippetMatch *p2 = &pSnippet->aMatch[jj];
101067	if( p2->iTerm==iLeft ){
101068	if( p2->iToken<=(p->iToken+nNear+1)
101069	&& p2->iToken>(p->iToken-nNear-nToken)
101070	){
101071	isOk = 1;
101072	}
101073	}
101074	}
101075	if( !isOk ){
101076	int kk;
101077	for(kk=0; kk<pLeft->pPhrase->nToken; kk++){
101078	pSnippet->aMatch[ii-kk].iTerm = -2;
101079	}
101080	return 1;
101081	}
101082	}
101083	}
101084	break;
101085	}
101086	}
101087
101088	if( trimSnippetOffsets(pExpr->pRight, pSnippet, piLeft) ){
101089	return 1;
101090	}
101091	}
101092	return 0;
101093	}
101094
101095	/*
101096	** Compute all offsets for the current row of the query.
101097	** If the offsets have already been computed, this routine is a no-op.
101098	*/
101099	static void snippetAllOffsets(fulltext_cursor *p){
101100	int nColumn;
101101	int iColumn, i;
101102	int iFirst, iLast;
101103	int iTerm = 0;
101104	fulltext_vtab *pFts = cursor_vtab(p);
101105
101106	if( p->snippet.nMatch \|\| p->pExpr==0 ){
101107	return;
101108	}
101109	nColumn = pFts->nColumn;
101110	iColumn = (p->iCursorType - QUERY_FULLTEXT);
101111	if( iColumn<0 \|\| iColumn>=nColumn ){
101112	/* Look for matches over all columns of the full-text index */
101113	iFirst = 0;
101114	iLast = nColumn-1;
101115	}else{
101116	/* Look for matches in the iColumn-th column of the index only */
101117	iFirst = iColumn;
101118	iLast = iColumn;
101119	}
101120	for(i=iFirst; i<=iLast; i++){
101121	const char *zDoc;
101122	int nDoc;
101123	zDoc = (const char*)sqlite3_column_text(p->pStmt, i+1);
101124	nDoc = sqlite3_column_bytes(p->pStmt, i+1);
101125	snippetOffsetsOfColumn(p, &p->snippet, i, zDoc, nDoc);
101126	}
101127
101128	while( trimSnippetOffsets(p->pExpr, &p->snippet, &iTerm) ){
101129	iTerm = 0;
101130	}
101131	}
101132
101133	/*
101134	** Convert the information in the aMatch[] array of the snippet
101135	** into the string zOffset[0..nOffset-1]. This string is used as
101136	** the return of the SQL offsets() function.
101137	*/
101138	static void snippetOffsetText(Snippet *p){
101139	int i;
101140	int cnt = 0;
101141	StringBuffer sb;
101142	char zBuf[200];
101143	if( p->zOffset ) return;
101144	initStringBuffer(&sb);
101145	for(i=0; i<p->nMatch; i++){
101146	struct snippetMatch *pMatch = &p->aMatch[i];
101147	if( pMatch->iTerm>=0 ){
101148	/* If snippetMatch.iTerm is less than 0, then the match was
101149	** discarded as part of processing the NEAR operator (see the
101150	** trimSnippetOffsetsForNear() function for details). Ignore
101151	** it in this case
101152	*/
101153	zBuf[0] = ' ';
101154	sqlite3_snprintf(sizeof(zBuf)-1, &zBuf[cnt>0], "%d %d %d %d",
101155	pMatch->iCol, pMatch->iTerm, pMatch->iStart, pMatch->nByte);
101156	append(&sb, zBuf);
101157	cnt++;
101158	}
101159	}
101160	p->zOffset = stringBufferData(&sb);
101161	p->nOffset = stringBufferLength(&sb);
101162	}
101163
101164	/*
101165	** zDoc[0..nDoc-1] is phrase of text. aMatch[0..nMatch-1] are a set
101166	** of matching words some of which might be in zDoc. zDoc is column
101167	** number iCol.
101168	**
101169	** iBreak is suggested spot in zDoc where we could begin or end an
101170	** excerpt. Return a value similar to iBreak but possibly adjusted
101171	** to be a little left or right so that the break point is better.
101172	*/
101173	static int wordBoundary(
101174	int iBreak, /* The suggested break point */
101175	const char zDoc, / Document text */
101176	int nDoc, /* Number of bytes in zDoc[] */
101177	struct snippetMatch aMatch, / Matching words */
101178	int nMatch, /* Number of entries in aMatch[] */
101179	int iCol /* The column number for zDoc[] */
101180	){
101181	int i;
101182	if( iBreak<=10 ){
101183	return 0;
101184	}
101185	if( iBreak>=nDoc-10 ){
101186	return nDoc;
101187	}
101188	for(i=0; i<nMatch && aMatch[i].iCol<iCol; i++){}
101189	while( i<nMatch && aMatch[i].iStart+aMatch[i].nByte<iBreak ){ i++; }
101190	if( i<nMatch ){
101191	if( aMatch[i].iStart<iBreak+10 ){
101192	return aMatch[i].iStart;
101193	}
101194	if( i>0 && aMatch[i-1].iStart+aMatch[i-1].nByte>=iBreak ){
101195	return aMatch[i-1].iStart;
101196	}
101197	}
101198	for(i=1; i<=10; i++){
101199	if( safe_isspace(zDoc[iBreak-i]) ){
101200	return iBreak - i + 1;
101201	}
101202	if( safe_isspace(zDoc[iBreak+i]) ){
101203	return iBreak + i + 1;
101204	}
101205	}
101206	return iBreak;
101207	}
101208
101209
101210
101211	/*
101212	** Allowed values for Snippet.aMatch[].snStatus
101213	*/
101214	#define SNIPPET_IGNORE 0 /* It is ok to omit this match from the snippet */
101215	#define SNIPPET_DESIRED 1 /* We want to include this match in the snippet */
101216
101217	/*
101218	** Generate the text of a snippet.
101219	*/
101220	static void snippetText(
101221	fulltext_cursor pCursor, / The cursor we need the snippet for */
101222	const char zStartMark, / Markup to appear before each match */
101223	const char zEndMark, / Markup to appear after each match */
101224	const char zEllipsis / Ellipsis mark */
101225	){
101226	int i, j;
101227	struct snippetMatch *aMatch;
101228	int nMatch;
101229	int nDesired;
101230	StringBuffer sb;
101231	int tailCol;
101232	int tailOffset;
101233	int iCol;
101234	int nDoc;
101235	const char *zDoc;
101236	int iStart, iEnd;
101237	int tailEllipsis = 0;
101238	int iMatch;
101239
101240
101241	sqlite3_free(pCursor->snippet.zSnippet);
101242	pCursor->snippet.zSnippet = 0;
101243	aMatch = pCursor->snippet.aMatch;
101244	nMatch = pCursor->snippet.nMatch;
101245	initStringBuffer(&sb);
101246
101247	for(i=0; i<nMatch; i++){
101248	aMatch[i].snStatus = SNIPPET_IGNORE;
101249	}
101250	nDesired = 0;
101251	for(i=0; i<FTS3_ROTOR_SZ; i++){
101252	for(j=0; j<nMatch; j++){
101253	if( aMatch[j].iTerm==i ){
101254	aMatch[j].snStatus = SNIPPET_DESIRED;
101255	nDesired++;
101256	break;
101257	}
101258	}
101259	}
101260
101261	iMatch = 0;
101262	tailCol = -1;
101263	tailOffset = 0;
101264	for(i=0; i<nMatch && nDesired>0; i++){
101265	if( aMatch[i].snStatus!=SNIPPET_DESIRED ) continue;
101266	nDesired--;
101267	iCol = aMatch[i].iCol;
101268	zDoc = (const char*)sqlite3_column_text(pCursor->pStmt, iCol+1);
101269	nDoc = sqlite3_column_bytes(pCursor->pStmt, iCol+1);
101270	iStart = aMatch[i].iStart - 40;
101271	iStart = wordBoundary(iStart, zDoc, nDoc, aMatch, nMatch, iCol);
101272	if( iStart<=10 ){
101273	iStart = 0;
101274	}
101275	if( iCol==tailCol && iStart<=tailOffset+20 ){
101276	iStart = tailOffset;
101277	}
101278	if( (iCol!=tailCol && tailCol>=0) \|\| iStart!=tailOffset ){
101279	trimWhiteSpace(&sb);
101280	appendWhiteSpace(&sb);
101281	append(&sb, zEllipsis);
101282	appendWhiteSpace(&sb);
101283	}
101284	iEnd = aMatch[i].iStart + aMatch[i].nByte + 40;
101285	iEnd = wordBoundary(iEnd, zDoc, nDoc, aMatch, nMatch, iCol);
101286	if( iEnd>=nDoc-10 ){
101287	iEnd = nDoc;
101288	tailEllipsis = 0;
101289	}else{
101290	tailEllipsis = 1;
101291	}
101292	while( iMatch<nMatch && aMatch[iMatch].iCol<iCol ){ iMatch++; }
101293	while( iStart<iEnd ){
101294	while( iMatch<nMatch && aMatch[iMatch].iStart<iStart
101295	&& aMatch[iMatch].iCol<=iCol ){
101296	iMatch++;
101297	}
101298	if( iMatch<nMatch && aMatch[iMatch].iStart<iEnd
101299	&& aMatch[iMatch].iCol==iCol ){
101300	nappend(&sb, &zDoc[iStart], aMatch[iMatch].iStart - iStart);
101301	iStart = aMatch[iMatch].iStart;
101302	append(&sb, zStartMark);
101303	nappend(&sb, &zDoc[iStart], aMatch[iMatch].nByte);
101304	append(&sb, zEndMark);
101305	iStart += aMatch[iMatch].nByte;
101306	for(j=iMatch+1; j<nMatch; j++){
101307	if( aMatch[j].iTerm==aMatch[iMatch].iTerm
101308	&& aMatch[j].snStatus==SNIPPET_DESIRED ){
101309	nDesired--;
101310	aMatch[j].snStatus = SNIPPET_IGNORE;
101311	}
101312	}
101313	}else{
101314	nappend(&sb, &zDoc[iStart], iEnd - iStart);
101315	iStart = iEnd;
101316	}
101317	}
101318	tailCol = iCol;
101319	tailOffset = iEnd;
101320	}
101321	trimWhiteSpace(&sb);
101322	if( tailEllipsis ){
101323	appendWhiteSpace(&sb);
101324	append(&sb, zEllipsis);
101325	}
101326	pCursor->snippet.zSnippet = stringBufferData(&sb);
101327	pCursor->snippet.nSnippet = stringBufferLength(&sb);
101328	}











































































































101329
101330
101331	/*
101332	** Close the cursor. For additional information see the documentation
101333	** on the xClose method of the virtual table interface.
101334	*/
101335	static int fulltextClose(sqlite3_vtab_cursor *pCursor){
101336	fulltext_cursor c = (fulltext_cursor ) pCursor;
101337	FTSTRACE(("FTS3 Close %p\n", c));
101338	sqlite3_finalize(c->pStmt);
101339	sqlite3Fts3ExprFree(c->pExpr);
101340	snippetClear(&c->snippet);
101341	if( c->result.nData!=0 ){
101342	dlrDestroy(&c->reader);
101343	}
101344	dataBufferDestroy(&c->result);
101345	sqlite3_free(c);
101346	return SQLITE_OK;
101347	}
101348
101349	static int fulltextNext(sqlite3_vtab_cursor *pCursor){
101350	fulltext_cursor c = (fulltext_cursor ) pCursor;
101351	int rc;
101352
101353	FTSTRACE(("FTS3 Next %p\n", pCursor));
101354	snippetClear(&c->snippet);
101355	if( c->iCursorType < QUERY_FULLTEXT ){
101356	/* TODO(shess) Handle SQLITE_SCHEMA AND SQLITE_BUSY. */
101357	rc = sqlite3_step(c->pStmt);
101358	switch( rc ){
101359	case SQLITE_ROW:
101360	c->eof = 0;
101361	return SQLITE_OK;
101362	case SQLITE_DONE:
101363	c->eof = 1;
101364	return SQLITE_OK;
101365	default:
101366	c->eof = 1;
101367	return rc;
101368	}
101369	} else { /* full-text query */
101370	rc = sqlite3_reset(c->pStmt);
101371	if( rc!=SQLITE_OK ) return rc;
101372
101373	if( c->result.nData==0 \|\| dlrAtEnd(&c->reader) ){
101374	c->eof = 1;
101375	return SQLITE_OK;
101376	}
101377	rc = sqlite3_bind_int64(c->pStmt, 1, dlrDocid(&c->reader));
101378	dlrStep(&c->reader);
101379	if( rc!=SQLITE_OK ) return rc;
101380	/* TODO(shess) Handle SQLITE_SCHEMA AND SQLITE_BUSY. */
101381	rc = sqlite3_step(c->pStmt);
101382	if( rc==SQLITE_ROW ){ /* the case we expect */
101383	c->eof = 0;
101384	return SQLITE_OK;
101385	}
101386	/* an error occurred; abort */
101387	return rc==SQLITE_DONE ? SQLITE_ERROR : rc;
101388	}
101389	}
101390
101391
101392	/* TODO(shess) If we pushed LeafReader to the top of the file, or to
101393	** another file, term_select() could be pushed above
101394	** docListOfTerm().
101395	*/
101396	static int termSelect(fulltext_vtab *v, int iColumn,
101397	const char *pTerm, int nTerm, int isPrefix,
101398	DocListType iType, DataBuffer *out);

































































































































































































































































































































































































































































































































































































































































































101399
101400	/*
101401	** Return a DocList corresponding to the phrase *pPhrase.
101402	**
101403	** The resulting DL_DOCIDS doclist is stored in pResult, which is
101404	** overwritten.
101405	*/
101406	static int docListOfPhrase(
101407	fulltext_vtab pTab, / The full text index */
101408	Fts3Phrase pPhrase, / Phrase to return a doclist corresponding to */
101409	DocListType eListType, /* Either DL_DOCIDS or DL_POSITIONS */
101410	DataBuffer pResult / Write the result here */

101411	){
101412	int ii;

101413	int rc = SQLITE_OK;

101414	int iCol = pPhrase->iColumn;
101415	DocListType eType = eListType;
101416	assert( eType==DL_POSITIONS \|\| eType==DL_DOCIDS );
101417	if( pPhrase->nToken>1 ){
101418	eType = DL_POSITIONS;
101419	}
101420
101421	/* This code should never be called with buffered updates. */
101422	assert( pTab->nPendingData<0 );
101423
101424	for(ii=0; rc==SQLITE_OK && ii<pPhrase->nToken; ii++){
101425	DataBuffer tmp;
101426	struct PhraseToken *p = &pPhrase->aToken[ii];
101427	rc = termSelect(pTab, iCol, p->z, p->n, p->isPrefix, eType, &tmp);
101428	if( rc==SQLITE_OK ){
101429	if( ii==0 ){
101430	*pResult = tmp;
101431	}else{
101432	DataBuffer res = *pResult;
101433	dataBufferInit(pResult, 0);
101434	if( ii==(pPhrase->nToken-1) ){
101435	eType = eListType;
101436	}
101437	docListPhraseMerge(
101438	res.pData, res.nData, tmp.pData, tmp.nData, 0, 0, eType, pResult
101439	);
101440	dataBufferDestroy(&res);
101441	dataBufferDestroy(&tmp);
101442	}
101443	}
101444	}
101445












101446	return rc;
101447	}
101448
101449	/*
101450	** Evaluate the full-text expression pExpr against fts3 table pTab. Write
101451	** the results into pRes.
101452	*/
101453	static int evalFts3Expr(
101454	fulltext_vtab pTab, / Fts3 Virtual table object */
101455	Fts3Expr pExpr, / Parsed fts3 expression */
101456	DataBuffer pRes / OUT: Write results of the expression here */

101457	){
101458	int rc = SQLITE_OK;
101459
101460	/* Initialize the output buffer. If this is an empty query (pExpr==0),
101461	** this is all that needs to be done. Empty queries produce empty
101462	** result sets.
101463	*/
101464	dataBufferInit(pRes, 0);
101465
101466	if( pExpr ){
101467	if( pExpr->eType==FTSQUERY_PHRASE ){
101468	DocListType eType = DL_DOCIDS;
101469	if( pExpr->pParent && pExpr->pParent->eType==FTSQUERY_NEAR ){
101470	eType = DL_POSITIONS;
101471	}
101472	rc = docListOfPhrase(pTab, pExpr->pPhrase, eType, pRes);
101473	}else{
101474	DataBuffer lhs;
101475	DataBuffer rhs;


101476
101477	dataBufferInit(&rhs, 0);
101478	if( SQLITE_OK==(rc = evalFts3Expr(pTab, pExpr->pLeft, &lhs))
101479	&& SQLITE_OK==(rc = evalFts3Expr(pTab, pExpr->pRight, &rhs))
101480	){
101481	switch( pExpr->eType ){
101482	case FTSQUERY_NEAR: {
101483	int nToken;
101484	Fts3Expr *pLeft;
101485	DocListType eType = DL_DOCIDS;





101486	if( pExpr->pParent && pExpr->pParent->eType==FTSQUERY_NEAR ){
101487	eType = DL_POSITIONS;
101488	}
101489	pLeft = pExpr->pLeft;
101490	while( pLeft->eType==FTSQUERY_NEAR ){
101491	pLeft=pLeft->pRight;
101492	}
101493	assert( pExpr->pRight->eType==FTSQUERY_PHRASE );
101494	assert( pLeft->eType==FTSQUERY_PHRASE );
101495	nToken = pLeft->pPhrase->nToken + pExpr->pRight->pPhrase->nToken;
101496	docListPhraseMerge(lhs.pData, lhs.nData, rhs.pData, rhs.nData,
101497	pExpr->nNear+1, nToken, eType, pRes
101498	);
101499	break;
101500	}
101501	case FTSQUERY_NOT: {
101502	docListExceptMerge(lhs.pData, lhs.nData, rhs.pData, rhs.nData,pRes);
101503	break;
101504	}
101505	case FTSQUERY_AND: {
101506	docListAndMerge(lhs.pData, lhs.nData, rhs.pData, rhs.nData, pRes);
101507	break;
101508	}
101509	case FTSQUERY_OR: {
101510	docListOrMerge(lhs.pData, lhs.nData, rhs.pData, rhs.nData, pRes);
101511	break;
101512	}
101513	}
101514	}
101515	dataBufferDestroy(&lhs);
101516	dataBufferDestroy(&rhs);
101517	}
101518	}
101519
101520	return rc;
101521	}
101522
101523	/* TODO(shess) Refactor the code to remove this forward decl. */
101524	static int flushPendingTerms(fulltext_vtab *v);
101525
101526	/* Perform a full-text query using the search expression in
101527	** zInput[0..nInput-1]. Return a list of matching documents
101528	** in pResult.
101529	**
101530	** Queries must match column iColumn. Or if iColumn>=nColumn
101531	** they are allowed to match against any column.
101532	*/
101533	static int fulltextQuery(
101534	fulltext_vtab v, / The full text index */
101535	int iColumn, /* Match against this column by default */
101536	const char zInput, / The query string */
101537	int nInput, /* Number of bytes in zInput[] */
101538	DataBuffer pResult, / Write the result doclist here */
101539	Fts3Expr *ppExpr / Put parsed query string here */
101540	){
101541	int rc;
101542
101543	/* TODO(shess) Instead of flushing pendingTerms, we could query for
101544	** the relevant term and merge the doclist into what we receive from
101545	** the database. Wait and see if this is a common issue, first.
101546	**
101547	** A good reason not to flush is to not generate update-related
101548	** error codes from here.
101549	*/
101550
101551	/* Flush any buffered updates before executing the query. */
101552	rc = flushPendingTerms(v);
101553	if( rc!=SQLITE_OK ){
101554	return rc;
101555	}
101556
101557	/* Parse the query passed to the MATCH operator. */
101558	rc = sqlite3Fts3ExprParse(v->pTokenizer,
101559	v->azColumn, v->nColumn, iColumn, zInput, nInput, ppExpr
101560	);
101561	if( rc!=SQLITE_OK ){
101562	assert( 0==(*ppExpr) );
101563	return rc;
101564	}
101565
101566	return evalFts3Expr(v, *ppExpr, pResult);
101567	}
101568
101569	/*
101570	** This is the xFilter interface for the virtual table. See
101571	** the virtual table xFilter method documentation for additional
101572	** information.
101573	**
101574	** If idxNum==QUERY_GENERIC then do a full table scan against
101575	** the %_content table.
101576	**
101577	** If idxNum==QUERY_DOCID then do a docid lookup for a single entry
101578	** in the %_content table.
101579	**
101580	** If idxNum>=QUERY_FULLTEXT then use the full text index. The
101581	** column on the left-hand side of the MATCH operator is column
101582	** number idxNum-QUERY_FULLTEXT, 0 indexed. argv[0] is the right-hand
101583	** side of the MATCH operator.
101584	*/
101585	/* TODO(shess) Upgrade the cursor initialization and destruction to
101586	** account for fulltextFilter() being called multiple times on the
101587	** same cursor. The current solution is very fragile. Apply fix to
101588	** fts3 as appropriate.
101589	*/
101590	static int fulltextFilter(
101591	sqlite3_vtab_cursor pCursor, / The cursor used for this query */
101592	int idxNum, const char idxStr, / Which indexing scheme to use */
101593	int argc, sqlite3_value *argv / Arguments for the indexing scheme */
101594	){
101595	fulltext_cursor c = (fulltext_cursor ) pCursor;
101596	fulltext_vtab *v = cursor_vtab(c);
101597	int rc;
101598
101599	FTSTRACE(("FTS3 Filter %p\n",pCursor));
101600
101601	/* If the cursor has a statement that was not prepared according to
101602	** idxNum, clear it. I believe all calls to fulltextFilter with a
101603	** given cursor will have the same idxNum , but in this case it's
101604	** easy to be safe.
101605	*/
101606	if( c->pStmt && c->iCursorType!=idxNum ){
101607	sqlite3_finalize(c->pStmt);
101608	c->pStmt = NULL;
101609	}
101610
101611	/* Get a fresh statement appropriate to idxNum. */
101612	/* TODO(shess): Add a prepared-statement cache in the vt structure.
101613	** The cache must handle multiple open cursors. Easier to cache the
101614	** statement variants at the vt to reduce malloc/realloc/free here.
101615	** Or we could have a StringBuffer variant which allowed stack
101616	** construction for small values.
101617	*/
101618	if( !c->pStmt ){
101619	StringBuffer sb;
101620	initStringBuffer(&sb);
101621	append(&sb, "SELECT docid, ");
101622	appendList(&sb, v->nColumn, v->azContentColumn);
101623	append(&sb, " FROM %_content");
101624	if( idxNum!=QUERY_GENERIC ) append(&sb, " WHERE docid = ?");
101625	rc = sql_prepare(v->db, v->zDb, v->zName, &c->pStmt,
101626	stringBufferData(&sb));
101627	stringBufferDestroy(&sb);
101628	if( rc!=SQLITE_OK ) return rc;
101629	c->iCursorType = idxNum;
101630	}else{
101631	sqlite3_reset(c->pStmt);
101632	assert( c->iCursorType==idxNum );
101633	}
101634
101635	switch( idxNum ){
101636	case QUERY_GENERIC:
101637	break;
101638
101639	case QUERY_DOCID:
101640	rc = sqlite3_bind_int64(c->pStmt, 1, sqlite3_value_int64(argv[0]));
101641	if( rc!=SQLITE_OK ) return rc;
101642	break;
101643
101644	default: /* full-text search */
101645	{
101646	int iCol = idxNum-QUERY_FULLTEXT;
101647	const char zQuery = (const char )sqlite3_value_text(argv[0]);
101648	assert( idxNum<=QUERY_FULLTEXT+v->nColumn);
101649	assert( argc==1 );
101650	if( c->result.nData!=0 ){
101651	/* This case happens if the same cursor is used repeatedly. */
101652	dlrDestroy(&c->reader);
101653	dataBufferReset(&c->result);
101654	}else{
101655	dataBufferInit(&c->result, 0);
101656	}
101657	rc = fulltextQuery(v, iCol, zQuery, -1, &c->result, &c->pExpr);
101658	if( rc!=SQLITE_OK ) return rc;
101659	if( c->result.nData!=0 ){
101660	dlrInit(&c->reader, DL_DOCIDS, c->result.pData, c->result.nData);
101661	}
101662	break;
101663	}
101664	}
101665
101666	return fulltextNext(pCursor);
101667	}
101668
101669	/* This is the xEof method of the virtual table. The SQLite core
101670	** calls this routine to find out if it has reached the end of
101671	** a query's results set.
101672	*/
101673	static int fulltextEof(sqlite3_vtab_cursor *pCursor){
101674	fulltext_cursor c = (fulltext_cursor ) pCursor;
101675	return c->eof;
101676	}
101677
101678	/* This is the xColumn method of the virtual table. The SQLite
101679	** core calls this method during a query when it needs the value
101680	** of a column from the virtual table. This method needs to use
101681	** one of the sqlite3_result_*() routines to store the requested
101682	** value back in the pContext.
101683	*/
101684	static int fulltextColumn(sqlite3_vtab_cursor *pCursor,
101685	sqlite3_context *pContext, int idxCol){
101686	fulltext_cursor c = (fulltext_cursor ) pCursor;
101687	fulltext_vtab *v = cursor_vtab(c);




101688
101689	if( idxCol<v->nColumn ){
101690	sqlite3_value *pVal = sqlite3_column_value(c->pStmt, idxCol+1);
101691	sqlite3_result_value(pContext, pVal);
101692	}else if( idxCol==v->nColumn ){
	@@ -101700,2884 +99365,227 @@
101700	sqlite3_result_value(pContext, pVal);
101701	}
101702	return SQLITE_OK;
101703	}
101704
101705	/* This is the xRowid method. The SQLite core calls this routine to
101706	** retrieve the rowid for the current row of the result set. fts3
101707	** exposes %_content.docid as the rowid for the virtual table. The

101708	** rowid should be written to *pRowid.
101709	*/
101710	static int fulltextRowid(sqlite3_vtab_cursor pCursor, sqlite_int64 pRowid){
101711	fulltext_cursor c = (fulltext_cursor ) pCursor;
101712
101713	*pRowid = sqlite3_column_int64(c->pStmt, 0);
101714	return SQLITE_OK;
101715	}
101716
101717	/* Add all terms in [zText] to pendingTerms table. If [iColumn] > 0,
101718	** we also store positions and offsets in the hash table using that
101719	** column number.
101720	*/
101721	static int buildTerms(fulltext_vtab *v, sqlite_int64 iDocid,
101722	const char *zText, int iColumn){
101723	sqlite3_tokenizer *pTokenizer = v->pTokenizer;
101724	sqlite3_tokenizer_cursor *pCursor;
101725	const char *pToken;
101726	int nTokenBytes;
101727	int iStartOffset, iEndOffset, iPosition;
101728	int rc;
101729
101730	rc = pTokenizer->pModule->xOpen(pTokenizer, zText, -1, &pCursor);
101731	if( rc!=SQLITE_OK ) return rc;
101732
101733	pCursor->pTokenizer = pTokenizer;
101734	while( SQLITE_OK==(rc=pTokenizer->pModule->xNext(pCursor,
101735	&pToken, &nTokenBytes,
101736	&iStartOffset, &iEndOffset,
101737	&iPosition)) ){
101738	DLCollector *p;
101739	int nData; /* Size of doclist before our update. */
101740
101741	/* Positions can't be negative; we use -1 as a terminator
101742	* internally. Token can't be NULL or empty. */
101743	if( iPosition<0 \|\| pToken == NULL \|\| nTokenBytes == 0 ){
101744	rc = SQLITE_ERROR;
101745	break;
101746	}
101747
101748	p = fts3HashFind(&v->pendingTerms, pToken, nTokenBytes);
101749	if( p==NULL ){
101750	nData = 0;
101751	p = dlcNew(iDocid, DL_DEFAULT);
101752	fts3HashInsert(&v->pendingTerms, pToken, nTokenBytes, p);
101753
101754	/* Overhead for our hash table entry, the key, and the value. */
101755	v->nPendingData += sizeof(struct fts3HashElem)+sizeof(*p)+nTokenBytes;
101756	}else{
101757	nData = p->b.nData;
101758	if( p->dlw.iPrevDocid!=iDocid ) dlcNext(p, iDocid);
101759	}
101760	if( iColumn>=0 ){
101761	dlcAddPos(p, iColumn, iPosition, iStartOffset, iEndOffset);
101762	}
101763
101764	/* Accumulate data added by dlcNew or dlcNext, and dlcAddPos. */
101765	v->nPendingData += p->b.nData-nData;
101766	}
101767
101768	/* TODO(shess) Check return? Should this be able to cause errors at
101769	** this point? Actually, same question about sqlite3_finalize(),
101770	** though one could argue that failure there means that the data is
101771	** not durable. ponder
101772	*/
101773	pTokenizer->pModule->xClose(pCursor);
101774	if( SQLITE_DONE == rc ) return SQLITE_OK;
101775	return rc;
101776	}
101777
101778	/* Add doclists for all terms in [pValues] to pendingTerms table. */
101779	static int insertTerms(fulltext_vtab *v, sqlite_int64 iDocid,
101780	sqlite3_value **pValues){
101781	int i;
101782	for(i = 0; i < v->nColumn ; ++i){
101783	char zText = (char)sqlite3_value_text(pValues[i]);
101784	int rc = buildTerms(v, iDocid, zText, i);
101785	if( rc!=SQLITE_OK ) return rc;
101786	}
101787	return SQLITE_OK;
101788	}
101789
101790	/* Add empty doclists for all terms in the given row's content to
101791	** pendingTerms.
101792	*/
101793	static int deleteTerms(fulltext_vtab *v, sqlite_int64 iDocid){
101794	const char **pValues;
101795	int i, rc;
101796
101797	/* TODO(shess) Should we allow such tables at all? */
101798	if( DL_DEFAULT==DL_DOCIDS ) return SQLITE_ERROR;
101799
101800	rc = content_select(v, iDocid, &pValues);
101801	if( rc!=SQLITE_OK ) return rc;
101802
101803	for(i = 0 ; i < v->nColumn; ++i) {
101804	rc = buildTerms(v, iDocid, pValues[i], -1);
101805	if( rc!=SQLITE_OK ) break;
101806	}
101807
101808	freeStringArray(v->nColumn, pValues);
101809	return SQLITE_OK;
101810	}
101811
101812	/* TODO(shess) Refactor the code to remove this forward decl. */
101813	static int initPendingTerms(fulltext_vtab *v, sqlite_int64 iDocid);
101814
101815	/* Insert a row into the %_content table; set *piDocid to be the ID of the
101816	** new row. Add doclists for terms to pendingTerms.
101817	*/
101818	static int index_insert(fulltext_vtab v, sqlite3_value pRequestDocid,
101819	sqlite3_value *pValues, sqlite_int64 piDocid){
101820	int rc;
101821
101822	rc = content_insert(v, pRequestDocid, pValues); /* execute an SQL INSERT */
101823	if( rc!=SQLITE_OK ) return rc;
101824
101825	/* docid column is an alias for rowid. */
101826	*piDocid = sqlite3_last_insert_rowid(v->db);
101827	rc = initPendingTerms(v, *piDocid);
101828	if( rc!=SQLITE_OK ) return rc;
101829
101830	return insertTerms(v, *piDocid, pValues);
101831	}
101832
101833	/* Delete a row from the %_content table; add empty doclists for terms
101834	** to pendingTerms.
101835	*/
101836	static int index_delete(fulltext_vtab *v, sqlite_int64 iRow){
101837	int rc = initPendingTerms(v, iRow);
101838	if( rc!=SQLITE_OK ) return rc;
101839
101840	rc = deleteTerms(v, iRow);
101841	if( rc!=SQLITE_OK ) return rc;
101842
101843	return content_delete(v, iRow); /* execute an SQL DELETE */
101844	}
101845
101846	/* Update a row in the %_content table; add delete doclists to
101847	** pendingTerms for old terms not in the new data, add insert doclists
101848	** to pendingTerms for terms in the new data.
101849	*/
101850	static int index_update(fulltext_vtab *v, sqlite_int64 iRow,
101851	sqlite3_value **pValues){
101852	int rc = initPendingTerms(v, iRow);
101853	if( rc!=SQLITE_OK ) return rc;
101854
101855	/* Generate an empty doclist for each term that previously appeared in this
101856	* row. */
101857	rc = deleteTerms(v, iRow);
101858	if( rc!=SQLITE_OK ) return rc;
101859
101860	rc = content_update(v, pValues, iRow); /* execute an SQL UPDATE */
101861	if( rc!=SQLITE_OK ) return rc;
101862
101863	/* Now add positions for terms which appear in the updated row. */
101864	return insertTerms(v, iRow, pValues);
101865	}
101866
101867	/*******************************************************************/
101868	/* InteriorWriter is used to collect terms and block references into
101869	** interior nodes in %_segments. See commentary at top of file for
101870	** format.
101871	*/
101872
101873	/* How large interior nodes can grow. */
101874	#define INTERIOR_MAX 2048
101875
101876	/* Minimum number of terms per interior node (except the root). This
101877	** prevents large terms from making the tree too skinny - must be >0
101878	** so that the tree always makes progress. Note that the min tree
101879	** fanout will be INTERIOR_MIN_TERMS+1.
101880	*/
101881	#define INTERIOR_MIN_TERMS 7
101882	#if INTERIOR_MIN_TERMS<1
101883	# error INTERIOR_MIN_TERMS must be greater than 0.
101884	#endif
101885
101886	/* ROOT_MAX controls how much data is stored inline in the segment
101887	** directory.
101888	*/
101889	/* TODO(shess) Push ROOT_MAX down to whoever is writing things. It's
101890	** only here so that interiorWriterRootInfo() and leafWriterRootInfo()
101891	** can both see it, but if the caller passed it in, we wouldn't even
101892	** need a define.
101893	*/
101894	#define ROOT_MAX 1024
101895	#if ROOT_MAX<VARINT_MAX*2
101896	# error ROOT_MAX must have enough space for a header.
101897	#endif
101898
101899	/* InteriorBlock stores a linked-list of interior blocks while a lower
101900	** layer is being constructed.
101901	*/
101902	typedef struct InteriorBlock {
101903	DataBuffer term; /* Leftmost term in block's subtree. */
101904	DataBuffer data; /* Accumulated data for the block. */
101905	struct InteriorBlock *next;
101906	} InteriorBlock;
101907
101908	static InteriorBlock *interiorBlockNew(int iHeight, sqlite_int64 iChildBlock,
101909	const char *pTerm, int nTerm){
101910	InteriorBlock *block = sqlite3_malloc(sizeof(InteriorBlock));
101911	char c[VARINT_MAX+VARINT_MAX];
101912	int n;
101913
101914	if( block ){
101915	memset(block, 0, sizeof(*block));
101916	dataBufferInit(&block->term, 0);
101917	dataBufferReplace(&block->term, pTerm, nTerm);
101918
101919	n = fts3PutVarint(c, iHeight);
101920	n += fts3PutVarint(c+n, iChildBlock);
101921	dataBufferInit(&block->data, INTERIOR_MAX);
101922	dataBufferReplace(&block->data, c, n);
101923	}
101924	return block;
101925	}
101926
101927	#ifndef NDEBUG
101928	/* Verify that the data is readable as an interior node. */
101929	static void interiorBlockValidate(InteriorBlock *pBlock){
101930	const char *pData = pBlock->data.pData;
101931	int nData = pBlock->data.nData;
101932	int n, iDummy;
101933	sqlite_int64 iBlockid;
101934
101935	assert( nData>0 );
101936	assert( pData!=0 );
101937	assert( pData+nData>pData );
101938
101939	/* Must lead with height of node as a varint(n), n>0 */
101940	n = fts3GetVarint32(pData, &iDummy);
101941	assert( n>0 );
101942	assert( iDummy>0 );
101943	assert( n<nData );
101944	pData += n;
101945	nData -= n;
101946
101947	/* Must contain iBlockid. */
101948	n = fts3GetVarint(pData, &iBlockid);
101949	assert( n>0 );
101950	assert( n<=nData );
101951	pData += n;
101952	nData -= n;
101953
101954	/* Zero or more terms of positive length */
101955	if( nData!=0 ){
101956	/* First term is not delta-encoded. */
101957	n = fts3GetVarint32(pData, &iDummy);
101958	assert( n>0 );
101959	assert( iDummy>0 );
101960	assert( n+iDummy>0);
101961	assert( n+iDummy<=nData );
101962	pData += n+iDummy;
101963	nData -= n+iDummy;
101964
101965	/* Following terms delta-encoded. */
101966	while( nData!=0 ){
101967	/* Length of shared prefix. */
101968	n = fts3GetVarint32(pData, &iDummy);
101969	assert( n>0 );
101970	assert( iDummy>=0 );
101971	assert( n<nData );
101972	pData += n;
101973	nData -= n;
101974
101975	/* Length and data of distinct suffix. */
101976	n = fts3GetVarint32(pData, &iDummy);
101977	assert( n>0 );
101978	assert( iDummy>0 );
101979	assert( n+iDummy>0);
101980	assert( n+iDummy<=nData );
101981	pData += n+iDummy;
101982	nData -= n+iDummy;
101983	}
101984	}
101985	}
101986	#define ASSERT_VALID_INTERIOR_BLOCK(x) interiorBlockValidate(x)
101987	#else
101988	#define ASSERT_VALID_INTERIOR_BLOCK(x) assert( 1 )
101989	#endif
101990
101991	typedef struct InteriorWriter {
101992	int iHeight; /* from 0 at leaves. */
101993	InteriorBlock first, last;
101994	struct InteriorWriter *parentWriter;
101995
101996	DataBuffer term; /* Last term written to block "last". */
101997	sqlite_int64 iOpeningChildBlock; /* First child block in block "last". */
101998	#ifndef NDEBUG
101999	sqlite_int64 iLastChildBlock; /* for consistency checks. */
102000	#endif
102001	} InteriorWriter;
102002
102003	/* Initialize an interior node where pTerm[nTerm] marks the leftmost
102004	** term in the tree. iChildBlock is the leftmost child block at the
102005	** next level down the tree.
102006	*/
102007	static void interiorWriterInit(int iHeight, const char *pTerm, int nTerm,
102008	sqlite_int64 iChildBlock,
102009	InteriorWriter *pWriter){
102010	InteriorBlock *block;
102011	assert( iHeight>0 );
102012	CLEAR(pWriter);
102013
102014	pWriter->iHeight = iHeight;
102015	pWriter->iOpeningChildBlock = iChildBlock;
102016	#ifndef NDEBUG
102017	pWriter->iLastChildBlock = iChildBlock;
102018	#endif
102019	block = interiorBlockNew(iHeight, iChildBlock, pTerm, nTerm);
102020	pWriter->last = pWriter->first = block;
102021	ASSERT_VALID_INTERIOR_BLOCK(pWriter->last);
102022	dataBufferInit(&pWriter->term, 0);
102023	}
102024
102025	/* Append the child node rooted at iChildBlock to the interior node,
102026	** with pTerm[nTerm] as the leftmost term in iChildBlock's subtree.
102027	*/
102028	static void interiorWriterAppend(InteriorWriter *pWriter,
102029	const char *pTerm, int nTerm,
102030	sqlite_int64 iChildBlock){
102031	char c[VARINT_MAX+VARINT_MAX];
102032	int n, nPrefix = 0;
102033
102034	ASSERT_VALID_INTERIOR_BLOCK(pWriter->last);
102035
102036	/* The first term written into an interior node is actually
102037	** associated with the second child added (the first child was added
102038	** in interiorWriterInit, or in the if clause at the bottom of this
102039	** function). That term gets encoded straight up, with nPrefix left
102040	** at 0.
102041	*/
102042	if( pWriter->term.nData==0 ){
102043	n = fts3PutVarint(c, nTerm);
102044	}else{
102045	while( nPrefix<pWriter->term.nData &&
102046	pTerm[nPrefix]==pWriter->term.pData[nPrefix] ){
102047	nPrefix++;
102048	}
102049
102050	n = fts3PutVarint(c, nPrefix);
102051	n += fts3PutVarint(c+n, nTerm-nPrefix);
102052	}
102053
102054	#ifndef NDEBUG
102055	pWriter->iLastChildBlock++;
102056	#endif
102057	assert( pWriter->iLastChildBlock==iChildBlock );
102058
102059	/* Overflow to a new block if the new term makes the current block
102060	** too big, and the current block already has enough terms.
102061	*/
102062	if( pWriter->last->data.nData+n+nTerm-nPrefix>INTERIOR_MAX &&
102063	iChildBlock-pWriter->iOpeningChildBlock>INTERIOR_MIN_TERMS ){
102064	pWriter->last->next = interiorBlockNew(pWriter->iHeight, iChildBlock,
102065	pTerm, nTerm);
102066	pWriter->last = pWriter->last->next;
102067	pWriter->iOpeningChildBlock = iChildBlock;
102068	dataBufferReset(&pWriter->term);
102069	}else{
102070	dataBufferAppend2(&pWriter->last->data, c, n,
102071	pTerm+nPrefix, nTerm-nPrefix);
102072	dataBufferReplace(&pWriter->term, pTerm, nTerm);
102073	}
102074	ASSERT_VALID_INTERIOR_BLOCK(pWriter->last);
102075	}
102076
102077	/* Free the space used by pWriter, including the linked-list of
102078	** InteriorBlocks, and parentWriter, if present.
102079	*/
102080	static int interiorWriterDestroy(InteriorWriter *pWriter){
102081	InteriorBlock *block = pWriter->first;
102082
102083	while( block!=NULL ){
102084	InteriorBlock *b = block;
102085	block = block->next;
102086	dataBufferDestroy(&b->term);
102087	dataBufferDestroy(&b->data);
102088	sqlite3_free(b);
102089	}
102090	if( pWriter->parentWriter!=NULL ){
102091	interiorWriterDestroy(pWriter->parentWriter);
102092	sqlite3_free(pWriter->parentWriter);
102093	}
102094	dataBufferDestroy(&pWriter->term);
102095	SCRAMBLE(pWriter);
102096	return SQLITE_OK;
102097	}
102098
102099	/* If pWriter can fit entirely in ROOT_MAX, return it as the root info
102100	** directly, leaving *piEndBlockid unchanged. Otherwise, flush
102101	** pWriter to %_segments, building a new layer of interior nodes, and
102102	** recursively ask for their root into.
102103	*/
102104	static int interiorWriterRootInfo(fulltext_vtab v, InteriorWriter pWriter,
102105	char *ppRootInfo, int pnRootInfo,
102106	sqlite_int64 *piEndBlockid){
102107	InteriorBlock *block = pWriter->first;
102108	sqlite_int64 iBlockid = 0;
102109	int rc;
102110
102111	/* If we can fit the segment inline */
102112	if( block==pWriter->last && block->data.nData<ROOT_MAX ){
102113	*ppRootInfo = block->data.pData;
102114	*pnRootInfo = block->data.nData;
102115	return SQLITE_OK;
102116	}
102117
102118	/* Flush the first block to %_segments, and create a new level of
102119	** interior node.
102120	*/
102121	ASSERT_VALID_INTERIOR_BLOCK(block);
102122	rc = block_insert(v, block->data.pData, block->data.nData, &iBlockid);
102123	if( rc!=SQLITE_OK ) return rc;
102124	*piEndBlockid = iBlockid;
102125
102126	pWriter->parentWriter = sqlite3_malloc(sizeof(*pWriter->parentWriter));
102127	interiorWriterInit(pWriter->iHeight+1,
102128	block->term.pData, block->term.nData,
102129	iBlockid, pWriter->parentWriter);
102130
102131	/* Flush additional blocks and append to the higher interior
102132	** node.
102133	*/
102134	for(block=block->next; block!=NULL; block=block->next){
102135	ASSERT_VALID_INTERIOR_BLOCK(block);
102136	rc = block_insert(v, block->data.pData, block->data.nData, &iBlockid);
102137	if( rc!=SQLITE_OK ) return rc;
102138	*piEndBlockid = iBlockid;
102139
102140	interiorWriterAppend(pWriter->parentWriter,
102141	block->term.pData, block->term.nData, iBlockid);
102142	}
102143
102144	/* Parent node gets the chance to be the root. */
102145	return interiorWriterRootInfo(v, pWriter->parentWriter,
102146	ppRootInfo, pnRootInfo, piEndBlockid);
102147	}
102148
102149	/****************************************************************/
102150	/* InteriorReader is used to read off the data from an interior node
102151	** (see comment at top of file for the format).
102152	*/
102153	typedef struct InteriorReader {
102154	const char *pData;
102155	int nData;
102156
102157	DataBuffer term; /* previous term, for decoding term delta. */
102158
102159	sqlite_int64 iBlockid;
102160	} InteriorReader;
102161
102162	static void interiorReaderDestroy(InteriorReader *pReader){
102163	dataBufferDestroy(&pReader->term);
102164	SCRAMBLE(pReader);
102165	}
102166
102167	/* TODO(shess) The assertions are great, but what if we're in NDEBUG
102168	** and the blob is empty or otherwise contains suspect data?
102169	*/
102170	static void interiorReaderInit(const char *pData, int nData,
102171	InteriorReader *pReader){
102172	int n, nTerm;
102173
102174	/* Require at least the leading flag byte */
102175	assert( nData>0 );
102176	assert( pData[0]!='\0' );
102177
102178	CLEAR(pReader);
102179
102180	/* Decode the base blockid, and set the cursor to the first term. */
102181	n = fts3GetVarint(pData+1, &pReader->iBlockid);
102182	assert( 1+n<=nData );
102183	pReader->pData = pData+1+n;
102184	pReader->nData = nData-(1+n);
102185
102186	/* A single-child interior node (such as when a leaf node was too
102187	** large for the segment directory) won't have any terms.
102188	** Otherwise, decode the first term.
102189	*/
102190	if( pReader->nData==0 ){
102191	dataBufferInit(&pReader->term, 0);
102192	}else{
102193	n = fts3GetVarint32(pReader->pData, &nTerm);
102194	dataBufferInit(&pReader->term, nTerm);
102195	dataBufferReplace(&pReader->term, pReader->pData+n, nTerm);
102196	assert( n+nTerm<=pReader->nData );
102197	pReader->pData += n+nTerm;
102198	pReader->nData -= n+nTerm;
102199	}
102200	}
102201
102202	static int interiorReaderAtEnd(InteriorReader *pReader){
102203	return pReader->term.nData==0;
102204	}
102205
102206	static sqlite_int64 interiorReaderCurrentBlockid(InteriorReader *pReader){
102207	return pReader->iBlockid;
102208	}
102209
102210	static int interiorReaderTermBytes(InteriorReader *pReader){
102211	assert( !interiorReaderAtEnd(pReader) );
102212	return pReader->term.nData;
102213	}
102214	static const char interiorReaderTerm(InteriorReader pReader){
102215	assert( !interiorReaderAtEnd(pReader) );
102216	return pReader->term.pData;
102217	}
102218
102219	/* Step forward to the next term in the node. */
102220	static void interiorReaderStep(InteriorReader *pReader){
102221	assert( !interiorReaderAtEnd(pReader) );
102222
102223	/* If the last term has been read, signal eof, else construct the
102224	** next term.
102225	*/
102226	if( pReader->nData==0 ){
102227	dataBufferReset(&pReader->term);
102228	}else{
102229	int n, nPrefix, nSuffix;
102230
102231	n = fts3GetVarint32(pReader->pData, &nPrefix);
102232	n += fts3GetVarint32(pReader->pData+n, &nSuffix);
102233
102234	/* Truncate the current term and append suffix data. */
102235	pReader->term.nData = nPrefix;
102236	dataBufferAppend(&pReader->term, pReader->pData+n, nSuffix);
102237
102238	assert( n+nSuffix<=pReader->nData );
102239	pReader->pData += n+nSuffix;
102240	pReader->nData -= n+nSuffix;
102241	}
102242	pReader->iBlockid++;
102243	}
102244
102245	/* Compare the current term to pTerm[nTerm], returning strcmp-style
102246	** results. If isPrefix, equality means equal through nTerm bytes.
102247	*/
102248	static int interiorReaderTermCmp(InteriorReader *pReader,
102249	const char *pTerm, int nTerm, int isPrefix){
102250	const char *pReaderTerm = interiorReaderTerm(pReader);
102251	int nReaderTerm = interiorReaderTermBytes(pReader);
102252	int c, n = nReaderTerm<nTerm ? nReaderTerm : nTerm;
102253
102254	if( n==0 ){
102255	if( nReaderTerm>0 ) return -1;
102256	if( nTerm>0 ) return 1;
102257	return 0;
102258	}
102259
102260	c = memcmp(pReaderTerm, pTerm, n);
102261	if( c!=0 ) return c;
102262	if( isPrefix && n==nTerm ) return 0;
102263	return nReaderTerm - nTerm;
102264	}
102265
102266	/****************************************************************/
102267	/* LeafWriter is used to collect terms and associated doclist data
102268	** into leaf blocks in %_segments (see top of file for format info).
102269	** Expected usage is:
102270	**
102271	** LeafWriter writer;
102272	** leafWriterInit(0, 0, &writer);
102273	** while( sorted_terms_left_to_process ){
102274	** // data is doclist data for that term.
102275	** rc = leafWriterStep(v, &writer, pTerm, nTerm, pData, nData);
102276	** if( rc!=SQLITE_OK ) goto err;
102277	** }
102278	** rc = leafWriterFinalize(v, &writer);
102279	**err:
102280	** leafWriterDestroy(&writer);
102281	** return rc;
102282	**
102283	** leafWriterStep() may write a collected leaf out to %_segments.
102284	** leafWriterFinalize() finishes writing any buffered data and stores
102285	** a root node in %_segdir. leafWriterDestroy() frees all buffers and
102286	** InteriorWriters allocated as part of writing this segment.
102287	**
102288	** TODO(shess) Document leafWriterStepMerge().
102289	*/
102290
102291	/* Put terms with data this big in their own block. */
102292	#define STANDALONE_MIN 1024
102293
102294	/* Keep leaf blocks below this size. */
102295	#define LEAF_MAX 2048
102296
102297	typedef struct LeafWriter {
102298	int iLevel;
102299	int idx;
102300	sqlite_int64 iStartBlockid; /* needed to create the root info */
102301	sqlite_int64 iEndBlockid; /* when we're done writing. */
102302
102303	DataBuffer term; /* previous encoded term */
102304	DataBuffer data; /* encoding buffer */
102305
102306	/* bytes of first term in the current node which distinguishes that
102307	** term from the last term of the previous node.
102308	*/
102309	int nTermDistinct;
102310
102311	InteriorWriter parentWriter; /* if we overflow */
102312	int has_parent;
102313	} LeafWriter;
102314
102315	static void leafWriterInit(int iLevel, int idx, LeafWriter *pWriter){
102316	CLEAR(pWriter);
102317	pWriter->iLevel = iLevel;
102318	pWriter->idx = idx;
102319
102320	dataBufferInit(&pWriter->term, 32);
102321
102322	/* Start out with a reasonably sized block, though it can grow. */
102323	dataBufferInit(&pWriter->data, LEAF_MAX);
102324	}
102325
102326	#ifndef NDEBUG
102327	/* Verify that the data is readable as a leaf node. */
102328	static void leafNodeValidate(const char *pData, int nData){
102329	int n, iDummy;
102330
102331	if( nData==0 ) return;
102332	assert( nData>0 );
102333	assert( pData!=0 );
102334	assert( pData+nData>pData );
102335
102336	/* Must lead with a varint(0) */
102337	n = fts3GetVarint32(pData, &iDummy);
102338	assert( iDummy==0 );
102339	assert( n>0 );
102340	assert( n<nData );
102341	pData += n;
102342	nData -= n;
102343
102344	/* Leading term length and data must fit in buffer. */
102345	n = fts3GetVarint32(pData, &iDummy);
102346	assert( n>0 );
102347	assert( iDummy>0 );
102348	assert( n+iDummy>0 );
102349	assert( n+iDummy<nData );
102350	pData += n+iDummy;
102351	nData -= n+iDummy;
102352
102353	/* Leading term's doclist length and data must fit. */
102354	n = fts3GetVarint32(pData, &iDummy);
102355	assert( n>0 );
102356	assert( iDummy>0 );
102357	assert( n+iDummy>0 );
102358	assert( n+iDummy<=nData );
102359	ASSERT_VALID_DOCLIST(DL_DEFAULT, pData+n, iDummy, NULL);
102360	pData += n+iDummy;
102361	nData -= n+iDummy;
102362
102363	/* Verify that trailing terms and doclists also are readable. */
102364	while( nData!=0 ){
102365	n = fts3GetVarint32(pData, &iDummy);
102366	assert( n>0 );
102367	assert( iDummy>=0 );
102368	assert( n<nData );
102369	pData += n;
102370	nData -= n;
102371	n = fts3GetVarint32(pData, &iDummy);
102372	assert( n>0 );
102373	assert( iDummy>0 );
102374	assert( n+iDummy>0 );
102375	assert( n+iDummy<nData );
102376	pData += n+iDummy;
102377	nData -= n+iDummy;
102378
102379	n = fts3GetVarint32(pData, &iDummy);
102380	assert( n>0 );
102381	assert( iDummy>0 );
102382	assert( n+iDummy>0 );
102383	assert( n+iDummy<=nData );
102384	ASSERT_VALID_DOCLIST(DL_DEFAULT, pData+n, iDummy, NULL);
102385	pData += n+iDummy;
102386	nData -= n+iDummy;
102387	}
102388	}
102389	#define ASSERT_VALID_LEAF_NODE(p, n) leafNodeValidate(p, n)
102390	#else
102391	#define ASSERT_VALID_LEAF_NODE(p, n) assert( 1 )
102392	#endif
102393
102394	/* Flush the current leaf node to %_segments, and adding the resulting
102395	** blockid and the starting term to the interior node which will
102396	** contain it.
102397	*/
102398	static int leafWriterInternalFlush(fulltext_vtab v, LeafWriter pWriter,
102399	int iData, int nData){
102400	sqlite_int64 iBlockid = 0;
102401	const char *pStartingTerm;
102402	int nStartingTerm, rc, n;
102403
102404	/* Must have the leading varint(0) flag, plus at least some
102405	** valid-looking data.
102406	*/
102407	assert( nData>2 );
102408	assert( iData>=0 );
102409	assert( iData+nData<=pWriter->data.nData );
102410	ASSERT_VALID_LEAF_NODE(pWriter->data.pData+iData, nData);
102411
102412	rc = block_insert(v, pWriter->data.pData+iData, nData, &iBlockid);
102413	if( rc!=SQLITE_OK ) return rc;
102414	assert( iBlockid!=0 );
102415
102416	/* Reconstruct the first term in the leaf for purposes of building
102417	** the interior node.
102418	*/
102419	n = fts3GetVarint32(pWriter->data.pData+iData+1, &nStartingTerm);
102420	pStartingTerm = pWriter->data.pData+iData+1+n;
102421	assert( pWriter->data.nData>iData+1+n+nStartingTerm );
102422	assert( pWriter->nTermDistinct>0 );
102423	assert( pWriter->nTermDistinct<=nStartingTerm );
102424	nStartingTerm = pWriter->nTermDistinct;
102425
102426	if( pWriter->has_parent ){
102427	interiorWriterAppend(&pWriter->parentWriter,
102428	pStartingTerm, nStartingTerm, iBlockid);
102429	}else{
102430	interiorWriterInit(1, pStartingTerm, nStartingTerm, iBlockid,
102431	&pWriter->parentWriter);
102432	pWriter->has_parent = 1;
102433	}
102434
102435	/* Track the span of this segment's leaf nodes. */
102436	if( pWriter->iEndBlockid==0 ){
102437	pWriter->iEndBlockid = pWriter->iStartBlockid = iBlockid;
102438	}else{
102439	pWriter->iEndBlockid++;
102440	assert( iBlockid==pWriter->iEndBlockid );
102441	}
102442
102443	return SQLITE_OK;
102444	}
102445	static int leafWriterFlush(fulltext_vtab v, LeafWriter pWriter){
102446	int rc = leafWriterInternalFlush(v, pWriter, 0, pWriter->data.nData);
102447	if( rc!=SQLITE_OK ) return rc;
102448
102449	/* Re-initialize the output buffer. */
102450	dataBufferReset(&pWriter->data);
102451
102452	return SQLITE_OK;
102453	}
102454
102455	/* Fetch the root info for the segment. If the entire leaf fits
102456	** within ROOT_MAX, then it will be returned directly, otherwise it
102457	** will be flushed and the root info will be returned from the
102458	** interior node. *piEndBlockid is set to the blockid of the last
102459	** interior or leaf node written to disk (0 if none are written at
102460	** all).
102461	*/
102462	static int leafWriterRootInfo(fulltext_vtab v, LeafWriter pWriter,
102463	char *ppRootInfo, int pnRootInfo,
102464	sqlite_int64 *piEndBlockid){
102465	/* we can fit the segment entirely inline */
102466	if( !pWriter->has_parent && pWriter->data.nData<ROOT_MAX ){
102467	*ppRootInfo = pWriter->data.pData;
102468	*pnRootInfo = pWriter->data.nData;
102469	*piEndBlockid = 0;
102470	return SQLITE_OK;
102471	}
102472
102473	/* Flush remaining leaf data. */
102474	if( pWriter->data.nData>0 ){
102475	int rc = leafWriterFlush(v, pWriter);
102476	if( rc!=SQLITE_OK ) return rc;
102477	}
102478
102479	/* We must have flushed a leaf at some point. */
102480	assert( pWriter->has_parent );
102481
102482	/* Tenatively set the end leaf blockid as the end blockid. If the
102483	** interior node can be returned inline, this will be the final
102484	** blockid, otherwise it will be overwritten by
102485	** interiorWriterRootInfo().
102486	*/
102487	*piEndBlockid = pWriter->iEndBlockid;
102488
102489	return interiorWriterRootInfo(v, &pWriter->parentWriter,
102490	ppRootInfo, pnRootInfo, piEndBlockid);
102491	}
102492
102493	/* Collect the rootInfo data and store it into the segment directory.
102494	** This has the effect of flushing the segment's leaf data to
102495	** %_segments, and also flushing any interior nodes to %_segments.
102496	*/
102497	static int leafWriterFinalize(fulltext_vtab v, LeafWriter pWriter){
102498	sqlite_int64 iEndBlockid;
102499	char *pRootInfo;
102500	int rc, nRootInfo;
102501
102502	rc = leafWriterRootInfo(v, pWriter, &pRootInfo, &nRootInfo, &iEndBlockid);
102503	if( rc!=SQLITE_OK ) return rc;
102504
102505	/* Don't bother storing an entirely empty segment. */
102506	if( iEndBlockid==0 && nRootInfo==0 ) return SQLITE_OK;
102507
102508	return segdir_set(v, pWriter->iLevel, pWriter->idx,
102509	pWriter->iStartBlockid, pWriter->iEndBlockid,
102510	iEndBlockid, pRootInfo, nRootInfo);
102511	}
102512
102513	static void leafWriterDestroy(LeafWriter *pWriter){
102514	if( pWriter->has_parent ) interiorWriterDestroy(&pWriter->parentWriter);
102515	dataBufferDestroy(&pWriter->term);
102516	dataBufferDestroy(&pWriter->data);
102517	}
102518
102519	/* Encode a term into the leafWriter, delta-encoding as appropriate.
102520	** Returns the length of the new term which distinguishes it from the
102521	** previous term, which can be used to set nTermDistinct when a node
102522	** boundary is crossed.
102523	*/
102524	static int leafWriterEncodeTerm(LeafWriter *pWriter,
102525	const char *pTerm, int nTerm){
102526	char c[VARINT_MAX+VARINT_MAX];
102527	int n, nPrefix = 0;
102528
102529	assert( nTerm>0 );
102530	while( nPrefix<pWriter->term.nData &&
102531	pTerm[nPrefix]==pWriter->term.pData[nPrefix] ){
102532	nPrefix++;
102533	/* Failing this implies that the terms weren't in order. */
102534	assert( nPrefix<nTerm );
102535	}
102536
102537	if( pWriter->data.nData==0 ){
102538	/* Encode the node header and leading term as:
102539	** varint(0)
102540	** varint(nTerm)
102541	** char pTerm[nTerm]
102542	*/
102543	n = fts3PutVarint(c, '\0');
102544	n += fts3PutVarint(c+n, nTerm);
102545	dataBufferAppend2(&pWriter->data, c, n, pTerm, nTerm);
102546	}else{
102547	/* Delta-encode the term as:
102548	** varint(nPrefix)
102549	** varint(nSuffix)
102550	** char pTermSuffix[nSuffix]
102551	*/
102552	n = fts3PutVarint(c, nPrefix);
102553	n += fts3PutVarint(c+n, nTerm-nPrefix);
102554	dataBufferAppend2(&pWriter->data, c, n, pTerm+nPrefix, nTerm-nPrefix);
102555	}
102556	dataBufferReplace(&pWriter->term, pTerm, nTerm);
102557
102558	return nPrefix+1;
102559	}
102560
102561	/* Used to avoid a memmove when a large amount of doclist data is in
102562	** the buffer. This constructs a node and term header before
102563	** iDoclistData and flushes the resulting complete node using
102564	** leafWriterInternalFlush().
102565	*/
102566	static int leafWriterInlineFlush(fulltext_vtab v, LeafWriter pWriter,
102567	const char *pTerm, int nTerm,
102568	int iDoclistData){
102569	char c[VARINT_MAX+VARINT_MAX];
102570	int iData, n = fts3PutVarint(c, 0);
102571	n += fts3PutVarint(c+n, nTerm);
102572
102573	/* There should always be room for the header. Even if pTerm shared
102574	** a substantial prefix with the previous term, the entire prefix
102575	** could be constructed from earlier data in the doclist, so there
102576	** should be room.
102577	*/
102578	assert( iDoclistData>=n+nTerm );
102579
102580	iData = iDoclistData-(n+nTerm);
102581	memcpy(pWriter->data.pData+iData, c, n);
102582	memcpy(pWriter->data.pData+iData+n, pTerm, nTerm);
102583
102584	return leafWriterInternalFlush(v, pWriter, iData, pWriter->data.nData-iData);
102585	}
102586
102587	/* Push pTerm[nTerm] along with the doclist data to the leaf layer of
102588	** %_segments.
102589	*/
102590	static int leafWriterStepMerge(fulltext_vtab v, LeafWriter pWriter,
102591	const char *pTerm, int nTerm,
102592	DLReader *pReaders, int nReaders){
102593	char c[VARINT_MAX+VARINT_MAX];
102594	int iTermData = pWriter->data.nData, iDoclistData;
102595	int i, nData, n, nActualData, nActual, rc, nTermDistinct;
102596
102597	ASSERT_VALID_LEAF_NODE(pWriter->data.pData, pWriter->data.nData);
102598	nTermDistinct = leafWriterEncodeTerm(pWriter, pTerm, nTerm);
102599
102600	/* Remember nTermDistinct if opening a new node. */
102601	if( iTermData==0 ) pWriter->nTermDistinct = nTermDistinct;
102602
102603	iDoclistData = pWriter->data.nData;
102604
102605	/* Estimate the length of the merged doclist so we can leave space
102606	** to encode it.
102607	*/
102608	for(i=0, nData=0; i<nReaders; i++){
102609	nData += dlrAllDataBytes(&pReaders[i]);
102610	}
102611	n = fts3PutVarint(c, nData);
102612	dataBufferAppend(&pWriter->data, c, n);
102613
102614	docListMerge(&pWriter->data, pReaders, nReaders);
102615	ASSERT_VALID_DOCLIST(DL_DEFAULT,
102616	pWriter->data.pData+iDoclistData+n,
102617	pWriter->data.nData-iDoclistData-n, NULL);
102618
102619	/* The actual amount of doclist data at this point could be smaller
102620	** than the length we encoded. Additionally, the space required to
102621	** encode this length could be smaller. For small doclists, this is
102622	** not a big deal, we can just use memmove() to adjust things.
102623	*/
102624	nActualData = pWriter->data.nData-(iDoclistData+n);
102625	nActual = fts3PutVarint(c, nActualData);
102626	assert( nActualData<=nData );
102627	assert( nActual<=n );
102628
102629	/* If the new doclist is big enough for force a standalone leaf
102630	** node, we can immediately flush it inline without doing the
102631	** memmove().
102632	*/
102633	/* TODO(shess) This test matches leafWriterStep(), which does this
102634	** test before it knows the cost to varint-encode the term and
102635	** doclist lengths. At some point, change to
102636	** pWriter->data.nData-iTermData>STANDALONE_MIN.
102637	*/
102638	if( nTerm+nActualData>STANDALONE_MIN ){
102639	/* Push leaf node from before this term. */
102640	if( iTermData>0 ){
102641	rc = leafWriterInternalFlush(v, pWriter, 0, iTermData);
102642	if( rc!=SQLITE_OK ) return rc;
102643
102644	pWriter->nTermDistinct = nTermDistinct;
102645	}
102646
102647	/* Fix the encoded doclist length. */
102648	iDoclistData += n - nActual;
102649	memcpy(pWriter->data.pData+iDoclistData, c, nActual);
102650
102651	/* Push the standalone leaf node. */
102652	rc = leafWriterInlineFlush(v, pWriter, pTerm, nTerm, iDoclistData);
102653	if( rc!=SQLITE_OK ) return rc;
102654
102655	/* Leave the node empty. */
102656	dataBufferReset(&pWriter->data);
102657
102658	return rc;
102659	}
102660
102661	/* At this point, we know that the doclist was small, so do the
102662	** memmove if indicated.
102663	*/
102664	if( nActual<n ){
102665	memmove(pWriter->data.pData+iDoclistData+nActual,
102666	pWriter->data.pData+iDoclistData+n,
102667	pWriter->data.nData-(iDoclistData+n));
102668	pWriter->data.nData -= n-nActual;
102669	}
102670
102671	/* Replace written length with actual length. */
102672	memcpy(pWriter->data.pData+iDoclistData, c, nActual);
102673
102674	/* If the node is too large, break things up. */
102675	/* TODO(shess) This test matches leafWriterStep(), which does this
102676	** test before it knows the cost to varint-encode the term and
102677	** doclist lengths. At some point, change to
102678	** pWriter->data.nData>LEAF_MAX.
102679	*/
102680	if( iTermData+nTerm+nActualData>LEAF_MAX ){
102681	/* Flush out the leading data as a node */
102682	rc = leafWriterInternalFlush(v, pWriter, 0, iTermData);
102683	if( rc!=SQLITE_OK ) return rc;
102684
102685	pWriter->nTermDistinct = nTermDistinct;
102686
102687	/* Rebuild header using the current term */
102688	n = fts3PutVarint(pWriter->data.pData, 0);
102689	n += fts3PutVarint(pWriter->data.pData+n, nTerm);
102690	memcpy(pWriter->data.pData+n, pTerm, nTerm);
102691	n += nTerm;
102692
102693	/* There should always be room, because the previous encoding
102694	** included all data necessary to construct the term.
102695	*/
102696	assert( n<iDoclistData );
102697	/* So long as STANDALONE_MIN is half or less of LEAF_MAX, the
102698	** following memcpy() is safe (as opposed to needing a memmove).
102699	*/
102700	assert( 2*STANDALONE_MIN<=LEAF_MAX );
102701	assert( n+pWriter->data.nData-iDoclistData<iDoclistData );
102702	memcpy(pWriter->data.pData+n,
102703	pWriter->data.pData+iDoclistData,
102704	pWriter->data.nData-iDoclistData);
102705	pWriter->data.nData -= iDoclistData-n;
102706	}
102707	ASSERT_VALID_LEAF_NODE(pWriter->data.pData, pWriter->data.nData);
102708
102709	return SQLITE_OK;
102710	}
102711
102712	/* Push pTerm[nTerm] along with the doclist data to the leaf layer of
102713	** %_segments.
102714	*/
102715	/* TODO(shess) Revise writeZeroSegment() so that doclists are
102716	** constructed directly in pWriter->data.
102717	*/
102718	static int leafWriterStep(fulltext_vtab v, LeafWriter pWriter,
102719	const char *pTerm, int nTerm,
102720	const char *pData, int nData){
102721	int rc;
102722	DLReader reader;
102723
102724	dlrInit(&reader, DL_DEFAULT, pData, nData);
102725	rc = leafWriterStepMerge(v, pWriter, pTerm, nTerm, &reader, 1);
102726	dlrDestroy(&reader);
102727
102728	return rc;
102729	}
102730
102731
102732	/****************************************************************/
102733	/* LeafReader is used to iterate over an individual leaf node. */
102734	typedef struct LeafReader {
102735	DataBuffer term; /* copy of current term. */
102736
102737	const char pData; / data for current term. */
102738	int nData;
102739	} LeafReader;
102740
102741	static void leafReaderDestroy(LeafReader *pReader){
102742	dataBufferDestroy(&pReader->term);
102743	SCRAMBLE(pReader);
102744	}
102745
102746	static int leafReaderAtEnd(LeafReader *pReader){
102747	return pReader->nData<=0;
102748	}
102749
102750	/* Access the current term. */
102751	static int leafReaderTermBytes(LeafReader *pReader){
102752	return pReader->term.nData;
102753	}
102754	static const char leafReaderTerm(LeafReader pReader){
102755	assert( pReader->term.nData>0 );
102756	return pReader->term.pData;
102757	}
102758
102759	/* Access the doclist data for the current term. */
102760	static int leafReaderDataBytes(LeafReader *pReader){
102761	int nData;
102762	assert( pReader->term.nData>0 );
102763	fts3GetVarint32(pReader->pData, &nData);
102764	return nData;
102765	}
102766	static const char leafReaderData(LeafReader pReader){
102767	int n, nData;
102768	assert( pReader->term.nData>0 );
102769	n = fts3GetVarint32(pReader->pData, &nData);
102770	return pReader->pData+n;
102771	}
102772
102773	static void leafReaderInit(const char *pData, int nData,
102774	LeafReader *pReader){
102775	int nTerm, n;
102776
102777	assert( nData>0 );
102778	assert( pData[0]=='\0' );
102779
102780	CLEAR(pReader);
102781
102782	/* Read the first term, skipping the header byte. */
102783	n = fts3GetVarint32(pData+1, &nTerm);
102784	dataBufferInit(&pReader->term, nTerm);
102785	dataBufferReplace(&pReader->term, pData+1+n, nTerm);
102786
102787	/* Position after the first term. */
102788	assert( 1+n+nTerm<nData );
102789	pReader->pData = pData+1+n+nTerm;
102790	pReader->nData = nData-1-n-nTerm;
102791	}
102792
102793	/* Step the reader forward to the next term. */
102794	static void leafReaderStep(LeafReader *pReader){
102795	int n, nData, nPrefix, nSuffix;
102796	assert( !leafReaderAtEnd(pReader) );
102797
102798	/* Skip previous entry's data block. */
102799	n = fts3GetVarint32(pReader->pData, &nData);
102800	assert( n+nData<=pReader->nData );
102801	pReader->pData += n+nData;
102802	pReader->nData -= n+nData;
102803
102804	if( !leafReaderAtEnd(pReader) ){
102805	/* Construct the new term using a prefix from the old term plus a
102806	** suffix from the leaf data.
102807	*/
102808	n = fts3GetVarint32(pReader->pData, &nPrefix);
102809	n += fts3GetVarint32(pReader->pData+n, &nSuffix);
102810	assert( n+nSuffix<pReader->nData );
102811	pReader->term.nData = nPrefix;
102812	dataBufferAppend(&pReader->term, pReader->pData+n, nSuffix);
102813
102814	pReader->pData += n+nSuffix;
102815	pReader->nData -= n+nSuffix;
102816	}
102817	}
102818
102819	/* strcmp-style comparison of pReader's current term against pTerm.
102820	** If isPrefix, equality means equal through nTerm bytes.
102821	*/
102822	static int leafReaderTermCmp(LeafReader *pReader,
102823	const char *pTerm, int nTerm, int isPrefix){
102824	int c, n = pReader->term.nData<nTerm ? pReader->term.nData : nTerm;
102825	if( n==0 ){
102826	if( pReader->term.nData>0 ) return -1;
102827	if(nTerm>0 ) return 1;
102828	return 0;
102829	}
102830
102831	c = memcmp(pReader->term.pData, pTerm, n);
102832	if( c!=0 ) return c;
102833	if( isPrefix && n==nTerm ) return 0;
102834	return pReader->term.nData - nTerm;
102835	}
102836
102837
102838	/****************************************************************/
102839	/* LeavesReader wraps LeafReader to allow iterating over the entire
102840	** leaf layer of the tree.
102841	*/
102842	typedef struct LeavesReader {
102843	int idx; /* Index within the segment. */
102844
102845	sqlite3_stmt pStmt; / Statement we're streaming leaves from. */
102846	int eof; /* we've seen SQLITE_DONE from pStmt. */
102847
102848	LeafReader leafReader; /* reader for the current leaf. */
102849	DataBuffer rootData; /* root data for inline. */
102850	} LeavesReader;
102851
102852	/* Access the current term. */
102853	static int leavesReaderTermBytes(LeavesReader *pReader){
102854	assert( !pReader->eof );
102855	return leafReaderTermBytes(&pReader->leafReader);
102856	}
102857	static const char leavesReaderTerm(LeavesReader pReader){
102858	assert( !pReader->eof );
102859	return leafReaderTerm(&pReader->leafReader);
102860	}
102861
102862	/* Access the doclist data for the current term. */
102863	static int leavesReaderDataBytes(LeavesReader *pReader){
102864	assert( !pReader->eof );
102865	return leafReaderDataBytes(&pReader->leafReader);
102866	}
102867	static const char leavesReaderData(LeavesReader pReader){
102868	assert( !pReader->eof );
102869	return leafReaderData(&pReader->leafReader);
102870	}
102871
102872	static int leavesReaderAtEnd(LeavesReader *pReader){
102873	return pReader->eof;
102874	}
102875
102876	/* loadSegmentLeaves() may not read all the way to SQLITE_DONE, thus
102877	** leaving the statement handle open, which locks the table.
102878	*/
102879	/* TODO(shess) This "solution" is not satisfactory. Really, there
102880	** should be check-in function for all statement handles which
102881	** arranges to call sqlite3_reset(). This most likely will require
102882	** modification to control flow all over the place, though, so for now
102883	** just punt.
102884	**
102885	** Note the the current system assumes that segment merges will run to
102886	** completion, which is why this particular probably hasn't arisen in
102887	** this case. Probably a brittle assumption.
102888	*/
102889	static int leavesReaderReset(LeavesReader *pReader){
102890	return sqlite3_reset(pReader->pStmt);
102891	}
102892
102893	static void leavesReaderDestroy(LeavesReader *pReader){
102894	/* If idx is -1, that means we're using a non-cached statement
102895	** handle in the optimize() case, so we need to release it.
102896	*/
102897	if( pReader->pStmt!=NULL && pReader->idx==-1 ){
102898	sqlite3_finalize(pReader->pStmt);
102899	}
102900	leafReaderDestroy(&pReader->leafReader);
102901	dataBufferDestroy(&pReader->rootData);
102902	SCRAMBLE(pReader);
102903	}
102904
102905	/* Initialize pReader with the given root data (if iStartBlockid==0
102906	** the leaf data was entirely contained in the root), or from the
102907	** stream of blocks between iStartBlockid and iEndBlockid, inclusive.
102908	*/
102909	static int leavesReaderInit(fulltext_vtab *v,
102910	int idx,
102911	sqlite_int64 iStartBlockid,
102912	sqlite_int64 iEndBlockid,
102913	const char *pRootData, int nRootData,
102914	LeavesReader *pReader){
102915	CLEAR(pReader);
102916	pReader->idx = idx;
102917
102918	dataBufferInit(&pReader->rootData, 0);
102919	if( iStartBlockid==0 ){
102920	/* Entire leaf level fit in root data. */
102921	dataBufferReplace(&pReader->rootData, pRootData, nRootData);
102922	leafReaderInit(pReader->rootData.pData, pReader->rootData.nData,
102923	&pReader->leafReader);
102924	}else{
102925	sqlite3_stmt *s;
102926	int rc = sql_get_leaf_statement(v, idx, &s);
102927	if( rc!=SQLITE_OK ) return rc;
102928
102929	rc = sqlite3_bind_int64(s, 1, iStartBlockid);
102930	if( rc!=SQLITE_OK ) return rc;
102931
102932	rc = sqlite3_bind_int64(s, 2, iEndBlockid);
102933	if( rc!=SQLITE_OK ) return rc;
102934
102935	rc = sqlite3_step(s);
102936	if( rc==SQLITE_DONE ){
102937	pReader->eof = 1;
102938	return SQLITE_OK;
102939	}
102940	if( rc!=SQLITE_ROW ) return rc;
102941
102942	pReader->pStmt = s;
102943	leafReaderInit(sqlite3_column_blob(pReader->pStmt, 0),
102944	sqlite3_column_bytes(pReader->pStmt, 0),
102945	&pReader->leafReader);
102946	}
102947	return SQLITE_OK;
102948	}
102949
102950	/* Step the current leaf forward to the next term. If we reach the
102951	** end of the current leaf, step forward to the next leaf block.
102952	*/
102953	static int leavesReaderStep(fulltext_vtab v, LeavesReader pReader){
102954	assert( !leavesReaderAtEnd(pReader) );
102955	leafReaderStep(&pReader->leafReader);
102956
102957	if( leafReaderAtEnd(&pReader->leafReader) ){
102958	int rc;
102959	if( pReader->rootData.pData ){
102960	pReader->eof = 1;
102961	return SQLITE_OK;
102962	}
102963	rc = sqlite3_step(pReader->pStmt);
102964	if( rc!=SQLITE_ROW ){
102965	pReader->eof = 1;
102966	return rc==SQLITE_DONE ? SQLITE_OK : rc;
102967	}
102968	leafReaderDestroy(&pReader->leafReader);
102969	leafReaderInit(sqlite3_column_blob(pReader->pStmt, 0),
102970	sqlite3_column_bytes(pReader->pStmt, 0),
102971	&pReader->leafReader);
102972	}
102973	return SQLITE_OK;
102974	}
102975
102976	/* Order LeavesReaders by their term, ignoring idx. Readers at eof
102977	** always sort to the end.
102978	*/
102979	static int leavesReaderTermCmp(LeavesReader lr1, LeavesReader lr2){
102980	if( leavesReaderAtEnd(lr1) ){
102981	if( leavesReaderAtEnd(lr2) ) return 0;
102982	return 1;
102983	}
102984	if( leavesReaderAtEnd(lr2) ) return -1;
102985
102986	return leafReaderTermCmp(&lr1->leafReader,
102987	leavesReaderTerm(lr2), leavesReaderTermBytes(lr2),
102988	0);
102989	}
102990
102991	/* Similar to leavesReaderTermCmp(), with additional ordering by idx
102992	** so that older segments sort before newer segments.
102993	*/
102994	static int leavesReaderCmp(LeavesReader lr1, LeavesReader lr2){
102995	int c = leavesReaderTermCmp(lr1, lr2);
102996	if( c!=0 ) return c;
102997	return lr1->idx-lr2->idx;
102998	}
102999
103000	/* Assume that pLr[1]..pLr[nLr] are sorted. Bubble pLr[0] into its
103001	** sorted position.
103002	*/
103003	static void leavesReaderReorder(LeavesReader *pLr, int nLr){
103004	while( nLr>1 && leavesReaderCmp(pLr, pLr+1)>0 ){
103005	LeavesReader tmp = pLr[0];
103006	pLr[0] = pLr[1];
103007	pLr[1] = tmp;
103008	nLr--;
103009	pLr++;
103010	}
103011	}
103012
103013	/* Initializes pReaders with the segments from level iLevel, returning
103014	** the number of segments in *piReaders. Leaves pReaders in sorted
103015	** order.
103016	*/
103017	static int leavesReadersInit(fulltext_vtab *v, int iLevel,
103018	LeavesReader pReaders, int piReaders){
103019	sqlite3_stmt *s;
103020	int i, rc = sql_get_statement(v, SEGDIR_SELECT_LEVEL_STMT, &s);
103021	if( rc!=SQLITE_OK ) return rc;
103022
103023	rc = sqlite3_bind_int(s, 1, iLevel);
103024	if( rc!=SQLITE_OK ) return rc;
103025
103026	i = 0;
103027	while( (rc = sqlite3_step(s))==SQLITE_ROW ){
103028	sqlite_int64 iStart = sqlite3_column_int64(s, 0);
103029	sqlite_int64 iEnd = sqlite3_column_int64(s, 1);
103030	const char *pRootData = sqlite3_column_blob(s, 2);
103031	int nRootData = sqlite3_column_bytes(s, 2);
103032
103033	assert( i<MERGE_COUNT );
103034	rc = leavesReaderInit(v, i, iStart, iEnd, pRootData, nRootData,
103035	&pReaders[i]);
103036	if( rc!=SQLITE_OK ) break;
103037
103038	i++;
103039	}
103040	if( rc!=SQLITE_DONE ){
103041	while( i-->0 ){
103042	leavesReaderDestroy(&pReaders[i]);
103043	}
103044	return rc;
103045	}
103046
103047	*piReaders = i;
103048
103049	/* Leave our results sorted by term, then age. */
103050	while( i-- ){
103051	leavesReaderReorder(pReaders+i, *piReaders-i);
103052	}
103053	return SQLITE_OK;
103054	}
103055
103056	/* Merge doclists from pReaders[nReaders] into a single doclist, which
103057	** is written to pWriter. Assumes pReaders is ordered oldest to
103058	** newest.
103059	*/
103060	/* TODO(shess) Consider putting this inline in segmentMerge(). */
103061	static int leavesReadersMerge(fulltext_vtab *v,
103062	LeavesReader *pReaders, int nReaders,
103063	LeafWriter *pWriter){
103064	DLReader dlReaders[MERGE_COUNT];
103065	const char *pTerm = leavesReaderTerm(pReaders);
103066	int i, nTerm = leavesReaderTermBytes(pReaders);
103067
103068	assert( nReaders<=MERGE_COUNT );
103069
103070	for(i=0; i<nReaders; i++){
103071	dlrInit(&dlReaders[i], DL_DEFAULT,
103072	leavesReaderData(pReaders+i),
103073	leavesReaderDataBytes(pReaders+i));
103074	}
103075
103076	return leafWriterStepMerge(v, pWriter, pTerm, nTerm, dlReaders, nReaders);
103077	}
103078
103079	/* Forward ref due to mutual recursion with segdirNextIndex(). */
103080	static int segmentMerge(fulltext_vtab *v, int iLevel);
103081
103082	/* Put the next available index at iLevel into *pidx. If iLevel
103083	** already has MERGE_COUNT segments, they are merged to a higher
103084	** level to make room.
103085	*/
103086	static int segdirNextIndex(fulltext_vtab v, int iLevel, int pidx){
103087	int rc = segdir_max_index(v, iLevel, pidx);
103088	if( rc==SQLITE_DONE ){ /* No segments at iLevel. */
103089	*pidx = 0;
103090	}else if( rc==SQLITE_ROW ){
103091	if( *pidx==(MERGE_COUNT-1) ){
103092	rc = segmentMerge(v, iLevel);
103093	if( rc!=SQLITE_OK ) return rc;
103094	*pidx = 0;
103095	}else{
103096	(*pidx)++;
103097	}
103098	}else{
103099	return rc;
103100	}
103101	return SQLITE_OK;
103102	}
103103
103104	/* Merge MERGE_COUNT segments at iLevel into a new segment at
103105	** iLevel+1. If iLevel+1 is already full of segments, those will be
103106	** merged to make room.
103107	*/
103108	static int segmentMerge(fulltext_vtab *v, int iLevel){
103109	LeafWriter writer;
103110	LeavesReader lrs[MERGE_COUNT];
103111	int i, rc, idx = 0;
103112
103113	/* Determine the next available segment index at the next level,
103114	** merging as necessary.
103115	*/
103116	rc = segdirNextIndex(v, iLevel+1, &idx);
103117	if( rc!=SQLITE_OK ) return rc;
103118
103119	/* TODO(shess) This assumes that we'll always see exactly
103120	** MERGE_COUNT segments to merge at a given level. That will be
103121	** broken if we allow the developer to request preemptive or
103122	** deferred merging.
103123	*/
103124	memset(&lrs, '\0', sizeof(lrs));
103125	rc = leavesReadersInit(v, iLevel, lrs, &i);
103126	if( rc!=SQLITE_OK ) return rc;
103127	assert( i==MERGE_COUNT );
103128
103129	leafWriterInit(iLevel+1, idx, &writer);
103130
103131	/* Since leavesReaderReorder() pushes readers at eof to the end,
103132	** when the first reader is empty, all will be empty.
103133	*/
103134	while( !leavesReaderAtEnd(lrs) ){
103135	/* Figure out how many readers share their next term. */
103136	for(i=1; i<MERGE_COUNT && !leavesReaderAtEnd(lrs+i); i++){
103137	if( 0!=leavesReaderTermCmp(lrs, lrs+i) ) break;
103138	}
103139
103140	rc = leavesReadersMerge(v, lrs, i, &writer);
103141	if( rc!=SQLITE_OK ) goto err;
103142
103143	/* Step forward those that were merged. */
103144	while( i-->0 ){
103145	rc = leavesReaderStep(v, lrs+i);
103146	if( rc!=SQLITE_OK ) goto err;
103147
103148	/* Reorder by term, then by age. */
103149	leavesReaderReorder(lrs+i, MERGE_COUNT-i);
103150	}
103151	}
103152
103153	for(i=0; i<MERGE_COUNT; i++){
103154	leavesReaderDestroy(&lrs[i]);
103155	}
103156
103157	rc = leafWriterFinalize(v, &writer);
103158	leafWriterDestroy(&writer);
103159	if( rc!=SQLITE_OK ) return rc;
103160
103161	/* Delete the merged segment data. */
103162	return segdir_delete(v, iLevel);
103163
103164	err:
103165	for(i=0; i<MERGE_COUNT; i++){
103166	leavesReaderDestroy(&lrs[i]);
103167	}
103168	leafWriterDestroy(&writer);
103169	return rc;
103170	}
103171
103172	/* Accumulate the union of acc and pData into acc. /
103173	static void docListAccumulateUnion(DataBuffer *acc,
103174	const char *pData, int nData) {
103175	DataBuffer tmp = *acc;
103176	dataBufferInit(acc, tmp.nData+nData);
103177	docListUnion(tmp.pData, tmp.nData, pData, nData, acc);
103178	dataBufferDestroy(&tmp);
103179	}
103180
103181	/* TODO(shess) It might be interesting to explore different merge
103182	** strategies, here. For instance, since this is a sorted merge, we
103183	** could easily merge many doclists in parallel. With some
103184	** comprehension of the storage format, we could merge all of the
103185	** doclists within a leaf node directly from the leaf node's storage.
103186	** It may be worthwhile to merge smaller doclists before larger
103187	** doclists, since they can be traversed more quickly - but the
103188	** results may have less overlap, making them more expensive in a
103189	** different way.
103190	*/
103191
103192	/* Scan pReader for pTerm/nTerm, and merge the term's doclist over
103193	** out (any doclists with duplicate docids overwrite those in out).
103194	** Internal function for loadSegmentLeaf().
103195	*/
103196	static int loadSegmentLeavesInt(fulltext_vtab v, LeavesReader pReader,
103197	const char *pTerm, int nTerm, int isPrefix,
103198	DataBuffer *out){
103199	/* doclist data is accumulated into pBuffers similar to how one does
103200	** increment in binary arithmetic. If index 0 is empty, the data is
103201	** stored there. If there is data there, it is merged and the
103202	** results carried into position 1, with further merge-and-carry
103203	** until an empty position is found.
103204	*/
103205	DataBuffer *pBuffers = NULL;
103206	int nBuffers = 0, nMaxBuffers = 0, rc;
103207
103208	assert( nTerm>0 );
103209
103210	for(rc=SQLITE_OK; rc==SQLITE_OK && !leavesReaderAtEnd(pReader);
103211	rc=leavesReaderStep(v, pReader)){
103212	/* TODO(shess) Really want leavesReaderTermCmp(), but that name is
103213	** already taken to compare the terms of two LeavesReaders. Think
103214	** on a better name. [Meanwhile, break encapsulation rather than
103215	** use a confusing name.]
103216	*/
103217	int c = leafReaderTermCmp(&pReader->leafReader, pTerm, nTerm, isPrefix);
103218	if( c>0 ) break; /* Past any possible matches. */
103219	if( c==0 ){
103220	const char *pData = leavesReaderData(pReader);
103221	int iBuffer, nData = leavesReaderDataBytes(pReader);
103222
103223	/* Find the first empty buffer. */
103224	for(iBuffer=0; iBuffer<nBuffers; ++iBuffer){
103225	if( 0==pBuffers[iBuffer].nData ) break;
103226	}
103227
103228	/* Out of buffers, add an empty one. */
103229	if( iBuffer==nBuffers ){
103230	if( nBuffers==nMaxBuffers ){
103231	DataBuffer *p;
103232	nMaxBuffers += 20;
103233
103234	/* Manual realloc so we can handle NULL appropriately. */
103235	p = sqlite3_malloc(nMaxBufferssizeof(pBuffers));
103236	if( p==NULL ){
103237	rc = SQLITE_NOMEM;
103238	break;
103239	}
103240
103241	if( nBuffers>0 ){
103242	assert(pBuffers!=NULL);
103243	memcpy(p, pBuffers, nBufferssizeof(pBuffers));
103244	sqlite3_free(pBuffers);
103245	}
103246	pBuffers = p;
103247	}
103248	dataBufferInit(&(pBuffers[nBuffers]), 0);
103249	nBuffers++;
103250	}
103251
103252	/* At this point, must have an empty at iBuffer. */
103253	assert(iBuffer<nBuffers && pBuffers[iBuffer].nData==0);
103254
103255	/* If empty was first buffer, no need for merge logic. */
103256	if( iBuffer==0 ){
103257	dataBufferReplace(&(pBuffers[0]), pData, nData);
103258	}else{
103259	/* pAcc is the empty buffer the merged data will end up in. */
103260	DataBuffer *pAcc = &(pBuffers[iBuffer]);
103261	DataBuffer *p = &(pBuffers[0]);
103262
103263	/* Handle position 0 specially to avoid need to prime pAcc
103264	** with pData/nData.
103265	*/
103266	dataBufferSwap(p, pAcc);
103267	docListAccumulateUnion(pAcc, pData, nData);
103268
103269	/* Accumulate remaining doclists into pAcc. */
103270	for(++p; p<pAcc; ++p){
103271	docListAccumulateUnion(pAcc, p->pData, p->nData);
103272
103273	/* dataBufferReset() could allow a large doclist to blow up
103274	** our memory requirements.
103275	*/
103276	if( p->nCapacity<1024 ){
103277	dataBufferReset(p);
103278	}else{
103279	dataBufferDestroy(p);
103280	dataBufferInit(p, 0);
103281	}
103282	}
103283	}
103284	}
103285	}
103286
103287	/* Union all the doclists together into out. /
103288	/* TODO(shess) What if out is big? Sigh. /
103289	if( rc==SQLITE_OK && nBuffers>0 ){
103290	int iBuffer;
103291	for(iBuffer=0; iBuffer<nBuffers; ++iBuffer){
103292	if( pBuffers[iBuffer].nData>0 ){
103293	if( out->nData==0 ){
103294	dataBufferSwap(out, &(pBuffers[iBuffer]));
103295	}else{
103296	docListAccumulateUnion(out, pBuffers[iBuffer].pData,
103297	pBuffers[iBuffer].nData);
103298	}
103299	}
103300	}
103301	}
103302
103303	while( nBuffers-- ){
103304	dataBufferDestroy(&(pBuffers[nBuffers]));
103305	}
103306	if( pBuffers!=NULL ) sqlite3_free(pBuffers);
103307
103308	return rc;
103309	}
103310
103311	/* Call loadSegmentLeavesInt() with pData/nData as input. */
103312	static int loadSegmentLeaf(fulltext_vtab v, const char pData, int nData,
103313	const char *pTerm, int nTerm, int isPrefix,
103314	DataBuffer *out){
103315	LeavesReader reader;
103316	int rc;
103317
103318	assert( nData>1 );
103319	assert( *pData=='\0' );
103320	rc = leavesReaderInit(v, 0, 0, 0, pData, nData, &reader);
103321	if( rc!=SQLITE_OK ) return rc;
103322
103323	rc = loadSegmentLeavesInt(v, &reader, pTerm, nTerm, isPrefix, out);
103324	leavesReaderReset(&reader);
103325	leavesReaderDestroy(&reader);
103326	return rc;
103327	}
103328
103329	/* Call loadSegmentLeavesInt() with the leaf nodes from iStartLeaf to
103330	** iEndLeaf (inclusive) as input, and merge the resulting doclist into
103331	** out.
103332	*/
103333	static int loadSegmentLeaves(fulltext_vtab *v,
103334	sqlite_int64 iStartLeaf, sqlite_int64 iEndLeaf,
103335	const char *pTerm, int nTerm, int isPrefix,
103336	DataBuffer *out){
103337	int rc;
103338	LeavesReader reader;
103339
103340	assert( iStartLeaf<=iEndLeaf );
103341	rc = leavesReaderInit(v, 0, iStartLeaf, iEndLeaf, NULL, 0, &reader);
103342	if( rc!=SQLITE_OK ) return rc;
103343
103344	rc = loadSegmentLeavesInt(v, &reader, pTerm, nTerm, isPrefix, out);
103345	leavesReaderReset(&reader);
103346	leavesReaderDestroy(&reader);
103347	return rc;
103348	}
103349
103350	/* Taking pData/nData as an interior node, find the sequence of child
103351	** nodes which could include pTerm/nTerm/isPrefix. Note that the
103352	** interior node terms logically come between the blocks, so there is
103353	** one more blockid than there are terms (that block contains terms >=
103354	** the last interior-node term).
103355	*/
103356	/* TODO(shess) The calling code may already know that the end child is
103357	** not worth calculating, because the end may be in a later sibling
103358	** node. Consider whether breaking symmetry is worthwhile. I suspect
103359	** it is not worthwhile.
103360	*/
103361	static void getChildrenContaining(const char *pData, int nData,
103362	const char *pTerm, int nTerm, int isPrefix,
103363	sqlite_int64 *piStartChild,
103364	sqlite_int64 *piEndChild){
103365	InteriorReader reader;
103366
103367	assert( nData>1 );
103368	assert( *pData!='\0' );
103369	interiorReaderInit(pData, nData, &reader);
103370
103371	/* Scan for the first child which could contain pTerm/nTerm. */
103372	while( !interiorReaderAtEnd(&reader) ){
103373	if( interiorReaderTermCmp(&reader, pTerm, nTerm, 0)>0 ) break;
103374	interiorReaderStep(&reader);
103375	}
103376	*piStartChild = interiorReaderCurrentBlockid(&reader);
103377
103378	/* Keep scanning to find a term greater than our term, using prefix
103379	** comparison if indicated. If isPrefix is false, this will be the
103380	** same blockid as the starting block.
103381	*/
103382	while( !interiorReaderAtEnd(&reader) ){
103383	if( interiorReaderTermCmp(&reader, pTerm, nTerm, isPrefix)>0 ) break;
103384	interiorReaderStep(&reader);
103385	}
103386	*piEndChild = interiorReaderCurrentBlockid(&reader);
103387
103388	interiorReaderDestroy(&reader);
103389
103390	/* Children must ascend, and if !prefix, both must be the same. */
103391	assert( piEndChild>=piStartChild );
103392	assert( isPrefix \|\| piStartChild==piEndChild );
103393	}
103394
103395	/* Read block at iBlockid and pass it with other params to
103396	** getChildrenContaining().
103397	*/
103398	static int loadAndGetChildrenContaining(
103399	fulltext_vtab *v,
103400	sqlite_int64 iBlockid,
103401	const char *pTerm, int nTerm, int isPrefix,
103402	sqlite_int64 piStartChild, sqlite_int64 piEndChild
103403	){
103404	sqlite3_stmt *s = NULL;
103405	int rc;
103406
103407	assert( iBlockid!=0 );
103408	assert( pTerm!=NULL );
103409	assert( nTerm!=0 ); /* TODO(shess) Why not allow this? */
103410	assert( piStartChild!=NULL );
103411	assert( piEndChild!=NULL );
103412
103413	rc = sql_get_statement(v, BLOCK_SELECT_STMT, &s);
103414	if( rc!=SQLITE_OK ) return rc;
103415
103416	rc = sqlite3_bind_int64(s, 1, iBlockid);
103417	if( rc!=SQLITE_OK ) return rc;
103418
103419	rc = sqlite3_step(s);
103420	if( rc==SQLITE_DONE ) return SQLITE_ERROR;
103421	if( rc!=SQLITE_ROW ) return rc;
103422
103423	getChildrenContaining(sqlite3_column_blob(s, 0), sqlite3_column_bytes(s, 0),
103424	pTerm, nTerm, isPrefix, piStartChild, piEndChild);
103425
103426	/* We expect only one row. We must execute another sqlite3_step()
103427	* to complete the iteration; otherwise the table will remain
103428	* locked. */
103429	rc = sqlite3_step(s);
103430	if( rc==SQLITE_ROW ) return SQLITE_ERROR;
103431	if( rc!=SQLITE_DONE ) return rc;
103432
103433	return SQLITE_OK;
103434	}
103435
103436	/* Traverse the tree represented by pData[nData] looking for
103437	** pTerm[nTerm], placing its doclist into *out. This is internal to
103438	** loadSegment() to make error-handling cleaner.
103439	*/
103440	static int loadSegmentInt(fulltext_vtab v, const char pData, int nData,
103441	sqlite_int64 iLeavesEnd,
103442	const char *pTerm, int nTerm, int isPrefix,
103443	DataBuffer *out){
103444	/* Special case where root is a leaf. */
103445	if( *pData=='\0' ){
103446	return loadSegmentLeaf(v, pData, nData, pTerm, nTerm, isPrefix, out);
103447	}else{
103448	int rc;
103449	sqlite_int64 iStartChild, iEndChild;
103450
103451	/* Process pData as an interior node, then loop down the tree
103452	** until we find the set of leaf nodes to scan for the term.
103453	*/
103454	getChildrenContaining(pData, nData, pTerm, nTerm, isPrefix,
103455	&iStartChild, &iEndChild);
103456	while( iStartChild>iLeavesEnd ){
103457	sqlite_int64 iNextStart, iNextEnd;
103458	rc = loadAndGetChildrenContaining(v, iStartChild, pTerm, nTerm, isPrefix,
103459	&iNextStart, &iNextEnd);
103460	if( rc!=SQLITE_OK ) return rc;
103461
103462	/* If we've branched, follow the end branch, too. */
103463	if( iStartChild!=iEndChild ){
103464	sqlite_int64 iDummy;
103465	rc = loadAndGetChildrenContaining(v, iEndChild, pTerm, nTerm, isPrefix,
103466	&iDummy, &iNextEnd);
103467	if( rc!=SQLITE_OK ) return rc;
103468	}
103469
103470	assert( iNextStart<=iNextEnd );
103471	iStartChild = iNextStart;
103472	iEndChild = iNextEnd;
103473	}
103474	assert( iStartChild<=iLeavesEnd );
103475	assert( iEndChild<=iLeavesEnd );
103476
103477	/* Scan through the leaf segments for doclists. */
103478	return loadSegmentLeaves(v, iStartChild, iEndChild,
103479	pTerm, nTerm, isPrefix, out);
103480	}
103481	}
103482
103483	/* Call loadSegmentInt() to collect the doclist for pTerm/nTerm, then
103484	** merge its doclist over *out (any duplicate doclists read from the
103485	** segment rooted at pData will overwrite those in *out).
103486	*/
103487	/* TODO(shess) Consider changing this to determine the depth of the
103488	** leaves using either the first characters of interior nodes (when
103489	** ==1, we're one level above the leaves), or the first character of
103490	** the root (which will describe the height of the tree directly).
103491	** Either feels somewhat tricky to me.
103492	*/
103493	/* TODO(shess) The current merge is likely to be slow for large
103494	** doclists (though it should process from newest/smallest to
103495	** oldest/largest, so it may not be that bad). It might be useful to
103496	** modify things to allow for N-way merging. This could either be
103497	** within a segment, with pairwise merges across segments, or across
103498	** all segments at once.
103499	*/
103500	static int loadSegment(fulltext_vtab v, const char pData, int nData,
103501	sqlite_int64 iLeavesEnd,
103502	const char *pTerm, int nTerm, int isPrefix,
103503	DataBuffer *out){
103504	DataBuffer result;
103505	int rc;
103506
103507	assert( nData>1 );
103508
103509	/* This code should never be called with buffered updates. */
103510	assert( v->nPendingData<0 );
103511
103512	dataBufferInit(&result, 0);
103513	rc = loadSegmentInt(v, pData, nData, iLeavesEnd,
103514	pTerm, nTerm, isPrefix, &result);
103515	if( rc==SQLITE_OK && result.nData>0 ){
103516	if( out->nData==0 ){
103517	DataBuffer tmp = *out;
103518	*out = result;
103519	result = tmp;
103520	}else{
103521	DataBuffer merged;
103522	DLReader readers[2];
103523
103524	dlrInit(&readers[0], DL_DEFAULT, out->pData, out->nData);
103525	dlrInit(&readers[1], DL_DEFAULT, result.pData, result.nData);
103526	dataBufferInit(&merged, out->nData+result.nData);
103527	docListMerge(&merged, readers, 2);
103528	dataBufferDestroy(out);
103529	*out = merged;
103530	dlrDestroy(&readers[0]);
103531	dlrDestroy(&readers[1]);
103532	}
103533	}
103534	dataBufferDestroy(&result);
103535	return rc;
103536	}
103537
103538	/* Scan the database and merge together the posting lists for the term
103539	** into *out.
103540	*/
103541	static int termSelect(
103542	fulltext_vtab *v,
103543	int iColumn,
103544	const char pTerm, int nTerm, / Term to query for */
103545	int isPrefix, /* True for a prefix search */
103546	DocListType iType,
103547	DataBuffer out / Write results here */
103548	){
103549	DataBuffer doclist;
103550	sqlite3_stmt *s;
103551	int rc = sql_get_statement(v, SEGDIR_SELECT_ALL_STMT, &s);
103552	if( rc!=SQLITE_OK ) return rc;
103553
103554	/* This code should never be called with buffered updates. */
103555	assert( v->nPendingData<0 );
103556
103557	dataBufferInit(&doclist, 0);
103558	dataBufferInit(out, 0);
103559
103560	/* Traverse the segments from oldest to newest so that newer doclist
103561	** elements for given docids overwrite older elements.
103562	*/
103563	while( (rc = sqlite3_step(s))==SQLITE_ROW ){
103564	const char *pData = sqlite3_column_blob(s, 2);
103565	const int nData = sqlite3_column_bytes(s, 2);
103566	const sqlite_int64 iLeavesEnd = sqlite3_column_int64(s, 1);
103567	rc = loadSegment(v, pData, nData, iLeavesEnd, pTerm, nTerm, isPrefix,
103568	&doclist);
103569	if( rc!=SQLITE_OK ) goto err;
103570	}
103571	if( rc==SQLITE_DONE ){
103572	if( doclist.nData!=0 ){
103573	/* TODO(shess) The old term_select_all() code applied the column
103574	** restrict as we merged segments, leading to smaller buffers.
103575	** This is probably worthwhile to bring back, once the new storage
103576	** system is checked in.
103577	*/
103578	if( iColumn==v->nColumn) iColumn = -1;
103579	docListTrim(DL_DEFAULT, doclist.pData, doclist.nData,
103580	iColumn, iType, out);
103581	}
103582	rc = SQLITE_OK;
103583	}
103584
103585	err:
103586	dataBufferDestroy(&doclist);
103587	return rc;
103588	}
103589
103590	/****************************************************************/
103591	/* Used to hold hashtable data for sorting. */
103592	typedef struct TermData {
103593	const char *pTerm;
103594	int nTerm;
103595	DLCollector *pCollector;
103596	} TermData;
103597
103598	/* Orders TermData elements in strcmp fashion ( <0 for less-than, 0
103599	** for equal, >0 for greater-than).
103600	*/
103601	static int termDataCmp(const void av, const void bv){
103602	const TermData a = (const TermData )av;
103603	const TermData b = (const TermData )bv;
103604	int n = a->nTerm<b->nTerm ? a->nTerm : b->nTerm;
103605	int c = memcmp(a->pTerm, b->pTerm, n);
103606	if( c!=0 ) return c;
103607	return a->nTerm-b->nTerm;
103608	}
103609
103610	/* Order pTerms data by term, then write a new level 0 segment using
103611	** LeafWriter.
103612	*/
103613	static int writeZeroSegment(fulltext_vtab v, fts3Hash pTerms){
103614	fts3HashElem *e;
103615	int idx, rc, i, n;
103616	TermData *pData;
103617	LeafWriter writer;
103618	DataBuffer dl;
103619
103620	/* Determine the next index at level 0, merging as necessary. */
103621	rc = segdirNextIndex(v, 0, &idx);
103622	if( rc!=SQLITE_OK ) return rc;
103623
103624	n = fts3HashCount(pTerms);
103625	pData = sqlite3_malloc(n*sizeof(TermData));
103626
103627	for(i = 0, e = fts3HashFirst(pTerms); e; i++, e = fts3HashNext(e)){
103628	assert( i<n );
103629	pData[i].pTerm = fts3HashKey(e);
103630	pData[i].nTerm = fts3HashKeysize(e);
103631	pData[i].pCollector = fts3HashData(e);
103632	}
103633	assert( i==n );
103634
103635	/* TODO(shess) Should we allow user-defined collation sequences,
103636	** here? I think we only need that once we support prefix searches.
103637	*/
103638	if( n>1 ) qsort(pData, n, sizeof(*pData), termDataCmp);
103639
103640	/* TODO(shess) Refactor so that we can write directly to the segment
103641	** DataBuffer, as happens for segment merges.
103642	*/
103643	leafWriterInit(0, idx, &writer);
103644	dataBufferInit(&dl, 0);
103645	for(i=0; i<n; i++){
103646	dataBufferReset(&dl);
103647	dlcAddDoclist(pData[i].pCollector, &dl);
103648	rc = leafWriterStep(v, &writer,
103649	pData[i].pTerm, pData[i].nTerm, dl.pData, dl.nData);
103650	if( rc!=SQLITE_OK ) goto err;
103651	}
103652	rc = leafWriterFinalize(v, &writer);
103653
103654	err:
103655	dataBufferDestroy(&dl);
103656	sqlite3_free(pData);
103657	leafWriterDestroy(&writer);
103658	return rc;
103659	}
103660
103661	/* If pendingTerms has data, free it. */
103662	static int clearPendingTerms(fulltext_vtab *v){
103663	if( v->nPendingData>=0 ){
103664	fts3HashElem *e;
103665	for(e=fts3HashFirst(&v->pendingTerms); e; e=fts3HashNext(e)){
103666	dlcDelete(fts3HashData(e));
103667	}
103668	fts3HashClear(&v->pendingTerms);
103669	v->nPendingData = -1;
103670	}
103671	return SQLITE_OK;
103672	}
103673
103674	/* If pendingTerms has data, flush it to a level-zero segment, and
103675	** free it.
103676	*/
103677	static int flushPendingTerms(fulltext_vtab *v){
103678	if( v->nPendingData>=0 ){
103679	int rc = writeZeroSegment(v, &v->pendingTerms);
103680	if( rc==SQLITE_OK ) clearPendingTerms(v);
103681	return rc;
103682	}
103683	return SQLITE_OK;
103684	}
103685
103686	/* If pendingTerms is "too big", or docid is out of order, flush it.
103687	** Regardless, be certain that pendingTerms is initialized for use.
103688	*/
103689	static int initPendingTerms(fulltext_vtab *v, sqlite_int64 iDocid){
103690	/* TODO(shess) Explore whether partially flushing the buffer on
103691	** forced-flush would provide better performance. I suspect that if
103692	** we ordered the doclists by size and flushed the largest until the
103693	** buffer was half empty, that would let the less frequent terms
103694	** generate longer doclists.
103695	*/
103696	if( iDocid<=v->iPrevDocid \|\| v->nPendingData>kPendingThreshold ){
103697	int rc = flushPendingTerms(v);
103698	if( rc!=SQLITE_OK ) return rc;
103699	}
103700	if( v->nPendingData<0 ){
103701	fts3HashInit(&v->pendingTerms, FTS3_HASH_STRING, 1);
103702	v->nPendingData = 0;
103703	}
103704	v->iPrevDocid = iDocid;
103705	return SQLITE_OK;
103706	}
103707
103708	/* This function implements the xUpdate callback; it is the top-level entry
103709	* point for inserting, deleting or updating a row in a full-text table. */
103710	static int fulltextUpdate(sqlite3_vtab pVtab, int nArg, sqlite3_value *ppArg,
103711	sqlite_int64 *pRowid){
103712	fulltext_vtab v = (fulltext_vtab ) pVtab;
103713	int rc;
103714
103715	FTSTRACE(("FTS3 Update %p\n", pVtab));
103716
103717	if( nArg<2 ){
103718	rc = index_delete(v, sqlite3_value_int64(ppArg[0]));
103719	if( rc==SQLITE_OK ){
103720	/* If we just deleted the last row in the table, clear out the
103721	** index data.
103722	*/
103723	rc = content_exists(v);
103724	if( rc==SQLITE_ROW ){
103725	rc = SQLITE_OK;
103726	}else if( rc==SQLITE_DONE ){
103727	/* Clear the pending terms so we don't flush a useless level-0
103728	** segment when the transaction closes.
103729	*/
103730	rc = clearPendingTerms(v);
103731	if( rc==SQLITE_OK ){
103732	rc = segdir_delete_all(v);
103733	}
103734	}
103735	}
103736	} else if( sqlite3_value_type(ppArg[0]) != SQLITE_NULL ){
103737	/* An update:
103738	* ppArg[0] = old rowid
103739	* ppArg[1] = new rowid
103740	* ppArg[2..2+v->nColumn-1] = values
103741	* ppArg[2+v->nColumn] = value for magic column (we ignore this)
103742	* ppArg[2+v->nColumn+1] = value for docid
103743	*/
103744	sqlite_int64 rowid = sqlite3_value_int64(ppArg[0]);
103745	if( sqlite3_value_type(ppArg[1]) != SQLITE_INTEGER \|\|
103746	sqlite3_value_int64(ppArg[1]) != rowid ){
103747	rc = SQLITE_ERROR; /* we don't allow changing the rowid */
103748	}else if( sqlite3_value_type(ppArg[2+v->nColumn+1]) != SQLITE_INTEGER \|\|
103749	sqlite3_value_int64(ppArg[2+v->nColumn+1]) != rowid ){
103750	rc = SQLITE_ERROR; /* we don't allow changing the docid */
103751	}else{
103752	assert( nArg==2+v->nColumn+2);
103753	rc = index_update(v, rowid, &ppArg[2]);
103754	}
103755	} else {
103756	/* An insert:
103757	* ppArg[1] = requested rowid
103758	* ppArg[2..2+v->nColumn-1] = values
103759	* ppArg[2+v->nColumn] = value for magic column (we ignore this)
103760	* ppArg[2+v->nColumn+1] = value for docid
103761	*/
103762	sqlite3_value *pRequestDocid = ppArg[2+v->nColumn+1];
103763	assert( nArg==2+v->nColumn+2);
103764	if( SQLITE_NULL != sqlite3_value_type(pRequestDocid) &&
103765	SQLITE_NULL != sqlite3_value_type(ppArg[1]) ){
103766	/* TODO(shess) Consider allowing this to work if the values are
103767	** identical. I'm inclined to discourage that usage, though,
103768	** given that both rowid and docid are special columns. Better
103769	** would be to define one or the other as the default winner,
103770	** but should it be fts3-centric (docid) or SQLite-centric
103771	** (rowid)?
103772	*/
103773	rc = SQLITE_ERROR;
103774	}else{
103775	if( SQLITE_NULL == sqlite3_value_type(pRequestDocid) ){
103776	pRequestDocid = ppArg[1];
103777	}
103778	rc = index_insert(v, pRequestDocid, &ppArg[2], pRowid);
103779	}
103780	}
103781
103782	return rc;
103783	}
103784
103785	static int fulltextSync(sqlite3_vtab *pVtab){
103786	FTSTRACE(("FTS3 xSync()\n"));
103787	return flushPendingTerms((fulltext_vtab *)pVtab);
103788	}
103789
103790	static int fulltextBegin(sqlite3_vtab *pVtab){
103791	fulltext_vtab v = (fulltext_vtab ) pVtab;
103792	FTSTRACE(("FTS3 xBegin()\n"));
103793
103794	/* Any buffered updates should have been cleared by the previous
103795	** transaction.
103796	*/
103797	assert( v->nPendingData<0 );
103798	return clearPendingTerms(v);
103799	}
103800
103801	static int fulltextCommit(sqlite3_vtab *pVtab){
103802	fulltext_vtab v = (fulltext_vtab ) pVtab;
103803	FTSTRACE(("FTS3 xCommit()\n"));
103804
103805	/* Buffered updates should have been cleared by fulltextSync(). */
103806	assert( v->nPendingData<0 );
103807	return clearPendingTerms(v);
103808	}
103809
103810	static int fulltextRollback(sqlite3_vtab *pVtab){
103811	FTSTRACE(("FTS3 xRollback()\n"));
103812	return clearPendingTerms((fulltext_vtab *)pVtab);
103813	}
103814
103815	/*
103816	** Implementation of the snippet() function for FTS3
103817	*/
103818	static void snippetFunc(
103819	sqlite3_context *pContext,
103820	int argc,
103821	sqlite3_value **argv
103822	){
103823	fulltext_cursor *pCursor;
103824	if( argc<1 ) return;
103825	if( sqlite3_value_type(argv[0])!=SQLITE_BLOB \|\|
103826	sqlite3_value_bytes(argv[0])!=sizeof(pCursor) ){
103827	sqlite3_result_error(pContext, "illegal first argument to html_snippet",-1);
103828	}else{
103829	const char *zStart = "<b>";
103830	const char *zEnd = "</b>";
103831	const char *zEllipsis = "<b>...</b>";
103832	memcpy(&pCursor, sqlite3_value_blob(argv[0]), sizeof(pCursor));
103833	if( argc>=2 ){
103834	zStart = (const char*)sqlite3_value_text(argv[1]);
103835	if( argc>=3 ){
103836	zEnd = (const char*)sqlite3_value_text(argv[2]);
103837	if( argc>=4 ){
103838	zEllipsis = (const char*)sqlite3_value_text(argv[3]);
103839	}
103840	}
103841	}
103842	snippetAllOffsets(pCursor);
103843	snippetText(pCursor, zStart, zEnd, zEllipsis);
103844	sqlite3_result_text(pContext, pCursor->snippet.zSnippet,
103845	pCursor->snippet.nSnippet, SQLITE_STATIC);
103846	}
103847	}
103848
103849	/*
103850	** Implementation of the offsets() function for FTS3
103851	*/
103852	static void snippetOffsetsFunc(
103853	sqlite3_context *pContext,
103854	int argc,
103855	sqlite3_value **argv
103856	){
103857	fulltext_cursor *pCursor;
103858	if( argc<1 ) return;
103859	if( sqlite3_value_type(argv[0])!=SQLITE_BLOB \|\|
103860	sqlite3_value_bytes(argv[0])!=sizeof(pCursor) ){
103861	sqlite3_result_error(pContext, "illegal first argument to offsets",-1);
103862	}else{
103863	memcpy(&pCursor, sqlite3_value_blob(argv[0]), sizeof(pCursor));
103864	snippetAllOffsets(pCursor);
103865	snippetOffsetText(&pCursor->snippet);
103866	sqlite3_result_text(pContext,
103867	pCursor->snippet.zOffset, pCursor->snippet.nOffset,
103868	SQLITE_STATIC);
103869	}
103870	}
103871
103872	/* OptLeavesReader is nearly identical to LeavesReader, except that
103873	** where LeavesReader is geared towards the merging of complete
103874	** segment levels (with exactly MERGE_COUNT segments), OptLeavesReader
103875	** is geared towards implementation of the optimize() function, and
103876	** can merge all segments simultaneously. This version may be
103877	** somewhat less efficient than LeavesReader because it merges into an
103878	** accumulator rather than doing an N-way merge, but since segment
103879	** size grows exponentially (so segment count logrithmically) this is
103880	** probably not an immediate problem.
103881	*/
103882	/* TODO(shess): Prove that assertion, or extend the merge code to
103883	** merge tree fashion (like the prefix-searching code does).
103884	*/
103885	/* TODO(shess): OptLeavesReader and LeavesReader could probably be
103886	** merged with little or no loss of performance for LeavesReader. The
103887	** merged code would need to handle >MERGE_COUNT segments, and would
103888	** also need to be able to optionally optimize away deletes.
103889	*/
103890	typedef struct OptLeavesReader {
103891	/* Segment number, to order readers by age. */
103892	int segment;
103893	LeavesReader reader;
103894	} OptLeavesReader;
103895
103896	static int optLeavesReaderAtEnd(OptLeavesReader *pReader){
103897	return leavesReaderAtEnd(&pReader->reader);
103898	}
103899	static int optLeavesReaderTermBytes(OptLeavesReader *pReader){
103900	return leavesReaderTermBytes(&pReader->reader);
103901	}
103902	static const char optLeavesReaderData(OptLeavesReader pReader){
103903	return leavesReaderData(&pReader->reader);
103904	}
103905	static int optLeavesReaderDataBytes(OptLeavesReader *pReader){
103906	return leavesReaderDataBytes(&pReader->reader);
103907	}
103908	static const char optLeavesReaderTerm(OptLeavesReader pReader){
103909	return leavesReaderTerm(&pReader->reader);
103910	}
103911	static int optLeavesReaderStep(fulltext_vtab v, OptLeavesReader pReader){
103912	return leavesReaderStep(v, &pReader->reader);
103913	}
103914	static int optLeavesReaderTermCmp(OptLeavesReader lr1, OptLeavesReader lr2){
103915	return leavesReaderTermCmp(&lr1->reader, &lr2->reader);
103916	}
103917	/* Order by term ascending, segment ascending (oldest to newest), with
103918	** exhausted readers to the end.
103919	*/
103920	static int optLeavesReaderCmp(OptLeavesReader lr1, OptLeavesReader lr2){
103921	int c = optLeavesReaderTermCmp(lr1, lr2);
103922	if( c!=0 ) return c;
103923	return lr1->segment-lr2->segment;
103924	}
103925	/* Bubble pLr[0] to appropriate place in pLr[1..nLr-1]. Assumes that
103926	** pLr[1..nLr-1] is already sorted.
103927	*/
103928	static void optLeavesReaderReorder(OptLeavesReader *pLr, int nLr){
103929	while( nLr>1 && optLeavesReaderCmp(pLr, pLr+1)>0 ){
103930	OptLeavesReader tmp = pLr[0];
103931	pLr[0] = pLr[1];
103932	pLr[1] = tmp;
103933	nLr--;
103934	pLr++;
103935	}
103936	}
103937
103938	/* optimize() helper function. Put the readers in order and iterate
103939	** through them, merging doclists for matching terms into pWriter.
103940	** Returns SQLITE_OK on success, or the SQLite error code which
103941	** prevented success.
103942	*/
103943	static int optimizeInternal(fulltext_vtab *v,
103944	OptLeavesReader *readers, int nReaders,
103945	LeafWriter *pWriter){
103946	int i, rc = SQLITE_OK;
103947	DataBuffer doclist, merged, tmp;
103948
103949	/* Order the readers. */
103950	i = nReaders;
103951	while( i-- > 0 ){
103952	optLeavesReaderReorder(&readers[i], nReaders-i);
103953	}
103954
103955	dataBufferInit(&doclist, LEAF_MAX);
103956	dataBufferInit(&merged, LEAF_MAX);
103957
103958	/* Exhausted readers bubble to the end, so when the first reader is
103959	** at eof, all are at eof.
103960	*/
103961	while( !optLeavesReaderAtEnd(&readers[0]) ){
103962
103963	/* Figure out how many readers share the next term. */
103964	for(i=1; i<nReaders && !optLeavesReaderAtEnd(&readers[i]); i++){
103965	if( 0!=optLeavesReaderTermCmp(&readers[0], &readers[i]) ) break;
103966	}
103967
103968	/* Special-case for no merge. */
103969	if( i==1 ){
103970	/* Trim deletions from the doclist. */
103971	dataBufferReset(&merged);
103972	docListTrim(DL_DEFAULT,
103973	optLeavesReaderData(&readers[0]),
103974	optLeavesReaderDataBytes(&readers[0]),
103975	-1, DL_DEFAULT, &merged);
103976	}else{
103977	DLReader dlReaders[MERGE_COUNT];
103978	int iReader, nReaders;
103979
103980	/* Prime the pipeline with the first reader's doclist. After
103981	** one pass index 0 will reference the accumulated doclist.
103982	*/
103983	dlrInit(&dlReaders[0], DL_DEFAULT,
103984	optLeavesReaderData(&readers[0]),
103985	optLeavesReaderDataBytes(&readers[0]));
103986	iReader = 1;
103987
103988	assert( iReader<i ); /* Must execute the loop at least once. */
103989	while( iReader<i ){
103990	/* Merge 16 inputs per pass. */
103991	for( nReaders=1; iReader<i && nReaders<MERGE_COUNT;
103992	iReader++, nReaders++ ){
103993	dlrInit(&dlReaders[nReaders], DL_DEFAULT,
103994	optLeavesReaderData(&readers[iReader]),
103995	optLeavesReaderDataBytes(&readers[iReader]));
103996	}
103997
103998	/* Merge doclists and swap result into accumulator. */
103999	dataBufferReset(&merged);
104000	docListMerge(&merged, dlReaders, nReaders);
104001	tmp = merged;
104002	merged = doclist;
104003	doclist = tmp;
104004
104005	while( nReaders-- > 0 ){
104006	dlrDestroy(&dlReaders[nReaders]);
104007	}
104008
104009	/* Accumulated doclist to reader 0 for next pass. */
104010	dlrInit(&dlReaders[0], DL_DEFAULT, doclist.pData, doclist.nData);
104011	}
104012
104013	/* Destroy reader that was left in the pipeline. */
104014	dlrDestroy(&dlReaders[0]);
104015
104016	/* Trim deletions from the doclist. */
104017	dataBufferReset(&merged);
104018	docListTrim(DL_DEFAULT, doclist.pData, doclist.nData,
104019	-1, DL_DEFAULT, &merged);
104020	}
104021
104022	/* Only pass doclists with hits (skip if all hits deleted). */
104023	if( merged.nData>0 ){
104024	rc = leafWriterStep(v, pWriter,
104025	optLeavesReaderTerm(&readers[0]),
104026	optLeavesReaderTermBytes(&readers[0]),
104027	merged.pData, merged.nData);
104028	if( rc!=SQLITE_OK ) goto err;
104029	}
104030
104031	/* Step merged readers to next term and reorder. */
104032	while( i-- > 0 ){
104033	rc = optLeavesReaderStep(v, &readers[i]);
104034	if( rc!=SQLITE_OK ) goto err;
104035
104036	optLeavesReaderReorder(&readers[i], nReaders-i);
104037	}
104038	}
104039
104040	err:
104041	dataBufferDestroy(&doclist);
104042	dataBufferDestroy(&merged);
104043	return rc;
104044	}
104045
104046	/* Implement optimize() function for FTS3. optimize(t) merges all
104047	** segments in the fts index into a single segment. 't' is the magic
104048	** table-named column.
104049	*/
104050	static void optimizeFunc(sqlite3_context *pContext,
104051	int argc, sqlite3_value **argv){
104052	fulltext_cursor *pCursor;
104053	if( argc>1 ){
104054	sqlite3_result_error(pContext, "excess arguments to optimize()",-1);
104055	}else if( sqlite3_value_type(argv[0])!=SQLITE_BLOB \|\|
104056	sqlite3_value_bytes(argv[0])!=sizeof(pCursor) ){
104057	sqlite3_result_error(pContext, "illegal first argument to optimize",-1);
104058	}else{
104059	fulltext_vtab *v;
104060	int i, rc, iMaxLevel;
104061	OptLeavesReader *readers;
104062	int nReaders;
104063	LeafWriter writer;
104064	sqlite3_stmt *s;
104065
104066	memcpy(&pCursor, sqlite3_value_blob(argv[0]), sizeof(pCursor));
104067	v = cursor_vtab(pCursor);
104068
104069	/* Flush any buffered updates before optimizing. */
104070	rc = flushPendingTerms(v);
104071	if( rc!=SQLITE_OK ) goto err;
104072
104073	rc = segdir_count(v, &nReaders, &iMaxLevel);
104074	if( rc!=SQLITE_OK ) goto err;
104075	if( nReaders==0 \|\| nReaders==1 ){
104076	sqlite3_result_text(pContext, "Index already optimal", -1,
104077	SQLITE_STATIC);
104078	return;
104079	}
104080
104081	rc = sql_get_statement(v, SEGDIR_SELECT_ALL_STMT, &s);
104082	if( rc!=SQLITE_OK ) goto err;
104083
104084	readers = sqlite3_malloc(nReaders*sizeof(readers[0]));
104085	if( readers==NULL ) goto err;
104086
104087	/* Note that there will already be a segment at this position
104088	** until we call segdir_delete() on iMaxLevel.
104089	*/
104090	leafWriterInit(iMaxLevel, 0, &writer);
104091
104092	i = 0;
104093	while( (rc = sqlite3_step(s))==SQLITE_ROW ){
104094	sqlite_int64 iStart = sqlite3_column_int64(s, 0);
104095	sqlite_int64 iEnd = sqlite3_column_int64(s, 1);
104096	const char *pRootData = sqlite3_column_blob(s, 2);
104097	int nRootData = sqlite3_column_bytes(s, 2);
104098
104099	assert( i<nReaders );
104100	rc = leavesReaderInit(v, -1, iStart, iEnd, pRootData, nRootData,
104101	&readers[i].reader);
104102	if( rc!=SQLITE_OK ) break;
104103
104104	readers[i].segment = i;
104105	i++;
104106	}
104107
104108	/* If we managed to successfully read them all, optimize them. */
104109	if( rc==SQLITE_DONE ){
104110	assert( i==nReaders );
104111	rc = optimizeInternal(v, readers, nReaders, &writer);
104112	}
104113
104114	while( i-- > 0 ){
104115	leavesReaderDestroy(&readers[i].reader);
104116	}
104117	sqlite3_free(readers);
104118
104119	/* If we've successfully gotten to here, delete the old segments
104120	** and flush the interior structure of the new segment.
104121	*/
104122	if( rc==SQLITE_OK ){
104123	for( i=0; i<=iMaxLevel; i++ ){
104124	rc = segdir_delete(v, i);
104125	if( rc!=SQLITE_OK ) break;
104126	}
104127
104128	if( rc==SQLITE_OK ) rc = leafWriterFinalize(v, &writer);
104129	}
104130
104131	leafWriterDestroy(&writer);
104132
104133	if( rc!=SQLITE_OK ) goto err;
104134
104135	sqlite3_result_text(pContext, "Index optimized", -1, SQLITE_STATIC);
104136	return;
104137
104138	/* TODO(shess): Error-handling needs to be improved along the
104139	** lines of the dump_ functions.
104140	*/
104141	err:
104142	{
104143	char buf[512];
104144	sqlite3_snprintf(sizeof(buf), buf, "Error in optimize: %s",
104145	sqlite3_errmsg(sqlite3_context_db_handle(pContext)));
104146	sqlite3_result_error(pContext, buf, -1);
104147	}
104148	}
104149	}
104150
104151	#ifdef SQLITE_TEST
104152	/* Generate an error of the form "<prefix>: <msg>". If msg is NULL,
104153	** pull the error from the context's db handle.
104154	*/
104155	static void generateError(sqlite3_context *pContext,
104156	const char prefix, const char msg){
104157	char buf[512];
104158	if( msg==NULL ) msg = sqlite3_errmsg(sqlite3_context_db_handle(pContext));
104159	sqlite3_snprintf(sizeof(buf), buf, "%s: %s", prefix, msg);
104160	sqlite3_result_error(pContext, buf, -1);
104161	}
104162
104163	/* Helper function to collect the set of terms in the segment into
104164	** pTerms. The segment is defined by the leaf nodes between
104165	** iStartBlockid and iEndBlockid, inclusive, or by the contents of
104166	** pRootData if iStartBlockid is 0 (in which case the entire segment
104167	** fit in a leaf).
104168	*/
104169	static int collectSegmentTerms(fulltext_vtab v, sqlite3_stmt s,
104170	fts3Hash *pTerms){
104171	const sqlite_int64 iStartBlockid = sqlite3_column_int64(s, 0);
104172	const sqlite_int64 iEndBlockid = sqlite3_column_int64(s, 1);
104173	const char *pRootData = sqlite3_column_blob(s, 2);
104174	const int nRootData = sqlite3_column_bytes(s, 2);
104175	LeavesReader reader;
104176	int rc = leavesReaderInit(v, 0, iStartBlockid, iEndBlockid,
104177	pRootData, nRootData, &reader);
104178	if( rc!=SQLITE_OK ) return rc;
104179
104180	while( rc==SQLITE_OK && !leavesReaderAtEnd(&reader) ){
104181	const char *pTerm = leavesReaderTerm(&reader);
104182	const int nTerm = leavesReaderTermBytes(&reader);
104183	void *oldValue = sqlite3Fts3HashFind(pTerms, pTerm, nTerm);
104184	void newValue = (void )((char *)oldValue+1);
104185
104186	/* From the comment before sqlite3Fts3HashInsert in fts3_hash.c,
104187	** the data value passed is returned in case of malloc failure.
104188	*/
104189	if( newValue==sqlite3Fts3HashInsert(pTerms, pTerm, nTerm, newValue) ){
104190	rc = SQLITE_NOMEM;
104191	}else{
104192	rc = leavesReaderStep(v, &reader);
104193	}
104194	}
104195
104196	leavesReaderDestroy(&reader);
104197	return rc;
104198	}
104199
104200	/* Helper function to build the result string for dump_terms(). */
104201	static int generateTermsResult(sqlite3_context pContext, fts3Hash pTerms){
104202	int iTerm, nTerms, nResultBytes, iByte;
104203	char *result;
104204	TermData *pData;
104205	fts3HashElem *e;
104206
104207	/* Iterate pTerms to generate an array of terms in pData for
104208	** sorting.
104209	*/
104210	nTerms = fts3HashCount(pTerms);
104211	assert( nTerms>0 );
104212	pData = sqlite3_malloc(nTerms*sizeof(TermData));
104213	if( pData==NULL ) return SQLITE_NOMEM;
104214
104215	nResultBytes = 0;
104216	for(iTerm = 0, e = fts3HashFirst(pTerms); e; iTerm++, e = fts3HashNext(e)){
104217	nResultBytes += fts3HashKeysize(e)+1; /* Term plus trailing space */
104218	assert( iTerm<nTerms );
104219	pData[iTerm].pTerm = fts3HashKey(e);
104220	pData[iTerm].nTerm = fts3HashKeysize(e);
104221	pData[iTerm].pCollector = fts3HashData(e); /* unused */
104222	}
104223	assert( iTerm==nTerms );
104224
104225	assert( nResultBytes>0 ); /* nTerms>0, nResultsBytes must be, too. */
104226	result = sqlite3_malloc(nResultBytes);
104227	if( result==NULL ){
104228	sqlite3_free(pData);
104229	return SQLITE_NOMEM;
104230	}
104231
104232	if( nTerms>1 ) qsort(pData, nTerms, sizeof(*pData), termDataCmp);
104233
104234	/* Read the terms in order to build the result. */
104235	iByte = 0;
104236	for(iTerm=0; iTerm<nTerms; ++iTerm){
104237	memcpy(result+iByte, pData[iTerm].pTerm, pData[iTerm].nTerm);
104238	iByte += pData[iTerm].nTerm;
104239	result[iByte++] = ' ';
104240	}
104241	assert( iByte==nResultBytes );
104242	assert( result[nResultBytes-1]==' ' );
104243	result[nResultBytes-1] = '\0';
104244
104245	/* Passes away ownership of result. */
104246	sqlite3_result_text(pContext, result, nResultBytes-1, sqlite3_free);
104247	sqlite3_free(pData);
104248	return SQLITE_OK;
104249	}
104250
104251	/* Implements dump_terms() for use in inspecting the fts3 index from
104252	** tests. TEXT result containing the ordered list of terms joined by
104253	** spaces. dump_terms(t, level, idx) dumps the terms for the segment
104254	** specified by level, idx (in %_segdir), while dump_terms(t) dumps
104255	** all terms in the index. In both cases t is the fts table's magic
104256	** table-named column.
104257	*/
104258	static void dumpTermsFunc(
104259	sqlite3_context *pContext,
104260	int argc, sqlite3_value **argv
104261	){
104262	fulltext_cursor *pCursor;
104263	if( argc!=3 && argc!=1 ){
104264	generateError(pContext, "dump_terms", "incorrect arguments");
104265	}else if( sqlite3_value_type(argv[0])!=SQLITE_BLOB \|\|
104266	sqlite3_value_bytes(argv[0])!=sizeof(pCursor) ){
104267	generateError(pContext, "dump_terms", "illegal first argument");
104268	}else{
104269	fulltext_vtab *v;
104270	fts3Hash terms;
104271	sqlite3_stmt *s = NULL;
104272	int rc;
104273
104274	memcpy(&pCursor, sqlite3_value_blob(argv[0]), sizeof(pCursor));
104275	v = cursor_vtab(pCursor);
104276
104277	/* If passed only the cursor column, get all segments. Otherwise
104278	** get the segment described by the following two arguments.
104279	*/
104280	if( argc==1 ){
104281	rc = sql_get_statement(v, SEGDIR_SELECT_ALL_STMT, &s);
104282	}else{
104283	rc = sql_get_statement(v, SEGDIR_SELECT_SEGMENT_STMT, &s);
104284	if( rc==SQLITE_OK ){
104285	rc = sqlite3_bind_int(s, 1, sqlite3_value_int(argv[1]));
104286	if( rc==SQLITE_OK ){
104287	rc = sqlite3_bind_int(s, 2, sqlite3_value_int(argv[2]));
104288	}
104289	}
104290	}
104291
104292	if( rc!=SQLITE_OK ){
104293	generateError(pContext, "dump_terms", NULL);
104294	return;
104295	}
104296
104297	/* Collect the terms for each segment. */
104298	sqlite3Fts3HashInit(&terms, FTS3_HASH_STRING, 1);
104299	while( (rc = sqlite3_step(s))==SQLITE_ROW ){
104300	rc = collectSegmentTerms(v, s, &terms);
104301	if( rc!=SQLITE_OK ) break;
104302	}
104303
104304	if( rc!=SQLITE_DONE ){
104305	sqlite3_reset(s);
104306	generateError(pContext, "dump_terms", NULL);
104307	}else{
104308	const int nTerms = fts3HashCount(&terms);
104309	if( nTerms>0 ){
104310	rc = generateTermsResult(pContext, &terms);
104311	if( rc==SQLITE_NOMEM ){
104312	generateError(pContext, "dump_terms", "out of memory");
104313	}else{
104314	assert( rc==SQLITE_OK );
104315	}
104316	}else if( argc==3 ){
104317	/* The specific segment asked for could not be found. */
104318	generateError(pContext, "dump_terms", "segment not found");
104319	}else{
104320	/* No segments found. */
104321	/* TODO(shess): It should be impossible to reach this. This
104322	** case can only happen for an empty table, in which case
104323	** SQLite has no rows to call this function on.
104324	*/
104325	sqlite3_result_null(pContext);
104326	}
104327	}
104328	sqlite3Fts3HashClear(&terms);
104329	}
104330	}
104331
104332	/* Expand the DL_DEFAULT doclist in pData into a text result in
104333	** pContext.
104334	*/
104335	static void createDoclistResult(sqlite3_context *pContext,
104336	const char *pData, int nData){
104337	DataBuffer dump;
104338	DLReader dlReader;
104339
104340	assert( pData!=NULL && nData>0 );
104341
104342	dataBufferInit(&dump, 0);
104343	dlrInit(&dlReader, DL_DEFAULT, pData, nData);
104344	for( ; !dlrAtEnd(&dlReader); dlrStep(&dlReader) ){
104345	char buf[256];
104346	PLReader plReader;
104347
104348	plrInit(&plReader, &dlReader);
104349	if( DL_DEFAULT==DL_DOCIDS \|\| plrAtEnd(&plReader) ){
104350	sqlite3_snprintf(sizeof(buf), buf, "[%lld] ", dlrDocid(&dlReader));
104351	dataBufferAppend(&dump, buf, strlen(buf));
104352	}else{
104353	int iColumn = plrColumn(&plReader);
104354
104355	sqlite3_snprintf(sizeof(buf), buf, "[%lld %d[",
104356	dlrDocid(&dlReader), iColumn);
104357	dataBufferAppend(&dump, buf, strlen(buf));
104358
104359	for( ; !plrAtEnd(&plReader); plrStep(&plReader) ){
104360	if( plrColumn(&plReader)!=iColumn ){
104361	iColumn = plrColumn(&plReader);
104362	sqlite3_snprintf(sizeof(buf), buf, "] %d[", iColumn);
104363	assert( dump.nData>0 );
104364	dump.nData--; /* Overwrite trailing space. */
104365	assert( dump.pData[dump.nData]==' ');
104366	dataBufferAppend(&dump, buf, strlen(buf));
104367	}
104368	if( DL_DEFAULT==DL_POSITIONS_OFFSETS ){
104369	sqlite3_snprintf(sizeof(buf), buf, "%d,%d,%d ",
104370	plrPosition(&plReader),
104371	plrStartOffset(&plReader), plrEndOffset(&plReader));
104372	}else if( DL_DEFAULT==DL_POSITIONS ){
104373	sqlite3_snprintf(sizeof(buf), buf, "%d ", plrPosition(&plReader));
104374	}else{
104375	assert( NULL=="Unhandled DL_DEFAULT value");
104376	}
104377	dataBufferAppend(&dump, buf, strlen(buf));
104378	}
104379	plrDestroy(&plReader);
104380
104381	assert( dump.nData>0 );
104382	dump.nData--; /* Overwrite trailing space. */
104383	assert( dump.pData[dump.nData]==' ');
104384	dataBufferAppend(&dump, "]] ", 3);
104385	}
104386	}
104387	dlrDestroy(&dlReader);
104388
104389	assert( dump.nData>0 );
104390	dump.nData--; /* Overwrite trailing space. */
104391	assert( dump.pData[dump.nData]==' ');
104392	dump.pData[dump.nData] = '\0';
104393	assert( dump.nData>0 );
104394
104395	/* Passes ownership of dump's buffer to pContext. */
104396	sqlite3_result_text(pContext, dump.pData, dump.nData, sqlite3_free);
104397	dump.pData = NULL;
104398	dump.nData = dump.nCapacity = 0;
104399	}
104400
104401	/* Implements dump_doclist() for use in inspecting the fts3 index from
104402	** tests. TEXT result containing a string representation of the
104403	** doclist for the indicated term. dump_doclist(t, term, level, idx)
104404	** dumps the doclist for term from the segment specified by level, idx
104405	** (in %_segdir), while dump_doclist(t, term) dumps the logical
104406	** doclist for the term across all segments. The per-segment doclist
104407	** can contain deletions, while the full-index doclist will not
104408	** (deletions are omitted).
104409	**
104410	** Result formats differ with the setting of DL_DEFAULTS. Examples:
104411	**
104412	** DL_DOCIDS: [1] [3] [7]
104413	** DL_POSITIONS: [1 0[0 4] 1[17]] [3 1[5]]
104414	** DL_POSITIONS_OFFSETS: [1 0[0,0,3 4,23,26] 1[17,102,105]] [3 1[5,20,23]]
104415	**
104416	** In each case the number after the outer '[' is the docid. In the
104417	** latter two cases, the number before the inner '[' is the column
104418	** associated with the values within. For DL_POSITIONS the numbers
104419	** within are the positions, for DL_POSITIONS_OFFSETS they are the
104420	** position, the start offset, and the end offset.
104421	*/
104422	static void dumpDoclistFunc(
104423	sqlite3_context *pContext,
104424	int argc, sqlite3_value **argv
104425	){
104426	fulltext_cursor *pCursor;
104427	if( argc!=2 && argc!=4 ){
104428	generateError(pContext, "dump_doclist", "incorrect arguments");
104429	}else if( sqlite3_value_type(argv[0])!=SQLITE_BLOB \|\|
104430	sqlite3_value_bytes(argv[0])!=sizeof(pCursor) ){
104431	generateError(pContext, "dump_doclist", "illegal first argument");
104432	}else if( sqlite3_value_text(argv[1])==NULL \|\|
104433	sqlite3_value_text(argv[1])[0]=='\0' ){
104434	generateError(pContext, "dump_doclist", "empty second argument");
104435	}else{
104436	const char pTerm = (const char )sqlite3_value_text(argv[1]);
104437	const int nTerm = strlen(pTerm);
104438	fulltext_vtab *v;
104439	int rc;
104440	DataBuffer doclist;
104441
104442	memcpy(&pCursor, sqlite3_value_blob(argv[0]), sizeof(pCursor));
104443	v = cursor_vtab(pCursor);
104444
104445	dataBufferInit(&doclist, 0);
104446
104447	/* termSelect() yields the same logical doclist that queries are
104448	** run against.
104449	*/
104450	if( argc==2 ){
104451	rc = termSelect(v, v->nColumn, pTerm, nTerm, 0, DL_DEFAULT, &doclist);
104452	}else{
104453	sqlite3_stmt *s = NULL;
104454
104455	/* Get our specific segment's information. */
104456	rc = sql_get_statement(v, SEGDIR_SELECT_SEGMENT_STMT, &s);
104457	if( rc==SQLITE_OK ){
104458	rc = sqlite3_bind_int(s, 1, sqlite3_value_int(argv[2]));
104459	if( rc==SQLITE_OK ){
104460	rc = sqlite3_bind_int(s, 2, sqlite3_value_int(argv[3]));
104461	}
104462	}
104463
104464	if( rc==SQLITE_OK ){
104465	rc = sqlite3_step(s);
104466
104467	if( rc==SQLITE_DONE ){
104468	dataBufferDestroy(&doclist);
104469	generateError(pContext, "dump_doclist", "segment not found");
104470	return;
104471	}
104472
104473	/* Found a segment, load it into doclist. */
104474	if( rc==SQLITE_ROW ){
104475	const sqlite_int64 iLeavesEnd = sqlite3_column_int64(s, 1);
104476	const char *pData = sqlite3_column_blob(s, 2);
104477	const int nData = sqlite3_column_bytes(s, 2);
104478
104479	/* loadSegment() is used by termSelect() to load each
104480	** segment's data.
104481	*/
104482	rc = loadSegment(v, pData, nData, iLeavesEnd, pTerm, nTerm, 0,
104483	&doclist);
104484	if( rc==SQLITE_OK ){
104485	rc = sqlite3_step(s);
104486
104487	/* Should not have more than one matching segment. */
104488	if( rc!=SQLITE_DONE ){
104489	sqlite3_reset(s);
104490	dataBufferDestroy(&doclist);
104491	generateError(pContext, "dump_doclist", "invalid segdir");
104492	return;
104493	}
104494	rc = SQLITE_OK;
104495	}
104496	}
104497	}
104498
104499	sqlite3_reset(s);
104500	}
104501
104502	if( rc==SQLITE_OK ){
104503	if( doclist.nData>0 ){
104504	createDoclistResult(pContext, doclist.pData, doclist.nData);
104505	}else{
104506	/* TODO(shess): This can happen if the term is not present, or
104507	** if all instances of the term have been deleted and this is
104508	** an all-index dump. It may be interesting to distinguish
104509	** these cases.
104510	*/
104511	sqlite3_result_text(pContext, "", 0, SQLITE_STATIC);
104512	}
104513	}else if( rc==SQLITE_NOMEM ){
104514	/* Handle out-of-memory cases specially because if they are
104515	** generated in fts3 code they may not be reflected in the db
104516	** handle.
104517	*/
104518	/* TODO(shess): Handle this more comprehensively.
104519	** sqlite3ErrStr() has what I need, but is internal.
104520	*/
104521	generateError(pContext, "dump_doclist", "out of memory");
104522	}else{
104523	generateError(pContext, "dump_doclist", NULL);
104524	}
104525
104526	dataBufferDestroy(&doclist);
104527	}
104528	}
104529	#endif
104530
104531	/*
104532	** This routine implements the xFindFunction method for the FTS3
104533	** virtual table.
104534	*/
104535	static int fulltextFindFunction(
104536	sqlite3_vtab *pVtab,
104537	int nArg,
104538	const char *zName,
104539	void (*pxFunc)(sqlite3_context,int,sqlite3_value**),
104540	void **ppArg
104541	){
104542	if( strcmp(zName,"snippet")==0 ){
104543	*pxFunc = snippetFunc;
104544	return 1;
104545	}else if( strcmp(zName,"offsets")==0 ){
104546	*pxFunc = snippetOffsetsFunc;
104547	return 1;
104548	}else if( strcmp(zName,"optimize")==0 ){
104549	*pxFunc = optimizeFunc;
104550	return 1;
104551	#ifdef SQLITE_TEST
104552	/* NOTE(shess): These functions are present only for testing
104553	** purposes. No particular effort is made to optimize their
104554	** execution or how they build their results.
104555	*/
104556	}else if( strcmp(zName,"dump_terms")==0 ){
104557	/* fprintf(stderr, "Found dump_terms\n"); */
104558	*pxFunc = dumpTermsFunc;
104559	return 1;
104560	}else if( strcmp(zName,"dump_doclist")==0 ){
104561	/* fprintf(stderr, "Found dump_doclist\n"); */
104562	*pxFunc = dumpDoclistFunc;
104563	return 1;
104564	#endif
104565	}
104566	return 0;
104567	}
104568
104569	/*
104570	** Rename an fts3 table.
104571	*/
104572	static int fulltextRename(
104573	sqlite3_vtab *pVtab,
104574	const char *zName
104575	){
104576	fulltext_vtab p = (fulltext_vtab )pVtab;
104577	int rc = SQLITE_NOMEM;
104578	char *zSql = sqlite3_mprintf(


104579	"ALTER TABLE %Q.'%q_content' RENAME TO '%q_content';"
104580	"ALTER TABLE %Q.'%q_segments' RENAME TO '%q_segments';"
104581	"ALTER TABLE %Q.'%q_segdir' RENAME TO '%q_segdir';"
104582	, p->zDb, p->zName, zName
104583	, p->zDb, p->zName, zName
	@@ -104590,33 +99598,38 @@
104590	return rc;
104591	}
104592
104593	static const sqlite3_module fts3Module = {
104594	/* iVersion */ 0,
104595	/* xCreate */ fulltextCreate,
104596	/* xConnect */ fulltextConnect,
104597	/* xBestIndex */ fulltextBestIndex,
104598	/* xDisconnect */ fulltextDisconnect,
104599	/* xDestroy */ fulltextDestroy,
104600	/* xOpen */ fulltextOpen,
104601	/* xClose */ fulltextClose,
104602	/* xFilter */ fulltextFilter,
104603	/* xNext */ fulltextNext,
104604	/* xEof */ fulltextEof,
104605	/* xColumn */ fulltextColumn,
104606	/* xRowid */ fulltextRowid,
104607	/* xUpdate */ fulltextUpdate,
104608	/* xBegin */ fulltextBegin,
104609	/* xSync */ fulltextSync,
104610	/* xCommit */ fulltextCommit,
104611	/* xRollback */ fulltextRollback,
104612	/* xFindFunction */ fulltextFindFunction,
104613	/* xRename */ fulltextRename,
104614	};
104615





104616	static void hashDestroy(void *p){
104617	fts3Hash pHash = (fts3Hash )p;
104618	sqlite3Fts3HashClear(pHash);
104619	sqlite3_free(pHash);
104620	}
104621
104622	/*
	@@ -104632,21 +99645,19 @@
104632	*/
104633	SQLITE_PRIVATE void sqlite3Fts3SimpleTokenizerModule(sqlite3_tokenizer_module const**ppModule);
104634	SQLITE_PRIVATE void sqlite3Fts3PorterTokenizerModule(sqlite3_tokenizer_module const**ppModule);
104635	SQLITE_PRIVATE void sqlite3Fts3IcuTokenizerModule(sqlite3_tokenizer_module const**ppModule);
104636
104637	SQLITE_PRIVATE int sqlite3Fts3InitHashTable(sqlite3 , fts3Hash , const char *);
104638
104639	/*
104640	** Initialise the fts3 extension. If this extension is built as part
104641	** of the sqlite library, then this function is called directly by
104642	** SQLite. If fts3 is built as a dynamically loadable extension, this
104643	** function is called by the sqlite3_extension_init() entry point.
104644	*/
104645	SQLITE_PRIVATE int sqlite3Fts3Init(sqlite3 *db){
104646	int rc = SQLITE_OK;
104647	fts3Hash *pHash = 0;
104648	const sqlite3_tokenizer_module *pSimple = 0;
104649	const sqlite3_tokenizer_module *pPorter = 0;
104650	const sqlite3_tokenizer_module *pIcu = 0;
104651
104652	sqlite3Fts3SimpleTokenizerModule(&pSimple);
	@@ -104654,11 +99665,11 @@
104654	#ifdef SQLITE_ENABLE_ICU
104655	sqlite3Fts3IcuTokenizerModule(&pIcu);
104656	#endif
104657
104658	/* Allocate and initialise the hash-table used to store tokenizers. */
104659	pHash = sqlite3_malloc(sizeof(fts3Hash));
104660	if( !pHash ){
104661	rc = SQLITE_NOMEM;
104662	}else{
104663	sqlite3Fts3HashInit(pHash, FTS3_HASH_STRING, 1);
104664	}
	@@ -104682,16 +99693,12 @@
104682	** module with sqlite.
104683	*/
104684	if( SQLITE_OK==rc
104685	&& SQLITE_OK==(rc = sqlite3Fts3InitHashTable(db, pHash, "fts3_tokenizer"))
104686	&& SQLITE_OK==(rc = sqlite3_overload_function(db, "snippet", -1))
104687	&& SQLITE_OK==(rc = sqlite3_overload_function(db, "offsets", -1))
104688	&& SQLITE_OK==(rc = sqlite3_overload_function(db, "optimize", -1))
104689	#ifdef SQLITE_TEST
104690	&& SQLITE_OK==(rc = sqlite3_overload_function(db, "dump_terms", -1))
104691	&& SQLITE_OK==(rc = sqlite3_overload_function(db, "dump_doclist", -1))
104692	#endif
104693	){
104694	return sqlite3_create_module_v2(
104695	db, "fts3", &fts3Module, (void *)pHash, hashDestroy
104696	);
104697	}
	@@ -104714,11 +99721,11 @@
104714	SQLITE_EXTENSION_INIT2(pApi)
104715	return sqlite3Fts3Init(db);
104716	}
104717	#endif
104718
104719	#endif /* !defined(SQLITE_CORE) \|\| defined(SQLITE_ENABLE_FTS3) */
104720
104721	/************ End of fts3.c **********************************************/
104722	/************ Begin file fts3_expr.c *************************************/
104723	/*
104724	** 2008 Nov 28
	@@ -104733,12 +99740,11 @@
104733	******************************************************************************
104734	**
104735	** This module contains code that implements a parser for fts3 query strings
104736	** (the right-hand argument to the MATCH operator). Because the supported
104737	** syntax is relatively simple, the whole tokenizer/parser system is
104738	** hand-coded. The public interface to this module is declared in source
104739	** code file "fts3_expr.h".
104740	*/
104741	#if !defined(SQLITE_CORE) \|\| defined(SQLITE_ENABLE_FTS3)
104742
104743	/*
104744	** By default, this module parses the legacy syntax that has been
	@@ -104760,11 +99766,33 @@
104760	** If compiled with SQLITE_TEST defined, then this module exports the
104761	** symbol "int sqlite3_fts3_enable_parentheses". Setting this variable
104762	** to zero causes the module to use the old syntax. If it is set to
104763	** non-zero the new syntax is activated. This is so both syntaxes can
104764	** be tested using a single build of testfixture.





















104765	*/

104766	#ifdef SQLITE_TEST
104767	SQLITE_API int sqlite3_fts3_enable_parentheses = 0;
104768	#else
104769	# ifdef SQLITE_ENABLE_FTS3_PARENTHESIS
104770	# define sqlite3_fts3_enable_parentheses 1
	@@ -105069,10 +100097,13 @@
105069	cNext = zInput[nKey];
105070	if( fts3isspace(cNext)
105071	\|\| cNext=='"' \|\| cNext=='(' \|\| cNext==')' \|\| cNext==0
105072	){
105073	pRet = (Fts3Expr *)sqlite3_malloc(sizeof(Fts3Expr));



105074	memset(pRet, 0, sizeof(Fts3Expr));
105075	pRet->eType = pKey->eType;
105076	pRet->nNear = nNear;
105077	*ppExpr = pRet;
105078	*pnConsumed = (zInput - z) + nKey;
	@@ -105643,10 +100674,125 @@
105643	** SQLite (in which case SQLITE_ENABLE_FTS3 is defined).
105644	*/
105645	#if !defined(SQLITE_CORE) \|\| defined(SQLITE_ENABLE_FTS3)
105646
105647



















































































































105648
105649	/*
105650	** Malloc and Free functions
105651	*/
105652	static void *fts3HashMalloc(int n){
	@@ -105668,11 +100814,11 @@
105668	** FTS3_HASH_BINARY or FTS3_HASH_STRING. The value of keyClass
105669	** determines what kind of key the hash table will use. "copyKey" is
105670	** true if the hash table should make its own private copy of keys and
105671	** false if it should just use the supplied pointer.
105672	*/
105673	SQLITE_PRIVATE void sqlite3Fts3HashInit(fts3Hash *pNew, int keyClass, int copyKey){
105674	assert( pNew!=0 );
105675	assert( keyClass>=FTS3_HASH_STRING && keyClass<=FTS3_HASH_BINARY );
105676	pNew->keyClass = keyClass;
105677	pNew->copyKey = copyKey;
105678	pNew->first = 0;
	@@ -105683,21 +100829,21 @@
105683
105684	/* Remove all entries from a hash table. Reclaim all memory.
105685	** Call this routine to delete a hash table or to reset a hash table
105686	** to the empty state.
105687	*/
105688	SQLITE_PRIVATE void sqlite3Fts3HashClear(fts3Hash *pH){
105689	fts3HashElem elem; / For looping over all elements of the table */
105690
105691	assert( pH!=0 );
105692	elem = pH->first;
105693	pH->first = 0;
105694	fts3HashFree(pH->ht);
105695	pH->ht = 0;
105696	pH->htsize = 0;
105697	while( elem ){
105698	fts3HashElem *next_elem = elem->next;
105699	if( pH->copyKey && elem->pKey ){
105700	fts3HashFree(elem->pKey);
105701	}
105702	fts3HashFree(elem);
105703	elem = next_elem;
	@@ -105776,15 +100922,15 @@
105776	}
105777
105778	/* Link an element into the hash table
105779	*/
105780	static void fts3HashInsertElement(
105781	fts3Hash pH, / The complete hash table */
105782	struct _fts3ht pEntry, / The entry into which pNew is inserted */
105783	fts3HashElem pNew / The element to be inserted */
105784	){
105785	fts3HashElem pHead; / First element already in pEntry */
105786	pHead = pEntry->chain;
105787	if( pHead ){
105788	pNew->next = pHead;
105789	pNew->prev = pHead->prev;
105790	if( pHead->prev ){ pHead->prev->next = pNew; }
	@@ -105803,13 +100949,13 @@
105803
105804	/* Resize the hash table so that it cantains "new_size" buckets.
105805	** "new_size" must be a power of 2. The hash table might fail
105806	** to resize if sqliteMalloc() fails.
105807	*/
105808	static void fts3Rehash(fts3Hash *pH, int new_size){
105809	struct _fts3ht new_ht; / The new hash table */
105810	fts3HashElem elem, next_elem; /* For looping over existing elements */
105811	int (xHash)(const void,int); /* The hash function */
105812
105813	assert( (new_size & (new_size-1))==0 );
105814	new_ht = (struct _fts3ht )fts3HashMalloc( new_sizesizeof(struct _fts3ht) );
105815	if( new_ht==0 ) return;
	@@ -105826,17 +100972,17 @@
105826
105827	/* This function (for internal use only) locates an element in an
105828	** hash table that matches the given key. The hash for this key has
105829	** already been computed and is passed as the 4th parameter.
105830	*/
105831	static fts3HashElem *fts3FindElementByHash(
105832	const fts3Hash pH, / The pH to be searched */
105833	const void pKey, / The key we are searching for */
105834	int nKey,
105835	int h /* The hash for this key. */
105836	){
105837	fts3HashElem elem; / Used to loop thru the element list */
105838	int count; /* Number of elements left to test */
105839	int (xCompare)(const void,int,const void,int); / comparison function */
105840
105841	if( pH->ht ){
105842	struct _fts3ht *pEntry = &pH->ht[h];
	@@ -105855,12 +101001,12 @@
105855
105856	/* Remove a single entry from the hash table given a pointer to that
105857	** element and a hash on the element's key.
105858	*/
105859	static void fts3RemoveElementByHash(
105860	fts3Hash pH, / The pH containing "elem" */
105861	fts3HashElem* elem, /* The element to be removed from the pH */
105862	int h /* Hash value for the element */
105863	){
105864	struct _fts3ht *pEntry;
105865	if( elem->prev ){
105866	elem->prev->next = elem->next;
	@@ -105892,13 +101038,13 @@
105892
105893	/* Attempt to locate an element of the hash table pH with a key
105894	** that matches pKey,nKey. Return the data for this element if it is
105895	** found, or NULL if there is no match.
105896	*/
105897	SQLITE_PRIVATE void sqlite3Fts3HashFind(const fts3Hash pH, const void *pKey, int nKey){
105898	int h; /* A hash on key */
105899	fts3HashElem elem; / The element that matches key */
105900	int (xHash)(const void,int); /* The hash function */
105901
105902	if( pH==0 \|\| pH->ht==0 ) return 0;
105903	xHash = ftsHashFunction(pH->keyClass);
105904	assert( xHash!=0 );
	@@ -105922,19 +101068,19 @@
105922	**
105923	** If the "data" parameter to this function is NULL, then the
105924	** element corresponding to "key" is removed from the hash table.
105925	*/
105926	SQLITE_PRIVATE void *sqlite3Fts3HashInsert(
105927	fts3Hash pH, / The hash table to insert into */
105928	const void pKey, / The key */
105929	int nKey, /* Number of bytes in the key */
105930	void data / The data */
105931	){
105932	int hraw; /* Raw hash value of the key */
105933	int h; /* the hash of the key modulo hash table size */
105934	fts3HashElem elem; / Used to loop thru the element list */
105935	fts3HashElem new_elem; / New element added to the pH */
105936	int (xHash)(const void,int); /* The hash function */
105937
105938	assert( pH!=0 );
105939	xHash = ftsHashFunction(pH->keyClass);
105940	assert( xHash!=0 );
	@@ -105957,11 +101103,11 @@
105957	if( pH->htsize==0 ){
105958	pH->count = 0;
105959	return data;
105960	}
105961	}
105962	new_elem = (fts3HashElem*)fts3HashMalloc( sizeof(fts3HashElem) );
105963	if( new_elem==0 ) return data;
105964	if( pH->copyKey && pKey!=0 ){
105965	new_elem->pKey = fts3HashMalloc( nKey );
105966	if( new_elem->pKey==0 ){
105967	fts3HashFree(new_elem);
	@@ -106014,10 +101160,162 @@
106014	*/
106015	#if !defined(SQLITE_CORE) \|\| defined(SQLITE_ENABLE_FTS3)
106016
106017
106018
























































































































































106019
106020	/*
106021	** Class derived from sqlite3_tokenizer
106022	*/
106023	typedef struct porter_tokenizer {
	@@ -106682,18 +101980,18 @@
106682	static void scalarFunc(
106683	sqlite3_context *context,
106684	int argc,
106685	sqlite3_value **argv
106686	){
106687	fts3Hash *pHash;
106688	void *pPtr = 0;
106689	const unsigned char *zName;
106690	int nName;
106691
106692	assert( argc==1 \|\| argc==2 );
106693
106694	pHash = (fts3Hash *)sqlite3_user_data(context);
106695
106696	zName = sqlite3_value_text(argv[0]);
106697	nName = sqlite3_value_bytes(argv[0])+1;
106698
106699	if( argc==2 ){
	@@ -106719,10 +102017,132 @@
106719	}
106720	}
106721
106722	sqlite3_result_blob(context, (void *)&pPtr, sizeof(pPtr), SQLITE_TRANSIENT);
106723	}


























































































































106724
106725	#ifdef SQLITE_TEST
106726
106727
106728	/*
	@@ -106754,11 +102174,11 @@
106754	static void testFunc(
106755	sqlite3_context *context,
106756	int argc,
106757	sqlite3_value **argv
106758	){
106759	fts3Hash *pHash;
106760	sqlite3_tokenizer_module *p;
106761	sqlite3_tokenizer *pTokenizer = 0;
106762	sqlite3_tokenizer_cursor *pCsr = 0;
106763
106764	const char *zErr = 0;
	@@ -106787,11 +102207,11 @@
106787
106788	if( argc==3 ){
106789	zArg = (const char *)sqlite3_value_text(argv[1]);
106790	}
106791
106792	pHash = (fts3Hash *)sqlite3_user_data(context);
106793	p = (sqlite3_tokenizer_module *)sqlite3Fts3HashFind(pHash, zName, nName+1);
106794
106795	if( !p ){
106796	char *zErr = sqlite3_mprintf("unknown tokenizer: %s", zName);
106797	sqlite3_result_error(context, zErr, -1);
	@@ -106956,11 +102376,11 @@
106956	** The third argument to this function, zName, is used as the name
106957	** of both the scalar and, if created, the virtual table.
106958	*/
106959	SQLITE_PRIVATE int sqlite3Fts3InitHashTable(
106960	sqlite3 *db,
106961	fts3Hash *pHash,
106962	const char *zName
106963	){
106964	int rc = SQLITE_OK;
106965	void p = (void )pHash;
106966	const int any = SQLITE_ANY;
	@@ -107219,10 +102639,2911 @@
107219	}
107220
107221	#endif /* !defined(SQLITE_CORE) \|\| defined(SQLITE_ENABLE_FTS3) */
107222
107223	/************ End of fts3_tokenizer1.c ***********************************/





















































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































107224	/************ Begin file rtree.c *****************************************/
107225	/*
107226	** 2001 September 15
107227	**
107228	** The author disclaims copyright to this source code. In place of
	@@ -107233,12 +105554,10 @@
107233	** May you share freely, never taking more than you give.
107234	**
107235	*************************************************************************
107236	** This file contains code for implementations of the r-tree and r*-tree
107237	** algorithms packaged as an SQLite virtual table module.
107238	**
107239	** $Id: rtree.c,v 1.14 2009/08/06 18:36:47 danielk1977 Exp $
107240	*/
107241
107242	#if !defined(SQLITE_CORE) \|\| defined(SQLITE_ENABLE_RTREE)
107243
107244	/*
107245

	--- src/sqlite3.c
	+++ src/sqlite3.c
	@@ -1,8 +1,8 @@
1	/******************************************************************************
2	** This file is an amalgamation of many separate C source files from SQLite
3	** version 3.6.21. By combining all the individual C code files into this
4	** single large file, the entire code can be compiled as a one translation
5	** unit. This allows many compilers to do optimizations that would not be
6	** possible if the files were compiled separately. Performance improvements
7	** of 5% are more are commonly seen when SQLite is compiled as a single
8	** translation unit.
	@@ -91,12 +91,10 @@
91	** May you share freely, never taking more than you give.
92	**
93	*************************************************************************
94	**
95	** This file defines various limits of what SQLite can process.


96	*/
97
98	/*
99	** The maximum length of a TEXT or BLOB in bytes. This also
100	** limits the size of a row in a table or index.
	@@ -277,15 +275,11 @@
275	** A value of 1 means that a trigger program will not be able to itself
276	** fire any triggers. A value of 0 means that no trigger programs at all
277	** may be executed.
278	*/
279	#ifndef SQLITE_MAX_TRIGGER_DEPTH



280	# define SQLITE_MAX_TRIGGER_DEPTH 1000

281	#endif
282
283	/************ End of sqliteLimit.h ***************************************/
284	/************ Continuing where we left off in sqliteInt.h ****************/
285
	@@ -647,13 +641,13 @@
641	** [sqlite3_libversion_number()], [sqlite3_sourceid()],
642	** [sqlite_version()] and [sqlite_source_id()].
643	**
644	** Requirements: [H10011] [H10014]
645	*/
646	#define SQLITE_VERSION "3.6.21"
647	#define SQLITE_VERSION_NUMBER 3006021
648	#define SQLITE_SOURCE_ID "2009-11-23 13:17:27 39214aee6553db76309851e7aa74fcc02d4f59b7"
649
650	/*
651	** CAPI3REF: Run-Time Library Version Numbers {H10020} <S60100>
652	** KEYWORDS: sqlite3_version
653	**
	@@ -6316,12 +6310,10 @@
6310	** May you share freely, never taking more than you give.
6311	**
6312	*************************************************************************
6313	** This is the header file for the generic hash-table implemenation
6314	** used in SQLite.


6315	*/
6316	#ifndef _SQLITE_HASH_H_
6317	#define _SQLITE_HASH_H_
6318
6319	/* Forward declarations of structures. */
	@@ -6920,12 +6912,10 @@
6912	**
6913	*************************************************************************
6914	** This header file defines the interface that the sqlite B-Tree file
6915	** subsystem. See comments in the source code for a detailed description
6916	** of what each interface routine does.


6917	*/
6918	#ifndef _BTREE_H_
6919	#define _BTREE_H_
6920
6921	/* TODO: This definition is just included so other modules compile. It
	@@ -7058,10 +7048,11 @@
7048	int wrFlag, /* 1 for writing. 0 for read-only */
7049	struct KeyInfo, / First argument to compare function */
7050	BtCursor pCursor / Space to write cursor structure */
7051	);
7052	SQLITE_PRIVATE int sqlite3BtreeCursorSize(void);
7053	SQLITE_PRIVATE void sqlite3BtreeCursorZero(BtCursor*);
7054
7055	SQLITE_PRIVATE int sqlite3BtreeCloseCursor(BtCursor*);
7056	SQLITE_PRIVATE int sqlite3BtreeMovetoUnpacked(
7057	BtCursor*,
7058	UnpackedRecord *pUnKey,
	@@ -7169,12 +7160,10 @@
7160	** Header file for the Virtual DataBase Engine (VDBE)
7161	**
7162	** This header defines the interface to the virtual database engine
7163	** or VDBE. The VDBE implements an abstract machine that runs a
7164	** simple program to access and modify the underlying database.


7165	*/
7166	#ifndef _SQLITE_VDBE_H_
7167	#define _SQLITE_VDBE_H_
7168
7169	/*
	@@ -7198,11 +7187,11 @@
7187	** as an instance of the following structure:
7188	*/
7189	struct VdbeOp {
7190	u8 opcode; /* What operation to perform */
7191	signed char p4type; /* One of the P4_xxx constants for p4 */
7192	u8 opflags; /* Mask of the OPFLG_* flags in opcodes.h */
7193	u8 p5; /* Fifth parameter is an unsigned character */
7194	int p1; /* First operand */
7195	int p2; /* Second parameter (often the jump destination) */
7196	int p3; /* The third parameter */
7197	union { /* fourth parameter */
	@@ -7476,26 +7465,27 @@
7465	#define OPFLG_JUMP 0x0001 /* jump: P2 holds jmp target */
7466	#define OPFLG_OUT2_PRERELEASE 0x0002 /* out2-prerelease: */
7467	#define OPFLG_IN1 0x0004 /* in1: P1 is an input */
7468	#define OPFLG_IN2 0x0008 /* in2: P2 is an input */
7469	#define OPFLG_IN3 0x0010 /* in3: P3 is an input */
7470	#define OPFLG_OUT2 0x0020 /* out2: P2 is an output */
7471	#define OPFLG_OUT3 0x0040 /* out3: P3 is an output */
7472	#define OPFLG_INITIALIZER {\
7473	/* 0 */ 0x00, 0x01, 0x05, 0x04, 0x04, 0x10, 0x00, 0x02,\
7474	/* 8 */ 0x02, 0x02, 0x02, 0x02, 0x00, 0x00, 0x24, 0x24,\
7475	/* 16 */ 0x00, 0x00, 0x00, 0x24, 0x04, 0x05, 0x04, 0x00,\
7476	/* 24 */ 0x00, 0x01, 0x05, 0x05, 0x00, 0x00, 0x00, 0x02,\
7477	/* 32 */ 0x00, 0x00, 0x00, 0x02, 0x10, 0x00, 0x00, 0x00,\
7478	/* 40 */ 0x00, 0x00, 0x00, 0x11, 0x11, 0x11, 0x11, 0x08,\
7479	/* 48 */ 0x11, 0x11, 0x11, 0x11, 0x02, 0x02, 0x00, 0x00,\
7480	/* 56 */ 0x00, 0x00, 0x00, 0x00, 0x02, 0x00, 0x01, 0x01,\
7481	/* 64 */ 0x01, 0x01, 0x01, 0x08, 0x4c, 0x4c, 0x00, 0x02,\
7482	/* 72 */ 0x01, 0x05, 0x05, 0x15, 0x15, 0x15, 0x15, 0x15,\
7483	/* 80 */ 0x15, 0x01, 0x4c, 0x4c, 0x4c, 0x4c, 0x4c, 0x4c,\
7484	/* 88 */ 0x4c, 0x4c, 0x4c, 0x4c, 0x02, 0x24, 0x02, 0x00,\
7485	/* 96 */ 0x02, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,\
7486	/* 104 */ 0x0c, 0x45, 0x15, 0x01, 0x02, 0x00, 0x01, 0x08,\
7487	/* 112 */ 0x05, 0x05, 0x05, 0x00, 0x00, 0x00, 0x01, 0x00,\
7488	/* 120 */ 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x01,\
7489	/* 128 */ 0x00, 0x00, 0x02, 0x02, 0x00, 0x00, 0x00, 0x00,\
7490	/* 136 */ 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, 0x04, 0x04,\
7491	/* 144 */ 0x04, 0x04,}
	@@ -7511,10 +7501,11 @@
7501	SQLITE_PRIVATE int sqlite3VdbeAddOp0(Vdbe*,int);
7502	SQLITE_PRIVATE int sqlite3VdbeAddOp1(Vdbe*,int,int);
7503	SQLITE_PRIVATE int sqlite3VdbeAddOp2(Vdbe*,int,int,int);
7504	SQLITE_PRIVATE int sqlite3VdbeAddOp3(Vdbe*,int,int,int,int);
7505	SQLITE_PRIVATE int sqlite3VdbeAddOp4(Vdbe,int,int,int,int,const char zP4,int);
7506	SQLITE_PRIVATE int sqlite3VdbeAddOp4Int(Vdbe*,int,int,int,int,int);
7507	SQLITE_PRIVATE int sqlite3VdbeAddOpList(Vdbe, int nOp, VdbeOpList const aOp);
7508	SQLITE_PRIVATE void sqlite3VdbeChangeP1(Vdbe*, int addr, int P1);
7509	SQLITE_PRIVATE void sqlite3VdbeChangeP2(Vdbe*, int addr, int P2);
7510	SQLITE_PRIVATE void sqlite3VdbeChangeP3(Vdbe*, int addr, int P3);
7511	SQLITE_PRIVATE void sqlite3VdbeChangeP5(Vdbe*, u8 P5);
	@@ -7579,12 +7570,10 @@
7570	**
7571	*************************************************************************
7572	** This header file defines the interface that the sqlite page cache
7573	** subsystem. The page cache subsystem reads and writes a file a page
7574	** at a time and provides a journal for rollback.


7575	*/
7576
7577	#ifndef _PAGER_H_
7578	#define _PAGER_H_
7579
	@@ -7750,12 +7739,10 @@
7739	** May you share freely, never taking more than you give.
7740	**
7741	*************************************************************************
7742	** This header file defines the interface that the sqlite page cache
7743	** subsystem.


7744	*/
7745
7746	#ifndef _PCACHE_H_
7747
7748	typedef struct PgHdr PgHdr;
	@@ -7918,12 +7905,10 @@
7905	** "os.c") attempt to abstract the underlying operating system so that
7906	** the SQLite library will work on both POSIX and windows systems.
7907	**
7908	** This header file is #include-ed by sqliteInt.h and thus ends up
7909	** being included by every source file.


7910	*/
7911	#ifndef _SQLITE_OS_H_
7912	#define _SQLITE_OS_H_
7913
7914	/*
	@@ -8198,12 +8183,10 @@
8183	** better organized.
8184	**
8185	** NOTE: source files should not #include this header file directly.
8186	** Source files should #include the sqliteInt.h file and let that file
8187	** include this one indirectly.


8188	*/
8189
8190
8191	/*
8192	** Figure out what version of the code to use. The choices are
	@@ -8576,10 +8559,11 @@
8559	#define SQLITE_FUNC_CASE 0x02 /* Case-sensitive LIKE-type function */
8560	#define SQLITE_FUNC_EPHEM 0x04 /* Ephemeral. Delete with VDBE */
8561	#define SQLITE_FUNC_NEEDCOLL 0x08 /* sqlite3GetFuncCollSeq() might be called */
8562	#define SQLITE_FUNC_PRIVATE 0x10 /* Allowed for internal use only */
8563	#define SQLITE_FUNC_COUNT 0x20 /* Built-in count() aggregate /
8564	#define SQLITE_FUNC_COALESCE 0x40 /* Built-in coalesce() or ifnull() function */
8565
8566	/*
8567	** The following three macros, FUNCTION(), LIKEFUNC() and AGGREGATE() are
8568	** used to create the initializers for the FuncDef structures.
8569	**
	@@ -10290,10 +10274,13 @@
10274	SQLITE_PRIVATE void sqlite3CloseSavepoints(sqlite3 *);
10275	SQLITE_PRIVATE int sqlite3ExprIsConstant(Expr*);
10276	SQLITE_PRIVATE int sqlite3ExprIsConstantNotJoin(Expr*);
10277	SQLITE_PRIVATE int sqlite3ExprIsConstantOrFunction(Expr*);
10278	SQLITE_PRIVATE int sqlite3ExprIsInteger(Expr, int);
10279	SQLITE_PRIVATE int sqlite3ExprCanBeNull(const Expr*);
10280	SQLITE_PRIVATE void sqlite3ExprCodeIsNullJump(Vdbe, const Expr, int, int);
10281	SQLITE_PRIVATE int sqlite3ExprNeedsNoAffinityChange(const Expr*, char);
10282	SQLITE_PRIVATE int sqlite3IsRowid(const char*);
10283	SQLITE_PRIVATE void sqlite3GenerateRowDelete(Parse, Table, int, int, int, Trigger *, int);
10284	SQLITE_PRIVATE void sqlite3GenerateRowIndexDelete(Parse, Table, int, int*);
10285	SQLITE_PRIVATE int sqlite3GenerateIndexKey(Parse, Index, int, int, int);
10286	SQLITE_PRIVATE void sqlite3GenerateConstraintChecks(Parse,Table,int,int,
	@@ -10461,10 +10448,11 @@
10448	SQLITE_PRIVATE char sqlite3Utf8to16(sqlite3 , u8, char , int, int );
10449	#endif
10450	SQLITE_PRIVATE int sqlite3ValueFromExpr(sqlite3 , Expr , u8, u8, sqlite3_value **);
10451	SQLITE_PRIVATE void sqlite3ValueApplyAffinity(sqlite3_value *, u8, u8);
10452	#ifndef SQLITE_AMALGAMATION
10453	SQLITE_PRIVATE const unsigned char sqlite3OpcodeProperty[];
10454	SQLITE_PRIVATE const unsigned char sqlite3UpperToLower[];
10455	SQLITE_PRIVATE const unsigned char sqlite3CtypeMap[];
10456	SQLITE_PRIVATE SQLITE_WSD struct Sqlite3Config sqlite3Config;
10457	SQLITE_PRIVATE SQLITE_WSD FuncDefHash sqlite3GlobalFunctions;
10458	SQLITE_PRIVATE int sqlite3PendingByte;
	@@ -10702,11 +10690,10 @@
10690	*************************************************************************
10691	**
10692	** This file contains definitions of global variables and contants.
10693	*/
10694

10695	/* An array to map all upper-case characters into their corresponding
10696	** lower-case character.
10697	**
10698	** SQLite only considers US-ASCII (or EBCDIC) characters. We do not
10699	** handle case conversions for the UTF character set since the tables
	@@ -10758,10 +10745,11 @@
10745	** isalpha() 0x02
10746	** isdigit() 0x04
10747	** isalnum() 0x06
10748	** isxdigit() 0x08
10749	** toupper() 0x20
10750	** SQLite identifier character 0x40
10751	**
10752	** Bit 0x20 is set if the mapped character requires translation to upper
10753	** case. i.e. if the character is a lower-case ASCII character.
10754	** If x is a lower-case ASCII character, then its upper-case equivalent
10755	** is (x - 0x20). Therefore toupper() can be implemented as:
	@@ -10768,10 +10756,15 @@
10756	**
10757	** (x & ~(map[x]&0x20))
10758	**
10759	** Standard function tolower() is implemented using the sqlite3UpperToLower[]
10760	** array. tolower() is used more often than toupper() by SQLite.
10761	**
10762	** Bit 0x40 is set if the character non-alphanumeric and can be used in an
10763	** SQLite identifier. Identifiers are alphanumerics, "_", "$", and any
10764	** non-ASCII UTF character. Hence the test for whether or not a character is
10765	** part of an identifier is 0x46.
10766	**
10767	** SQLite's versions are identical to the standard versions assuming a
10768	** locale of "C". They are implemented as macros in sqliteInt.h.
10769	*/
10770	#ifdef SQLITE_ASCII
	@@ -10778,41 +10771,41 @@
10771	SQLITE_PRIVATE const unsigned char sqlite3CtypeMap[256] = {
10772	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 00..07 ........ */
10773	0x00, 0x01, 0x01, 0x01, 0x01, 0x01, 0x00, 0x00, /* 08..0f ........ */
10774	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 10..17 ........ */
10775	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 18..1f ........ */
10776	0x01, 0x00, 0x00, 0x00, 0x40, 0x00, 0x00, 0x00, /* 20..27 !"#$%&' */
10777	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 28..2f ()+,-./ /
10778	0x0c, 0x0c, 0x0c, 0x0c, 0x0c, 0x0c, 0x0c, 0x0c, /* 30..37 01234567 */
10779	0x0c, 0x0c, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 38..3f 89:;<=>? */
10780
10781	0x00, 0x0a, 0x0a, 0x0a, 0x0a, 0x0a, 0x0a, 0x02, /* 40..47 @ABCDEFG */
10782	0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, /* 48..4f HIJKLMNO */
10783	0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, /* 50..57 PQRSTUVW */
10784	0x02, 0x02, 0x02, 0x00, 0x00, 0x00, 0x00, 0x40, /* 58..5f XYZ[\]^_ */
10785	0x00, 0x2a, 0x2a, 0x2a, 0x2a, 0x2a, 0x2a, 0x22, /* 60..67 `abcdefg */
10786	0x22, 0x22, 0x22, 0x22, 0x22, 0x22, 0x22, 0x22, /* 68..6f hijklmno */
10787	0x22, 0x22, 0x22, 0x22, 0x22, 0x22, 0x22, 0x22, /* 70..77 pqrstuvw */
10788	0x22, 0x22, 0x22, 0x00, 0x00, 0x00, 0x00, 0x00, /* 78..7f xyz{\|}~. */
10789
10790	0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, /* 80..87 ........ */
10791	0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, /* 88..8f ........ */
10792	0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, /* 90..97 ........ */
10793	0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, /* 98..9f ........ */
10794	0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, /* a0..a7 ........ */
10795	0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, /* a8..af ........ */
10796	0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, /* b0..b7 ........ */
10797	0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, /* b8..bf ........ */
10798
10799	0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, /* c0..c7 ........ */
10800	0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, /* c8..cf ........ */
10801	0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, /* d0..d7 ........ */
10802	0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, /* d8..df ........ */
10803	0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, /* e0..e7 ........ */
10804	0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, /* e8..ef ........ */
10805	0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, /* f0..f7 ........ */
10806	0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40 /* f8..ff ........ */
10807	};
10808	#endif
10809
10810
10811
	@@ -10877,10 +10870,18 @@
10870	** Changing the pending byte during operating results in undefined
10871	** and dileterious behavior.
10872	*/
10873	SQLITE_PRIVATE int sqlite3PendingByte = 0x40000000;
10874
10875	/*
10876	** Properties of opcodes. The OPFLG_INITIALIZER macro is
10877	** created by mkopcodeh.awk during compilation. Data is obtained
10878	** from the comments following the "case OP_xxxx:" statements in
10879	** the vdbe.c file.
10880	*/
10881	SQLITE_PRIVATE const unsigned char sqlite3OpcodeProperty[] = OPFLG_INITIALIZER;
10882
10883	/************ End of global.c ********************************************/
10884	/************ Begin file status.c ****************************************/
10885	/*
10886	** 2008 June 18
10887	**
	@@ -10893,12 +10894,10 @@
10894	**
10895	*************************************************************************
10896	**
10897	** This module implements the sqlite3_status() interface and related
10898	** functionality.


10899	*/
10900
10901	/*
10902	** Variables in which to record status information.
10903	*/
	@@ -11021,12 +11020,10 @@
11020	**
11021	** There is only one exported symbol in this file - the function
11022	** sqlite3RegisterDateTimeFunctions() found at the bottom of the file.
11023	** All other code has file scope.
11024	**


11025	** SQLite processes all times and dates as Julian Day numbers. The
11026	** dates and times are stored as the number of days since noon
11027	** in Greenwich on November 24, 4714 B.C. according to the Gregorian
11028	** calendar system.
11029	**
	@@ -12125,12 +12122,10 @@
12122	**
12123	******************************************************************************
12124	**
12125	** This file contains OS interface code that is common to all
12126	** architectures.


12127	*/
12128	#define _SQLITE_OS_C_ 1
12129	#undef _SQLITE_OS_C_
12130
12131	/*
	@@ -12252,10 +12247,11 @@
12247	sqlite3_vfs *pVfs,
12248	const char *zPath,
12249	int nPathOut,
12250	char *zPathOut
12251	){
12252	zPathOut[0] = 0;
12253	return pVfs->xFullPathname(pVfs, zPath, nPathOut, zPathOut);
12254	}
12255	#ifndef SQLITE_OMIT_LOAD_EXTENSION
12256	SQLITE_PRIVATE void sqlite3OsDlOpen(sqlite3_vfs pVfs, const char *zPath){
12257	return pVfs->xDlOpen(pVfs, zPath);
	@@ -12423,14 +12419,10 @@
12419	** May you find forgiveness for yourself and forgive others.
12420	** May you share freely, never taking more than you give.
12421	**
12422	*************************************************************************
12423	**




12424	** This file contains code to support the concept of "benign"
12425	** malloc failures (when the xMalloc() or xRealloc() method of the
12426	** sqlite3_mem_methods structure fails to allocate a block of memory
12427	** and returns 0).
12428	**
	@@ -12521,12 +12513,10 @@
12513	** This file contains a no-op memory allocation drivers for use when
12514	** SQLITE_ZERO_MALLOC is defined. The allocation drivers implemented
12515	** here always fail. SQLite will not operate with these drivers. These
12516	** are merely placeholders. Real drivers must be substituted using
12517	** sqlite3_config() before SQLite will operate.


12518	*/
12519
12520	/*
12521	** This version of the memory allocator is the default. It is
12522	** used when no other memory allocator is specified using compile-time
	@@ -12585,12 +12575,10 @@
12575	** SQLite will use the standard C-library malloc/realloc/free interface
12576	** to obtain the memory it needs.
12577	**
12578	** This file contains implementations of the low-level memory allocation
12579	** routines specified in the sqlite3_mem_methods object.


12580	*/
12581
12582	/*
12583	** This version of the memory allocator is the default. It is
12584	** used when no other memory allocator is specified using compile-time
	@@ -12734,12 +12722,10 @@
12722	** information to each allocation in order to help detect and fix memory
12723	** leaks and memory usage errors.
12724	**
12725	** This file contains implementations of the low-level memory allocation
12726	** routines specified in the sqlite3_mem_methods object.


12727	*/
12728
12729	/*
12730	** This version of the memory allocator is used only if the
12731	** SQLITE_MEMDEBUG macro is defined
	@@ -13183,12 +13169,10 @@
13169	** the amount of memory available to SQLite is fixed and cannot
13170	** be changed.
13171	**
13172	** This version of the memory allocation subsystem is included
13173	** in the build only if SQLITE_ENABLE_MEMSYS3 is defined.


13174	*/
13175
13176	/*
13177	** This version of the memory allocator is only built into the library
13178	** SQLITE_ENABLE_MEMSYS3 is defined. Defining this symbol does not
	@@ -14442,13 +14426,10 @@
14426	**
14427	*************************************************************************
14428	** This file contains the C functions that implement mutexes.
14429	**
14430	** This file contains code that is common across all mutex implementations.



14431	*/
14432
14433	#if defined(SQLITE_DEBUG) && !defined(SQLITE_MUTEX_OMIT)
14434	/*
14435	** For debugging purposes, record when the mutex subsystem is initialized
	@@ -14609,12 +14590,10 @@
14590	** interface.
14591	**
14592	** If compiled with SQLITE_DEBUG, then additional logic is inserted
14593	** that does error checking on mutexes to make sure they are being
14594	** called correctly.


14595	*/
14596
14597
14598	#if defined(SQLITE_MUTEX_NOOP) && !defined(SQLITE_DEBUG)
14599	/*
	@@ -14783,12 +14762,10 @@
14762	** May you find forgiveness for yourself and forgive others.
14763	** May you share freely, never taking more than you give.
14764	**
14765	*************************************************************************
14766	** This file contains the C functions that implement mutexes for OS/2


14767	*/
14768
14769	/*
14770	** The code in this file is only used if SQLITE_MUTEX_OS2 is defined.
14771	** See the mutex.h file for details.
	@@ -15058,12 +15035,10 @@
15035	** May you find forgiveness for yourself and forgive others.
15036	** May you share freely, never taking more than you give.
15037	**
15038	*************************************************************************
15039	** This file contains the C functions that implement mutexes for pthreads


15040	*/
15041
15042	/*
15043	** The code in this file is only used if we are compiling threadsafe
15044	** under unix with pthreads.
	@@ -15389,12 +15364,10 @@
15364	** May you find forgiveness for yourself and forgive others.
15365	** May you share freely, never taking more than you give.
15366	**
15367	*************************************************************************
15368	** This file contains the C functions that implement mutexes for win32


15369	*/
15370
15371	/*
15372	** The code in this file is only used if we are compiling multithreaded
15373	** on a win32 system.
	@@ -15676,12 +15649,10 @@
15649	** May you share freely, never taking more than you give.
15650	**
15651	*************************************************************************
15652	**
15653	** Memory allocation functions used throughout sqlite.


15654	*/
15655
15656	/*
15657	** This routine runs when the memory allocator sees that the
15658	** total memory allocation is about to exceed the soft heap
	@@ -16383,12 +16354,10 @@
16354	** the public domain. The original comments are included here for
16355	** completeness. They are very out-of-date but might be useful as
16356	** an historical reference. Most of the "enhancements" have been backed
16357	** out so that the functionality is now the same as standard printf().
16358	**


16359	**************************************************************************
16360	**
16361	** The following modules is an enhanced replacement for the "printf" subroutines
16362	** found in the standard C library. The following enhancements are
16363	** supported:
	@@ -17352,12 +17321,10 @@
17321	** This file contains code to implement a pseudo-random number
17322	** generator (PRNG) for SQLite.
17323	**
17324	** Random numbers are used by some of the database backends in order
17325	** to generate random integer keys for tables or random filenames.


17326	*/
17327
17328
17329	/* All threads share a single random number generator.
17330	** This structure is the current state of the generator.
	@@ -17499,12 +17466,10 @@
17466	**
17467	*************************************************************************
17468	** This file contains routines used to translate between UTF-8,
17469	** UTF-16, UTF-16BE, and UTF-16LE.
17470	**


17471	** Notes on UTF-8:
17472	**
17473	** Byte-0 Byte-1 Byte-2 Byte-3 Value
17474	** 0xxxxxxx 00000000 00000000 0xxxxxxx
17475	** 110yyyyy 10xxxxxx 00000000 00000yyy yyxxxxxx
	@@ -17540,12 +17505,10 @@
17505	** This is the header file for information that is private to the
17506	** VDBE. This information used to all be at the top of the single
17507	** source code file "vdbe.c". When that file became too big (over
17508	** 6000 lines long) it was split up into several smaller files and
17509	** this header information was factored out.


17510	*/
17511	#ifndef _VDBEINT_H_
17512	#define _VDBEINT_H_
17513
17514	/*
	@@ -17907,11 +17870,10 @@
17870	SQLITE_PRIVATE int sqlite3VdbeMemFromBtree(BtCursor,int,int,int,Mem);
17871	SQLITE_PRIVATE void sqlite3VdbeMemRelease(Mem *p);
17872	SQLITE_PRIVATE void sqlite3VdbeMemReleaseExternal(Mem *p);
17873	SQLITE_PRIVATE int sqlite3VdbeMemFinalize(Mem, FuncDef);
17874	SQLITE_PRIVATE const char *sqlite3OpcodeName(int);

17875	SQLITE_PRIVATE int sqlite3VdbeMemGrow(Mem *pMem, int n, int preserve);
17876	SQLITE_PRIVATE int sqlite3VdbeCloseStatement(Vdbe *, int);
17877	SQLITE_PRIVATE void sqlite3VdbeFrameDelete(VdbeFrame*);
17878	SQLITE_PRIVATE int sqlite3VdbeFrameRestore(VdbeFrame *);
17879	SQLITE_PRIVATE void sqlite3VdbeMemStoreType(Mem *pMem);
	@@ -19574,12 +19536,10 @@
19536	** May you share freely, never taking more than you give.
19537	**
19538	*************************************************************************
19539	** This is the implementation of generic hash-tables
19540	** used in SQLite.


19541	*/
19542
19543	/* Turn bulk memory into a hash table object by initializing the
19544	** fields of the Hash structure.
19545	**
	@@ -20011,12 +19971,10 @@
19971	** May you share freely, never taking more than you give.
19972	**
19973	******************************************************************************
19974	**
19975	** This file contains code that is specific to OS/2.


19976	*/
19977
19978
19979	#if SQLITE_OS_OS2
19980
	@@ -20074,12 +20032,10 @@
20032	** all of the platform-specific files (os_*.c) and is #included into those
20033	** files.
20034	**
20035	** This file should be #included by the os_*.c files only. It is not a
20036	** general purpose header file.


20037	*/
20038	#ifndef _OS_COMMON_H_
20039	#define _OS_COMMON_H_
20040
20041	/*
	@@ -20136,12 +20092,10 @@
20092	**
20093	******************************************************************************
20094	**
20095	** This file contains inline asm code for retrieving "high-performance"
20096	** counters for x86 class CPUs.


20097	*/
20098	#ifndef _HWTIME_H_
20099	#define _HWTIME_H_
20100
20101	/*
	@@ -21603,12 +21557,10 @@
21557	** all of the platform-specific files (os_*.c) and is #included into those
21558	** files.
21559	**
21560	** This file should be #included by the os_*.c files only. It is not a
21561	** general purpose header file.


21562	*/
21563	#ifndef _OS_COMMON_H_
21564	#define _OS_COMMON_H_
21565
21566	/*
	@@ -21665,12 +21617,10 @@
21617	**
21618	******************************************************************************
21619	**
21620	** This file contains inline asm code for retrieving "high-performance"
21621	** counters for x86 class CPUs.


21622	*/
21623	#ifndef _HWTIME_H_
21624	#define _HWTIME_H_
21625
21626	/*
	@@ -24685,10 +24635,23 @@
24635	rc = ftruncate(((unixFile*)id)->h, (off_t)nByte);
24636	if( rc ){
24637	((unixFile*)id)->lastErrno = errno;
24638	return SQLITE_IOERR_TRUNCATE;
24639	}else{
24640	#ifndef NDEBUG
24641	/* If we are doing a normal write to a database file (as opposed to
24642	** doing a hot-journal rollback or a write to some file other than a
24643	** normal database file) and we truncate the file to zero length,
24644	** that effectively updates the change counter. This might happen
24645	** when restoring a database using the backup API from a zero-length
24646	** source.
24647	*/
24648	if( ((unixFile*)id)->inNormalWrite && nByte==0 ){
24649	((unixFile*)id)->transCntrChng = 1;
24650	}
24651	#endif
24652
24653	return SQLITE_OK;
24654	}
24655	}
24656
24657	/*
	@@ -27042,12 +27005,10 @@
27005	** all of the platform-specific files (os_*.c) and is #included into those
27006	** files.
27007	**
27008	** This file should be #included by the os_*.c files only. It is not a
27009	** general purpose header file.


27010	*/
27011	#ifndef _OS_COMMON_H_
27012	#define _OS_COMMON_H_
27013
27014	/*
	@@ -27104,12 +27065,10 @@
27065	**
27066	******************************************************************************
27067	**
27068	** This file contains inline asm code for retrieving "high-performance"
27069	** counters for x86 class CPUs.


27070	*/
27071	#ifndef _HWTIME_H_
27072	#define _HWTIME_H_
27073
27074	/*
	@@ -29147,12 +29106,10 @@
29106	** 5 and 500 set operations per Bitvec object, though the number of sets can
29107	** sometimes grow into tens of thousands or larger. The size of the
29108	** Bitvec object is the number of pages in the database file at the
29109	** start of a transaction, and is thus usually less than a few thousand,
29110	** but can be as large as 2 billion for a really big database.


29111	*/
29112
29113	/* Size of the Bitvec structure in bytes. */
29114	#define BITVEC_SZ (sizeof(void)128) /* 512 on 32bit. 1024 on 64bit */
29115
	@@ -29536,12 +29493,10 @@
29493	** May you find forgiveness for yourself and forgive others.
29494	** May you share freely, never taking more than you give.
29495	**
29496	*************************************************************************
29497	** This file implements that page cache.


29498	*/
29499
29500	/*
29501	** A complete page cache is an instance of this structure.
29502	*/
	@@ -30119,12 +30074,10 @@
30074	** This file implements the default page cache implementation (the
30075	** sqlite3_pcache interface). It also contains part of the implementation
30076	** of the SQLITE_CONFIG_PAGECACHE and sqlite3_release_memory() features.
30077	** If the default page cache implementation is overriden, then neither of
30078	** these two features are available.


30079	*/
30080
30081
30082	typedef struct PCache1 PCache1;
30083	typedef struct PgHdr1 PgHdr1;
	@@ -30770,19 +30723,11 @@
30723
30724	h = iNew%pCache->nHash;
30725	pPage->iKey = iNew;
30726	pPage->pNext = pCache->apHash[h];
30727	pCache->apHash[h] = pPage;
30728	if( iNew>pCache->iMaxKey ){








30729	pCache->iMaxKey = iNew;
30730	}
30731
30732	pcache1LeaveMutex();
30733	}
	@@ -30954,12 +30899,10 @@
30899	** of the first SMALLEST is O(NlogN). Second and subsequent SMALLEST
30900	** primitives are constant time. The cost of DESTROY is O(N).
30901	**
30902	** There is an added cost of O(N) when switching between TEST and
30903	** SMALLEST primitives.


30904	*/
30905
30906
30907	/*
30908	** Target size for allocation chunks.
	@@ -31338,12 +31281,10 @@
31281	** atomic commit and rollback through the use of a journal file that
31282	** is separate from the database file. The pager also implements file
31283	** locking to prevent two processes from writing the same database
31284	** file simultaneously, or one process from reading the database while
31285	** another is writing.


31286	*/
31287	#ifndef SQLITE_OMIT_DISKIO
31288
31289	/*
31290	** Macros for troubleshooting. Normally turned off
	@@ -32233,14 +32174,14 @@
32174	return rc;
32175	}
32176
32177	/* Check that the values read from the page-size and sector-size fields
32178	** are within range. To be 'in range', both values need to be a power
32179	** of two greater than or equal to 512 or 32, and not greater than their
32180	** respective compile time maximum limits.
32181	*/
32182	if( iPageSize<512 \|\| iSectorSize<32
32183	\|\| iPageSize>SQLITE_MAX_PAGE_SIZE \|\| iSectorSize>MAX_SECTOR_SIZE
32184	\|\| ((iPageSize-1)&iPageSize)!=0 \|\| ((iSectorSize-1)&iSectorSize)!=0
32185	){
32186	/* If the either the page-size or sector-size in the journal-header is
32187	** invalid, then the process that wrote the journal-header must have
	@@ -32743,28 +32684,28 @@
32684	){
32685	int rc;
32686	PgHdr pPg; / An existing page in the cache */
32687	Pgno pgno; /* The page number of a page in journal */
32688	u32 cksum; /* Checksum used for sanity checking */
32689	char aData; / Temporary storage for the page */
32690	sqlite3_file jfd; / The file descriptor for the journal file */
32691
32692	assert( (isMainJrnl&~1)==0 ); /* isMainJrnl is 0 or 1 */
32693	assert( (isSavepnt&~1)==0 ); /* isSavepnt is 0 or 1 */
32694	assert( isMainJrnl \|\| pDone ); /* pDone always used on sub-journals */
32695	assert( isSavepnt \|\| pDone==0 ); /* pDone never used on non-savepoint */
32696
32697	aData = pPager->pTmpSpace;
32698	assert( aData ); /* Temp storage must have already been allocated */
32699
32700	/* Read the page number and page data from the journal or sub-journal
32701	** file. Return an error code to the caller if an IO error occurs.
32702	*/
32703	jfd = isMainJrnl ? pPager->jfd : pPager->sjfd;
32704	rc = read32bits(jfd, *pOffset, &pgno);
32705	if( rc!=SQLITE_OK ) return rc;
32706	rc = sqlite3OsRead(jfd, (u8)aData, pPager->pageSize, (pOffset)+4);
32707	if( rc!=SQLITE_OK ) return rc;
32708	pOffset += pPager->pageSize + 4 + isMainJrnl4;
32709
32710	/* Sanity checking on the page. This is more important that I originally
32711	** thought. If a power failure occurs while the journal is being written,
	@@ -32779,11 +32720,11 @@
32720	return SQLITE_OK;
32721	}
32722	if( isMainJrnl ){
32723	rc = read32bits(jfd, (*pOffset)-4, &cksum);
32724	if( rc ) return rc;
32725	if( !isSavepnt && pager_cksum(pPager, (u8*)aData)!=cksum ){
32726	return SQLITE_DONE;
32727	}
32728	}
32729
32730	if( pDone && (rc = sqlite3BitvecSet(pDone, pgno))!=SQLITE_OK ){
	@@ -32825,27 +32766,27 @@
32766	** Do not attempt to write if database file has never been opened.
32767	*/
32768	pPg = pager_lookup(pPager, pgno);
32769	assert( pPg \|\| !MEMDB );
32770	PAGERTRACE(("PLAYBACK %d page %d hash(%08x) %s\n",
32771	PAGERID(pPager), pgno, pager_datahash(pPager->pageSize, (u8*)aData),
32772	(isMainJrnl?"main-journal":"sub-journal")
32773	));
32774	if( (pPager->state>=PAGER_EXCLUSIVE)
32775	&& (pPg==0 \|\| 0==(pPg->flags&PGHDR_NEED_SYNC))
32776	&& isOpen(pPager->fd)
32777	&& !isUnsync
32778	){
32779	i64 ofst = (pgno-1)*(i64)pPager->pageSize;
32780	rc = sqlite3OsWrite(pPager->fd, (u8*)aData, pPager->pageSize, ofst);
32781	if( pgno>pPager->dbFileSize ){
32782	pPager->dbFileSize = pgno;
32783	}
32784	if( pPager->pBackup ){
32785	CODEC1(pPager, aData, pgno, 3, rc=SQLITE_NOMEM);
32786	sqlite3BackupUpdate(pPager->pBackup, pgno, (u8*)aData);
32787	CODEC2(pPager, aData, pgno, 7, rc=SQLITE_NOMEM, aData);
32788	}
32789	}else if( !isMainJrnl && pPg==0 ){
32790	/* If this is a rollback of a savepoint and data was not written to
32791	** the database and the page is not in-memory, there is a potential
32792	** problem. When the page is next fetched by the b-tree layer, it
	@@ -32876,11 +32817,11 @@
32817	** of an internal error resulting in an automatic call to
32818	** sqlite3PagerRollback().
32819	*/
32820	void *pData;
32821	pData = pPg->pData;
32822	memcpy(pData, (u8*)aData, pPager->pageSize);
32823	pPager->xReiniter(pPg);
32824	if( isMainJrnl && (!isSavepnt \|\| *pOffset<=pPager->journalHdr) ){
32825	/* If the contents of this page were just restored from the main
32826	** journal file, then its content must be as they were when the
32827	** transaction was first opened. In this case we can mark the page
	@@ -33101,12 +33042,12 @@
33042	** master journal pointers within created journal files.
33043	**
33044	** For temporary files the effective sector size is always 512 bytes.
33045	**
33046	** Otherwise, for non-temporary files, the effective sector size is
33047	** the value returned by the xSectorSize() method rounded up to 32 if
33048	** it is less than 32, or rounded down to MAX_SECTOR_SIZE if it
33049	** is greater than MAX_SECTOR_SIZE.
33050	*/
33051	static void setSectorSize(Pager *pPager){
33052	assert( isOpen(pPager->fd) \|\| pPager->tempFile );
33053
	@@ -33115,11 +33056,11 @@
33056	** may not have been opened yet, in which case the OsSectorSize()
33057	** call will segfault.
33058	*/
33059	pPager->sectorSize = sqlite3OsSectorSize(pPager->fd);
33060	}
33061	if( pPager->sectorSize<32 ){
33062	pPager->sectorSize = 512;
33063	}
33064	if( pPager->sectorSize>MAX_SECTOR_SIZE ){
33065	assert( MAX_SECTOR_SIZE>=512 );
33066	pPager->sectorSize = MAX_SECTOR_SIZE;
	@@ -34193,11 +34134,13 @@
34134	** than Pager.dbSize, this means sqlite3PagerTruncateImage() was called to
34135	** make the file smaller (presumably by auto-vacuum code). Do not write
34136	** any such pages to the file.
34137	**
34138	** Also, do not write out any page that has the PGHDR_DONT_WRITE flag
34139	** set (set by sqlite3PagerDontWrite()). Note that if compiled with
34140	** SQLITE_SECURE_DELETE the PGHDR_DONT_WRITE bit is never set and so
34141	** the second test is always true.
34142	*/
34143	if( pgno<=pPager->dbSize && 0==(pList->flags&PGHDR_DONT_WRITE) ){
34144	i64 offset = (pgno-1)(i64)pPager->pageSize; / Offset to write */
34145	char pData; / Data to write */
34146
	@@ -35702,10 +35645,11 @@
35645	SQLITE_PRIVATE int sqlite3PagerIswriteable(DbPage *pPg){
35646	return pPg->flags&PGHDR_DIRTY;
35647	}
35648	#endif
35649
35650	#ifndef SQLITE_SECURE_DELETE
35651	/*
35652	** A call to this routine tells the pager that it is not necessary to
35653	** write the information on page pPg back to the disk, even though
35654	** that page might be marked as dirty. This happens, for example, when
35655	** the page has been added as a leaf of the freelist and so its
	@@ -35727,10 +35671,11 @@
35671	#ifdef SQLITE_CHECK_PAGES
35672	pPg->pageHash = pager_pagehash(pPg);
35673	#endif
35674	}
35675	}
35676	#endif /* !defined(SQLITE_SECURE_DELETE) */
35677
35678	/*
35679	** This routine is called to increment the value of the database file
35680	** change-counter, stored as a 4-byte big-endian integer starting at
35681	** byte offset 24 of the pager file.
	@@ -35766,11 +35711,11 @@
35711	#else
35712	# define DIRECT_MODE isDirectMode
35713	#endif
35714
35715	assert( pPager->state>=PAGER_RESERVED );
35716	if( !pPager->changeCountDone && pPager->dbSize>0 ){
35717	PgHdr pPgHdr; / Reference to page 1 */
35718	u32 change_counter; /* Initial value of change-counter field */
35719
35720	assert( !pPager->tempFile && isOpen(pPager->fd) );
35721
	@@ -36418,10 +36363,18 @@
36363	Pgno needSyncPgno = 0; /* Old value of pPg->pgno, if sync is required */
36364	int rc; /* Return code */
36365	Pgno origPgno; /* The original page number */
36366
36367	assert( pPg->nRef>0 );
36368
36369	/* In order to be able to rollback, an in-memory database must journal
36370	** the page we are moving from.
36371	*/
36372	if( MEMDB ){
36373	rc = sqlite3PagerWrite(pPg);
36374	if( rc ) return rc;
36375	}
36376
36377	/* If the page being moved is dirty and has not been saved by the latest
36378	** savepoint, then save the current contents of the page into the
36379	** sub-journal now. This is required to handle the following scenario:
36380	**
	@@ -36437,11 +36390,11 @@
36390	**
36391	** subjournalPage() may need to allocate space to store pPg->pgno into
36392	** one or more savepoint bitvecs. This is the reason this function
36393	** may return SQLITE_NOMEM.
36394	*/
36395	if( pPg->flags&PGHDR_DIRTY
36396	&& subjRequiresPage(pPg)
36397	&& SQLITE_OK!=(rc = subjournalPage(pPg))
36398	){
36399	return rc;
36400	}
	@@ -36472,11 +36425,18 @@
36425	pPg->flags &= ~PGHDR_NEED_SYNC;
36426	pPgOld = pager_lookup(pPager, pgno);
36427	assert( !pPgOld \|\| pPgOld->nRef==1 );
36428	if( pPgOld ){
36429	pPg->flags \|= (pPgOld->flags&PGHDR_NEED_SYNC);
36430	if( MEMDB ){
36431	/* Do not discard pages from an in-memory database since we might
36432	** need to rollback later. Just move the page out of the way. */
36433	assert( pPager->dbSizeValid );
36434	sqlite3PcacheMove(pPgOld, pPager->dbSize+1);
36435	}else{
36436	sqlite3PcacheDrop(pPgOld);
36437	}
36438	}
36439
36440	origPgno = pPg->pgno;
36441	sqlite3PcacheMove(pPg, pgno);
36442	sqlite3PcacheMakeDirty(pPg);
	@@ -36517,22 +36477,16 @@
36477	sqlite3PagerUnref(pPgHdr);
36478	}
36479
36480	/*
36481	** For an in-memory database, make sure the original page continues
36482	** to exist, in case the transaction needs to roll back. Use pPgOld
36483	** as the original page since it has already been allocated.

36484	*/
36485	if( MEMDB ){
36486	sqlite3PcacheMove(pPgOld, origPgno);
36487	sqlite3PagerUnref(pPgOld);





36488	}
36489
36490	return SQLITE_OK;
36491	}
36492	#endif
	@@ -36655,12 +36609,10 @@
36609	** May you find forgiveness for yourself and forgive others.
36610	** May you share freely, never taking more than you give.
36611	**
36612	*************************************************************************
36613	**


36614	** This file contains code used to implement mutexes on Btree objects.
36615	** This code really belongs in btree.c. But btree.c is getting too
36616	** big and we want to break it down some. This packaged seemed like
36617	** a good breakout.
36618	*/
	@@ -36675,12 +36627,10 @@
36627	** May you do good and not evil.
36628	** May you find forgiveness for yourself and forgive others.
36629	** May you share freely, never taking more than you give.
36630	**
36631	*************************************************************************


36632	** This file implements a external (disk-based) database using BTrees.
36633	** For a detailed discussion of BTrees, refer to
36634	**
36635	** Donald E. Knuth, THE ART OF COMPUTER PROGRAMMING, Volume 3:
36636	** "Sorting And Searching", pages 473-480. Addison-Wesley
	@@ -37660,12 +37610,10 @@
37610	** May you do good and not evil.
37611	** May you find forgiveness for yourself and forgive others.
37612	** May you share freely, never taking more than you give.
37613	**
37614	*************************************************************************


37615	** This file implements a external (disk-based) database using BTrees.
37616	** See the header comment on "btreeInt.h" for additional information.
37617	** Including a description of file format and an overview of operation.
37618	*/
37619
	@@ -40930,12 +40878,12 @@
40878	**
40879	** No checking is done to make sure that page iTable really is the
40880	** root page of a b-tree. If it is not, then the cursor acquired
40881	** will not work correctly.
40882	**
40883	** It is assumed that the sqlite3BtreeCursorZero() has been called
40884	** on pCur to initialize the memory space prior to invoking this routine.
40885	*/
40886	static int btreeCursor(
40887	Btree p, / The btree */
40888	int iTable, /* Root page of table to open */
40889	int wrFlag, /* 1 to write. 0 read-only */
	@@ -41004,11 +40952,23 @@
40952	** sufficient storage to hold a cursor. The BtCursor object is opaque
40953	** to users so they cannot do the sizeof() themselves - they must call
40954	** this routine.
40955	*/
40956	SQLITE_PRIVATE int sqlite3BtreeCursorSize(void){
40957	return ROUND8(sizeof(BtCursor));
40958	}
40959
40960	/*
40961	** Initialize memory that will be converted into a BtCursor object.
40962	**
40963	** The simple approach here would be to memset() the entire object
40964	** to zero. But it turns out that the apPage[] and aiIdx[] arrays
40965	** do not need to be zeroed and they are large, so we can save a lot
40966	** of run-time by skipping the initialization of those elements.
40967	*/
40968	SQLITE_PRIVATE void sqlite3BtreeCursorZero(BtCursor *p){
40969	memset(p, 0, offsetof(BtCursor, iPage));
40970	}
40971
40972	/*
40973	** Set the cached rowid value of every cursor in the same database file
40974	** as pCur and having the same root page number as pCur. The value is
	@@ -45489,12 +45449,10 @@
45449	** May you share freely, never taking more than you give.
45450	**
45451	*************************************************************************
45452	** This file contains the implementation of the sqlite3_backup_XXX()
45453	** API functions and the related features.


45454	*/
45455
45456	/* Macro to find the minimum of two numeric values.
45457	*/
45458	#ifndef MIN
	@@ -46123,12 +46081,10 @@
46081	**
46082	** This file contains code use to manipulate "Mem" structure. A "Mem"
46083	** stores a single value in the VDBE. Mem is an opaque structure visible
46084	** only within the VDBE. Interface routines refer to a Mem using the
46085	** name sqlite_value


46086	*/
46087
46088	/*
46089	** Call sqlite3VdbeMemExpandBlob() on the supplied value (type Mem*)
46090	** P if required.
	@@ -46863,13 +46819,10 @@
46819	SQLITE_PRIVATE int sqlite3MemCompare(const Mem pMem1, const Mem pMem2, const CollSeq *pColl){
46820	int rc;
46821	int f1, f2;
46822	int combined_flags;
46823



46824	f1 = pMem1->flags;
46825	f2 = pMem2->flags;
46826	combined_flags = f1\|f2;
46827	assert( (combined_flags & MEM_RowSet)==0 );
46828
	@@ -47116,11 +47069,11 @@
47069	*ppVal = 0;
47070	return SQLITE_OK;
47071	}
47072	op = pExpr->op;
47073	if( op==TK_REGISTER ){
47074	op = pExpr->op2; /* This only happens with SQLITE_ENABLE_STAT2 */
47075	}
47076
47077	if( op==TK_STRING \|\| op==TK_FLOAT \|\| op==TK_INTEGER ){
47078	pVal = sqlite3ValueNew(db);
47079	if( pVal==0 ) goto no_mem;
	@@ -47229,12 +47182,10 @@
47182	*************************************************************************
47183	** This file contains code used for creating, destroying, and populating
47184	** a VDBE (or an "sqlite3_stmt" as it is known to the outside world.) Prior
47185	** to version 2.8.7, all this code was combined into the vdbe.c source file.
47186	** But that file was getting too big so this subroutines were split out.


47187	*/
47188
47189
47190
47191	/*
	@@ -47410,10 +47361,26 @@
47361	){
47362	int addr = sqlite3VdbeAddOp3(p, op, p1, p2, p3);
47363	sqlite3VdbeChangeP4(p, addr, zP4, p4type);
47364	return addr;
47365	}
47366
47367	/*
47368	** Add an opcode that includes the p4 value as an integer.
47369	*/
47370	SQLITE_PRIVATE int sqlite3VdbeAddOp4Int(
47371	Vdbe p, / Add the opcode to this VM */
47372	int op, /* The new opcode */
47373	int p1, /* The P1 operand */
47374	int p2, /* The P2 operand */
47375	int p3, /* The P3 operand */
47376	int p4 /* The P4 operand as an integer */
47377	){
47378	int addr = sqlite3VdbeAddOp3(p, op, p1, p2, p3);
47379	sqlite3VdbeChangeP4(p, addr, SQLITE_INT_TO_PTR(p4), P4_INT32);
47380	return addr;
47381	}
47382
47383	/*
47384	** Create a new symbolic label for an instruction that has yet to be
47385	** coded. The symbolic label is really just a negative number. The
47386	** label can be used as the P2 value of an operation. Later, when
	@@ -47586,10 +47553,12 @@
47553	** This routine is called once after all opcodes have been inserted.
47554	**
47555	** Variable *pMaxFuncArgs is set to the maximum value of any P2 argument
47556	** to an OP_Function, OP_AggStep or OP_VFilter opcode. This is used by
47557	** sqlite3VdbeMakeReady() to size the Vdbe.apArg[] array.
47558	**
47559	** The Op.opflags field is set on all opcodes.
47560	*/
47561	static void resolveP2Values(Vdbe p, int pMaxFuncArgs){
47562	int i;
47563	int nMaxArgs = *pMaxFuncArgs;
47564	Op *pOp;
	@@ -47596,29 +47565,28 @@
47565	int *aLabel = p->aLabel;
47566	p->readOnly = 1;
47567	for(pOp=p->aOp, i=p->nOp-1; i>=0; i--, pOp++){
47568	u8 opcode = pOp->opcode;
47569
47570	pOp->opflags = sqlite3OpcodeProperty[opcode];
47571	if( opcode==OP_Function \|\| opcode==OP_AggStep ){
47572	if( pOp->p5>nMaxArgs ) nMaxArgs = pOp->p5;
47573	}else if( opcode==OP_Transaction && pOp->p2!=0 ){
47574	p->readOnly = 0;
47575	#ifndef SQLITE_OMIT_VIRTUALTABLE
47576	}else if( opcode==OP_VUpdate ){
47577	if( pOp->p2>nMaxArgs ) nMaxArgs = pOp->p2;




47578	}else if( opcode==OP_VFilter ){
47579	int n;
47580	assert( p->nOp - i >= 3 );
47581	assert( pOp[-1].opcode==OP_Integer );
47582	n = pOp[-1].p1;
47583	if( n>nMaxArgs ) nMaxArgs = n;
47584	#endif
47585	}
47586
47587	if( (pOp->opflags & OPFLG_JUMP)!=0 && pOp->p2<0 ){
47588	assert( -1-pOp->p2<p->nLabel );
47589	pOp->p2 = aLabel[-1-pOp->p2];
47590	}
47591	}
47592	sqlite3DbFree(p->db, p->aLabel);
	@@ -47676,11 +47644,11 @@
47644	for(i=0; i<nOp; i++, pIn++){
47645	int p2 = pIn->p2;
47646	VdbeOp *pOut = &p->aOp[i+addr];
47647	pOut->opcode = pIn->opcode;
47648	pOut->p1 = pIn->p1;
47649	if( p2<0 && (sqlite3OpcodeProperty[pOut->opcode] & OPFLG_JUMP)!=0 ){
47650	pOut->p2 = addr + ADDR(p2);
47651	}else{
47652	pOut->p2 = p2;
47653	}
47654	pOut->p3 = pIn->p3;
	@@ -49986,13 +49954,21 @@
49954	Mem mem1;
49955
49956	pKeyInfo = pPKey2->pKeyInfo;
49957	mem1.enc = pKeyInfo->enc;
49958	mem1.db = pKeyInfo->db;
49959	/* mem1.flags = 0; // Will be initialized by sqlite3VdbeSerialGet() */
49960	VVA_ONLY( mem1.zMalloc = 0; ) /* Only needed by assert() statements */
49961
49962	/* Compilers may complain that mem1.u.i is potentially uninitialized.
49963	** We could initialize it, as shown here, to silence those complaints.
49964	** But in fact, mem1.u.i will never actually be used initialized, and doing
49965	** the unnecessary initialization has a measurable negative performance
49966	** impact, since this routine is a very high runner. And so, we choose
49967	** to ignore the compiler warnings and leave this variable uninitialized.
49968	*/
49969	/* mem1.u.i = 0; // not needed, here to silence compiler warning */
49970
49971	idx1 = getVarint32(aKey1, szHdr1);
49972	d1 = szHdr1;
49973	if( pPKey2->flags & UNPACKED_IGNORE_ROWID ){
49974	szHdr1--;
	@@ -50012,51 +49988,56 @@
49988	/* Do the comparison
49989	*/
49990	rc = sqlite3MemCompare(&mem1, &pPKey2->aMem[i],
49991	i<nField ? pKeyInfo->aColl[i] : 0);
49992	if( rc!=0 ){
49993	assert( mem1.zMalloc==0 ); /* See comment below */
49994
49995	/* Invert the result if we are using DESC sort order. */
49996	if( pKeyInfo->aSortOrder && i<nField && pKeyInfo->aSortOrder[i] ){
49997	rc = -rc;
49998	}
49999
50000	/* If the PREFIX_SEARCH flag is set and all fields except the final
50001	** rowid field were equal, then clear the PREFIX_SEARCH flag and set
50002	** pPKey2->rowid to the value of the rowid field in (pKey1, nKey1).
50003	** This is used by the OP_IsUnique opcode.
50004	*/
50005	if( (pPKey2->flags & UNPACKED_PREFIX_SEARCH) && i==(pPKey2->nField-1) ){
50006	assert( idx1==szHdr1 && rc );
50007	assert( mem1.flags & MEM_Int );
50008	pPKey2->flags &= ~UNPACKED_PREFIX_SEARCH;
50009	pPKey2->rowid = mem1.u.i;
50010	}
50011
50012	return rc;
50013	}
50014	i++;
50015	}
50016
50017	/* No memory allocation is ever used on mem1. Prove this using
50018	** the following assert(). If the assert() fails, it indicates a
50019	** memory leak and a need to call sqlite3VdbeMemRelease(&mem1).
50020	*/
50021	assert( mem1.zMalloc==0 );
50022
50023	/* rc==0 here means that one of the keys ran out of fields and
50024	** all the fields up to that point were equal. If the UNPACKED_INCRKEY
50025	** flag is set, then break the tie by treating key2 as larger.
50026	** If the UPACKED_PREFIX_MATCH flag is set, then keys with common prefixes
50027	** are considered to be equal. Otherwise, the longer key is the
50028	** larger. As it happens, the pPKey2 will always be the longer
50029	** if there is a difference.
50030	*/
50031	assert( rc==0 );
50032	if( pPKey2->flags & UNPACKED_INCRKEY ){
50033	rc = -1;
50034	}else if( pPKey2->flags & UNPACKED_PREFIX_MATCH ){
50035	/* Leave rc==0 */
50036	}else if( idx1<szHdr1 ){
50037	rc = 1;
50038	}














50039	return rc;
50040	}
50041
50042
50043	/*
	@@ -50272,12 +50253,10 @@
50253	**
50254	*************************************************************************
50255	**
50256	** This file contains code use to implement APIs that are part of the
50257	** VDBE.


50258	*/
50259
50260	#ifndef SQLITE_OMIT_DEPRECATED
50261	/*
50262	** Return TRUE (non-zero) of the statement supplied as an argument needs
	@@ -51528,12 +51507,10 @@
51507	** Various scripts scan this source file in order to generate HTML
51508	** documentation, headers files, or other derived files. The formatting
51509	** of the code in this file is, therefore, important. See other comments
51510	** in this file for details. If in doubt, do not deviate from existing
51511	** commenting and indentation practices when changing or adding code.


51512	*/
51513
51514	/*
51515	** The following global variable is incremented every time a cursor
51516	** moves, either by the OP_SeekXX, OP_Next, or OP_Prev opcodes. The test
	@@ -51656,27 +51633,10 @@
51633	}else{
51634	pMem->type = SQLITE_BLOB;
51635	}
51636	}
51637

















51638	/*
51639	** Allocate VdbeCursor number iCur. Return a pointer to it. Return NULL
51640	** if we run out of memory.
51641	*/
51642	static VdbeCursor *allocateCursor(
	@@ -51707,11 +51667,11 @@
51667	Mem *pMem = &p->aMem[p->nMem-iCur];
51668
51669	int nByte;
51670	VdbeCursor *pCx = 0;
51671	nByte =
51672	ROUND8(sizeof(VdbeCursor)) +
51673	(isBtreeCursor?sqlite3BtreeCursorSize():0) +
51674	2nFieldsizeof(u32);
51675
51676	assert( iCur<p->nCursor );
51677	if( p->apCsr[iCur] ){
	@@ -51718,19 +51678,20 @@
51678	sqlite3VdbeFreeCursor(p, p->apCsr[iCur]);
51679	p->apCsr[iCur] = 0;
51680	}
51681	if( SQLITE_OK==sqlite3VdbeMemGrow(pMem, nByte, 0) ){
51682	p->apCsr[iCur] = pCx = (VdbeCursor*)pMem->z;
51683	memset(pCx, 0, sizeof(VdbeCursor));
51684	pCx->iDb = iDb;
51685	pCx->nField = nField;
51686	if( nField ){
51687	pCx->aType = (u32 *)&pMem->z[ROUND8(sizeof(VdbeCursor))];
51688	}
51689	if( isBtreeCursor ){
51690	pCx->pCursor = (BtCursor*)
51691	&pMem->z[ROUND8(sizeof(VdbeCursor))+2nFieldsizeof(u32)];
51692	sqlite3BtreeCursorZero(pCx->pCursor);
51693	}
51694	}
51695	return pCx;
51696	}
51697
	@@ -51968,12 +51929,10 @@
51929	**
51930	******************************************************************************
51931	**
51932	** This file contains inline asm code for retrieving "high-performance"
51933	** counters for x86 class CPUs.


51934	*/
51935	#ifndef _HWTIME_H_
51936	#define _HWTIME_H_
51937
51938	/*
	@@ -52132,28 +52091,31 @@
52091	*/
52092	SQLITE_PRIVATE int sqlite3VdbeExec(
52093	Vdbe p / The VDBE */
52094	){
52095	int pc; /* The program counter */
52096	Op aOp = p->aOp; / Copy of p->aOp */
52097	Op pOp; / Current operation */
52098	int rc = SQLITE_OK; /* Value to return */
52099	sqlite3 db = p->db; / The database */
52100	u8 resetSchemaOnFault = 0; /* Reset schema after an error if true */
52101	u8 encoding = ENC(db); /* The database encoding */
52102	#ifndef SQLITE_OMIT_PROGRESS_CALLBACK
52103	u8 checkProgress; /* True if progress callbacks are enabled */
52104	int nProgressOps = 0; /* Opcodes executed since progress callback. */
52105	#endif
52106	Mem aMem = p->aMem; / Copy of p->aMem */
52107	Mem pIn1 = 0; / 1st input operand */
52108	Mem pIn2 = 0; / 2nd input operand */
52109	Mem pIn3 = 0; / 3rd input operand */
52110	Mem pOut = 0; / Output operand */

52111	int iCompare = 0; /* Result of last OP_Compare operation */
52112	int aPermute = 0; / Permutation of columns for OP_Compare */
52113	#ifdef VDBE_PROFILE
52114	u64 start; /* CPU clock count at start of opcode */
52115	int origPc; /* Program counter at start of opcode */
52116	#endif



52117	/********************************************************************
52118	** Automatically generated code
52119	**
52120	** The following union is automatically generated by the
52121	** vdbe-compress.tcl script. The purpose of this union is to
	@@ -52245,14 +52207,12 @@
52207	int szHdr; /* Size of the header size field at start of record */
52208	int avail; /* Number of bytes of available data */
52209	Mem pReg; / PseudoTable input register */
52210	} am;
52211	struct OP_Affinity_stack_vars {
52212	const char zAffinity; / The affinity to be applied */
52213	char cAff; /* A single character of affinity */


52214	} an;
52215	struct OP_MakeRecord_stack_vars {
52216	u8 zNewRecord; / A buffer to hold the data for the new record */
52217	Mem pRec; / The new record */
52218	u64 nData; /* Number of bytes of data space */
	@@ -52333,18 +52293,19 @@
52293	struct OP_Found_stack_vars {
52294	int alreadyExists;
52295	VdbeCursor *pC;
52296	int res;
52297	UnpackedRecord *pIdxKey;
52298	UnpackedRecord r;
52299	char aTempRec[ROUND8(sizeof(UnpackedRecord)) + sizeof(Mem)*3 + 7];
52300	} bb;
52301	struct OP_IsUnique_stack_vars {
52302	u16 ii;
52303	VdbeCursor *pCx;
52304	BtCursor *pCrsr;
52305	u16 nField;
52306	Mem *aMx;
52307	UnpackedRecord r; /* B-Tree index search key */
52308	i64 R; /* Rowid stored in register P3 */
52309	} bc;
52310	struct OP_NotExists_stack_vars {
52311	VdbeCursor *pC;
	@@ -52453,63 +52414,58 @@
52414	int j; /* Loop counter */
52415	int nErr; /* Number of errors reported */
52416	char z; / Text of the error report */
52417	Mem pnErr; / Register keeping track of errors remaining */
52418	} bv;




52419	struct OP_RowSetRead_stack_vars {

52420	i64 val;
52421	} bw;
52422	struct OP_RowSetTest_stack_vars {
52423	int iSet;
52424	int exists;
52425	} bx;
52426	struct OP_Program_stack_vars {
52427	int nMem; /* Number of memory registers for sub-program */
52428	int nByte; /* Bytes of runtime space required for sub-program */
52429	Mem pRt; / Register to allocate runtime space */
52430	Mem pMem; / Used to iterate through memory cells */
52431	Mem pEnd; / Last memory cell in new array */
52432	VdbeFrame pFrame; / New vdbe frame to execute in */
52433	SubProgram pProgram; / Sub-program to execute */
52434	void t; / Token identifying trigger */
52435	} by;
52436	struct OP_Param_stack_vars {
52437	VdbeFrame *pFrame;
52438	Mem *pIn;
52439	} bz;
52440	struct OP_MemMax_stack_vars {
52441	Mem *pIn1;
52442	VdbeFrame *pFrame;
52443	} ca;
52444	struct OP_AggStep_stack_vars {
52445	int n;
52446	int i;
52447	Mem *pMem;
52448	Mem *pRec;
52449	sqlite3_context ctx;
52450	sqlite3_value **apVal;
52451	} cb;
52452	struct OP_AggFinal_stack_vars {
52453	Mem *pMem;
52454	} cc;
52455	struct OP_IncrVacuum_stack_vars {
52456	Btree *pBt;
52457	} cd;
52458	struct OP_VBegin_stack_vars {
52459	VTable *pVTab;
52460	} ce;
52461	struct OP_VOpen_stack_vars {
52462	VdbeCursor *pCur;
52463	sqlite3_vtab_cursor *pVtabCursor;
52464	sqlite3_vtab *pVtab;
52465	sqlite3_module *pModule;
52466	} cf;
52467	struct OP_VFilter_stack_vars {
52468	int nArg;
52469	int iQuery;
52470	const sqlite3_module *pModule;
52471	Mem *pQuery;
	@@ -52518,44 +52474,44 @@
52474	sqlite3_vtab *pVtab;
52475	VdbeCursor *pCur;
52476	int res;
52477	int i;
52478	Mem **apArg;
52479	} cg;
52480	struct OP_VColumn_stack_vars {
52481	sqlite3_vtab *pVtab;
52482	const sqlite3_module *pModule;
52483	Mem *pDest;
52484	sqlite3_context sContext;
52485	} ch;
52486	struct OP_VNext_stack_vars {
52487	sqlite3_vtab *pVtab;
52488	const sqlite3_module *pModule;
52489	int res;
52490	VdbeCursor *pCur;
52491	} ci;
52492	struct OP_VRename_stack_vars {
52493	sqlite3_vtab *pVtab;
52494	Mem *pName;
52495	} cj;
52496	struct OP_VUpdate_stack_vars {
52497	sqlite3_vtab *pVtab;
52498	sqlite3_module *pModule;
52499	int nArg;
52500	int i;
52501	sqlite_int64 rowid;
52502	Mem **apArg;
52503	Mem *pX;
52504	} ck;
52505	struct OP_Pagecount_stack_vars {
52506	int p1;
52507	int nPage;
52508	Pager *pPager;
52509	} cl;
52510	struct OP_Trace_stack_vars {
52511	char *zTrace;
52512	} cm;
52513	} u;
52514	/* End automatically generated code
52515	********************************************************************/
52516
52517	assert( p->magic==VDBE_MAGIC_RUN ); /* sqlite3_step() verifies this */
	@@ -52571,20 +52527,23 @@
52527	assert( p->explain==0 );
52528	p->pResultSet = 0;
52529	db->busyHandler.nBusy = 0;
52530	CHECK_FOR_INTERRUPT;
52531	sqlite3VdbeIOTraceSql(p);
52532	#ifndef SQLITE_OMIT_PROGRESS_CALLBACK
52533	checkProgress = db->xProgress!=0;
52534	#endif
52535	#ifdef SQLITE_DEBUG
52536	sqlite3BeginBenignMalloc();
52537	if( p->pc==0
52538	&& ((p->db->flags & SQLITE_VdbeListing) \|\| fileExists(db, "vdbe_explain"))
52539	){
52540	int i;
52541	printf("VDBE Program Listing:\n");
52542	sqlite3VdbePrintSql(p);
52543	for(i=0; i<p->nOp; i++){
52544	sqlite3VdbePrintOp(stdout, i, &aOp[i]);
52545	}
52546	}
52547	if( fileExists(db, "vdbe_trace") ){
52548	p->trace = stdout;
52549	}
	@@ -52595,11 +52554,11 @@
52554	if( db->mallocFailed ) goto no_mem;
52555	#ifdef VDBE_PROFILE
52556	origPc = pc;
52557	start = sqlite3Hwtime();
52558	#endif
52559	pOp = &aOp[pc];
52560
52561	/* Only allow tracing if SQLITE_DEBUG is defined.
52562	*/
52563	#ifdef SQLITE_DEBUG
52564	if( p->trace ){
	@@ -52636,11 +52595,11 @@
52595	** of VDBE ops have been executed (either since this invocation of
52596	** sqlite3VdbeExec() or since last time the progress callback was called).
52597	** If the progress callback returns non-zero, exit the virtual machine with
52598	** a return code SQLITE_ABORT.
52599	*/
52600	if( checkProgress ){
52601	if( db->nProgressOps==nProgressOps ){
52602	int prc;
52603	if( sqlite3SafetyOff(db) ) goto abort_due_to_misuse;
52604	prc =db->xProgress(db->pProgressArg);
52605	if( sqlite3SafetyOn(db) ) goto abort_due_to_misuse;
	@@ -52652,70 +52611,51 @@
52611	}
52612	nProgressOps++;
52613	}
52614	#endif
52615
52616	/* On any opcode with the "out2-prerelase" tag, free any
52617	** external allocations out of mem[p2] and set mem[p2] to be
52618	** an undefined integer. Opcodes will either fill in the integer
52619	** value or convert mem[p2] to a different type.
52620	*/
52621	assert( pOp->opflags==sqlite3OpcodeProperty[pOp->opcode] );
52622	if( pOp->opflags & OPFLG_OUT2_PRERELEASE ){
52623	assert( pOp->p2>0 );
52624	assert( pOp->p2<=p->nMem );
52625	pOut = &aMem[pOp->p2];
52626	sqlite3VdbeMemReleaseExternal(pOut);
52627	pOut->flags = MEM_Int;
52628	}
52629
52630	/* Sanity checking on other operands */
52631	#ifdef SQLITE_DEBUG
52632	if( (pOp->opflags & OPFLG_IN1)!=0 ){










52633	assert( pOp->p1>0 );
52634	assert( pOp->p1<=p->nMem );
52635	REGISTER_TRACE(pOp->p1, &aMem[pOp->p1]);
52636	}
52637	if( (pOp->opflags & OPFLG_IN2)!=0 ){
52638	assert( pOp->p2>0 );
52639	assert( pOp->p2<=p->nMem );
52640	REGISTER_TRACE(pOp->p2, &aMem[pOp->p2]);
52641	}
52642	if( (pOp->opflags & OPFLG_IN3)!=0 ){
52643	assert( pOp->p3>0 );
52644	assert( pOp->p3<=p->nMem );
52645	REGISTER_TRACE(pOp->p3, &aMem[pOp->p3]);
52646	}
52647	if( (pOp->opflags & OPFLG_OUT2)!=0 ){
52648	assert( pOp->p2>0 );
52649	assert( pOp->p2<=p->nMem );
52650	}
52651	if( (pOp->opflags & OPFLG_OUT3)!=0 ){
52652	assert( pOp->p3>0 );
52653	assert( pOp->p3<=p->nMem );
52654	}
52655	#endif
52656









52657	switch( pOp->opcode ){
52658
52659	/*****************************************************************************
52660	** What follows is a massive switch statement where each case implements a
52661	** separate instruction in the virtual machine. If we follow the usual
	@@ -52767,14 +52707,12 @@
52707	/* Opcode: Gosub P1 P2 * * *
52708	**
52709	** Write the current address onto register P1
52710	** and then jump to address P2.
52711	*/
52712	case OP_Gosub: { /* jump, in1 */
52713	pIn1 = &aMem[pOp->p1];


52714	assert( (pIn1->flags & MEM_Dyn)==0 );
52715	pIn1->flags = MEM_Int;
52716	pIn1->u.i = pc;
52717	REGISTER_TRACE(pOp->p1, pIn1);
52718	pc = pOp->p2 - 1;
	@@ -52784,10 +52722,11 @@
52722	/* Opcode: Return P1 * * * *
52723	**
52724	** Jump to the next instruction after the address in register P1.
52725	*/
52726	case OP_Return: { /* in1 */
52727	pIn1 = &aMem[pOp->p1];
52728	assert( pIn1->flags & MEM_Int );
52729	pc = (int)pIn1->u.i;
52730	break;
52731	}
52732
	@@ -52797,10 +52736,11 @@
52736	*/
52737	case OP_Yield: { /* in1 */
52738	#if 0 /* local variables moved into u.aa */
52739	int pcDest;
52740	#endif /* local variables moved into u.aa */
52741	pIn1 = &aMem[pOp->p1];
52742	assert( (pIn1->flags & MEM_Dyn)==0 );
52743	pIn1->flags = MEM_Int;
52744	u.aa.pcDest = (int)pIn1->u.i;
52745	pIn1->u.i = pc;
52746	REGISTER_TRACE(pOp->p1, pIn1);
	@@ -52813,10 +52753,11 @@
52753	** Check the value in register P3. If is is NULL then Halt using
52754	** parameter P1, P2, and P4 as if this were a Halt instruction. If the
52755	** value in register P3 is not NULL, then this routine is a no-op.
52756	*/
52757	case OP_HaltIfNull: { /* in3 */
52758	pIn3 = &aMem[pOp->p3];
52759	if( (pIn3->flags & MEM_Null)==0 ) break;
52760	/* Fall through into OP_Halt */
52761	}
52762
52763	/* Opcode: Halt P1 P2 * P4 *
	@@ -52852,10 +52793,12 @@
52793	** instruction is set to OE_Ignore, then the sub-program is throwing
52794	** an IGNORE exception. In this case jump to the address specified
52795	** as the p2 of the calling OP_Program. */
52796	pc = p->aOp[pc].p2-1;
52797	}
52798	aOp = p->aOp;
52799	aMem = p->aMem;
52800	break;
52801	}
52802
52803	p->rc = pOp->p1;
52804	p->errorAction = (u8)pOp->p2;
	@@ -52878,11 +52821,10 @@
52821	/* Opcode: Integer P1 P2 * * *
52822	**
52823	** The 32-bit integer value P1 is written into register P2.
52824	*/
52825	case OP_Integer: { /* out2-prerelease */

52826	pOut->u.i = pOp->p1;
52827	break;
52828	}
52829
52830	/* Opcode: Int64 * P2 * P4 *
	@@ -52890,11 +52832,10 @@
52832	** P4 is a pointer to a 64-bit integer value.
52833	** Write that value into register P2.
52834	*/
52835	case OP_Int64: { /* out2-prerelease */
52836	assert( pOp->p4.pI64!=0 );

52837	pOut->u.i = *pOp->p4.pI64;
52838	break;
52839	}
52840
52841	/* Opcode: Real * P2 * P4 *
	@@ -52960,10 +52901,11 @@
52901	/* Opcode: Null * P2 * * *
52902	**
52903	** Write a NULL into register P2.
52904	*/
52905	case OP_Null: { /* out2-prerelease */
52906	pOut->flags = MEM_Null;
52907	break;
52908	}
52909
52910
52911	/* Opcode: Blob P1 P2 * P4
	@@ -53009,11 +52951,11 @@
52951	while( u.ab.n-- > 0 ){
52952	u.ab.pVar = &p->aVar[u.ab.p1++];
52953	if( sqlite3VdbeMemTooBig(u.ab.pVar) ){
52954	goto too_big;
52955	}
52956	pOut = &aMem[u.ab.p2++];
52957	sqlite3VdbeMemReleaseExternal(pOut);
52958	pOut->flags = MEM_Null;
52959	sqlite3VdbeMemShallowCopy(pOut, u.ab.pVar, MEM_Static);
52960	UPDATE_MAX_BLOBSIZE(pOut);
52961	}
	@@ -53039,15 +52981,15 @@
52981	u.ac.p1 = pOp->p1;
52982	u.ac.p2 = pOp->p2;
52983	assert( u.ac.n>0 && u.ac.p1>0 && u.ac.p2>0 );
52984	assert( u.ac.p1+u.ac.n<=u.ac.p2 \|\| u.ac.p2+u.ac.n<=u.ac.p1 );
52985
52986	pIn1 = &aMem[u.ac.p1];
52987	pOut = &aMem[u.ac.p2];
52988	while( u.ac.n-- ){
52989	assert( pOut<=&aMem[p->nMem] );
52990	assert( pIn1<=&aMem[p->nMem] );
52991	u.ac.zMalloc = pOut->zMalloc;
52992	pOut->zMalloc = 0;
52993	sqlite3VdbeMemMove(pOut, pIn1);
52994	pIn1->zMalloc = u.ac.zMalloc;
52995	REGISTER_TRACE(u.ac.p2++, pOut);
	@@ -53062,14 +53004,13 @@
53004	** Make a copy of register P1 into register P2.
53005	**
53006	** This instruction makes a deep copy of the value. A duplicate
53007	** is made of any string or blob constant. See also OP_SCopy.
53008	*/
53009	case OP_Copy: { /* in1, out2 */
53010	pIn1 = &aMem[pOp->p1];
53011	pOut = &aMem[pOp->p2];

53012	assert( pOut!=pIn1 );
53013	sqlite3VdbeMemShallowCopy(pOut, pIn1, MEM_Ephem);
53014	Deephemeralize(pOut);
53015	REGISTER_TRACE(pOp->p2, pOut);
53016	break;
	@@ -53085,15 +53026,13 @@
53026	** Worse, if the original is deallocated, the copy becomes invalid.
53027	** Thus the program must guarantee that the original will not change
53028	** during the lifetime of the copy. Use OP_Copy to make a complete
53029	** copy.
53030	*/
53031	case OP_SCopy: { /* in1, out2 */
53032	pIn1 = &aMem[pOp->p1];
53033	pOut = &aMem[pOp->p2];


53034	assert( pOut!=pIn1 );
53035	sqlite3VdbeMemShallowCopy(pOut, pIn1, MEM_Ephem);
53036	REGISTER_TRACE(pOp->p2, pOut);
53037	break;
53038	}
	@@ -53150,11 +53089,11 @@
53089
53090	/* Make sure the results of the current row are \000 terminated
53091	** and have an assigned type. The results are de-ephemeralized as
53092	** as side effect.
53093	*/
53094	u.ad.pMem = p->pResultSet = &aMem[pOp->p1];
53095	for(u.ad.i=0; u.ad.i<pOp->p2; u.ad.i++){
53096	sqlite3VdbeMemNulTerminate(&u.ad.pMem[u.ad.i]);
53097	sqlite3VdbeMemStoreType(&u.ad.pMem[u.ad.i]);
53098	REGISTER_TRACE(pOp->p1+u.ad.i, &u.ad.pMem[u.ad.i]);
53099	}
	@@ -53182,10 +53121,13 @@
53121	case OP_Concat: { /* same as TK_CONCAT, in1, in2, out3 */
53122	#if 0 /* local variables moved into u.ae */
53123	i64 nByte;
53124	#endif /* local variables moved into u.ae */
53125
53126	pIn1 = &aMem[pOp->p1];
53127	pIn2 = &aMem[pOp->p2];
53128	pOut = &aMem[pOp->p3];
53129	assert( pIn1!=pOut );
53130	if( (pIn1->flags \| pIn2->flags) & MEM_Null ){
53131	sqlite3VdbeMemSetNull(pOut);
53132	break;
53133	}
	@@ -53257,12 +53199,15 @@
53199	i64 iB; /* Integer value of right operand */
53200	double rA; /* Real value of left operand */
53201	double rB; /* Real value of right operand */
53202	#endif /* local variables moved into u.af */
53203
53204	pIn1 = &aMem[pOp->p1];
53205	applyNumericAffinity(pIn1);
53206	pIn2 = &aMem[pOp->p2];
53207	applyNumericAffinity(pIn2);
53208	pOut = &aMem[pOp->p3];
53209	u.af.flags = pIn1->flags \| pIn2->flags;
53210	if( (u.af.flags & MEM_Null)!=0 ) goto arithmetic_result_is_null;
53211	if( (pIn1->flags & pIn2->flags & MEM_Int)==MEM_Int ){
53212	u.af.iA = pIn1->u.i;
53213	u.af.iB = pIn2->u.i;
	@@ -53375,11 +53320,11 @@
53320	u.ag.apVal = p->apArg;
53321	assert( u.ag.apVal \|\| u.ag.n==0 );
53322
53323	assert( u.ag.n==0 \|\| (pOp->p2>0 && pOp->p2+u.ag.n<=p->nMem+1) );
53324	assert( pOp->p3<pOp->p2 \|\| pOp->p3>=pOp->p2+u.ag.n );
53325	u.ag.pArg = &aMem[pOp->p2];
53326	for(u.ag.i=0; u.ag.i<u.ag.n; u.ag.i++, u.ag.pArg++){
53327	u.ag.apVal[u.ag.i] = u.ag.pArg;
53328	sqlite3VdbeMemStoreType(u.ag.pArg);
53329	REGISTER_TRACE(pOp->p2, u.ag.pArg);
53330	}
	@@ -53392,11 +53337,11 @@
53337	u.ag.ctx.pVdbeFunc = (VdbeFunc*)pOp->p4.pVdbeFunc;
53338	u.ag.ctx.pFunc = u.ag.ctx.pVdbeFunc->pFunc;
53339	}
53340
53341	assert( pOp->p3>0 && pOp->p3<=p->nMem );
53342	pOut = &aMem[pOp->p3];
53343	u.ag.ctx.s.flags = MEM_Null;
53344	u.ag.ctx.s.db = db;
53345	u.ag.ctx.s.xDel = 0;
53346	u.ag.ctx.s.zMalloc = 0;
53347
	@@ -53407,11 +53352,11 @@
53352	sqlite3VdbeMemMove(&u.ag.ctx.s, pOut);
53353	MemSetTypeFlag(&u.ag.ctx.s, MEM_Null);
53354
53355	u.ag.ctx.isError = 0;
53356	if( u.ag.ctx.pFunc->flags & SQLITE_FUNC_NEEDCOLL ){
53357	assert( pOp>aOp );
53358	assert( pOp[-1].p4type==P4_COLLSEQ );
53359	assert( pOp[-1].opcode==OP_CollSeq );
53360	u.ag.ctx.pColl = pOp[-1].p4.pColl;
53361	}
53362	if( sqlite3SafetyOff(db) ) goto abort_due_to_misuse;
	@@ -53493,10 +53438,13 @@
53438	#if 0 /* local variables moved into u.ah */
53439	i64 a;
53440	i64 b;
53441	#endif /* local variables moved into u.ah */
53442
53443	pIn1 = &aMem[pOp->p1];
53444	pIn2 = &aMem[pOp->p2];
53445	pOut = &aMem[pOp->p3];
53446	if( (pIn1->flags \| pIn2->flags) & MEM_Null ){
53447	sqlite3VdbeMemSetNull(pOut);
53448	break;
53449	}
53450	u.ah.a = sqlite3VdbeIntValue(pIn2);
	@@ -53519,10 +53467,11 @@
53467	** The result is always an integer.
53468	**
53469	** To force any register to be an integer, just add 0.
53470	*/
53471	case OP_AddImm: { /* in1 */
53472	pIn1 = &aMem[pOp->p1];
53473	sqlite3VdbeMemIntegerify(pIn1);
53474	pIn1->u.i += pOp->p2;
53475	break;
53476	}
53477
	@@ -53532,10 +53481,11 @@
53481	** in P1 is not an integer and cannot be converted into an integer
53482	** without data loss, then jump immediately to P2, or if P2==0
53483	** raise an SQLITE_MISMATCH exception.
53484	*/
53485	case OP_MustBeInt: { /* jump, in1 */
53486	pIn1 = &aMem[pOp->p1];
53487	applyAffinity(pIn1, SQLITE_AFF_NUMERIC, encoding);
53488	if( (pIn1->flags & MEM_Int)==0 ){
53489	if( pOp->p2==0 ){
53490	rc = SQLITE_MISMATCH;
53491	goto abort_due_to_error;
	@@ -53556,10 +53506,11 @@
53506	** has REAL affinity. Such column values may still be stored as
53507	** integers, for space efficiency, but after extraction we want them
53508	** to have only a real value.
53509	*/
53510	case OP_RealAffinity: { /* in1 */
53511	pIn1 = &aMem[pOp->p1];
53512	if( pIn1->flags & MEM_Int ){
53513	sqlite3VdbeMemRealify(pIn1);
53514	}
53515	break;
53516	}
	@@ -53573,10 +53524,11 @@
53524	** are afterwards simply interpreted as text.
53525	**
53526	** A NULL value is not changed by this routine. It remains NULL.
53527	*/
53528	case OP_ToText: { /* same as TK_TO_TEXT, in1 */
53529	pIn1 = &aMem[pOp->p1];
53530	if( pIn1->flags & MEM_Null ) break;
53531	assert( MEM_Str==(MEM_Blob>>3) );
53532	pIn1->flags \|= (pIn1->flags&MEM_Blob)>>3;
53533	applyAffinity(pIn1, SQLITE_AFF_TEXT, encoding);
53534	rc = ExpandBlob(pIn1);
	@@ -53594,10 +53546,11 @@
53546	** to the underlying data.
53547	**
53548	** A NULL value is not changed by this routine. It remains NULL.
53549	*/
53550	case OP_ToBlob: { /* same as TK_TO_BLOB, in1 */
53551	pIn1 = &aMem[pOp->p1];
53552	if( pIn1->flags & MEM_Null ) break;
53553	if( (pIn1->flags & MEM_Blob)==0 ){
53554	applyAffinity(pIn1, SQLITE_AFF_TEXT, encoding);
53555	assert( pIn1->flags & MEM_Str \|\| db->mallocFailed );
53556	MemSetTypeFlag(pIn1, MEM_Blob);
	@@ -53617,10 +53570,11 @@
53570	** is possible.
53571	**
53572	** A NULL value is not changed by this routine. It remains NULL.
53573	*/
53574	case OP_ToNumeric: { /* same as TK_TO_NUMERIC, in1 */
53575	pIn1 = &aMem[pOp->p1];
53576	if( (pIn1->flags & (MEM_Null\|MEM_Int\|MEM_Real))==0 ){
53577	sqlite3VdbeMemNumerify(pIn1);
53578	}
53579	break;
53580	}
	@@ -53634,10 +53588,11 @@
53588	** equivalent of atoi() and store 0 if no such conversion is possible.
53589	**
53590	** A NULL value is not changed by this routine. It remains NULL.
53591	*/
53592	case OP_ToInt: { /* same as TK_TO_INT, in1 */
53593	pIn1 = &aMem[pOp->p1];
53594	if( (pIn1->flags & MEM_Null)==0 ){
53595	sqlite3VdbeMemIntegerify(pIn1);
53596	}
53597	break;
53598	}
	@@ -53651,10 +53606,11 @@
53606	** equivalent of atoi() and store 0.0 if no such conversion is possible.
53607	**
53608	** A NULL value is not changed by this routine. It remains NULL.
53609	*/
53610	case OP_ToReal: { /* same as TK_TO_REAL, in1 */
53611	pIn1 = &aMem[pOp->p1];
53612	if( (pIn1->flags & MEM_Null)==0 ){
53613	sqlite3VdbeMemRealify(pIn1);
53614	}
53615	break;
53616	}
	@@ -53741,10 +53697,12 @@
53697	#if 0 /* local variables moved into u.ai */
53698	int res; /* Result of the comparison of pIn1 against pIn3 */
53699	char affinity; /* Affinity to use for comparison */
53700	#endif /* local variables moved into u.ai */
53701
53702	pIn1 = &aMem[pOp->p1];
53703	pIn3 = &aMem[pOp->p3];
53704	if( (pIn1->flags \| pIn3->flags)&MEM_Null ){
53705	/* One or both operands are NULL */
53706	if( pOp->p5 & SQLITE_NULLEQ ){
53707	/* If SQLITE_NULLEQ is set (which will only happen if the operator is
53708	** OP_Eq or OP_Ne) then take the jump or not depending on whether
	@@ -53756,11 +53714,11 @@
53714	/* SQLITE_NULLEQ is clear and at least one operand is NULL,
53715	** then the result is always NULL.
53716	** The jump is taken if the SQLITE_JUMPIFNULL bit is set.
53717	*/
53718	if( pOp->p5 & SQLITE_STOREP2 ){
53719	pOut = &aMem[pOp->p2];
53720	MemSetTypeFlag(pOut, MEM_Null);
53721	REGISTER_TRACE(pOp->p2, pOut);
53722	}else if( pOp->p5 & SQLITE_JUMPIFNULL ){
53723	pc = pOp->p2-1;
53724	}
	@@ -53788,11 +53746,11 @@
53746	case OP_Gt: u.ai.res = u.ai.res>0; break;
53747	default: u.ai.res = u.ai.res>=0; break;
53748	}
53749
53750	if( pOp->p5 & SQLITE_STOREP2 ){
53751	pOut = &aMem[pOp->p2];
53752	MemSetTypeFlag(pOut, MEM_Int);
53753	pOut->u.i = u.ai.res;
53754	REGISTER_TRACE(pOp->p2, pOut);
53755	}else if( u.ai.res ){
53756	pc = pOp->p2-1;
	@@ -53859,16 +53817,16 @@
53817	assert( u.aj.p2>0 && u.aj.p2+u.aj.n<=p->nMem+1 );
53818	}
53819	#endif /* SQLITE_DEBUG */
53820	for(u.aj.i=0; u.aj.i<u.aj.n; u.aj.i++){
53821	u.aj.idx = aPermute ? aPermute[u.aj.i] : u.aj.i;
53822	REGISTER_TRACE(u.aj.p1+u.aj.idx, &aMem[u.aj.p1+u.aj.idx]);
53823	REGISTER_TRACE(u.aj.p2+u.aj.idx, &aMem[u.aj.p2+u.aj.idx]);
53824	assert( u.aj.i<u.aj.pKeyInfo->nField );
53825	u.aj.pColl = u.aj.pKeyInfo->aColl[u.aj.i];
53826	u.aj.bRev = u.aj.pKeyInfo->aSortOrder[u.aj.i];
53827	iCompare = sqlite3MemCompare(&aMem[u.aj.p1+u.aj.idx], &aMem[u.aj.p2+u.aj.idx], u.aj.pColl);
53828	if( iCompare ){
53829	if( u.aj.bRev ) iCompare = -iCompare;
53830	break;
53831	}
53832	}
	@@ -53916,15 +53874,17 @@
53874	#if 0 /* local variables moved into u.ak */
53875	int v1; /* Left operand: 0==FALSE, 1==TRUE, 2==UNKNOWN or NULL */
53876	int v2; /* Right operand: 0==FALSE, 1==TRUE, 2==UNKNOWN or NULL */
53877	#endif /* local variables moved into u.ak */
53878
53879	pIn1 = &aMem[pOp->p1];
53880	if( pIn1->flags & MEM_Null ){
53881	u.ak.v1 = 2;
53882	}else{
53883	u.ak.v1 = sqlite3VdbeIntValue(pIn1)!=0;
53884	}
53885	pIn2 = &aMem[pOp->p2];
53886	if( pIn2->flags & MEM_Null ){
53887	u.ak.v2 = 2;
53888	}else{
53889	u.ak.v2 = sqlite3VdbeIntValue(pIn2)!=0;
53890	}
	@@ -53933,10 +53893,11 @@
53893	u.ak.v1 = and_logic[u.ak.v1*3+u.ak.v2];
53894	}else{
53895	static const unsigned char or_logic[] = { 0, 1, 2, 1, 1, 1, 2, 1, 2 };
53896	u.ak.v1 = or_logic[u.ak.v1*3+u.ak.v2];
53897	}
53898	pOut = &aMem[pOp->p3];
53899	if( u.ak.v1==2 ){
53900	MemSetTypeFlag(pOut, MEM_Null);
53901	}else{
53902	pOut->u.i = u.ak.v1;
53903	MemSetTypeFlag(pOut, MEM_Int);
	@@ -53948,12 +53909,13 @@
53909	**
53910	** Interpret the value in register P1 as a boolean value. Store the
53911	** boolean complement in register P2. If the value in register P1 is
53912	** NULL, then a NULL is stored in P2.
53913	*/
53914	case OP_Not: { /* same as TK_NOT, in1, out2 */
53915	pIn1 = &aMem[pOp->p1];
53916	pOut = &aMem[pOp->p2];
53917	if( pIn1->flags & MEM_Null ){
53918	sqlite3VdbeMemSetNull(pOut);
53919	}else{
53920	sqlite3VdbeMemSetInt64(pOut, !sqlite3VdbeIntValue(pIn1));
53921	}
	@@ -53964,12 +53926,13 @@
53926	**
53927	** Interpret the content of register P1 as an integer. Store the
53928	** ones-complement of the P1 value into register P2. If P1 holds
53929	** a NULL then store a NULL in P2.
53930	*/
53931	case OP_BitNot: { /* same as TK_BITNOT, in1, out2 */
53932	pIn1 = &aMem[pOp->p1];
53933	pOut = &aMem[pOp->p2];
53934	if( pIn1->flags & MEM_Null ){
53935	sqlite3VdbeMemSetNull(pOut);
53936	}else{
53937	sqlite3VdbeMemSetInt64(pOut, ~sqlite3VdbeIntValue(pIn1));
53938	}
	@@ -53991,10 +53954,11 @@
53954	case OP_If: /* jump, in1 */
53955	case OP_IfNot: { /* jump, in1 */
53956	#if 0 /* local variables moved into u.al */
53957	int c;
53958	#endif /* local variables moved into u.al */
53959	pIn1 = &aMem[pOp->p1];
53960	if( pIn1->flags & MEM_Null ){
53961	u.al.c = pOp->p3;
53962	}else{
53963	#ifdef SQLITE_OMIT_FLOATING_POINT
53964	u.al.c = sqlite3VdbeIntValue(pIn1)!=0;
	@@ -54012,10 +53976,11 @@
53976	/* Opcode: IsNull P1 P2 * * *
53977	**
53978	** Jump to P2 if the value in register P1 is NULL.
53979	*/
53980	case OP_IsNull: { /* same as TK_ISNULL, jump, in1 */
53981	pIn1 = &aMem[pOp->p1];
53982	if( (pIn1->flags & MEM_Null)!=0 ){
53983	pc = pOp->p2 - 1;
53984	}
53985	break;
53986	}
	@@ -54023,10 +53988,11 @@
53988	/* Opcode: NotNull P1 P2 * * *
53989	**
53990	** Jump to P2 if the value in register P1 is not NULL.
53991	*/
53992	case OP_NotNull: { /* same as TK_NOTNULL, jump, in1 */
53993	pIn1 = &aMem[pOp->p1];
53994	if( (pIn1->flags & MEM_Null)==0 ){
53995	pc = pOp->p2 - 1;
53996	}
53997	break;
53998	}
	@@ -54081,11 +54047,11 @@
54047	u.am.p2 = pOp->p2;
54048	u.am.pC = 0;
54049	memset(&u.am.sMem, 0, sizeof(u.am.sMem));
54050	assert( u.am.p1<p->nCursor );
54051	assert( pOp->p3>0 && pOp->p3<=p->nMem );
54052	u.am.pDest = &aMem[pOp->p3];
54053	MemSetTypeFlag(u.am.pDest, MEM_Null);
54054	u.am.zRec = 0;
54055
54056	/* This block sets the variable u.am.payloadSize to be the total number of
54057	** bytes in the record.
	@@ -54127,11 +54093,11 @@
54093	assert( sqlite3BtreeCursorIsValid(u.am.pCrsr) );
54094	rc = sqlite3BtreeDataSize(u.am.pCrsr, &u.am.payloadSize);
54095	assert( rc==SQLITE_OK ); /* DataSize() cannot fail */
54096	}
54097	}else if( u.am.pC->pseudoTableReg>0 ){
54098	u.am.pReg = &aMem[u.am.pC->pseudoTableReg];
54099	assert( u.am.pReg->flags & MEM_Blob );
54100	u.am.payloadSize = u.am.pReg->n;
54101	u.am.zRec = u.am.pReg->z;
54102	u.am.pC->cacheStatus = (pOp->p5&OPFLAG_CLEARCACHE) ? CACHE_STALE : p->cacheCtr;
54103	assert( u.am.payloadSize==0 \|\| u.am.zRec!=0 );
	@@ -54339,22 +54305,23 @@
54305	** string indicates the column affinity that should be used for the nth
54306	** memory cell in the range.
54307	*/
54308	case OP_Affinity: {
54309	#if 0 /* local variables moved into u.an */
54310	const char zAffinity; / The affinity to be applied */
54311	char cAff; /* A single character of affinity */


54312	#endif /* local variables moved into u.an */
54313
54314	u.an.zAffinity = pOp->p4.z;
54315	assert( u.an.zAffinity!=0 );
54316	assert( u.an.zAffinity[pOp->p2]==0 );
54317	pIn1 = &aMem[pOp->p1];
54318	while( (u.an.cAff = *(u.an.zAffinity++))!=0 ){
54319	assert( pIn1 <= &p->aMem[p->nMem] );
54320	ExpandBlob(pIn1);
54321	applyAffinity(pIn1, u.an.cAff, encoding);
54322	pIn1++;
54323	}
54324	break;
54325	}
54326
54327	/* Opcode: MakeRecord P1 P2 P3 P4 *
	@@ -54414,11 +54381,11 @@
54381	u.ao.nByte = 0; /* Data space required for this record */
54382	u.ao.nZero = 0; /* Number of zero bytes at the end of the record */
54383	u.ao.nField = pOp->p1;
54384	u.ao.zAffinity = pOp->p4.z;
54385	assert( u.ao.nField>0 && pOp->p2>0 && pOp->p2+u.ao.nField<=p->nMem+1 );
54386	u.ao.pData0 = &aMem[u.ao.nField];
54387	u.ao.nField = pOp->p2;
54388	u.ao.pLast = &u.ao.pData0[u.ao.nField-1];
54389	u.ao.file_format = p->minWriteFileFormat;
54390
54391	/* Loop through the elements that will make up the record to figure
	@@ -54458,11 +54425,11 @@
54425	** the new record. The output register (pOp->p3) is not allowed to
54426	** be one of the input registers (because the following call to
54427	** sqlite3VdbeMemGrow() could clobber the value before it is used).
54428	*/
54429	assert( pOp->p3<pOp->p1 \|\| pOp->p3>=pOp->p1+pOp->p2 );
54430	pOut = &aMem[pOp->p3];
54431	if( sqlite3VdbeMemGrow(pOut, (int)u.ao.nByte, 0) ){
54432	goto no_mem;
54433	}
54434	u.ao.zNewRecord = (u8 *)pOut->z;
54435
	@@ -54507,11 +54474,10 @@
54474	if( u.ap.pCrsr ){
54475	rc = sqlite3BtreeCount(u.ap.pCrsr, &u.ap.nEntry);
54476	}else{
54477	u.ap.nEntry = 0;
54478	}

54479	pOut->u.i = u.ap.nEntry;
54480	break;
54481	}
54482	#endif
54483
	@@ -54836,11 +54802,10 @@
54802	assert( db->aDb[u.at.iDb].pBt!=0 );
54803	assert( (p->btreeMask & (1<<u.at.iDb))!=0 );
54804
54805	sqlite3BtreeGetMeta(db->aDb[u.at.iDb].pBt, u.at.iCookie, (u32 *)&u.at.iMeta);
54806	pOut->u.i = u.at.iMeta;

54807	break;
54808	}
54809
54810	/* Opcode: SetCookie P1 P2 P3 * *
54811	**
	@@ -54859,10 +54824,11 @@
54824	assert( pOp->p2<SQLITE_N_BTREE_META );
54825	assert( pOp->p1>=0 && pOp->p1<db->nDb );
54826	assert( (p->btreeMask & (1<<pOp->p1))!=0 );
54827	u.au.pDb = &db->aDb[pOp->p1];
54828	assert( u.au.pDb->pBt!=0 );
54829	pIn3 = &aMem[pOp->p3];
54830	sqlite3VdbeMemIntegerify(pIn3);
54831	/* See note about index shifting on OP_ReadCookie */
54832	rc = sqlite3BtreeUpdateMeta(u.au.pDb->pBt, pOp->p2, (int)pIn3->u.i);
54833	if( pOp->p2==BTREE_SCHEMA_VERSION ){
54834	/* When the schema cookie changes, record the new cookie internally */
	@@ -55021,11 +54987,11 @@
54987	u.aw.wrFlag = 0;
54988	}
54989	if( pOp->p5 ){
54990	assert( u.aw.p2>0 );
54991	assert( u.aw.p2<=p->nMem );
54992	pIn2 = &aMem[u.aw.p2];
54993	sqlite3VdbeMemIntegerify(pIn2);
54994	u.aw.p2 = (int)pIn2->u.i;
54995	/* The u.aw.p2 value always comes from a prior OP_CreateTable opcode and
54996	** that opcode will always set the u.aw.p2 value to 2 or more or else fail.
54997	** If there were a failure, the prepared statement would have halted
	@@ -55243,17 +55209,21 @@
55209	assert( pOp->p1>=0 && pOp->p1<p->nCursor );
55210	assert( pOp->p2!=0 );
55211	u.az.pC = p->apCsr[pOp->p1];
55212	assert( u.az.pC!=0 );
55213	assert( u.az.pC->pseudoTableReg==0 );
55214	assert( OP_SeekLe == OP_SeekLt+1 );
55215	assert( OP_SeekGe == OP_SeekLt+2 );
55216	assert( OP_SeekGt == OP_SeekLt+3 );
55217	if( u.az.pC->pCursor!=0 ){
55218	u.az.oc = pOp->opcode;
55219	u.az.pC->nullRow = 0;
55220	if( u.az.pC->isTable ){
55221	/* The input value in P3 might be of any type: integer, real, string,
55222	** blob, or NULL. But it needs to be an integer before we can do
55223	** the seek, so covert it. */
55224	pIn3 = &aMem[pOp->p3];
55225	applyNumericAffinity(pIn3);
55226	u.az.iKey = sqlite3VdbeIntValue(pIn3);
55227	u.az.pC->rowidIsValid = 0;
55228
55229	/* If the P3 value could not be converted into an integer without
	@@ -55272,16 +55242,16 @@
55242	if( u.az.iKey==SMALLEST_INT64 && (pIn3->r<(double)u.az.iKey \|\| pIn3->r>0) ){
55243	/* The P3 value is too large in magnitude to be expressed as an
55244	** integer. */
55245	u.az.res = 1;
55246	if( pIn3->r<0 ){
55247	if( u.az.oc>=OP_SeekGe ){ assert( u.az.oc==OP_SeekGe \|\| u.az.oc==OP_SeekGt );
55248	rc = sqlite3BtreeFirst(u.az.pC->pCursor, &u.az.res);
55249	if( rc!=SQLITE_OK ) goto abort_due_to_error;
55250	}
55251	}else{
55252	if( u.az.oc<=OP_SeekLe ){ assert( u.az.oc==OP_SeekLt \|\| u.az.oc==OP_SeekLe );
55253	rc = sqlite3BtreeLast(u.az.pC->pCursor, &u.az.res);
55254	if( rc!=SQLITE_OK ) goto abort_due_to_error;
55255	}
55256	}
55257	if( u.az.res ){
	@@ -55309,16 +55279,26 @@
55279	u.az.nField = pOp->p4.i;
55280	assert( pOp->p4type==P4_INT32 );
55281	assert( u.az.nField>0 );
55282	u.az.r.pKeyInfo = u.az.pC->pKeyInfo;
55283	u.az.r.nField = (u16)u.az.nField;
55284
55285	/* The next line of code computes as follows, only faster:
55286	** if( u.az.oc==OP_SeekGt \|\| u.az.oc==OP_SeekLe ){
55287	** u.az.r.flags = UNPACKED_INCRKEY;
55288	** }else{
55289	** u.az.r.flags = 0;
55290	** }
55291	*/
55292	u.az.r.flags = UNPACKED_INCRKEY * (1 & (u.az.oc - OP_SeekLt));
55293	assert( u.az.oc!=OP_SeekGt \|\| u.az.r.flags==UNPACKED_INCRKEY );
55294	assert( u.az.oc!=OP_SeekLe \|\| u.az.r.flags==UNPACKED_INCRKEY );
55295	assert( u.az.oc!=OP_SeekGe \|\| u.az.r.flags==0 );
55296	assert( u.az.oc!=OP_SeekLt \|\| u.az.r.flags==0 );
55297
55298	u.az.r.aMem = &aMem[pOp->p3];
55299	ExpandBlob(u.az.r.aMem);
55300	rc = sqlite3BtreeMovetoUnpacked(u.az.pC->pCursor, &u.az.r, 0, 0, &u.az.res);
55301	if( rc!=SQLITE_OK ){
55302	goto abort_due_to_error;
55303	}
55304	u.az.pC->rowidIsValid = 0;
	@@ -55326,11 +55306,11 @@
55306	u.az.pC->deferredMoveto = 0;
55307	u.az.pC->cacheStatus = CACHE_STALE;
55308	#ifdef SQLITE_TEST
55309	sqlite3_search_count++;
55310	#endif
55311	if( u.az.oc>=OP_SeekGe ){ assert( u.az.oc==OP_SeekGe \|\| u.az.oc==OP_SeekGt );
55312	if( u.az.res<0 \|\| (u.az.res==0 && u.az.oc==OP_SeekGt) ){
55313	rc = sqlite3BtreeNext(u.az.pC->pCursor, &u.az.res);
55314	if( rc!=SQLITE_OK ) goto abort_due_to_error;
55315	u.az.pC->rowidIsValid = 0;
55316	}else{
	@@ -55381,45 +55361,40 @@
55361	u.ba.pC = p->apCsr[pOp->p1];
55362	assert( u.ba.pC!=0 );
55363	if( ALWAYS(u.ba.pC->pCursor!=0) ){
55364	assert( u.ba.pC->isTable );
55365	u.ba.pC->nullRow = 0;
55366	pIn2 = &aMem[pOp->p2];
55367	u.ba.pC->movetoTarget = sqlite3VdbeIntValue(pIn2);
55368	u.ba.pC->rowidIsValid = 0;
55369	u.ba.pC->deferredMoveto = 1;
55370	}
55371	break;
55372	}
55373
55374
55375	/* Opcode: Found P1 P2 P3 P4 *
55376	**
55377	** If P4==0 then register P3 holds a blob constructed by MakeRecord. If
55378	** P4>0 then register P3 is the first of P4 registers that form an unpacked
55379	** record.
55380	**
55381	** Cursor P1 is on an index btree. If the record identified by P3 and P4
55382	** is a prefix of any entry in P1 then a jump is made to P2 and
55383	** P1 is left pointing at the matching entry.
55384	*/
55385	/* Opcode: NotFound P1 P2 P3 P4 *
55386	**
55387	** If P4==0 then register P3 holds a blob constructed by MakeRecord. If
55388	** P4>0 then register P3 is the first of P4 registers that form an unpacked
55389	** record.
55390	**
55391	** Cursor P1 is on an index btree. If the record identified by P3 and P4
55392	** is not the prefix of any entry in P1 then a jump is made to P2. If P1
55393	** does contain an entry whose prefix matches the P3/P4 record then control
55394	** falls through to the next instruction and P1 is left pointing at the
55395	** matching entry.






55396	**
55397	** See also: Found, NotExists, IsUnique
55398	*/
55399	case OP_NotFound: /* jump, in3 */
55400	case OP_Found: { /* jump, in3 */
	@@ -55426,36 +55401,47 @@
55401	#if 0 /* local variables moved into u.bb */
55402	int alreadyExists;
55403	VdbeCursor *pC;
55404	int res;
55405	UnpackedRecord *pIdxKey;
55406	UnpackedRecord r;
55407	char aTempRec[ROUND8(sizeof(UnpackedRecord)) + sizeof(Mem)*3 + 7];
55408	#endif /* local variables moved into u.bb */
55409
55410	#ifdef SQLITE_TEST
55411	sqlite3_found_count++;
55412	#endif
55413
55414	u.bb.alreadyExists = 0;
55415	assert( pOp->p1>=0 && pOp->p1<p->nCursor );
55416	assert( pOp->p4type==P4_INT32 );
55417	u.bb.pC = p->apCsr[pOp->p1];
55418	assert( u.bb.pC!=0 );
55419	pIn3 = &aMem[pOp->p3];
55420	if( ALWAYS(u.bb.pC->pCursor!=0) ){
55421
55422	assert( u.bb.pC->isTable==0 );
55423	if( pOp->p4.i>0 ){
55424	u.bb.r.pKeyInfo = u.bb.pC->pKeyInfo;
55425	u.bb.r.nField = pOp->p4.i;
55426	u.bb.r.aMem = pIn3;
55427	u.bb.r.flags = UNPACKED_PREFIX_MATCH;
55428	u.bb.pIdxKey = &u.bb.r;
55429	}else{
55430	assert( pIn3->flags & MEM_Blob );
55431	ExpandBlob(pIn3);
55432	u.bb.pIdxKey = sqlite3VdbeRecordUnpack(u.bb.pC->pKeyInfo, pIn3->n, pIn3->z,
55433	u.bb.aTempRec, sizeof(u.bb.aTempRec));
55434	if( u.bb.pIdxKey==0 ){
55435	goto no_mem;
55436	}
55437	u.bb.pIdxKey->flags \|= UNPACKED_PREFIX_MATCH;
55438	}
55439	rc = sqlite3BtreeMovetoUnpacked(u.bb.pC->pCursor, u.bb.pIdxKey, 0, 0, &u.bb.res);
55440	if( pOp->p4.i==0 ){
55441	sqlite3VdbeDeleteUnpackedRecord(u.bb.pIdxKey);
55442	}
55443	if( rc!=SQLITE_OK ){
55444	break;
55445	}
55446	u.bb.alreadyExists = (u.bb.res==0);
55447	u.bb.pC->deferredMoveto = 0;
	@@ -55469,13 +55455,14 @@
55455	break;
55456	}
55457
55458	/* Opcode: IsUnique P1 P2 P3 P4 *
55459	**
55460	** Cursor P1 is open on an index b-tree - that is to say, a btree which
55461	** no data and where the key are records generated by OP_MakeRecord with
55462	** the list field being the integer ROWID of the entry that the index
55463	** entry refers to.
55464	**
55465	** The P3 register contains an integer record number. Call this record
55466	** number R. Register P4 is the first in a set of N contiguous registers
55467	** that make up an unpacked index key that can be used with cursor P1.
55468	** The value of N can be inferred from the cursor. N includes the rowid
	@@ -55498,16 +55485,17 @@
55485	#if 0 /* local variables moved into u.bc */
55486	u16 ii;
55487	VdbeCursor *pCx;
55488	BtCursor *pCrsr;
55489	u16 nField;
55490	Mem *aMx;
55491	UnpackedRecord r; /* B-Tree index search key */
55492	i64 R; /* Rowid stored in register P3 */
55493	#endif /* local variables moved into u.bc */
55494
55495	pIn3 = &aMem[pOp->p3];
55496	u.bc.aMx = &aMem[pOp->p4.i];
55497	/* Assert that the values of parameters P1 and P4 are in range. */
55498	assert( pOp->p4type==P4_INT32 );
55499	assert( pOp->p4.i>0 && pOp->p4.i<=p->nMem );
55500	assert( pOp->p1>=0 && pOp->p1<p->nCursor );
55501
	@@ -55519,24 +55507,24 @@
55507	u.bc.pCrsr = u.bc.pCx->pCursor;
55508
55509	/* If any of the values are NULL, take the jump. */
55510	u.bc.nField = u.bc.pCx->pKeyInfo->nField;
55511	for(u.bc.ii=0; u.bc.ii<u.bc.nField; u.bc.ii++){
55512	if( u.bc.aMx[u.bc.ii].flags & MEM_Null ){
55513	pc = pOp->p2 - 1;
55514	u.bc.pCrsr = 0;
55515	break;
55516	}
55517	}
55518	assert( (u.bc.aMx[u.bc.nField].flags & MEM_Null)==0 );
55519
55520	if( u.bc.pCrsr!=0 ){
55521	/* Populate the index search key. */
55522	u.bc.r.pKeyInfo = u.bc.pCx->pKeyInfo;
55523	u.bc.r.nField = u.bc.nField + 1;
55524	u.bc.r.flags = UNPACKED_PREFIX_SEARCH;
55525	u.bc.r.aMem = u.bc.aMx;
55526
55527	/* Extract the value of u.bc.R from register P3. */
55528	sqlite3VdbeMemIntegerify(pIn3);
55529	u.bc.R = pIn3->u.i;
55530
	@@ -55573,10 +55561,11 @@
55561	BtCursor *pCrsr;
55562	int res;
55563	u64 iKey;
55564	#endif /* local variables moved into u.bd */
55565
55566	pIn3 = &aMem[pOp->p3];
55567	assert( pIn3->flags & MEM_Int );
55568	assert( pOp->p1>=0 && pOp->p1<p->nCursor );
55569	u.bd.pC = p->apCsr[pOp->p1];
55570	assert( u.bd.pC!=0 );
55571	assert( u.bd.pC->isTable );
	@@ -55616,11 +55605,10 @@
55605	*/
55606	case OP_Sequence: { /* out2-prerelease */
55607	assert( pOp->p1>=0 && pOp->p1<p->nCursor );
55608	assert( p->apCsr[pOp->p1]!=0 );
55609	pOut->u.i = p->apCsr[pOp->p1]->seqCount++;

55610	break;
55611	}
55612
55613
55614	/* Opcode: NewRowid P1 P2 P3 * *
	@@ -55712,11 +55700,11 @@
55700	assert( pOp->p3<=u.be.pFrame->nMem );
55701	u.be.pMem = &u.be.pFrame->aMem[pOp->p3];
55702	}else{
55703	/* Assert that P3 is a valid memory cell. */
55704	assert( pOp->p3<=p->nMem );
55705	u.be.pMem = &aMem[pOp->p3];
55706	}
55707
55708	REGISTER_TRACE(pOp->p3, u.be.pMem);
55709	sqlite3VdbeMemIntegerify(u.be.pMem);
55710	assert( (u.be.pMem->flags & MEM_Int)!=0 ); /* mem(P3) holds an integer */
	@@ -55755,11 +55743,10 @@
55743	}
55744	u.be.pC->rowidIsValid = 0;
55745	u.be.pC->deferredMoveto = 0;
55746	u.be.pC->cacheStatus = CACHE_STALE;
55747	}

55748	pOut->u.i = u.be.v;
55749	break;
55750	}
55751
55752	/* Opcode: Insert P1 P2 P3 P4 P5
	@@ -55818,21 +55805,21 @@
55805	const char zDb; / database name - used by the update hook */
55806	const char zTbl; / Table name - used by the opdate hook */
55807	int op; /* Opcode for update hook: SQLITE_UPDATE or SQLITE_INSERT */
55808	#endif /* local variables moved into u.bf */
55809
55810	u.bf.pData = &aMem[pOp->p2];
55811	assert( pOp->p1>=0 && pOp->p1<p->nCursor );
55812	u.bf.pC = p->apCsr[pOp->p1];
55813	assert( u.bf.pC!=0 );
55814	assert( u.bf.pC->pCursor!=0 );
55815	assert( u.bf.pC->pseudoTableReg==0 );
55816	assert( u.bf.pC->isTable );
55817	REGISTER_TRACE(pOp->p2, u.bf.pData);
55818
55819	if( pOp->opcode==OP_Insert ){
55820	u.bf.pKey = &aMem[pOp->p3];
55821	assert( u.bf.pKey->flags & MEM_Int );
55822	REGISTER_TRACE(pOp->p3, u.bf.pKey);
55823	u.bf.iKey = u.bf.pKey->u.i;
55824	}else{
55825	assert( pOp->opcode==OP_InsertInt );
	@@ -55980,11 +55967,11 @@
55967	BtCursor *pCrsr;
55968	u32 n;
55969	i64 n64;
55970	#endif /* local variables moved into u.bh */
55971
55972	pOut = &aMem[pOp->p2];
55973
55974	/* Note that RowKey and RowData are really exactly the same instruction */
55975	assert( pOp->p1>=0 && pOp->p1<p->nCursor );
55976	u.bh.pC = p->apCsr[pOp->p1];
55977	assert( u.bh.pC->isTable \|\| pOp->opcode==OP_RowKey );
	@@ -56055,11 +56042,11 @@
56042	assert( pOp->p1>=0 && pOp->p1<p->nCursor );
56043	u.bi.pC = p->apCsr[pOp->p1];
56044	assert( u.bi.pC!=0 );
56045	assert( u.bi.pC->pseudoTableReg==0 );
56046	if( u.bi.pC->nullRow ){
56047	pOut->flags = MEM_Null;
56048	break;
56049	}else if( u.bi.pC->deferredMoveto ){
56050	u.bi.v = u.bi.pC->movetoTarget;
56051	#ifndef SQLITE_OMIT_VIRTUALTABLE
56052	}else if( u.bi.pC->pVtabCursor ){
	@@ -56083,11 +56070,10 @@
56070	rc = sqlite3BtreeKeySize(u.bi.pC->pCursor, &u.bi.v);
56071	assert( rc==SQLITE_OK ); /* Always so because of CursorMoveto() above */
56072	}
56073	}
56074	pOut->u.i = u.bi.v;

56075	break;
56076	}
56077
56078	/* Opcode: NullRow P1 * * * *
56079	**
	@@ -56278,10 +56264,11 @@
56264	#endif /* local variables moved into u.bn */
56265
56266	assert( pOp->p1>=0 && pOp->p1<p->nCursor );
56267	u.bn.pC = p->apCsr[pOp->p1];
56268	assert( u.bn.pC!=0 );
56269	pIn2 = &aMem[pOp->p2];
56270	assert( pIn2->flags & MEM_Blob );
56271	u.bn.pCrsr = u.bn.pC->pCursor;
56272	if( ALWAYS(u.bn.pCrsr!=0) ){
56273	assert( u.bn.pC->isTable==0 );
56274	rc = ExpandBlob(pIn2);
	@@ -56320,11 +56307,11 @@
56307	u.bo.pCrsr = u.bo.pC->pCursor;
56308	if( ALWAYS(u.bo.pCrsr!=0) ){
56309	u.bo.r.pKeyInfo = u.bo.pC->pKeyInfo;
56310	u.bo.r.nField = (u16)pOp->p3;
56311	u.bo.r.flags = 0;
56312	u.bo.r.aMem = &aMem[pOp->p2];
56313	rc = sqlite3BtreeMovetoUnpacked(u.bo.pCrsr, &u.bo.r, 0, 0, &u.bo.res);
56314	if( rc==SQLITE_OK && u.bo.res==0 ){
56315	rc = sqlite3BtreeDelete(u.bo.pCrsr);
56316	}
56317	assert( u.bo.pC->deferredMoveto==0 );
	@@ -56350,10 +56337,11 @@
56337
56338	assert( pOp->p1>=0 && pOp->p1<p->nCursor );
56339	u.bp.pC = p->apCsr[pOp->p1];
56340	assert( u.bp.pC!=0 );
56341	u.bp.pCrsr = u.bp.pC->pCursor;
56342	pOut->flags = MEM_Null;
56343	if( ALWAYS(u.bp.pCrsr!=0) ){
56344	rc = sqlite3VdbeCursorMoveto(u.bp.pC);
56345	if( NEVER(rc) ) goto abort_due_to_error;
56346	assert( u.bp.pC->deferredMoveto==0 );
56347	assert( u.bp.pC->isTable==0 );
	@@ -56360,12 +56348,12 @@
56348	if( !u.bp.pC->nullRow ){
56349	rc = sqlite3VdbeIdxRowid(db, u.bp.pCrsr, &u.bp.rowid);
56350	if( rc!=SQLITE_OK ){
56351	goto abort_due_to_error;
56352	}

56353	pOut->u.i = u.bp.rowid;
56354	pOut->flags = MEM_Int;
56355	}
56356	}
56357	break;
56358	}
56359
	@@ -56393,12 +56381,12 @@
56381	** Otherwise fall through to the next instruction.
56382	**
56383	** If P5 is non-zero then the key value is increased by an epsilon prior
56384	** to the comparison. This makes the opcode work like IdxLE.
56385	*/
56386	case OP_IdxLT: /* jump */
56387	case OP_IdxGE: { /* jump */
56388	#if 0 /* local variables moved into u.bq */
56389	VdbeCursor *pC;
56390	int res;
56391	UnpackedRecord r;
56392	#endif /* local variables moved into u.bq */
	@@ -56415,11 +56403,11 @@
56403	if( pOp->p5 ){
56404	u.bq.r.flags = UNPACKED_INCRKEY \| UNPACKED_IGNORE_ROWID;
56405	}else{
56406	u.bq.r.flags = UNPACKED_IGNORE_ROWID;
56407	}
56408	u.bq.r.aMem = &aMem[pOp->p3];
56409	rc = sqlite3VdbeIdxKeyCompare(u.bq.pC, &u.bq.r, &u.bq.res);
56410	if( pOp->opcode==OP_IdxLT ){
56411	u.bq.res = -u.bq.res;
56412	}else{
56413	assert( pOp->opcode==OP_IdxGE );
	@@ -56467,23 +56455,25 @@
56455	}
56456	}
56457	#else
56458	u.br.iCnt = db->activeVdbeCnt;
56459	#endif
56460	pOut->flags = MEM_Null;
56461	if( u.br.iCnt>1 ){
56462	rc = SQLITE_LOCKED;
56463	p->errorAction = OE_Abort;
56464	}else{
56465	u.br.iDb = pOp->p3;
56466	assert( u.br.iCnt==1 );
56467	assert( (p->btreeMask & (1<<u.br.iDb))!=0 );
56468	rc = sqlite3BtreeDropTable(db->aDb[u.br.iDb].pBt, pOp->p1, &u.br.iMoved);
56469	pOut->flags = MEM_Int;
56470	pOut->u.i = u.br.iMoved;
56471	#ifndef SQLITE_OMIT_AUTOVACUUM
56472	if( rc==SQLITE_OK && u.br.iMoved!=0 ){
56473	sqlite3RootPageMoved(&db->aDb[u.br.iDb], u.br.iMoved, pOp->p1);
56474	resetSchemaOnFault = 1;
56475	}
56476	#endif
56477	}
56478	break;
56479	}
	@@ -56517,11 +56507,11 @@
56507	db->aDb[pOp->p2].pBt, pOp->p1, (pOp->p3 ? &u.bs.nChange : 0)
56508	);
56509	if( pOp->p3 ){
56510	p->nChange += u.bs.nChange;
56511	if( pOp->p3>0 ){
56512	aMem[pOp->p3].u.i += u.bs.nChange;
56513	}
56514	}
56515	break;
56516	}
56517
	@@ -56566,11 +56556,10 @@
56556	}else{
56557	u.bt.flags = BTREE_ZERODATA;
56558	}
56559	rc = sqlite3BtreeCreateTable(u.bt.pDb->pBt, &u.bt.pgno, u.bt.flags);
56560	pOut->u.i = u.bt.pgno;

56561	break;
56562	}
56563
56564	/* Opcode: ParseSchema P1 P2 * P4 *
56565	**
	@@ -56733,14 +56722,14 @@
56722	u.bv.nRoot = pOp->p2;
56723	assert( u.bv.nRoot>0 );
56724	u.bv.aRoot = sqlite3DbMallocRaw(db, sizeof(int)*(u.bv.nRoot+1) );
56725	if( u.bv.aRoot==0 ) goto no_mem;
56726	assert( pOp->p3>0 && pOp->p3<=p->nMem );
56727	u.bv.pnErr = &aMem[pOp->p3];
56728	assert( (u.bv.pnErr->flags & MEM_Int)!=0 );
56729	assert( (u.bv.pnErr->flags & (MEM_Str\|MEM_Blob))==0 );
56730	pIn1 = &aMem[pOp->p1];
56731	for(u.bv.j=0; u.bv.j<u.bv.nRoot; u.bv.j++){
56732	u.bv.aRoot[u.bv.j] = (int)sqlite3VdbeIntValue(&pIn1[u.bv.j]);
56733	}
56734	u.bv.aRoot[u.bv.j] = 0;
56735	assert( pOp->p5<db->nDb );
	@@ -56768,53 +56757,43 @@
56757	** Insert the integer value held by register P2 into a boolean index
56758	** held in register P1.
56759	**
56760	** An assertion fails if P2 is not an integer.
56761	*/
56762	case OP_RowSetAdd: { /* in1, in2 */
56763	pIn1 = &aMem[pOp->p1];
56764	pIn2 = &aMem[pOp->p2];
56765	assert( (pIn2->flags & MEM_Int)!=0 );
56766	if( (pIn1->flags & MEM_RowSet)==0 ){
56767	sqlite3VdbeMemSetRowSet(pIn1);
56768	if( (pIn1->flags & MEM_RowSet)==0 ) goto no_mem;
56769	}
56770	sqlite3RowSetInsert(pIn1->u.pRowSet, pIn2->u.i);






56771	break;
56772	}
56773
56774	/* Opcode: RowSetRead P1 P2 P3 * *
56775	**
56776	** Extract the smallest value from boolean index P1 and put that value into
56777	** register P3. Or, if boolean index P1 is initially empty, leave P3
56778	** unchanged and jump to instruction P2.
56779	*/
56780	case OP_RowSetRead: { /* jump, in1, out3 */
56781	#if 0 /* local variables moved into u.bw */

56782	i64 val;
56783	#endif /* local variables moved into u.bw */

56784	CHECK_FOR_INTERRUPT;
56785	pIn1 = &aMem[pOp->p1];
56786	if( (pIn1->flags & MEM_RowSet)==0
56787	\|\| sqlite3RowSetNext(pIn1->u.pRowSet, &u.bw.val)==0

56788	){
56789	/* The boolean index is empty */
56790	sqlite3VdbeMemSetNull(pIn1);
56791	pc = pOp->p2 - 1;
56792	}else{
56793	/* A value was pulled from the index */
56794	sqlite3VdbeMemSetInt64(&aMem[pOp->p3], u.bw.val);

56795	}
56796	break;
56797	}
56798
56799	/* Opcode: RowSetTest P1 P2 P3 P4
	@@ -56839,16 +56818,18 @@
56818	** inserted, there is no need to search to see if the same value was
56819	** previously inserted as part of set X (only if it was previously
56820	** inserted as part of some other set).
56821	*/
56822	case OP_RowSetTest: { /* jump, in1, in3 */
56823	#if 0 /* local variables moved into u.bx */
56824	int iSet;
56825	int exists;
56826	#endif /* local variables moved into u.bx */
56827
56828	pIn1 = &aMem[pOp->p1];
56829	pIn3 = &aMem[pOp->p3];
56830	u.bx.iSet = pOp->p4.i;
56831	assert( pIn3->flags&MEM_Int );
56832
56833	/* If there is anything other than a rowset object in memory cell P1,
56834	** delete it now and initialize P1 with an empty rowset
56835	*/
	@@ -56856,21 +56837,21 @@
56837	sqlite3VdbeMemSetRowSet(pIn1);
56838	if( (pIn1->flags & MEM_RowSet)==0 ) goto no_mem;
56839	}
56840
56841	assert( pOp->p4type==P4_INT32 );
56842	assert( u.bx.iSet==-1 \|\| u.bx.iSet>=0 );
56843	if( u.bx.iSet ){
56844	u.bx.exists = sqlite3RowSetTest(pIn1->u.pRowSet,
56845	(u8)(u.bx.iSet>=0 ? u.bx.iSet & 0xf : 0xff),
56846	pIn3->u.i);
56847	if( u.bx.exists ){
56848	pc = pOp->p2 - 1;
56849	break;
56850	}
56851	}
56852	if( u.bx.iSet>=0 ){
56853	sqlite3RowSetInsert(pIn1->u.pRowSet, pIn3->u.i);
56854	}
56855	break;
56856	}
56857
	@@ -56889,24 +56870,24 @@
56870	** memory required by the sub-vdbe at runtime.
56871	**
56872	** P4 is a pointer to the VM containing the trigger program.
56873	*/
56874	case OP_Program: { /* jump */
56875	#if 0 /* local variables moved into u.by */
56876	int nMem; /* Number of memory registers for sub-program */
56877	int nByte; /* Bytes of runtime space required for sub-program */
56878	Mem pRt; / Register to allocate runtime space */
56879	Mem pMem; / Used to iterate through memory cells */
56880	Mem pEnd; / Last memory cell in new array */
56881	VdbeFrame pFrame; / New vdbe frame to execute in */
56882	SubProgram pProgram; / Sub-program to execute */
56883	void t; / Token identifying trigger */
56884	#endif /* local variables moved into u.by */
56885
56886	u.by.pProgram = pOp->p4.pProgram;
56887	u.by.pRt = &aMem[pOp->p3];
56888	assert( u.by.pProgram->nOp>0 );
56889
56890	/* If the p5 flag is clear, then recursive invocation of triggers is
56891	** disabled for backwards compatibility (p5 is set if this sub-program
56892	** is really a trigger, not a foreign key action, and the flag set
56893	** and cleared by the "PRAGMA recursive_triggers" command is clear).
	@@ -56916,79 +56897,79 @@
56897	** SubProgram (if the trigger may be executed with more than one different
56898	** ON CONFLICT algorithm). SubProgram structures associated with a
56899	** single trigger all have the same value for the SubProgram.token
56900	** variable. */
56901	if( pOp->p5 ){
56902	u.by.t = u.by.pProgram->token;
56903	for(u.by.pFrame=p->pFrame; u.by.pFrame && u.by.pFrame->token!=u.by.t; u.by.pFrame=u.by.pFrame->pParent);
56904	if( u.by.pFrame ) break;
56905	}
56906
56907	if( p->nFrame>=db->aLimit[SQLITE_LIMIT_TRIGGER_DEPTH] ){
56908	rc = SQLITE_ERROR;
56909	sqlite3SetString(&p->zErrMsg, db, "too many levels of trigger recursion");
56910	break;
56911	}
56912
56913	/* Register u.by.pRt is used to store the memory required to save the state
56914	** of the current program, and the memory required at runtime to execute
56915	** the trigger program. If this trigger has been fired before, then u.by.pRt
56916	** is already allocated. Otherwise, it must be initialized. */
56917	if( (u.by.pRt->flags&MEM_Frame)==0 ){
56918	/* SubProgram.nMem is set to the number of memory cells used by the
56919	** program stored in SubProgram.aOp. As well as these, one memory
56920	** cell is required for each cursor used by the program. Set local
56921	** variable u.by.nMem (and later, VdbeFrame.nChildMem) to this value.
56922	*/
56923	u.by.nMem = u.by.pProgram->nMem + u.by.pProgram->nCsr;
56924	u.by.nByte = ROUND8(sizeof(VdbeFrame))
56925	+ u.by.nMem * sizeof(Mem)
56926	+ u.by.pProgram->nCsr * sizeof(VdbeCursor *);
56927	u.by.pFrame = sqlite3DbMallocZero(db, u.by.nByte);
56928	if( !u.by.pFrame ){
56929	goto no_mem;
56930	}
56931	sqlite3VdbeMemRelease(u.by.pRt);
56932	u.by.pRt->flags = MEM_Frame;
56933	u.by.pRt->u.pFrame = u.by.pFrame;
56934
56935	u.by.pFrame->v = p;
56936	u.by.pFrame->nChildMem = u.by.nMem;
56937	u.by.pFrame->nChildCsr = u.by.pProgram->nCsr;
56938	u.by.pFrame->pc = pc;
56939	u.by.pFrame->aMem = p->aMem;
56940	u.by.pFrame->nMem = p->nMem;
56941	u.by.pFrame->apCsr = p->apCsr;
56942	u.by.pFrame->nCursor = p->nCursor;
56943	u.by.pFrame->aOp = p->aOp;
56944	u.by.pFrame->nOp = p->nOp;
56945	u.by.pFrame->token = u.by.pProgram->token;
56946
56947	u.by.pEnd = &VdbeFrameMem(u.by.pFrame)[u.by.pFrame->nChildMem];
56948	for(u.by.pMem=VdbeFrameMem(u.by.pFrame); u.by.pMem!=u.by.pEnd; u.by.pMem++){
56949	u.by.pMem->flags = MEM_Null;
56950	u.by.pMem->db = db;
56951	}
56952	}else{
56953	u.by.pFrame = u.by.pRt->u.pFrame;
56954	assert( u.by.pProgram->nMem+u.by.pProgram->nCsr==u.by.pFrame->nChildMem );
56955	assert( u.by.pProgram->nCsr==u.by.pFrame->nChildCsr );
56956	assert( pc==u.by.pFrame->pc );
56957	}
56958
56959	p->nFrame++;
56960	u.by.pFrame->pParent = p->pFrame;
56961	u.by.pFrame->lastRowid = db->lastRowid;
56962	u.by.pFrame->nChange = p->nChange;
56963	p->nChange = 0;
56964	p->pFrame = u.by.pFrame;
56965	p->aMem = aMem = &VdbeFrameMem(u.by.pFrame)[-1];
56966	p->nMem = u.by.pFrame->nChildMem;
56967	p->nCursor = (u16)u.by.pFrame->nChildCsr;
56968	p->apCsr = (VdbeCursor **)&aMem[p->nMem+1];
56969	p->aOp = aOp = u.by.pProgram->aOp;
56970	p->nOp = u.by.pProgram->nOp;
56971	pc = -1;
56972
56973	break;
56974	}
56975
	@@ -57003,17 +56984,17 @@
56984	** The address of the cell in the parent frame is determined by adding
56985	** the value of the P1 argument to the value of the P1 argument to the
56986	** calling OP_Program instruction.
56987	*/
56988	case OP_Param: { /* out2-prerelease */
56989	#if 0 /* local variables moved into u.bz */
56990	VdbeFrame *pFrame;
56991	Mem *pIn;
56992	#endif /* local variables moved into u.bz */
56993	u.bz.pFrame = p->pFrame;
56994	u.bz.pIn = &u.bz.pFrame->aMem[pOp->p1 + u.bz.pFrame->aOp[u.bz.pFrame->pc].p1];
56995	sqlite3VdbeMemShallowCopy(pOut, u.bz.pIn, MEM_Ephem);
56996	break;
56997	}
56998
56999	#endif /* #ifndef SQLITE_OMIT_TRIGGER */
57000
	@@ -57065,24 +57046,25 @@
57046	**
57047	** This instruction throws an error if the memory cell is not initially
57048	** an integer.
57049	*/
57050	case OP_MemMax: { /* in2 */
57051	#if 0 /* local variables moved into u.ca */
57052	Mem *pIn1;
57053	VdbeFrame *pFrame;
57054	#endif /* local variables moved into u.ca */
57055	if( p->pFrame ){
57056	for(u.ca.pFrame=p->pFrame; u.ca.pFrame->pParent; u.ca.pFrame=u.ca.pFrame->pParent);
57057	u.ca.pIn1 = &u.ca.pFrame->aMem[pOp->p1];
57058	}else{
57059	u.ca.pIn1 = &aMem[pOp->p1];
57060	}
57061	sqlite3VdbeMemIntegerify(u.ca.pIn1);
57062	pIn2 = &aMem[pOp->p2];
57063	sqlite3VdbeMemIntegerify(pIn2);
57064	if( u.ca.pIn1->u.i<pIn2->u.i){
57065	u.ca.pIn1->u.i = pIn2->u.i;
57066	}
57067	break;
57068	}
57069	#endif /* SQLITE_OMIT_AUTOINCREMENT */
57070
	@@ -57092,10 +57074,11 @@
57074	**
57075	** It is illegal to use this instruction on a register that does
57076	** not contain an integer. An assertion fault will result if you try.
57077	*/
57078	case OP_IfPos: { /* jump, in1 */
57079	pIn1 = &aMem[pOp->p1];
57080	assert( pIn1->flags&MEM_Int );
57081	if( pIn1->u.i>0 ){
57082	pc = pOp->p2 - 1;
57083	}
57084	break;
	@@ -57107,26 +57090,30 @@
57090	**
57091	** It is illegal to use this instruction on a register that does
57092	** not contain an integer. An assertion fault will result if you try.
57093	*/
57094	case OP_IfNeg: { /* jump, in1 */
57095	pIn1 = &aMem[pOp->p1];
57096	assert( pIn1->flags&MEM_Int );
57097	if( pIn1->u.i<0 ){
57098	pc = pOp->p2 - 1;
57099	}
57100	break;
57101	}
57102
57103	/* Opcode: IfZero P1 P2 P3 * *
57104	**
57105	** The register P1 must contain an integer. Add literal P3 to the
57106	** value in register P1. If the result is exactly 0, jump to P2.
57107	**
57108	** It is illegal to use this instruction on a register that does
57109	** not contain an integer. An assertion fault will result if you try.
57110	*/
57111	case OP_IfZero: { /* jump, in1 */
57112	pIn1 = &aMem[pOp->p1];
57113	assert( pIn1->flags&MEM_Int );
57114	pIn1->u.i += pOp->p3;
57115	if( pIn1->u.i==0 ){
57116	pc = pOp->p2 - 1;
57117	}
57118	break;
57119	}
	@@ -57140,51 +57127,51 @@
57127	**
57128	** The P5 arguments are taken from register P2 and its
57129	** successors.
57130	*/
57131	case OP_AggStep: {
57132	#if 0 /* local variables moved into u.cb */
57133	int n;
57134	int i;
57135	Mem *pMem;
57136	Mem *pRec;
57137	sqlite3_context ctx;
57138	sqlite3_value **apVal;
57139	#endif /* local variables moved into u.cb */
57140
57141	u.cb.n = pOp->p5;
57142	assert( u.cb.n>=0 );
57143	u.cb.pRec = &aMem[pOp->p2];
57144	u.cb.apVal = p->apArg;
57145	assert( u.cb.apVal \|\| u.cb.n==0 );
57146	for(u.cb.i=0; u.cb.i<u.cb.n; u.cb.i++, u.cb.pRec++){
57147	u.cb.apVal[u.cb.i] = u.cb.pRec;
57148	sqlite3VdbeMemStoreType(u.cb.pRec);
57149	}
57150	u.cb.ctx.pFunc = pOp->p4.pFunc;
57151	assert( pOp->p3>0 && pOp->p3<=p->nMem );
57152	u.cb.ctx.pMem = u.cb.pMem = &aMem[pOp->p3];
57153	u.cb.pMem->n++;
57154	u.cb.ctx.s.flags = MEM_Null;
57155	u.cb.ctx.s.z = 0;
57156	u.cb.ctx.s.zMalloc = 0;
57157	u.cb.ctx.s.xDel = 0;
57158	u.cb.ctx.s.db = db;
57159	u.cb.ctx.isError = 0;
57160	u.cb.ctx.pColl = 0;
57161	if( u.cb.ctx.pFunc->flags & SQLITE_FUNC_NEEDCOLL ){
57162	assert( pOp>p->aOp );
57163	assert( pOp[-1].p4type==P4_COLLSEQ );
57164	assert( pOp[-1].opcode==OP_CollSeq );
57165	u.cb.ctx.pColl = pOp[-1].p4.pColl;
57166	}
57167	(u.cb.ctx.pFunc->xStep)(&u.cb.ctx, u.cb.n, u.cb.apVal);
57168	if( u.cb.ctx.isError ){
57169	sqlite3SetString(&p->zErrMsg, db, "%s", sqlite3_value_text(&u.cb.ctx.s));
57170	rc = u.cb.ctx.isError;
57171	}
57172	sqlite3VdbeMemRelease(&u.cb.ctx.s);
57173	break;
57174	}
57175
57176	/* Opcode: AggFinal P1 P2 * P4 *
57177	**
	@@ -57197,23 +57184,23 @@
57184	** functions that can take varying numbers of arguments. The
57185	** P4 argument is only needed for the degenerate case where
57186	** the step function was not previously called.
57187	*/
57188	case OP_AggFinal: {
57189	#if 0 /* local variables moved into u.cc */
57190	Mem *pMem;
57191	#endif /* local variables moved into u.cc */
57192	assert( pOp->p1>0 && pOp->p1<=p->nMem );
57193	u.cc.pMem = &aMem[pOp->p1];
57194	assert( (u.cc.pMem->flags & ~(MEM_Null\|MEM_Agg))==0 );
57195	rc = sqlite3VdbeMemFinalize(u.cc.pMem, pOp->p4.pFunc);
57196	if( rc ){
57197	sqlite3SetString(&p->zErrMsg, db, "%s", sqlite3_value_text(u.cc.pMem));
57198	}
57199	sqlite3VdbeChangeEncoding(u.cc.pMem, encoding);
57200	UPDATE_MAX_BLOBSIZE(u.cc.pMem);
57201	if( sqlite3VdbeMemTooBig(u.cc.pMem) ){
57202	goto too_big;
57203	}
57204	break;
57205	}
57206
	@@ -57239,18 +57226,18 @@
57226	** Perform a single step of the incremental vacuum procedure on
57227	** the P1 database. If the vacuum has finished, jump to instruction
57228	** P2. Otherwise, fall through to the next instruction.
57229	*/
57230	case OP_IncrVacuum: { /* jump */
57231	#if 0 /* local variables moved into u.cd */
57232	Btree *pBt;
57233	#endif /* local variables moved into u.cd */
57234
57235	assert( pOp->p1>=0 && pOp->p1<db->nDb );
57236	assert( (p->btreeMask & (1<<pOp->p1))!=0 );
57237	u.cd.pBt = db->aDb[pOp->p1].pBt;
57238	rc = sqlite3BtreeIncrVacuum(u.cd.pBt);
57239	if( rc==SQLITE_DONE ){
57240	pc = pOp->p2 - 1;
57241	rc = SQLITE_OK;
57242	}
57243	break;
	@@ -57316,19 +57303,19 @@
57303	** Also, whether or not P4 is set, check that this is not being called from
57304	** within a callback to a virtual table xSync() method. If it is, the error
57305	** code will be set to SQLITE_LOCKED.
57306	*/
57307	case OP_VBegin: {
57308	#if 0 /* local variables moved into u.ce */
57309	VTable *pVTab;
57310	#endif /* local variables moved into u.ce */
57311	u.ce.pVTab = pOp->p4.pVtab;
57312	rc = sqlite3VtabBegin(db, u.ce.pVTab);
57313	if( u.ce.pVTab ){
57314	sqlite3DbFree(db, p->zErrMsg);
57315	p->zErrMsg = u.ce.pVTab->pVtab->zErrMsg;
57316	u.ce.pVTab->pVtab->zErrMsg = 0;
57317	}
57318	break;
57319	}
57320	#endif /* SQLITE_OMIT_VIRTUALTABLE */
57321
	@@ -57364,40 +57351,40 @@
57351	** P4 is a pointer to a virtual table object, an sqlite3_vtab structure.
57352	** P1 is a cursor number. This opcode opens a cursor to the virtual
57353	** table and stores that cursor in P1.
57354	*/
57355	case OP_VOpen: {
57356	#if 0 /* local variables moved into u.cf */
57357	VdbeCursor *pCur;
57358	sqlite3_vtab_cursor *pVtabCursor;
57359	sqlite3_vtab *pVtab;
57360	sqlite3_module *pModule;
57361	#endif /* local variables moved into u.cf */
57362
57363	u.cf.pCur = 0;
57364	u.cf.pVtabCursor = 0;
57365	u.cf.pVtab = pOp->p4.pVtab->pVtab;
57366	u.cf.pModule = (sqlite3_module *)u.cf.pVtab->pModule;
57367	assert(u.cf.pVtab && u.cf.pModule);
57368	if( sqlite3SafetyOff(db) ) goto abort_due_to_misuse;
57369	rc = u.cf.pModule->xOpen(u.cf.pVtab, &u.cf.pVtabCursor);
57370	sqlite3DbFree(db, p->zErrMsg);
57371	p->zErrMsg = u.cf.pVtab->zErrMsg;
57372	u.cf.pVtab->zErrMsg = 0;
57373	if( sqlite3SafetyOn(db) ) goto abort_due_to_misuse;
57374	if( SQLITE_OK==rc ){
57375	/* Initialize sqlite3_vtab_cursor base class */
57376	u.cf.pVtabCursor->pVtab = u.cf.pVtab;
57377
57378	/* Initialise vdbe cursor object */
57379	u.cf.pCur = allocateCursor(p, pOp->p1, 0, -1, 0);
57380	if( u.cf.pCur ){
57381	u.cf.pCur->pVtabCursor = u.cf.pVtabCursor;
57382	u.cf.pCur->pModule = u.cf.pVtabCursor->pVtab->pModule;
57383	}else{
57384	db->mallocFailed = 1;
57385	u.cf.pModule->xClose(u.cf.pVtabCursor);
57386	}
57387	}
57388	break;
57389	}
57390	#endif /* SQLITE_OMIT_VIRTUALTABLE */
	@@ -57420,11 +57407,11 @@
57407	** xFilter as argv. Register P3+2 becomes argv[0] when passed to xFilter.
57408	**
57409	** A jump is made to P2 if the result set after filtering would be empty.
57410	*/
57411	case OP_VFilter: { /* jump */
57412	#if 0 /* local variables moved into u.cg */
57413	int nArg;
57414	int iQuery;
57415	const sqlite3_module *pModule;
57416	Mem *pQuery;
57417	Mem *pArgc;
	@@ -57432,52 +57419,52 @@
57419	sqlite3_vtab *pVtab;
57420	VdbeCursor *pCur;
57421	int res;
57422	int i;
57423	Mem **apArg;
57424	#endif /* local variables moved into u.cg */
57425
57426	u.cg.pQuery = &aMem[pOp->p3];
57427	u.cg.pArgc = &u.cg.pQuery[1];
57428	u.cg.pCur = p->apCsr[pOp->p1];
57429	REGISTER_TRACE(pOp->p3, u.cg.pQuery);
57430	assert( u.cg.pCur->pVtabCursor );
57431	u.cg.pVtabCursor = u.cg.pCur->pVtabCursor;
57432	u.cg.pVtab = u.cg.pVtabCursor->pVtab;
57433	u.cg.pModule = u.cg.pVtab->pModule;
57434
57435	/* Grab the index number and argc parameters */
57436	assert( (u.cg.pQuery->flags&MEM_Int)!=0 && u.cg.pArgc->flags==MEM_Int );
57437	u.cg.nArg = (int)u.cg.pArgc->u.i;
57438	u.cg.iQuery = (int)u.cg.pQuery->u.i;
57439
57440	/* Invoke the xFilter method */
57441	{
57442	u.cg.res = 0;
57443	u.cg.apArg = p->apArg;
57444	for(u.cg.i = 0; u.cg.i<u.cg.nArg; u.cg.i++){
57445	u.cg.apArg[u.cg.i] = &u.cg.pArgc[u.cg.i+1];
57446	sqlite3VdbeMemStoreType(u.cg.apArg[u.cg.i]);
57447	}
57448
57449	if( sqlite3SafetyOff(db) ) goto abort_due_to_misuse;
57450	p->inVtabMethod = 1;
57451	rc = u.cg.pModule->xFilter(u.cg.pVtabCursor, u.cg.iQuery, pOp->p4.z, u.cg.nArg, u.cg.apArg);
57452	p->inVtabMethod = 0;
57453	sqlite3DbFree(db, p->zErrMsg);
57454	p->zErrMsg = u.cg.pVtab->zErrMsg;
57455	u.cg.pVtab->zErrMsg = 0;
57456	if( rc==SQLITE_OK ){
57457	u.cg.res = u.cg.pModule->xEof(u.cg.pVtabCursor);
57458	}
57459	if( sqlite3SafetyOn(db) ) goto abort_due_to_misuse;
57460
57461	if( u.cg.res ){
57462	pc = pOp->p2 - 1;
57463	}
57464	}
57465	u.cg.pCur->nullRow = 0;
57466
57467	break;
57468	}
57469	#endif /* SQLITE_OMIT_VIRTUALTABLE */
57470
	@@ -57487,60 +57474,60 @@
57474	** Store the value of the P2-th column of
57475	** the row of the virtual-table that the
57476	** P1 cursor is pointing to into register P3.
57477	*/
57478	case OP_VColumn: {
57479	#if 0 /* local variables moved into u.ch */
57480	sqlite3_vtab *pVtab;
57481	const sqlite3_module *pModule;
57482	Mem *pDest;
57483	sqlite3_context sContext;
57484	#endif /* local variables moved into u.ch */
57485
57486	VdbeCursor *pCur = p->apCsr[pOp->p1];
57487	assert( pCur->pVtabCursor );
57488	assert( pOp->p3>0 && pOp->p3<=p->nMem );
57489	u.ch.pDest = &aMem[pOp->p3];
57490	if( pCur->nullRow ){
57491	sqlite3VdbeMemSetNull(u.ch.pDest);
57492	break;
57493	}
57494	u.ch.pVtab = pCur->pVtabCursor->pVtab;
57495	u.ch.pModule = u.ch.pVtab->pModule;
57496	assert( u.ch.pModule->xColumn );
57497	memset(&u.ch.sContext, 0, sizeof(u.ch.sContext));
57498
57499	/* The output cell may already have a buffer allocated. Move
57500	** the current contents to u.ch.sContext.s so in case the user-function
57501	** can use the already allocated buffer instead of allocating a
57502	** new one.
57503	*/
57504	sqlite3VdbeMemMove(&u.ch.sContext.s, u.ch.pDest);
57505	MemSetTypeFlag(&u.ch.sContext.s, MEM_Null);
57506
57507	if( sqlite3SafetyOff(db) ) goto abort_due_to_misuse;
57508	rc = u.ch.pModule->xColumn(pCur->pVtabCursor, &u.ch.sContext, pOp->p2);
57509	sqlite3DbFree(db, p->zErrMsg);
57510	p->zErrMsg = u.ch.pVtab->zErrMsg;
57511	u.ch.pVtab->zErrMsg = 0;
57512	if( u.ch.sContext.isError ){
57513	rc = u.ch.sContext.isError;
57514	}
57515
57516	/* Copy the result of the function to the P3 register. We
57517	** do this regardless of whether or not an error occurred to ensure any
57518	** dynamic allocation in u.ch.sContext.s (a Mem struct) is released.
57519	*/
57520	sqlite3VdbeChangeEncoding(&u.ch.sContext.s, encoding);
57521	REGISTER_TRACE(pOp->p3, u.ch.pDest);
57522	sqlite3VdbeMemMove(u.ch.pDest, &u.ch.sContext.s);
57523	UPDATE_MAX_BLOBSIZE(u.ch.pDest);
57524
57525	if( sqlite3SafetyOn(db) ){
57526	goto abort_due_to_misuse;
57527	}
57528	if( sqlite3VdbeMemTooBig(u.ch.pDest) ){
57529	goto too_big;
57530	}
57531	break;
57532	}
57533	#endif /* SQLITE_OMIT_VIRTUALTABLE */
	@@ -57551,46 +57538,46 @@
57538	** Advance virtual table P1 to the next row in its result set and
57539	** jump to instruction P2. Or, if the virtual table has reached
57540	** the end of its result set, then fall through to the next instruction.
57541	*/
57542	case OP_VNext: { /* jump */
57543	#if 0 /* local variables moved into u.ci */
57544	sqlite3_vtab *pVtab;
57545	const sqlite3_module *pModule;
57546	int res;
57547	VdbeCursor *pCur;
57548	#endif /* local variables moved into u.ci */
57549
57550	u.ci.res = 0;
57551	u.ci.pCur = p->apCsr[pOp->p1];
57552	assert( u.ci.pCur->pVtabCursor );
57553	if( u.ci.pCur->nullRow ){
57554	break;
57555	}
57556	u.ci.pVtab = u.ci.pCur->pVtabCursor->pVtab;
57557	u.ci.pModule = u.ci.pVtab->pModule;
57558	assert( u.ci.pModule->xNext );
57559
57560	/* Invoke the xNext() method of the module. There is no way for the
57561	** underlying implementation to return an error if one occurs during
57562	** xNext(). Instead, if an error occurs, true is returned (indicating that
57563	** data is available) and the error code returned when xColumn or
57564	** some other method is next invoked on the save virtual table cursor.
57565	*/
57566	if( sqlite3SafetyOff(db) ) goto abort_due_to_misuse;
57567	p->inVtabMethod = 1;
57568	rc = u.ci.pModule->xNext(u.ci.pCur->pVtabCursor);
57569	p->inVtabMethod = 0;
57570	sqlite3DbFree(db, p->zErrMsg);
57571	p->zErrMsg = u.ci.pVtab->zErrMsg;
57572	u.ci.pVtab->zErrMsg = 0;
57573	if( rc==SQLITE_OK ){
57574	u.ci.res = u.ci.pModule->xEof(u.ci.pCur->pVtabCursor);
57575	}
57576	if( sqlite3SafetyOn(db) ) goto abort_due_to_misuse;
57577
57578	if( !u.ci.res ){
57579	/* If there is data, jump to P2 */
57580	pc = pOp->p2 - 1;
57581	}
57582	break;
57583	}
	@@ -57602,25 +57589,25 @@
57589	** P4 is a pointer to a virtual table object, an sqlite3_vtab structure.
57590	** This opcode invokes the corresponding xRename method. The value
57591	** in register P1 is passed as the zName argument to the xRename method.
57592	*/
57593	case OP_VRename: {
57594	#if 0 /* local variables moved into u.cj */
57595	sqlite3_vtab *pVtab;
57596	Mem *pName;
57597	#endif /* local variables moved into u.cj */
57598
57599	u.cj.pVtab = pOp->p4.pVtab->pVtab;
57600	u.cj.pName = &aMem[pOp->p1];
57601	assert( u.cj.pVtab->pModule->xRename );
57602	REGISTER_TRACE(pOp->p1, u.cj.pName);
57603	assert( u.cj.pName->flags & MEM_Str );
57604	if( sqlite3SafetyOff(db) ) goto abort_due_to_misuse;
57605	rc = u.cj.pVtab->pModule->xRename(u.cj.pVtab, u.cj.pName->z);
57606	sqlite3DbFree(db, p->zErrMsg);
57607	p->zErrMsg = u.cj.pVtab->zErrMsg;
57608	u.cj.pVtab->zErrMsg = 0;
57609	if( sqlite3SafetyOn(db) ) goto abort_due_to_misuse;
57610
57611	break;
57612	}
57613	#endif
	@@ -57648,41 +57635,41 @@
57635	** P1 is a boolean flag. If it is set to true and the xUpdate call
57636	** is successful, then the value returned by sqlite3_last_insert_rowid()
57637	** is set to the value of the rowid for the row just inserted.
57638	*/
57639	case OP_VUpdate: {
57640	#if 0 /* local variables moved into u.ck */
57641	sqlite3_vtab *pVtab;
57642	sqlite3_module *pModule;
57643	int nArg;
57644	int i;
57645	sqlite_int64 rowid;
57646	Mem **apArg;
57647	Mem *pX;
57648	#endif /* local variables moved into u.ck */
57649
57650	u.ck.pVtab = pOp->p4.pVtab->pVtab;
57651	u.ck.pModule = (sqlite3_module *)u.ck.pVtab->pModule;
57652	u.ck.nArg = pOp->p2;
57653	assert( pOp->p4type==P4_VTAB );
57654	if( ALWAYS(u.ck.pModule->xUpdate) ){
57655	u.ck.apArg = p->apArg;
57656	u.ck.pX = &aMem[pOp->p3];
57657	for(u.ck.i=0; u.ck.i<u.ck.nArg; u.ck.i++){
57658	sqlite3VdbeMemStoreType(u.ck.pX);
57659	u.ck.apArg[u.ck.i] = u.ck.pX;
57660	u.ck.pX++;
57661	}
57662	if( sqlite3SafetyOff(db) ) goto abort_due_to_misuse;
57663	rc = u.ck.pModule->xUpdate(u.ck.pVtab, u.ck.nArg, u.ck.apArg, &u.ck.rowid);
57664	sqlite3DbFree(db, p->zErrMsg);
57665	p->zErrMsg = u.ck.pVtab->zErrMsg;
57666	u.ck.pVtab->zErrMsg = 0;
57667	if( sqlite3SafetyOn(db) ) goto abort_due_to_misuse;
57668	if( rc==SQLITE_OK && pOp->p1 ){
57669	assert( u.ck.nArg>1 && u.ck.apArg[0] && (u.ck.apArg[0]->flags&MEM_Null) );
57670	db->lastRowid = u.ck.rowid;
57671	}
57672	p->nChange++;
57673	}
57674	break;
57675	}
	@@ -57692,25 +57679,24 @@
57679	/* Opcode: Pagecount P1 P2 * * *
57680	**
57681	** Write the current number of pages in database P1 to memory cell P2.
57682	*/
57683	case OP_Pagecount: { /* out2-prerelease */
57684	#if 0 /* local variables moved into u.cl */
57685	int p1;
57686	int nPage;
57687	Pager *pPager;
57688	#endif /* local variables moved into u.cl */
57689
57690	u.cl.p1 = pOp->p1;
57691	u.cl.pPager = sqlite3BtreePager(db->aDb[u.cl.p1].pBt);
57692	rc = sqlite3PagerPagecount(u.cl.pPager, &u.cl.nPage);
57693	/* OP_Pagecount is always called from within a read transaction. The
57694	** page count has already been successfully read and cached. So the
57695	** sqlite3PagerPagecount() call above cannot fail. */
57696	if( ALWAYS(rc==SQLITE_OK) ){
57697	pOut->u.i = u.cl.nPage;

57698	}
57699	break;
57700	}
57701	#endif
57702
	@@ -57719,22 +57705,22 @@
57705	**
57706	** If tracing is enabled (by the sqlite3_trace()) interface, then
57707	** the UTF-8 string contained in P4 is emitted on the trace callback.
57708	*/
57709	case OP_Trace: {
57710	#if 0 /* local variables moved into u.cm */
57711	char *zTrace;
57712	#endif /* local variables moved into u.cm */
57713
57714	u.cm.zTrace = (pOp->p4.z ? pOp->p4.z : p->zSql);
57715	if( u.cm.zTrace ){
57716	if( db->xTrace ){
57717	db->xTrace(db->pTraceArg, u.cm.zTrace);
57718	}
57719	#ifdef SQLITE_DEBUG
57720	if( (db->flags & SQLITE_SqlTrace)!=0 ){
57721	sqlite3DebugPrintf("SQL-trace: %s\n", u.cm.zTrace);
57722	}
57723	#endif /* SQLITE_DEBUG */
57724	}
57725	break;
57726	}
	@@ -57769,11 +57755,11 @@
57755	u64 elapsed = sqlite3Hwtime() - start;
57756	pOp->cycles += elapsed;
57757	pOp->cnt++;
57758	#if 0
57759	fprintf(stdout, "%10llu ", elapsed);
57760	sqlite3VdbePrintOp(stdout, origPc, &aOp[origPc]);
57761	#endif
57762	}
57763	#endif
57764
57765	/* The following code adds nothing to the actual functionality
	@@ -57785,15 +57771,15 @@
57771	assert( pc>=-1 && pc<p->nOp );
57772
57773	#ifdef SQLITE_DEBUG
57774	if( p->trace ){
57775	if( rc!=0 ) fprintf(p->trace,"rc=%d\n",rc);
57776	if( pOp->opflags & (OPFLG_OUT2_PRERELEASE\|OPFLG_OUT2) ){
57777	registerTrace(p->trace, pOp->p2, &aMem[pOp->p2]);
57778	}
57779	if( pOp->opflags & OPFLG_OUT3 ){
57780	registerTrace(p->trace, pOp->p3, &aMem[pOp->p3]);
57781	}
57782	}
57783	#endif /* SQLITE_DEBUG */
57784	#endif /* NDEBUG */
57785	} /* The end of the for(;;) loop the loops through opcodes */
	@@ -57805,10 +57791,11 @@
57791	assert( rc );
57792	p->rc = rc;
57793	sqlite3VdbeHalt(p);
57794	if( rc==SQLITE_IOERR_NOMEM ) db->mallocFailed = 1;
57795	rc = SQLITE_ERROR;
57796	if( resetSchemaOnFault ) sqlite3ResetInternalSchema(db, 0);
57797
57798	/* This is the only way out of this procedure. We have to
57799	** release the mutexes on btrees that were acquired at the
57800	** top. */
57801	vdbe_return:
	@@ -57872,12 +57859,10 @@
57859	** May you share freely, never taking more than you give.
57860	**
57861	*************************************************************************
57862	**
57863	** This file contains code used to implement incremental BLOB I/O.


57864	*/
57865
57866
57867	#ifndef SQLITE_OMIT_INCRBLOB
57868
	@@ -58266,16 +58251,10 @@
58251	** May you find forgiveness for yourself and forgive others.
58252	** May you share freely, never taking more than you give.
58253	**
58254	*************************************************************************
58255	**






58256	** This file implements a special kind of sqlite3_file object used
58257	** by SQLite to create journal files if the atomic-write optimization
58258	** is enabled.
58259	**
58260	** The distinctive characteristic of this sqlite3_file is that the
	@@ -58286,11 +58265,11 @@
58265	**
58266	** 1) The in-memory representation grows too large for the allocated
58267	** buffer, or
58268	** 2) The sqlite3JournalCreate() function is called.
58269	*/
58270	#ifdef SQLITE_ENABLE_ATOMIC_WRITE
58271
58272
58273	/*
58274	** A JournalFile object is a subclass of sqlite3_file used by
58275	** as an open file handle for journal files.
	@@ -58511,12 +58490,10 @@
58490	*************************************************************************
58491	**
58492	** This file contains code use to implement an in-memory rollback journal.
58493	** The in-memory rollback journal is used to journal transactions for
58494	** ":memory:" databases and when the journal_mode=MEMORY pragma is used.


58495	*/
58496
58497	/* Forward references to internal structures */
58498	typedef struct MemJournal MemJournal;
58499	typedef struct FilePoint FilePoint;
	@@ -58770,12 +58747,10 @@
58747	** May you share freely, never taking more than you give.
58748	**
58749	*************************************************************************
58750	** This file contains routines used for walking the parser tree for
58751	** an SQL statement.


58752	*/
58753
58754
58755	/*
58756	** Walk an expression tree. Invoke the callback once for each node
	@@ -58910,12 +58885,10 @@
58885	*************************************************************************
58886	**
58887	** This file contains routines used for walking the parser tree and
58888	** resolve all identifiers by associating them with a particular
58889	** table and column.


58890	*/
58891
58892	/*
58893	** Turn the pExpr expression into an alias for the iCol-th column of the
58894	** result set in pEList.
	@@ -61342,10 +61315,98 @@
61315	p->flags \|= EP_IntValue;
61316	p->u.iValue = *pValue;
61317	}
61318	return rc;
61319	}
61320
61321	/*
61322	** Return FALSE if there is no chance that the expression can be NULL.
61323	**
61324	** If the expression might be NULL or if the expression is too complex
61325	** to tell return TRUE.
61326	**
61327	** This routine is used as an optimization, to skip OP_IsNull opcodes
61328	** when we know that a value cannot be NULL. Hence, a false positive
61329	** (returning TRUE when in fact the expression can never be NULL) might
61330	** be a small performance hit but is otherwise harmless. On the other
61331	** hand, a false negative (returning FALSE when the result could be NULL)
61332	** will likely result in an incorrect answer. So when in doubt, return
61333	** TRUE.
61334	*/
61335	SQLITE_PRIVATE int sqlite3ExprCanBeNull(const Expr *p){
61336	u8 op;
61337	while( p->op==TK_UPLUS \|\| p->op==TK_UMINUS ){ p = p->pLeft; }
61338	op = p->op;
61339	if( op==TK_REGISTER ) op = p->op2;
61340	switch( op ){
61341	case TK_INTEGER:
61342	case TK_STRING:
61343	case TK_FLOAT:
61344	case TK_BLOB:
61345	return 0;
61346	default:
61347	return 1;
61348	}
61349	}
61350
61351	/*
61352	** Generate an OP_IsNull instruction that tests register iReg and jumps
61353	** to location iDest if the value in iReg is NULL. The value in iReg
61354	** was computed by pExpr. If we can look at pExpr at compile-time and
61355	** determine that it can never generate a NULL, then the OP_IsNull operation
61356	** can be omitted.
61357	*/
61358	SQLITE_PRIVATE void sqlite3ExprCodeIsNullJump(
61359	Vdbe v, / The VDBE under construction */
61360	const Expr pExpr, / Only generate OP_IsNull if this expr can be NULL */
61361	int iReg, /* Test the value in this register for NULL */
61362	int iDest /* Jump here if the value is null */
61363	){
61364	if( sqlite3ExprCanBeNull(pExpr) ){
61365	sqlite3VdbeAddOp2(v, OP_IsNull, iReg, iDest);
61366	}
61367	}
61368
61369	/*
61370	** Return TRUE if the given expression is a constant which would be
61371	** unchanged by OP_Affinity with the affinity given in the second
61372	** argument.
61373	**
61374	** This routine is used to determine if the OP_Affinity operation
61375	** can be omitted. When in doubt return FALSE. A false negative
61376	** is harmless. A false positive, however, can result in the wrong
61377	** answer.
61378	*/
61379	SQLITE_PRIVATE int sqlite3ExprNeedsNoAffinityChange(const Expr *p, char aff){
61380	u8 op;
61381	if( aff==SQLITE_AFF_NONE ) return 1;
61382	while( p->op==TK_UPLUS \|\| p->op==TK_UMINUS ){ p = p->pLeft; }
61383	op = p->op;
61384	if( op==TK_REGISTER ) op = p->op2;
61385	switch( op ){
61386	case TK_INTEGER: {
61387	return aff==SQLITE_AFF_INTEGER \|\| aff==SQLITE_AFF_NUMERIC;
61388	}
61389	case TK_FLOAT: {
61390	return aff==SQLITE_AFF_REAL \|\| aff==SQLITE_AFF_NUMERIC;
61391	}
61392	case TK_STRING: {
61393	return aff==SQLITE_AFF_TEXT;
61394	}
61395	case TK_BLOB: {
61396	return 1;
61397	}
61398	case TK_COLUMN: {
61399	assert( p->iTable>=0 ); /* p cannot be part of a CHECK constraint */
61400	return p->iColumn<0
61401	&& (aff==SQLITE_AFF_INTEGER \|\| aff==SQLITE_AFF_NUMERIC);
61402	}
61403	default: {
61404	return 0;
61405	}
61406	}
61407	}
61408
61409	/*
61410	** Return TRUE if the given string is a row-id column name.
61411	*/
61412	SQLITE_PRIVATE int sqlite3IsRowid(const char *z){
	@@ -61430,20 +61491,20 @@
61491	** be found with <column> as its left-most column.
61492	**
61493	** When the b-tree is being used for membership tests, the calling function
61494	** needs to know whether or not the structure contains an SQL NULL
61495	** value in order to correctly evaluate expressions like "X IN (Y, Z)".
61496	** If there is any chance that the (...) might contain a NULL value at
61497	** runtime, then a register is allocated and the register number written
61498	** to *prNotFound. If there is no chance that the (...) contains a
61499	** NULL value, then *prNotFound is left unchanged.
61500	**
61501	** If a register is allocated and its location stored in *prNotFound, then
61502	** its initial value is NULL. If the (...) does not remain constant
61503	** for the duration of the query (i.e. the SELECT within the (...)
61504	** is a correlated subquery) then the value of the allocated register is
61505	** reset to NULL each time the subquery is rerun. This allows the
61506	** caller to use vdbe code equivalent to the following:
61507	**
61508	** if( register==NULL ){
61509	** has_null = <test if data structure contains null>
61510	** register = 1
	@@ -61631,11 +61692,11 @@
61692	}
61693
61694	affinity = sqlite3ExprAffinity(pLeft);
61695
61696	/* Whether this is an 'x IN(SELECT...)' or an 'x IN(<exprlist>)'
61697	** expression it is handled the same way. An ephemeral table is
61698	** filled with single-field index keys representing the results
61699	** from the SELECT or the <exprlist>.
61700	**
61701	** If the 'x' expression is a column value, or the SELECT...
61702	** statement returns a column value, then the affinity of that
	@@ -61779,10 +61840,132 @@
61840
61841	return rReg;
61842	}
61843	#endif /* SQLITE_OMIT_SUBQUERY */
61844
61845	#ifndef SQLITE_OMIT_SUBQUERY
61846	/*
61847	** Generate code for an IN expression.
61848	**
61849	** x IN (SELECT ...)
61850	** x IN (value, value, ...)
61851	**
61852	** The left-hand side (LHS) is a scalar expression. The right-hand side (RHS)
61853	** is an array of zero or more values. The expression is true if the LHS is
61854	** contained within the RHS. The value of the expression is unknown (NULL)
61855	** if the LHS is NULL or if the LHS is not contained within the RHS and the
61856	** RHS contains one or more NULL values.
61857	**
61858	** This routine generates code will jump to destIfFalse if the LHS is not
61859	** contained within the RHS. If due to NULLs we cannot determine if the LHS
61860	** is contained in the RHS then jump to destIfNull. If the LHS is contained
61861	** within the RHS then fall through.
61862	*/
61863	static void sqlite3ExprCodeIN(
61864	Parse pParse, / Parsing and code generating context */
61865	Expr pExpr, / The IN expression */
61866	int destIfFalse, /* Jump here if LHS is not contained in the RHS */
61867	int destIfNull /* Jump here if the results are unknown due to NULLs */
61868	){
61869	int rRhsHasNull = 0; /* Register that is true if RHS contains NULL values */
61870	char affinity; /* Comparison affinity to use */
61871	int eType; /* Type of the RHS */
61872	int r1; /* Temporary use register */
61873	Vdbe v; / Statement under construction */
61874
61875	/* Compute the RHS. After this step, the table with cursor
61876	** pExpr->iTable will contains the values that make up the RHS.
61877	*/
61878	v = pParse->pVdbe;
61879	assert( v!=0 ); /* OOM detected prior to this routine */
61880	VdbeNoopComment((v, "begin IN expr"));
61881	eType = sqlite3FindInIndex(pParse, pExpr, &rRhsHasNull);
61882
61883	/* Figure out the affinity to use to create a key from the results
61884	** of the expression. affinityStr stores a static string suitable for
61885	** P4 of OP_MakeRecord.
61886	*/
61887	affinity = comparisonAffinity(pExpr);
61888
61889	/* Code the LHS, the <expr> from "<expr> IN (...)".
61890	*/
61891	sqlite3ExprCachePush(pParse);
61892	r1 = sqlite3GetTempReg(pParse);
61893	sqlite3ExprCode(pParse, pExpr->pLeft, r1);
61894	sqlite3VdbeAddOp2(v, OP_IsNull, r1, destIfNull);
61895
61896
61897	if( eType==IN_INDEX_ROWID ){
61898	/* In this case, the RHS is the ROWID of table b-tree
61899	*/
61900	sqlite3VdbeAddOp2(v, OP_MustBeInt, r1, destIfFalse);
61901	sqlite3VdbeAddOp3(v, OP_NotExists, pExpr->iTable, destIfFalse, r1);
61902	}else{
61903	/* In this case, the RHS is an index b-tree.
61904	*/
61905	sqlite3VdbeAddOp4(v, OP_Affinity, r1, 1, 0, &affinity, 1);
61906
61907	/* If the set membership test fails, then the result of the
61908	** "x IN (...)" expression must be either 0 or NULL. If the set
61909	** contains no NULL values, then the result is 0. If the set
61910	** contains one or more NULL values, then the result of the
61911	** expression is also NULL.
61912	*/
61913	if( rRhsHasNull==0 \|\| destIfFalse==destIfNull ){
61914	/* This branch runs if it is known at compile time that the RHS
61915	** cannot contain NULL values. This happens as the result
61916	** of a "NOT NULL" constraint in the database schema.
61917	**
61918	** Also run this branch if NULL is equivalent to FALSE
61919	** for this particular IN operator.
61920	*/
61921	sqlite3VdbeAddOp4Int(v, OP_NotFound, pExpr->iTable, destIfFalse, r1, 1);
61922
61923	}else{
61924	/* In this branch, the RHS of the IN might contain a NULL and
61925	** the presence of a NULL on the RHS makes a difference in the
61926	** outcome.
61927	*/
61928	int j1, j2, j3;
61929
61930	/* First check to see if the LHS is contained in the RHS. If so,
61931	** then the presence of NULLs in the RHS does not matter, so jump
61932	** over all of the code that follows.
61933	*/
61934	j1 = sqlite3VdbeAddOp4Int(v, OP_Found, pExpr->iTable, 0, r1, 1);
61935
61936	/* Here we begin generating code that runs if the LHS is not
61937	** contained within the RHS. Generate additional code that
61938	** tests the RHS for NULLs. If the RHS contains a NULL then
61939	** jump to destIfNull. If there are no NULLs in the RHS then
61940	** jump to destIfFalse.
61941	*/
61942	j2 = sqlite3VdbeAddOp1(v, OP_NotNull, rRhsHasNull);
61943	j3 = sqlite3VdbeAddOp4Int(v, OP_Found, pExpr->iTable, 0, rRhsHasNull, 1);
61944	sqlite3VdbeAddOp2(v, OP_Integer, -1, rRhsHasNull);
61945	sqlite3VdbeJumpHere(v, j3);
61946	sqlite3VdbeAddOp2(v, OP_AddImm, rRhsHasNull, 1);
61947	sqlite3VdbeJumpHere(v, j2);
61948
61949	/* Jump to the appropriate target depending on whether or not
61950	** the RHS contains a NULL
61951	*/
61952	sqlite3VdbeAddOp2(v, OP_If, rRhsHasNull, destIfNull);
61953	sqlite3VdbeAddOp2(v, OP_Goto, 0, destIfFalse);
61954
61955	/* The OP_Found at the top of this branch jumps here when true,
61956	** causing the overall IN expression evaluation to fall through.
61957	*/
61958	sqlite3VdbeJumpHere(v, j1);
61959	}
61960	}
61961	sqlite3ReleaseTempReg(pParse, r1);
61962	sqlite3ExprCachePop(pParse, 1);
61963	VdbeComment((v, "end IN expr"));
61964	}
61965	#endif /* SQLITE_OMIT_SUBQUERY */
61966
61967	/*
61968	** Duplicate an 8-byte value
61969	*/
61970	static char dup8bytes(Vdbe v, const char *in){
61971	char *out = sqlite3DbMallocRaw(sqlite3VdbeDb(v), 8);
	@@ -62479,10 +62662,31 @@
62662	pDef = sqlite3FindFunction(db, zId, nId, nFarg, enc, 0);
62663	if( pDef==0 ){
62664	sqlite3ErrorMsg(pParse, "unknown function: %.*s()", nId, zId);
62665	break;
62666	}
62667
62668	/* Attempt a direct implementation of the built-in COALESCE() and
62669	** IFNULL() functions. This avoids unnecessary evalation of
62670	** arguments past the first non-NULL argument.
62671	*/
62672	if( pDef->flags & SQLITE_FUNC_COALESCE ){
62673	int endCoalesce = sqlite3VdbeMakeLabel(v);
62674	assert( nFarg>=2 );
62675	sqlite3ExprCode(pParse, pFarg->a[0].pExpr, target);
62676	for(i=1; i<nFarg; i++){
62677	sqlite3VdbeAddOp2(v, OP_NotNull, target, endCoalesce);
62678	sqlite3ExprCacheRemove(pParse, target);
62679	sqlite3ExprCachePush(pParse);
62680	sqlite3ExprCode(pParse, pFarg->a[i].pExpr, target);
62681	sqlite3ExprCachePop(pParse, 1);
62682	}
62683	sqlite3VdbeResolveLabel(v, endCoalesce);
62684	break;
62685	}
62686
62687
62688	if( pFarg ){
62689	r1 = sqlite3GetTempRange(pParse, nFarg);
62690	sqlite3ExprCachePush(pParse); /* Ticket 2ea2425d34be */
62691	sqlite3ExprCodeExprList(pParse, pFarg, r1, 1);
62692	sqlite3ExprCachePop(pParse, 1); /* Ticket 2ea2425d34be */
	@@ -62536,99 +62740,23 @@
62740	testcase( op==TK_SELECT );
62741	inReg = sqlite3CodeSubselect(pParse, pExpr, 0, 0);
62742	break;
62743	}
62744	case TK_IN: {
62745	int destIfFalse = sqlite3VdbeMakeLabel(v);
62746	int destIfNull = sqlite3VdbeMakeLabel(v);
62747	sqlite3VdbeAddOp2(v, OP_Null, 0, target);
62748	sqlite3ExprCodeIN(pParse, pExpr, destIfFalse, destIfNull);
62749	sqlite3VdbeAddOp2(v, OP_Integer, 1, target);
62750	sqlite3VdbeResolveLabel(v, destIfFalse);
62751	sqlite3VdbeAddOp2(v, OP_AddImm, target, 0);
62752	sqlite3VdbeResolveLabel(v, destIfNull);
62753	break;
62754	}
62755	#endif /* SQLITE_OMIT_SUBQUERY */
62756
62757












































































62758	/*
62759	** x BETWEEN y AND z
62760	**
62761	** This is equivalent to
62762	**
	@@ -63054,10 +63182,66 @@
63182	sqlite3ExprHardCopy(pParse, target, n);
63183	}
63184	}
63185	return n;
63186	}
63187
63188	/*
63189	** Generate code for a BETWEEN operator.
63190	**
63191	** x BETWEEN y AND z
63192	**
63193	** The above is equivalent to
63194	**
63195	** x>=y AND x<=z
63196	**
63197	** Code it as such, taking care to do the common subexpression
63198	** elementation of x.
63199	*/
63200	static void exprCodeBetween(
63201	Parse pParse, / Parsing and code generating context */
63202	Expr pExpr, / The BETWEEN expression */
63203	int dest, /* Jump here if the jump is taken */
63204	int jumpIfTrue, /* Take the jump if the BETWEEN is true */
63205	int jumpIfNull /* Take the jump if the BETWEEN is NULL */
63206	){
63207	Expr exprAnd; /* The AND operator in x>=y AND x<=z */
63208	Expr compLeft; /* The x>=y term */
63209	Expr compRight; /* The x<=z term */
63210	Expr exprX; /* The x subexpression */
63211	int regFree1 = 0; /* Temporary use register */
63212
63213	assert( !ExprHasProperty(pExpr, EP_xIsSelect) );
63214	exprX = *pExpr->pLeft;
63215	exprAnd.op = TK_AND;
63216	exprAnd.pLeft = &compLeft;
63217	exprAnd.pRight = &compRight;
63218	compLeft.op = TK_GE;
63219	compLeft.pLeft = &exprX;
63220	compLeft.pRight = pExpr->x.pList->a[0].pExpr;
63221	compRight.op = TK_LE;
63222	compRight.pLeft = &exprX;
63223	compRight.pRight = pExpr->x.pList->a[1].pExpr;
63224	exprX.iTable = sqlite3ExprCodeTemp(pParse, &exprX, &regFree1);
63225	exprX.op = TK_REGISTER;
63226	if( jumpIfTrue ){
63227	sqlite3ExprIfTrue(pParse, &exprAnd, dest, jumpIfNull);
63228	}else{
63229	sqlite3ExprIfFalse(pParse, &exprAnd, dest, jumpIfNull);
63230	}
63231	sqlite3ReleaseTempReg(pParse, regFree1);
63232
63233	/* Ensure adequate test coverage */
63234	testcase( jumpIfTrue==0 && jumpIfNull==0 && regFree1==0 );
63235	testcase( jumpIfTrue==0 && jumpIfNull==0 && regFree1!=0 );
63236	testcase( jumpIfTrue==0 && jumpIfNull!=0 && regFree1==0 );
63237	testcase( jumpIfTrue==0 && jumpIfNull!=0 && regFree1!=0 );
63238	testcase( jumpIfTrue!=0 && jumpIfNull==0 && regFree1==0 );
63239	testcase( jumpIfTrue!=0 && jumpIfNull==0 && regFree1!=0 );
63240	testcase( jumpIfTrue!=0 && jumpIfNull!=0 && regFree1==0 );
63241	testcase( jumpIfTrue!=0 && jumpIfNull!=0 && regFree1!=0 );
63242	}
63243
63244	/*
63245	** Generate code for a boolean expression such that a jump is made
63246	** to the label "dest" if the expression is true but execution
63247	** continues straight thru if the expression is false.
	@@ -63154,40 +63338,20 @@
63338	sqlite3VdbeAddOp2(v, op, r1, dest);
63339	testcase( regFree1==0 );
63340	break;
63341	}
63342	case TK_BETWEEN: {




























63343	testcase( jumpIfNull==0 );
63344	exprCodeBetween(pParse, pExpr, dest, 1, jumpIfNull);
63345	break;
63346	}
63347	case TK_IN: {
63348	int destIfFalse = sqlite3VdbeMakeLabel(v);
63349	int destIfNull = jumpIfNull ? dest : destIfFalse;
63350	sqlite3ExprCodeIN(pParse, pExpr, destIfFalse, destIfNull);
63351	sqlite3VdbeAddOp2(v, OP_Goto, 0, dest);
63352	sqlite3VdbeResolveLabel(v, destIfFalse);
63353	break;
63354	}
63355	default: {
63356	r1 = sqlite3ExprCodeTemp(pParse, pExpr, &regFree1);
63357	sqlite3VdbeAddOp3(v, OP_If, r1, dest, jumpIfNull!=0);
	@@ -63267,10 +63431,11 @@
63431	sqlite3VdbeResolveLabel(v, d2);
63432	sqlite3ExprCachePop(pParse, 1);
63433	break;
63434	}
63435	case TK_NOT: {
63436	testcase( jumpIfNull==0 );
63437	sqlite3ExprIfTrue(pParse, pExpr->pLeft, dest, jumpIfNull);
63438	break;
63439	}
63440	case TK_LT:
63441	case TK_LE:
	@@ -63314,40 +63479,22 @@
63479	sqlite3VdbeAddOp2(v, op, r1, dest);
63480	testcase( regFree1==0 );
63481	break;
63482	}
63483	case TK_BETWEEN: {




























63484	testcase( jumpIfNull==0 );
63485	exprCodeBetween(pParse, pExpr, dest, 0, jumpIfNull);
63486	break;
63487	}
63488	case TK_IN: {
63489	if( jumpIfNull ){
63490	sqlite3ExprCodeIN(pParse, pExpr, dest, dest);
63491	}else{
63492	int destIfNull = sqlite3VdbeMakeLabel(v);
63493	sqlite3ExprCodeIN(pParse, pExpr, dest, destIfNull);
63494	sqlite3VdbeResolveLabel(v, destIfNull);
63495	}
63496	break;
63497	}
63498	default: {
63499	r1 = sqlite3ExprCodeTemp(pParse, pExpr, &regFree1);
63500	sqlite3VdbeAddOp3(v, OP_IfNot, r1, dest, jumpIfNull!=0);
	@@ -63692,12 +63839,10 @@
63839	** May you share freely, never taking more than you give.
63840	**
63841	*************************************************************************
63842	** This file contains C code routines that used to generate VDBE code
63843	** that implements the ALTER TABLE command.


63844	*/
63845
63846	/*
63847	** The code in this file only exists if we are not omitting the
63848	** ALTER TABLE logic from the build.
	@@ -64488,12 +64633,10 @@
64633	** May you find forgiveness for yourself and forgive others.
64634	** May you share freely, never taking more than you give.
64635	**
64636	*************************************************************************
64637	** This file contains code associated with the ANALYZE command.


64638	*/
64639	#ifndef SQLITE_OMIT_ANALYZE
64640
64641	/*
64642	** This routine generates code that opens the sqlite_stat1 table for
	@@ -65143,12 +65286,10 @@
65286	** May you find forgiveness for yourself and forgive others.
65287	** May you share freely, never taking more than you give.
65288	**
65289	*************************************************************************
65290	** This file contains code used to implement the ATTACH and DETACH commands.


65291	*/
65292
65293	#ifndef SQLITE_OMIT_ATTACH
65294	/*
65295	** Resolve an expression that was part of an ATTACH or DETACH statement. This
	@@ -65685,12 +65826,10 @@
65826	*************************************************************************
65827	** This file contains code used to implement the sqlite3_set_authorizer()
65828	** API. This facility is an optional feature of the library. Embedded
65829	** systems that do not need this facility may omit it by recompiling
65830	** the library with -DSQLITE_OMIT_AUTHORIZATION=1


65831	*/
65832
65833	/*
65834	** All of the code in this file may be omitted by defining a single
65835	** macro.
	@@ -65946,12 +66085,10 @@
66085	** DROP INDEX
66086	** creating ID lists
66087	** BEGIN TRANSACTION
66088	** COMMIT
66089	** ROLLBACK


66090	*/
66091
66092	/*
66093	** This routine is called when a new SQL statement is beginning to
66094	** be parsed. Initialize the pParse structure as needed.
	@@ -69645,12 +69782,10 @@
69782	**
69783	*************************************************************************
69784	**
69785	** This file contains functions used to access the internal hash tables
69786	** of user defined functions and collation sequences.


69787	*/
69788
69789
69790	/*
69791	** Invoke the 'collation needed' callback to request a collation sequence
	@@ -70098,12 +70233,10 @@
70233	** May you share freely, never taking more than you give.
70234	**
70235	*************************************************************************
70236	** This file contains C code routines that are called by the parser
70237	** in order to generate code for DELETE FROM statements.


70238	*/
70239
70240	/*
70241	** Look up every table that is named in pSrc. If any table is not found,
70242	** add an error message to pParse->zErrMsg and return NULL. If all tables
	@@ -70722,11 +70855,10 @@
70855	}
70856	sqlite3ReleaseTempRange(pParse, regBase, nCol+1);
70857	return regBase;
70858	}
70859

70860	/************ End of delete.c ********************************************/
70861	/************ Begin file func.c ******************************************/
70862	/*
70863	** 2002 February 23
70864	**
	@@ -70880,10 +71012,12 @@
71012	** p1 is 1-indexed. So substr(x,1,1) returns the first character
71013	** of x. If x is text, then we actually count UTF-8 characters.
71014	** If x is a blob, then we count bytes.
71015	**
71016	** If p1 is negative, then we begin abs(p1) from the end of x[].
71017	**
71018	** If p2 is negative, return the p2 characters preceeding p1.
71019	*/
71020	static void substrFunc(
71021	sqlite3_context *context,
71022	int argc,
71023	sqlite3_value **argv
	@@ -70900,24 +71034,26 @@
71034	\|\| (argc==3 && sqlite3_value_type(argv[2])==SQLITE_NULL)
71035	){
71036	return;
71037	}
71038	p0type = sqlite3_value_type(argv[0]);
71039	p1 = sqlite3_value_int(argv[1]);
71040	if( p0type==SQLITE_BLOB ){
71041	len = sqlite3_value_bytes(argv[0]);
71042	z = sqlite3_value_blob(argv[0]);
71043	if( z==0 ) return;
71044	assert( len==sqlite3_value_bytes(argv[0]) );
71045	}else{
71046	z = sqlite3_value_text(argv[0]);
71047	if( z==0 ) return;
71048	len = 0;
71049	if( p1<0 ){
71050	for(z2=z; *z2; len++){
71051	SQLITE_SKIP_UTF8(z2);
71052	}
71053	}
71054	}

71055	if( argc==3 ){
71056	p2 = sqlite3_value_int(argv[2]);
71057	if( p2<0 ){
71058	p2 = -p2;
71059	negP2 = 1;
	@@ -70943,14 +71079,10 @@
71079	p2 += p1;
71080	p1 = 0;
71081	}
71082	}
71083	assert( p1>=0 && p2>=0 );




71084	if( p0type!=SQLITE_BLOB ){
71085	while( *z && p1 ){
71086	SQLITE_SKIP_UTF8(z);
71087	p1--;
71088	}
	@@ -70957,10 +71089,14 @@
71089	for(z2=z; *z2 && p2; p2--){
71090	SQLITE_SKIP_UTF8(z2);
71091	}
71092	sqlite3_result_text(context, (char*)z, (int)(z2-z), SQLITE_TRANSIENT);
71093	}else{
71094	if( p1+p2>len ){
71095	p2 = len-p1;
71096	if( p2<0 ) p2 = 0;
71097	}
71098	sqlite3_result_blob(context, (char*)&z[p1], (int)p2, SQLITE_TRANSIENT);
71099	}
71100	}
71101
71102	/*
	@@ -71058,10 +71194,18 @@
71194	sqlite3_result_text(context, (char *)z1, -1, sqlite3_free);
71195	}
71196	}
71197	}
71198
71199
71200	#if 0 /* This function is never used. */
71201	/*
71202	** The COALESCE() and IFNULL() functions used to be implemented as shown
71203	** here. But now they are implemented as VDBE code so that unused arguments
71204	** do not have to be computed. This legacy implementation is retained as
71205	** comment.
71206	*/
71207	/*
71208	** Implementation of the IFNULL(), NVL(), and COALESCE() functions.
71209	** All three do the same thing. They return the first non-NULL
71210	** argument.
71211	*/
	@@ -71076,10 +71220,12 @@
71220	sqlite3_result_value(context, argv[i]);
71221	break;
71222	}
71223	}
71224	}
71225	#endif /* NOT USED */
71226	#define ifnullFunc versionFunc /* Substitute function - never called */
71227
71228	/*
71229	** Implementation of random(). Return a random integer.
71230	*/
71231	static void randomFunc(
	@@ -72160,14 +72306,16 @@
72306	FUNCTION(round, 2, 0, 0, roundFunc ),
72307	#endif
72308	FUNCTION(upper, 1, 0, 0, upperFunc ),
72309	FUNCTION(lower, 1, 0, 0, lowerFunc ),
72310	FUNCTION(coalesce, 1, 0, 0, 0 ),

72311	FUNCTION(coalesce, 0, 0, 0, 0 ),
72312	/* FUNCTION(coalesce, -1, 0, 0, ifnullFunc ), */
72313	{-1,SQLITE_UTF8,SQLITE_FUNC_COALESCE,0,0,ifnullFunc,0,0,"coalesce",0},
72314	FUNCTION(hex, 1, 0, 0, hexFunc ),
72315	/* FUNCTION(ifnull, 2, 0, 0, ifnullFunc ), */
72316	{2,SQLITE_UTF8,SQLITE_FUNC_COALESCE,0,0,ifnullFunc,0,0,"ifnull",0},
72317	FUNCTION(random, 0, 0, 0, randomFunc ),
72318	FUNCTION(randomblob, 1, 0, 0, randomBlob ),
72319	FUNCTION(nullif, 2, 0, 1, nullifFunc ),
72320	FUNCTION(sqlite_version, 0, 0, 0, versionFunc ),
72321	FUNCTION(sqlite_source_id, 0, 0, 0, sourceidFunc ),
	@@ -72613,11 +72761,11 @@
72761	sqlite3VdbeAddOp2(v, OP_Goto, 0, iOk);
72762	}
72763
72764	sqlite3VdbeAddOp3(v, OP_MakeRecord, regTemp, nCol, regRec);
72765	sqlite3VdbeChangeP4(v, -1, sqlite3IndexAffinityStr(v, pIdx), 0);
72766	sqlite3VdbeAddOp4Int(v, OP_Found, iCur, iOk, regRec, 0);
72767
72768	sqlite3ReleaseTempReg(pParse, regRec);
72769	sqlite3ReleaseTempRange(pParse, regTemp, nCol);
72770	}
72771	}
	@@ -73419,12 +73567,10 @@
73567	** May you share freely, never taking more than you give.
73568	**
73569	*************************************************************************
73570	** This file contains C code routines that are called by the parser
73571	** to handle INSERT statements in SQLite.


73572	*/
73573
73574	/*
73575	** Generate code that will open a table for reading.
73576	*/
	@@ -75240,12 +75386,10 @@
75386	*************************************************************************
75387	** Main file for the SQLite library. The routines in this file
75388	** implement the programmer interface to the library. Routines in
75389	** other files are for internal use by SQLite and should not be
75390	** accessed by users of the library.


75391	*/
75392
75393
75394	/*
75395	** Execute SQL code. Return one of the SQLITE_ success/failure
	@@ -75386,12 +75530,10 @@
75530	** May you share freely, never taking more than you give.
75531	**
75532	*************************************************************************
75533	** This file contains code used to dynamically load extensions into
75534	** the SQLite library.


75535	*/
75536
75537	#ifndef SQLITE_CORE
75538	#define SQLITE_CORE 1 /* Disable the API redefinition in sqlite3ext.h */
75539	#endif
	@@ -75411,12 +75553,10 @@
75553	** This header file defines the SQLite interface for use by
75554	** shared libraries that want to be imported as extensions into
75555	** an SQLite instance. Shared libraries that intend to be loaded
75556	** as extensions by SQLite should #include this file instead of
75557	** sqlite3.h.


75558	*/
75559	#ifndef _SQLITE3EXT_H_
75560	#define _SQLITE3EXT_H_
75561
75562	typedef struct sqlite3_api_routines sqlite3_api_routines;
	@@ -76380,12 +76520,10 @@
76520	** May you find forgiveness for yourself and forgive others.
76521	** May you share freely, never taking more than you give.
76522	**
76523	*************************************************************************
76524	** This file contains code used to implement the PRAGMA command.


76525	*/
76526
76527	/* Ignore this whole file if pragmas are disabled
76528	*/
76529	#if !defined(SQLITE_OMIT_PRAGMA)
	@@ -77499,10 +77637,11 @@
77637	sqlite3VdbeAddOp2(v, OP_Integer, 0, 2); /* reg(2) will count entries */
77638	loopTop = sqlite3VdbeAddOp2(v, OP_Rewind, 1, 0);
77639	sqlite3VdbeAddOp2(v, OP_AddImm, 2, 1); /* increment entry count */
77640	for(j=0, pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext, j++){
77641	int jmp2;
77642	int r1;
77643	static const VdbeOpList idxErr[] = {
77644	{ OP_AddImm, 1, -1, 0},
77645	{ OP_String8, 0, 3, 0}, /* 1 */
77646	{ OP_Rowid, 1, 4, 0},
77647	{ OP_String8, 0, 5, 0}, /* 3 */
	@@ -77512,12 +77651,12 @@
77651	{ OP_Concat, 6, 3, 3},
77652	{ OP_ResultRow, 3, 1, 0},
77653	{ OP_IfPos, 1, 0, 0}, /* 9 */
77654	{ OP_Halt, 0, 0, 0},
77655	};
77656	r1 = sqlite3GenerateIndexKey(pParse, pIdx, 1, 3, 0);
77657	jmp2 = sqlite3VdbeAddOp4Int(v, OP_Found, j+2, 0, r1, pIdx->nColumn+1);
77658	addr = sqlite3VdbeAddOpList(v, ArraySize(idxErr), idxErr);
77659	sqlite3VdbeChangeP4(v, addr+1, "rowid ", P4_STATIC);
77660	sqlite3VdbeChangeP4(v, addr+3, " missing from index ", P4_STATIC);
77661	sqlite3VdbeChangeP4(v, addr+4, pIdx->zName, P4_STATIC);
77662	sqlite3VdbeJumpHere(v, addr+9);
	@@ -77820,12 +77959,10 @@
77959	**
77960	*************************************************************************
77961	** This file contains the implementation of the sqlite3_prepare()
77962	** interface, and routines that contribute to loading the database schema
77963	** from disk.


77964	*/
77965
77966	/*
77967	** Fill the InitData structure with an error message that indicates
77968	** that the database is corrupt.
	@@ -78694,12 +78831,10 @@
78831	** May you share freely, never taking more than you give.
78832	**
78833	*************************************************************************
78834	** This file contains C code routines that are called by the parser
78835	** to handle SELECT statements in SQLite.


78836	*/
78837
78838
78839	/*
78840	** Delete all the content of a Select structure but do not deallocate
	@@ -79123,12 +79258,12 @@
79258	Vdbe *v;
79259	int r1;
79260
79261	v = pParse->pVdbe;
79262	r1 = sqlite3GetTempReg(pParse);
79263	sqlite3VdbeAddOp4Int(v, OP_Found, iTab, addrRepeat, iMem, N);
79264	sqlite3VdbeAddOp3(v, OP_MakeRecord, iMem, N, r1);

79265	sqlite3VdbeAddOp2(v, OP_IdxInsert, iTab, r1);
79266	sqlite3ReleaseTempReg(pParse, r1);
79267	}
79268
79269	/*
	@@ -79365,12 +79500,11 @@
79500	/* Jump to the end of the loop if the LIMIT is reached.
79501	*/
79502	if( p->iLimit ){
79503	assert( pOrderBy==0 ); /* If there is an ORDER BY, the call to
79504	** pushOntoSorter() would have cleared p->iLimit */
79505	sqlite3VdbeAddOp3(v, OP_IfZero, p->iLimit, iBreak, -1);

79506	}
79507	}
79508
79509	/*
79510	** Given an expression list, generate a KeyInfo structure that records
	@@ -79992,11 +80126,11 @@
80126	*/
80127	static void computeLimitRegisters(Parse pParse, Select p, int iBreak){
80128	Vdbe *v = 0;
80129	int iLimit = 0;
80130	int iOffset;
80131	int addr1, n;
80132	if( p->iLimit ) return;
80133
80134	/*
80135	** "LIMIT -1" always shows all rows. There is some
80136	** contraversy about what the correct behavior should be.
	@@ -80007,14 +80141,22 @@
80141	assert( p->pOffset==0 \|\| p->pLimit!=0 );
80142	if( p->pLimit ){
80143	p->iLimit = iLimit = ++pParse->nMem;
80144	v = sqlite3GetVdbe(pParse);
80145	if( NEVER(v==0) ) return; /* VDBE should have already been allocated */
80146	if( sqlite3ExprIsInteger(p->pLimit, &n) ){
80147	sqlite3VdbeAddOp2(v, OP_Integer, n, iLimit);
80148	VdbeComment((v, "LIMIT counter"));
80149	if( n==0 ){
80150	sqlite3VdbeAddOp2(v, OP_Goto, 0, iBreak);
80151	}
80152	}else{
80153	sqlite3ExprCode(pParse, p->pLimit, iLimit);
80154	sqlite3VdbeAddOp1(v, OP_MustBeInt, iLimit);
80155	VdbeComment((v, "LIMIT counter"));
80156	sqlite3VdbeAddOp2(v, OP_IfZero, iLimit, iBreak);
80157	}
80158	if( p->pOffset ){
80159	p->iOffset = iOffset = ++pParse->nMem;
80160	pParse->nMem++; /* Allocate an extra register for limit+offset */
80161	sqlite3ExprCode(pParse, p->pOffset, iOffset);
80162	sqlite3VdbeAddOp1(v, OP_MustBeInt, iOffset);
	@@ -80346,11 +80488,11 @@
80488	iCont = sqlite3VdbeMakeLabel(v);
80489	computeLimitRegisters(pParse, p, iBreak);
80490	sqlite3VdbeAddOp2(v, OP_Rewind, tab1, iBreak);
80491	r1 = sqlite3GetTempReg(pParse);
80492	iStart = sqlite3VdbeAddOp2(v, OP_RowKey, tab1, r1);
80493	sqlite3VdbeAddOp4Int(v, OP_NotFound, tab2, iCont, r1, 0);
80494	sqlite3ReleaseTempReg(pParse, r1);
80495	selectInnerLoop(pParse, p, p->pEList, tab1, p->pEList->nExpr,
80496	0, -1, &dest, iCont, iBreak);
80497	sqlite3VdbeResolveLabel(v, iCont);
80498	sqlite3VdbeAddOp2(v, OP_Next, tab1, iStart);
	@@ -80565,12 +80707,11 @@
80707	}
80708
80709	/* Jump to the end of the loop if the LIMIT is reached.
80710	*/
80711	if( p->iLimit ){
80712	sqlite3VdbeAddOp3(v, OP_IfZero, p->iLimit, iBreak, -1);

80713	}
80714
80715	/* Generate the subroutine return
80716	*/
80717	sqlite3VdbeResolveLabel(v, iContinue);
	@@ -82934,12 +83075,10 @@
83075	** interface routines. These are just wrappers around the main
83076	** interface routine of sqlite3_exec().
83077	**
83078	** These routines are in a separate files so that they will not be linked
83079	** if they are not used.


83080	*/
83081
83082	#ifndef SQLITE_OMIT_GET_TABLE
83083
83084	/*
	@@ -83126,13 +83265,11 @@
83265	** May you do good and not evil.
83266	** May you find forgiveness for yourself and forgive others.
83267	** May you share freely, never taking more than you give.
83268	**
83269	*************************************************************************
83270	** This file contains the implementation for TRIGGERs


83271	*/
83272
83273	#ifndef SQLITE_OMIT_TRIGGER
83274	/*
83275	** Delete a linked list of TriggerStep structures.
	@@ -84210,12 +84347,10 @@
84347	** May you share freely, never taking more than you give.
84348	**
84349	*************************************************************************
84350	** This file contains C code routines that are called by the parser
84351	** to handle UPDATE statements.


84352	*/
84353
84354	#ifndef SQLITE_OMIT_VIRTUALTABLE
84355	/* Forward declaration */
84356	static void updateVirtualTable(
	@@ -84843,12 +84978,10 @@
84978	*************************************************************************
84979	** This file contains code used to implement the VACUUM command.
84980	**
84981	** Most of the code in this file may be omitted by defining the
84982	** SQLITE_OMIT_VACUUM macro.


84983	*/
84984
84985	#if !defined(SQLITE_OMIT_VACUUM) && !defined(SQLITE_OMIT_ATTACH)
84986	/*
84987	** Execute zSql on database db. Return an error code.
	@@ -85149,12 +85282,10 @@
85282	** May you find forgiveness for yourself and forgive others.
85283	** May you share freely, never taking more than you give.
85284	**
85285	*************************************************************************
85286	** This file contains code used to help implement virtual tables.


85287	*/
85288	#ifndef SQLITE_OMIT_VIRTUALTABLE
85289
85290	/*
85291	** The actual function that does the work of creating a new module.
	@@ -86118,12 +86249,10 @@
86249	** the WHERE clause of SQL statements. This module is responsible for
86250	** generating the code that loops through a table looking for applicable
86251	** rows. Indices are selected and used to speed the search when doing
86252	** so is applicable. Because this module is responsible for selecting
86253	** indices, you might also think of this module as the "query optimizer".


86254	*/
86255
86256	/*
86257	** Trace output macros
86258	*/
	@@ -88698,20 +88827,43 @@
88827
88828	/*
88829	** Code an OP_Affinity opcode to apply the column affinity string zAff
88830	** to the n registers starting at base.
88831	**
88832	** As an optimization, SQLITE_AFF_NONE entries (which are no-ops) at the
88833	** beginning and end of zAff are ignored. If all entries in zAff are
88834	** SQLITE_AFF_NONE, then no code gets generated.
88835	**
88836	** This routine makes its own copy of zAff so that the caller is free
88837	** to modify zAff after this routine returns.
88838	*/
88839	static void codeApplyAffinity(Parse pParse, int base, int n, char zAff){
88840	Vdbe *v = pParse->pVdbe;
88841	if( zAff==0 ){
88842	assert( pParse->db->mallocFailed );
88843	return;
88844	}
88845	assert( v!=0 );
88846
88847	/* Adjust base and n to skip over SQLITE_AFF_NONE entries at the beginning
88848	** and end of the affinity string.
88849	*/
88850	while( n>0 && zAff[0]==SQLITE_AFF_NONE ){
88851	n--;
88852	base++;
88853	zAff++;
88854	}
88855	while( n>1 && zAff[n-1]==SQLITE_AFF_NONE ){
88856	n--;
88857	}
88858
88859	/* Code the OP_Affinity opcode if there is anything left to do. */
88860	if( n>0 ){
88861	sqlite3VdbeAddOp2(v, OP_Affinity, base, n);
88862	sqlite3VdbeChangeP4(v, -1, zAff, n);
88863	sqlite3ExprCacheAffinityChange(pParse, base, n);
88864	}
88865	}
88866
88867
88868	/*
88869	** Generate code for a single equality term of the WHERE clause. An equality
	@@ -88778,11 +88930,11 @@
88930	return iReg;
88931	}
88932
88933	/*
88934	** Generate code that will evaluate all == and IN constraints for an
88935	** index.
88936	**
88937	** For example, consider table t1(a,b,c,d,e,f) with index i1(a,b,c).
88938	** Suppose the WHERE clause is this: a==5 AND b IN (1,2,3) AND c>5 AND c<10
88939	** The index has as many as three equality constraints, but in this
88940	** example, the third "c" value is an inequality. So only two
	@@ -88790,11 +88942,12 @@
88942	** a==5 and b IN (1,2,3). The current values for a and b will be stored
88943	** in consecutive registers and the index of the first register is returned.
88944	**
88945	** In the example above nEq==2. But this subroutine works for any value
88946	** of nEq including 0. If nEq==0, this routine is nearly a no-op.
88947	** The only thing it does is allocate the pLevel->iMem memory cell and
88948	** compute the affinity string.
88949	**
88950	** This routine always allocates at least one memory cell and returns
88951	** the index of that memory cell. The code that
88952	** calls this routine will use that memory cell to store the termination
88953	** key value of the loop. If one or more IN operators appear, then
	@@ -88868,15 +89021,19 @@
89021	}
89022	}
89023	testcase( pTerm->eOperator & WO_ISNULL );
89024	testcase( pTerm->eOperator & WO_IN );
89025	if( (pTerm->eOperator & (WO_ISNULL\|WO_IN))==0 ){
89026	Expr *pRight = pTerm->pExpr->pRight;
89027	sqlite3ExprCodeIsNullJump(v, pRight, regBase+j, pLevel->addrBrk);
89028	if( zAff ){
89029	if( sqlite3CompareAffinity(pRight, zAff[j])==SQLITE_AFF_NONE ){
89030	zAff[j] = SQLITE_AFF_NONE;
89031	}
89032	if( sqlite3ExprNeedsNoAffinityChange(pRight, zAff[j]) ){
89033	zAff[j] = SQLITE_AFF_NONE;
89034	}
89035	}
89036	}
89037	}
89038	*pzAff = zAff;
89039	return regBase;
	@@ -89198,19 +89355,22 @@
89355	/* Seek the index cursor to the start of the range. */
89356	nConstraint = nEq;
89357	if( pRangeStart ){
89358	Expr *pRight = pRangeStart->pExpr->pRight;
89359	sqlite3ExprCode(pParse, pRight, regBase+nEq);
89360	sqlite3ExprCodeIsNullJump(v, pRight, regBase+nEq, addrNxt);
89361	if( zAff ){
89362	if( sqlite3CompareAffinity(pRight, zAff[nConstraint])==SQLITE_AFF_NONE){
89363	/* Since the comparison is to be performed with no conversions
89364	** applied to the operands, set the affinity to apply to pRight to
89365	** SQLITE_AFF_NONE. */
89366	zAff[nConstraint] = SQLITE_AFF_NONE;
89367	}
89368	if( sqlite3ExprNeedsNoAffinityChange(pRight, zAff[nConstraint]) ){
89369	zAff[nConstraint] = SQLITE_AFF_NONE;
89370	}
89371	}
89372	nConstraint++;
89373	}else if( isMinQuery ){
89374	sqlite3VdbeAddOp2(v, OP_Null, 0, regBase+nEq);
89375	nConstraint++;
89376	startEq = 0;
	@@ -89223,34 +89383,36 @@
89383	testcase( op==OP_Last );
89384	testcase( op==OP_SeekGt );
89385	testcase( op==OP_SeekGe );
89386	testcase( op==OP_SeekLe );
89387	testcase( op==OP_SeekLt );
89388	sqlite3VdbeAddOp4Int(v, op, iIdxCur, addrNxt, regBase, nConstraint);

89389
89390	/* Load the value for the inequality constraint at the end of the
89391	** range (if any).
89392	*/
89393	nConstraint = nEq;
89394	if( pRangeEnd ){
89395	Expr *pRight = pRangeEnd->pExpr->pRight;
89396	sqlite3ExprCacheRemove(pParse, regBase+nEq);
89397	sqlite3ExprCode(pParse, pRight, regBase+nEq);
89398	sqlite3ExprCodeIsNullJump(v, pRight, regBase+nEq, addrNxt);
89399	if( zAff ){
89400	if( sqlite3CompareAffinity(pRight, zAff[nConstraint])==SQLITE_AFF_NONE){
89401	/* Since the comparison is to be performed with no conversions
89402	** applied to the operands, set the affinity to apply to pRight to
89403	** SQLITE_AFF_NONE. */
89404	zAff[nConstraint] = SQLITE_AFF_NONE;
89405	}
89406	if( sqlite3ExprNeedsNoAffinityChange(pRight, zAff[nConstraint]) ){
89407	zAff[nConstraint] = SQLITE_AFF_NONE;
89408	}
89409	}
89410	codeApplyAffinity(pParse, regBase, nEq+1, zAff);
89411	nConstraint++;
89412	}
89413	sqlite3DbFree(pParse->db, zAff);
89414
89415	/* Top of the loop body */
89416	pLevel->p2 = sqlite3VdbeCurrentAddr(v);
89417
89418	/* Check if the index cursor is past the end of the range. */
	@@ -89257,12 +89419,11 @@
89419	op = aEndOp[(pRangeEnd \|\| nEq) * (1 + bRev)];
89420	testcase( op==OP_Noop );
89421	testcase( op==OP_IdxGE );
89422	testcase( op==OP_IdxLT );
89423	if( op!=OP_Noop ){
89424	sqlite3VdbeAddOp4Int(v, op, iIdxCur, addrNxt, regBase, nConstraint);

89425	sqlite3VdbeChangeP5(v, endEq!=bRev ?1:0);
89426	}
89427
89428	/* If there are inequality constraints, check that the value
89429	** of the table column that the inequality contrains is not NULL.
	@@ -89387,13 +89548,12 @@
89548	if( (wctrlFlags & WHERE_DUPLICATES_OK)==0 ){
89549	int iSet = ((ii==pOrWc->nTerm-1)?-1:ii);
89550	int r;
89551	r = sqlite3ExprCodeGetColumn(pParse, pTabItem->pTab, -1, iCur,
89552	regRowid, 0);
89553	sqlite3VdbeAddOp4Int(v, OP_RowSetTest, regRowset,
89554	sqlite3VdbeCurrentAddr(v)+2, r, iSet);

89555	}
89556	sqlite3VdbeAddOp2(v, OP_Gosub, regReturn, iLoopBody);
89557
89558	/* Finish the loop through table entries that match term pOrTerm. */
89559	sqlite3WhereEnd(pSubWInfo);
	@@ -89920,11 +90080,12 @@
90080	sqlite3OpenTable(pParse, pTabItem->iCursor, iDb, pTab, op);
90081	if( !pWInfo->okOnePass && pTab->nCol<BMS ){
90082	Bitmask b = pTabItem->colUsed;
90083	int n = 0;
90084	for(; b; b=b>>1, n++){}
90085	sqlite3VdbeChangeP4(v, sqlite3VdbeCurrentAddr(v)-1,
90086	SQLITE_INT_TO_PTR(n), P4_INT32);
90087	assert( n<=pTab->nCol );
90088	}
90089	}else{
90090	sqlite3TableLock(pParse, iDb, pTab->tnum, 0, pTab->zName);
90091	}
	@@ -90048,11 +90209,15 @@
90209	}
90210	sqlite3VdbeResolveLabel(v, pLevel->addrBrk);
90211	if( pLevel->iLeftJoin ){
90212	int addr;
90213	addr = sqlite3VdbeAddOp1(v, OP_IfPos, pLevel->iLeftJoin);
90214	assert( (pLevel->plan.wsFlags & WHERE_IDX_ONLY)==0
90215	\|\| (pLevel->plan.wsFlags & WHERE_INDEXED)!=0 );
90216	if( (pLevel->plan.wsFlags & WHERE_IDX_ONLY)==0 ){
90217	sqlite3VdbeAddOp1(v, OP_NullRow, pTabList->a[i].iCursor);
90218	}
90219	if( pLevel->iIdxCur>=0 ){
90220	sqlite3VdbeAddOp1(v, OP_NullRow, pLevel->iIdxCur);
90221	}
90222	if( pLevel->op==OP_Return ){
90223	sqlite3VdbeAddOp2(v, OP_Gosub, pLevel->p1, pLevel->addrFirst);
	@@ -90099,11 +90264,10 @@
90264	*/
90265	if( (pLevel->plan.wsFlags & WHERE_INDEXED)!=0 && !db->mallocFailed){
90266	int k, j, last;
90267	VdbeOp *pOp;
90268	Index *pIdx = pLevel->plan.u.pIdx;

90269
90270	assert( pIdx!=0 );
90271	pOp = sqlite3VdbeGetOp(v, pWInfo->iTop);
90272	last = sqlite3VdbeCurrentAddr(v);
90273	for(k=pWInfo->iTop; k<last; k++, pOp++){
	@@ -90114,16 +90278,15 @@
90278	pOp->p2 = j;
90279	pOp->p1 = pLevel->iIdxCur;
90280	break;
90281	}
90282	}
90283	assert( (pLevel->plan.wsFlags & WHERE_IDX_ONLY)==0
90284	\|\| j<pIdx->nColumn );
90285	}else if( pOp->opcode==OP_Rowid ){
90286	pOp->p1 = pLevel->iIdxCur;
90287	pOp->opcode = OP_IdxRowid;


90288	}
90289	}
90290	}
90291	}
90292
	@@ -90239,10 +90402,21 @@
90402	){
90403	pOut->pExpr = sqlite3PExpr(pParse, op, pOperand->pExpr, 0, 0);
90404	pOut->zStart = pOperand->zStart;
90405	pOut->zEnd = &pPostOp->z[pPostOp->n];
90406	}
90407
90408	/* A routine to convert a binary TK_IS or TK_ISNOT expression into a
90409	** unary TK_ISNULL or TK_NOTNULL expression. */
90410	static void binaryToUnaryIfNull(Parse pParse, Expr pY, Expr *pA, int op){
90411	sqlite3 *db = pParse->db;
90412	if( db->mallocFailed==0 && pY->op==TK_NULL ){
90413	pA->op = op;
90414	sqlite3ExprDelete(db, pA->pRight);
90415	pA->pRight = 0;
90416	}
90417	}
90418
90419	/* Construct an expression node for a unary prefix operator
90420	*/
90421	static void spanUnaryPrefix(
90422	ExprSpan pOut, / Write the new expression node here */
	@@ -92857,21 +93031,17 @@
93031	{spanUnaryPostfix(&yygotominor.yy346,pParse,TK_NOTNULL,&yymsp[-2].minor.yy346,&yymsp[0].minor.yy0);}
93032	break;
93033	case 216: /* expr ::= expr IS expr */
93034	{
93035	spanBinaryExpr(&yygotominor.yy346,pParse,TK_IS,&yymsp[-2].minor.yy346,&yymsp[0].minor.yy346);
93036	binaryToUnaryIfNull(pParse, yymsp[0].minor.yy346.pExpr, yygotominor.yy346.pExpr, TK_ISNULL);


93037	}
93038	break;
93039	case 217: /* expr ::= expr IS NOT expr */
93040	{
93041	spanBinaryExpr(&yygotominor.yy346,pParse,TK_ISNOT,&yymsp[-3].minor.yy346,&yymsp[0].minor.yy346);
93042	binaryToUnaryIfNull(pParse, yymsp[0].minor.yy346.pExpr, yygotominor.yy346.pExpr, TK_NOTNULL);


93043	}
93044	break;
93045	case 218: /* expr ::= NOT expr */
93046	case 219: /* expr ::= BITNOT expr */ yytestcase(yyruleno==219);
93047	{spanUnaryPrefix(&yygotominor.yy346,pParse,yymsp[-1].major,&yymsp[0].minor.yy346,&yymsp[-1].minor.yy0);}
	@@ -93551,12 +93721,10 @@
93721	** An tokenizer for SQL
93722	**
93723	** This file contains C code that splits an SQL input string up into
93724	** individual tokens and sends those tokens one-by-one over to the
93725	** parser for analysis.


93726	*/
93727
93728	/*
93729	** The charMap() macro maps alphabetic characters into their
93730	** lower-case ASCII equivalent. On ASCII machines, this is just
	@@ -93892,20 +94060,11 @@
94060	** middle of identfiers. But many SQL implementations do.
94061	** SQLite will allow '$' in identifiers for compatibility.
94062	** But the feature is undocumented.
94063	*/
94064	#ifdef SQLITE_ASCII
94065	#define IdChar(C) ((sqlite3CtypeMap[(unsigned char)C]&0x46)!=0)









94066	#endif
94067	#ifdef SQLITE_EBCDIC
94068	SQLITE_PRIVATE const char sqlite3IsEbcdicIdChar[] = {
94069	/* x0 x1 x2 x3 x4 x5 x6 x7 x8 x9 xA xB xC xD xE xF */
94070	0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, /* 4x */
	@@ -94361,22 +94520,19 @@
94520	**
94521	** This file contains C code that implements the sqlite3_complete() API.
94522	** This code used to be part of the tokenizer.c source file. But by
94523	** separating it out, the code will be automatically omitted from
94524	** static links that do not use it.


94525	*/
94526	#ifndef SQLITE_OMIT_COMPLETE
94527
94528	/*
94529	** This is defined in tokenize.c. We just have to import the definition.
94530	*/
94531	#ifndef SQLITE_AMALGAMATION
94532	#ifdef SQLITE_ASCII
94533	#define IdChar(C) ((sqlite3CtypeMap[(unsigned char)C]&0x46)!=0)

94534	#endif
94535	#ifdef SQLITE_EBCDIC
94536	SQLITE_PRIVATE const char sqlite3IsEbcdicIdChar[];
94537	#define IdChar(C) (((c=C)>=0x42 && sqlite3IsEbcdicIdChar[c-0x40]))
94538	#endif
	@@ -94529,11 +94685,13 @@
94685	if( *zSql==0 ) return 0;
94686	token = tkOTHER;
94687	break;
94688	}
94689	default: {
94690	#ifdef SQLITE_EBCDIC
94691	unsigned char c;
94692	#endif
94693	if( IdChar((u8)*zSql) ){
94694	/* Keywords and unquoted identifiers */
94695	int nId;
94696	for(nId=1; IdChar(zSql[nId]); nId++){}
94697	#ifdef SQLITE_OMIT_TRIGGER
	@@ -97015,12 +97173,10 @@
97173	**
97174	*************************************************************************
97175	**
97176	** This file contains the implementation of the sqlite3_unlock_notify()
97177	** API method and its associated functionality.


97178	*/
97179
97180	/* Omit this entire file if SQLITE_ENABLE_UNLOCK_NOTIFY is not defined. */
97181	#ifdef SQLITE_ENABLE_UNLOCK_NOTIFY
97182
	@@ -97615,380 +97771,17 @@
97771
97772	#if defined(SQLITE_ENABLE_FTS3) && !defined(SQLITE_CORE)
97773	# define SQLITE_CORE 1
97774	#endif
97775
97776	#include "fts3Int.h"
97777
97778












































































































































































































































































































































































97779	#ifndef SQLITE_CORE
97780	SQLITE_EXTENSION_INIT1
97781	#endif
97782
97783
97784
97785	/* TODO(shess) MAN, this thing needs some refactoring. At minimum, it
97786	** would be nice to order the file better, perhaps something along the
97787	** lines of:
	@@ -98006,29 +97799,10 @@
97799	# define FTSTRACE(A) printf A; fflush(stdout)
97800	#else
97801	# define FTSTRACE(A)
97802	#endif
97803



















97804	typedef enum DocListType {
97805	DL_DOCIDS, /* docids only */
97806	DL_POSITIONS, /* docids + positions */
97807	DL_POSITIONS_OFFSETS /* docids + positions + offsets */
97808	} DocListType;
	@@ -98050,15 +97824,10 @@
97824	POS_END = 0, /* end of this position list */
97825	POS_COLUMN, /* followed by new column number */
97826	POS_BASE
97827	};
97828





97829	/* utility functions */
97830
97831	/* CLEAR() and SCRAMBLE() abstract memset() on a pointer to a single
97832	** record to prevent errors of the form:
97833	**
	@@ -98073,2225 +97842,71 @@
97842	# define SCRAMBLE(b) memset(b, 0x55, sizeof(*(b)))
97843	#else
97844	# define SCRAMBLE(b)
97845	#endif
97846
97847	/*
97848	** Write a 64-bit variable-length integer to memory starting at p[0].
97849	** The length of data written will be between 1 and FTS3_VARINT_MAX bytes.
97850	** The number of bytes written is returned.
97851	*/
97852	SQLITE_PRIVATE int sqlite3Fts3PutVarint(char *p, sqlite_int64 v){

97853	unsigned char q = (unsigned char ) p;
97854	sqlite_uint64 vu = v;
97855	do{
97856	*q++ = (unsigned char) ((vu & 0x7f) \| 0x80);
97857	vu >>= 7;
97858	}while( vu!=0 );
97859	q[-1] &= 0x7f; /* turn off high bit in final byte */
97860	assert( q - (unsigned char *)p <= FTS3_VARINT_MAX );
97861	return (int) (q - (unsigned char *)p);
97862	}
97863
97864	/*
97865	** Read a 64-bit variable-length integer from memory starting at p[0].
97866	** Return the number of bytes read, or 0 on error.
97867	** The value is stored in *v.
97868	*/
97869	SQLITE_PRIVATE int sqlite3Fts3GetVarint(const char p, sqlite_int64 v){
97870	const unsigned char q = (const unsigned char ) p;
97871	sqlite_uint64 x = 0, y = 1;
97872	while( (*q & 0x80) == 0x80 ){
97873	x += y * (*q++ & 0x7f);
97874	y <<= 7;
97875	if( q - (unsigned char )p >= FTS3_VARINT_MAX ){ / bad data */
97876	assert( 0 );
97877	return 0;
97878	}
97879	}
97880	x += y * (*q++);
97881	*v = (sqlite_int64) x;
97882	return (int) (q - (unsigned char *)p);
97883	}
97884
97885	/*
97886	** Similar to sqlite3Fts3GetVarint(), except that the output is truncated to a
97887	** 32-bit integer before it is returned.
97888	*/
97889	SQLITE_PRIVATE int sqlite3Fts3GetVarint32(const char p, int pi){
97890	sqlite_int64 i;
97891	int ret = sqlite3Fts3GetVarint(p, &i);
97892	*pi = (int) i;
97893	assert( *pi==i );
97894	return ret;
97895	}
97896
97897	/*
97898	** Return the number of bytes required to store the value passed as the
97899	** first argument in varint form.
97900	*/
97901	SQLITE_PRIVATE int sqlite3Fts3VarintLen(sqlite3_uint64 v){
97902	int i = 0;
97903	do{
97904	i++;
97905	v >>= 7;
97906	}while( v!=0 );
97907	return i;















































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































97908	}
97909
97910	/*
97911	** Convert an SQL-style quoted string into a normal string by removing
97912	** the quote characters. The conversion is done in-place. If the
	@@ -100303,14 +97918,14 @@
97918	** "abc" becomes abc
97919	** 'xyz' becomes xyz
97920	** [pqr] becomes pqr
97921	** `mno` becomes mno
97922	*/
97923	SQLITE_PRIVATE void sqlite3Fts3Dequote(char *z){
97924	int quote;
97925	int i, j;
97926
97927	quote = z[0];
97928	switch( quote ){
97929	case '\'': break;
97930	case '"': break;
97931	case '`': break; /* For MySQL compatibility */
	@@ -100330,1363 +97945,1413 @@
97945	z[j++] = z[i];
97946	}
97947	}
97948	}
97949
97950	static void fts3GetDeltaVarint(char *pp, sqlite3_int64 pVal){
97951	sqlite3_int64 iVal;
97952	pp += sqlite3Fts3GetVarint(pp, &iVal);
97953	*pVal += iVal;
97954	}
97955
97956	static void fts3GetDeltaVarint2(char *pp, char pEnd, sqlite3_int64 *pVal){
97957	if( *pp>=pEnd ){
97958	*pp = 0;
97959	}else{
97960	fts3GetDeltaVarint(pp, pVal);
97961	}
97962	}
97963
97964
97965	/*
97966	** The Fts3Cursor.eType member is always set to one of the following.
97967	*/
97968	#define FTS3_FULLSCAN_SEARCH 0 /* Linear scan of %_content table */
97969	#define FTS3_DOCID_SEARCH 1 /* Lookup by rowid on %_content table */
97970	#define FTS3_FULLTEXT_SEARCH 2 /* Full-text index search */
97971
97972	static Fts3Table cursor_vtab(Fts3Cursor c){
97973	return (Fts3Table *) c->base.pVtab;
97974	}
97975
97976	/*
97977	** The xDisconnect() virtual table method.
97978	*/
97979	static int fts3DisconnectMethod(sqlite3_vtab *pVtab){
97980	Fts3Table p = (Fts3Table )pVtab;
97981	int i;
97982
97983	assert( p->nPendingData==0 );
97984
97985	/* Free any prepared statements held */
97986	for(i=0; i<SizeofArray(p->aStmt); i++){
97987	sqlite3_finalize(p->aStmt[i]);
97988	}
97989	for(i=0; i<p->nLeavesStmt; i++){
97990	sqlite3_finalize(p->aLeavesStmt[i]);
97991	}
97992	sqlite3_free(p->zSelectLeaves);
97993	sqlite3_free(p->aLeavesStmt);
97994
97995	/* Invoke the tokenizer destructor to free the tokenizer. */
97996	p->pTokenizer->pModule->xDestroy(p->pTokenizer);
97997
97998	sqlite3_free(p);
97999	return SQLITE_OK;
98000	}
98001
98002	/*
98003	** The xDestroy() virtual table method.
98004	*/
98005	static int fts3DestroyMethod(sqlite3_vtab *pVtab){
98006	int rc; /* Return code */
98007	Fts3Table p = (Fts3Table )pVtab;
98008
98009	/* Create a script to drop the underlying three storage tables. */
98010	char *zSql = sqlite3_mprintf(
98011	"DROP TABLE IF EXISTS %Q.'%q_content';"
98012	"DROP TABLE IF EXISTS %Q.'%q_segments';"
98013	"DROP TABLE IF EXISTS %Q.'%q_segdir';",
98014	p->zDb, p->zName, p->zDb, p->zName, p->zDb, p->zName
98015	);
98016
98017	/* If malloc has failed, set rc to SQLITE_NOMEM. Otherwise, try to
98018	** execute the SQL script created above.
98019	*/
98020	if( zSql ){
98021	rc = sqlite3_exec(p->db, zSql, 0, 0, 0);
98022	sqlite3_free(zSql);
98023	}else{
98024	rc = SQLITE_NOMEM;
98025	}
98026
98027	/* If everything has worked, invoke fts3DisconnectMethod() to free the
98028	** memory associated with the Fts3Table structure and return SQLITE_OK.
98029	** Otherwise, return an SQLite error code.
98030	*/
98031	return (rc==SQLITE_OK ? fts3DisconnectMethod(pVtab) : rc);
98032	}
98033
98034
98035	/*
98036	** Invoke sqlite3_declare_vtab() to declare the schema for the FTS3 table
98037	** passed as the first argument. This is done as part of the xConnect()
98038	** and xCreate() methods.
98039	*/
98040	static int fts3DeclareVtab(Fts3Table *p){
98041	int i; /* Iterator variable */
98042	int rc; /* Return code */
98043	char zSql; / SQL statement passed to declare_vtab() */
98044	char zCols; / List of user defined columns */
98045
98046	/* Create a list of user columns for the virtual table */
98047	zCols = sqlite3_mprintf("%Q, ", p->azColumn[0]);
98048	for(i=1; zCols && i<p->nColumn; i++){
98049	zCols = sqlite3_mprintf("%z%Q, ", zCols, p->azColumn[i]);
98050	}
98051
98052	/* Create the whole "CREATE TABLE" statement to pass to SQLite */
98053	zSql = sqlite3_mprintf(
98054	"CREATE TABLE x(%s %Q HIDDEN, docid HIDDEN)", zCols, p->zName
98055	);
98056
98057	if( !zCols \|\| !zSql ){
98058	rc = SQLITE_NOMEM;
98059	}else{
98060	rc = sqlite3_declare_vtab(p->db, zSql);
98061	}
98062
98063	sqlite3_free(zSql);
98064	sqlite3_free(zCols);
98065	return rc;
98066	}
98067
98068	/*
98069	** Create the backing store tables (%_content, %_segments and %_segdir)
98070	** required by the FTS3 table passed as the only argument. This is done
98071	** as part of the vtab xCreate() method.
98072	*/
98073	static int fts3CreateTables(Fts3Table *p){
98074	int rc; /* Return code */
98075	int i; /* Iterator variable */
98076	char zContentCols; / Columns of %_content table */
98077	char zSql; / SQL script to create required tables */
98078
98079	/* Create a list of user columns for the content table */
98080	zContentCols = sqlite3_mprintf("docid INTEGER PRIMARY KEY");
98081	for(i=0; zContentCols && i<p->nColumn; i++){
98082	char *z = p->azColumn[i];
98083	zContentCols = sqlite3_mprintf("%z, 'c%d%q'", zContentCols, i, z);
98084	}
98085
98086	/* Create the whole SQL script */
98087	zSql = sqlite3_mprintf(
98088	"CREATE TABLE %Q.'%q_content'(%s);"
98089	"CREATE TABLE %Q.'%q_segments'(blockid INTEGER PRIMARY KEY, block BLOB);"
98090	"CREATE TABLE %Q.'%q_segdir'("
98091	"level INTEGER,"
98092	"idx INTEGER,"
98093	"start_block INTEGER,"
98094	"leaves_end_block INTEGER,"
98095	"end_block INTEGER,"
98096	"root BLOB,"
98097	"PRIMARY KEY(level, idx)"
98098	");",
98099	p->zDb, p->zName, zContentCols, p->zDb, p->zName, p->zDb, p->zName
98100	);
98101
98102	/* Unless a malloc() failure has occurred, execute the SQL script to
98103	** create the tables used to store data for this FTS3 virtual table.
98104	*/
98105	if( zContentCols==0 \|\| zSql==0 ){
98106	rc = SQLITE_NOMEM;
98107	}else{
98108	rc = sqlite3_exec(p->db, zSql, 0, 0, 0);
98109	}
98110
98111	sqlite3_free(zSql);
98112	sqlite3_free(zContentCols);
98113	return rc;
98114	}
98115
98116	/*
98117	** This function is the implementation of both the xConnect and xCreate
98118	** methods of the FTS3 virtual table.
98119	**
98120	** The argv[] array contains the following:
98121	**
98122	** argv[0] -> module name
98123	** argv[1] -> database name
98124	** argv[2] -> table name
98125	** argv[...] -> "column name" fields...
98126	*/
98127	int fts3InitVtab(
98128	int isCreate, /* True for xCreate, false for xConnect */
98129	sqlite3 db, / The SQLite database connection */
98130	void pAux, / Hash table containing tokenizers */
98131	int argc, /* Number of elements in argv array */
98132	const char * const argv, / xCreate/xConnect argument array */
98133	sqlite3_vtab *ppVTab, / Write the resulting vtab structure here */
98134	char *pzErr / Write any error message here */
98135	){
98136	Fts3Hash pHash = (Fts3Hash )pAux;
98137	Fts3Table p; / Pointer to allocated vtab */
98138	int rc; /* Return code */
98139	int i;
98140	int nByte;



































































































































































































































































































































































































































































































































































































































































































































98141	int iCol;
98142	int nString = 0;
98143	int nCol = 0;
98144	char *zCsr;
98145	int nDb;
98146	int nName;
98147
98148	const char *zTokenizer = 0;
98149	sqlite3_tokenizer pTokenizer; / Tokenizer for this table */
98150
98151	nDb = strlen(argv[1]) + 1;
98152	nName = strlen(argv[2]) + 1;
98153	for(i=3; i<argc; i++){
98154	char const *z = argv[i];
98155	rc = sqlite3Fts3InitTokenizer(pHash, z, &pTokenizer, &zTokenizer, pzErr);
98156	if( rc!=SQLITE_OK ){
98157	return rc;
98158	}
98159	if( z!=zTokenizer ){
98160	nString += strlen(z) + 1;
98161	}
98162	}
98163	nCol = argc - 3 - (zTokenizer!=0);
98164	if( zTokenizer==0 ){
98165	rc = sqlite3Fts3InitTokenizer(pHash, 0, &pTokenizer, 0, pzErr);
98166	if( rc!=SQLITE_OK ){
98167	return rc;
98168	}
98169	assert( pTokenizer );
98170	}
98171
98172	/* Allocate and populate the Fts3Table structure. */
98173	nByte = sizeof(Fts3Table) + /* Fts3Table */
98174	nCol * sizeof(char ) + / azColumn */
98175	nName + /* zName */
98176	nDb + /* zDb */
98177	nString; /* Space for azColumn strings */
98178	p = (Fts3Table*)sqlite3_malloc(nByte);
98179	if( p==0 ){
98180	rc = SQLITE_NOMEM;
98181	goto fts3_init_out;
98182	}
98183	memset(p, 0, nByte);
98184
98185	p->db = db;
98186	p->nColumn = nCol;
98187	p->nPendingData = 0;
98188	p->azColumn = (char **)&p[1];
98189	p->pTokenizer = pTokenizer;
98190	zCsr = (char *)&p->azColumn[nCol];
98191
98192	fts3HashInit(&p->pendingTerms, FTS3_HASH_STRING, 1);
98193
98194	/* Fill in the zName and zDb fields of the vtab structure. */
98195	p->zName = zCsr;
98196	memcpy(zCsr, argv[2], nName);
98197	zCsr += nName;
98198	p->zDb = zCsr;
98199	memcpy(zCsr, argv[1], nDb);
98200	zCsr += nDb;
98201
98202	/* Fill in the azColumn array */
98203	iCol = 0;
98204	for(i=3; i<argc; i++){
98205	if( argv[i]!=zTokenizer ){
98206	char *z;
98207	int n;
98208	z = (char *)sqlite3Fts3NextToken(argv[i], &n);
98209	memcpy(zCsr, z, n);
98210	zCsr[n] = '\0';
98211	sqlite3Fts3Dequote(zCsr);
98212	p->azColumn[iCol++] = zCsr;
98213	zCsr += n+1;
98214	assert( zCsr <= &((char *)p)[nByte] );
98215	}
98216	}
98217
98218	/* If this is an xCreate call, create the underlying tables in the
98219	** database. TODO: For xConnect(), it could verify that said tables exist.
98220	*/
98221	if( isCreate ){
98222	rc = fts3CreateTables(p);
98223	if( rc!=SQLITE_OK ) goto fts3_init_out;
98224	}
98225
98226	rc = fts3DeclareVtab(p);
98227	if( rc!=SQLITE_OK ) goto fts3_init_out;
98228
98229	*ppVTab = &p->base;
98230
98231	fts3_init_out:
98232	if( rc!=SQLITE_OK ){
98233	if( p ) fts3DisconnectMethod((sqlite3_vtab *)p);
98234	else if( pTokenizer ) pTokenizer->pModule->xDestroy(pTokenizer);
98235	}
98236	return rc;
98237	}
98238
98239	/*
98240	** The xConnect() and xCreate() methods for the virtual table. All the
98241	** work is done in function fts3InitVtab().
98242	*/
98243	static int fts3ConnectMethod(
98244	sqlite3 db, / Database connection */
98245	void pAux, / Pointer to tokenizer hash table */
98246	int argc, /* Number of elements in argv array */
98247	const char * const argv, / xCreate/xConnect argument array */
98248	sqlite3_vtab *ppVtab, / OUT: New sqlite3_vtab object */
98249	char *pzErr / OUT: sqlite3_malloc'd error message */
98250	){
98251	return fts3InitVtab(0, db, pAux, argc, argv, ppVtab, pzErr);
98252	}
98253	static int fts3CreateMethod(
98254	sqlite3 db, / Database connection */
98255	void pAux, / Pointer to tokenizer hash table */
98256	int argc, /* Number of elements in argv array */
98257	const char * const argv, / xCreate/xConnect argument array */
98258	sqlite3_vtab *ppVtab, / OUT: New sqlite3_vtab object */
98259	char *pzErr / OUT: sqlite3_malloc'd error message */
98260	){
98261	return fts3InitVtab(1, db, pAux, argc, argv, ppVtab, pzErr);
98262	}
98263
98264	/*
98265	** Implementation of the xBestIndex method for FTS3 tables. There
98266	** are three possible strategies, in order of preference:
98267	**
98268	** 1. Direct lookup by rowid or docid.
98269	** 2. Full-text search using a MATCH operator on a non-docid column.
98270	** 3. Linear scan of %_content table.
98271	*/
98272	static int fts3BestIndexMethod(sqlite3_vtab pVTab, sqlite3_index_info pInfo){
98273	Fts3Table p = (Fts3Table )pVTab;
98274	int i; /* Iterator variable */
98275	int iCons = -1; /* Index of constraint to use */
98276
98277	/* By default use a full table scan. This is an expensive option,
98278	** so search through the constraints to see if a more efficient
98279	** strategy is possible.
98280	*/
98281	pInfo->idxNum = FTS3_FULLSCAN_SEARCH;
98282	pInfo->estimatedCost = 500000;
98283	for(i=0; i<pInfo->nConstraint; i++){
98284	struct sqlite3_index_constraint *pCons = &pInfo->aConstraint[i];
98285	if( pCons->usable==0 ) continue;
98286
98287	/* A direct lookup on the rowid or docid column. This is the best
98288	** strategy in all cases. Assign a cost of 1.0 and return early.
98289	*/
98290	if( pCons->op==SQLITE_INDEX_CONSTRAINT_EQ
98291	&& (pCons->iColumn<0 \|\| pCons->iColumn==p->nColumn+1 )
98292	){
98293	pInfo->idxNum = FTS3_DOCID_SEARCH;
98294	pInfo->estimatedCost = 1.0;
98295	iCons = i;
98296	break;
98297	}
98298
98299	/* A MATCH constraint. Use a full-text search.
98300	**
98301	** If there is more than one MATCH constraint available, use the first
98302	** one encountered. If there is both a MATCH constraint and a direct
98303	** rowid/docid lookup, prefer the rowid/docid strategy.
98304	*/
98305	if( iCons<0
98306	&& pCons->op==SQLITE_INDEX_CONSTRAINT_MATCH
98307	&& pCons->iColumn>=0 && pCons->iColumn<=p->nColumn
98308	){
98309	pInfo->idxNum = FTS3_FULLTEXT_SEARCH + pCons->iColumn;
98310	pInfo->estimatedCost = 2.0;
98311	iCons = i;
98312	}
98313	}
98314
98315	if( iCons>=0 ){
98316	pInfo->aConstraintUsage[iCons].argvIndex = 1;
98317	pInfo->aConstraintUsage[iCons].omit = 1;
98318	}
98319	return SQLITE_OK;
98320	}
98321
98322	/*
98323	** Implementation of xOpen method.
98324	*/
98325	static int fts3OpenMethod(sqlite3_vtab pVTab, sqlite3_vtab_cursor *ppCsr){
98326	sqlite3_vtab_cursor pCsr; / Allocated cursor */
98327
98328	/* Allocate a buffer large enough for an Fts3Cursor structure. If the
98329	** allocation succeeds, zero it and return SQLITE_OK. Otherwise,
98330	** if the allocation fails, return SQLITE_NOMEM.
98331	*/
98332	ppCsr = pCsr = (sqlite3_vtab_cursor )sqlite3_malloc(sizeof(Fts3Cursor));
98333	if( !pCsr ){
98334	return SQLITE_NOMEM;
98335	}
98336	memset(pCsr, 0, sizeof(Fts3Cursor));
98337	return SQLITE_OK;
98338	}
98339
98340	/****************************************************************/
98341	/****************************************************************/
98342	/****************************************************************/
98343	/****************************************************************/
98344
98345
98346	/*
98347	** Close the cursor. For additional information see the documentation
98348	** on the xClose method of the virtual table interface.
98349	*/
98350	static int fulltextClose(sqlite3_vtab_cursor *pCursor){
98351	Fts3Cursor pCsr = (Fts3Cursor )pCursor;
98352	sqlite3_finalize(pCsr->pStmt);
98353	sqlite3Fts3ExprFree(pCsr->pExpr);
98354	sqlite3_free(pCsr->aDoclist);
98355	sqlite3_free(pCsr);
98356	return SQLITE_OK;
98357	}
98358
98359	static int fts3CursorSeek(Fts3Cursor *pCsr){
98360	if( pCsr->isRequireSeek ){
98361	pCsr->isRequireSeek = 0;
98362	sqlite3_bind_int64(pCsr->pStmt, 1, pCsr->iPrevId);
98363	if( SQLITE_ROW==sqlite3_step(pCsr->pStmt) ){
98364	return SQLITE_OK;
98365	}else{
98366	int rc;
98367	pCsr->isEof = 1;
98368	if( SQLITE_OK==(rc = sqlite3_reset(pCsr->pStmt)) ){
98369	rc = SQLITE_ERROR;
98370	}
98371	return rc;
98372	}
98373	}else{
98374	return SQLITE_OK;
98375	}
98376	}
98377
98378	static int fts3NextMethod(sqlite3_vtab_cursor *pCursor){
98379	int rc = SQLITE_OK; /* Return code */
98380	Fts3Cursor pCsr = (Fts3Cursor )pCursor;
98381
98382	if( pCsr->aDoclist==0 ){
98383	if( SQLITE_ROW!=sqlite3_step(pCsr->pStmt) ){
98384	pCsr->isEof = 1;
98385	rc = sqlite3_reset(pCsr->pStmt);
98386	}
98387	}else if( pCsr->pNextId>=&pCsr->aDoclist[pCsr->nDoclist] ){
98388	pCsr->isEof = 1;
98389	}else{
98390	sqlite3_reset(pCsr->pStmt);
98391	fts3GetDeltaVarint(&pCsr->pNextId, &pCsr->iPrevId);
98392	pCsr->isRequireSeek = 1;
98393	}
98394	return rc;
98395	}
98396
98397
98398	/*
98399	** The buffer pointed to by argument zNode (size nNode bytes) contains the
98400	** root node of a b-tree segment. The segment is guaranteed to be at least
98401	** one level high (i.e. the root node is not also a leaf). If successful,
98402	** this function locates the leaf node of the segment that may contain the
98403	** term specified by arguments zTerm and nTerm and writes its block number
98404	** to *piLeaf.
98405	**
98406	** It is possible that the returned leaf node does not contain the specified
98407	** term. However, if the segment does contain said term, it is stored on
98408	** the identified leaf node. Because this function only inspects interior
98409	** segment nodes (and never loads leaf nodes into memory), it is not possible
98410	** to be sure.
98411	**
98412	** If an error occurs, an error code other than SQLITE_OK is returned.
98413	*/
98414	static int fts3SelectLeaf(
98415	Fts3Table p, / Virtual table handle */
98416	const char zTerm, / Term to select leaves for */
98417	int nTerm, /* Size of term zTerm in bytes */
98418	const char zNode, / Buffer containing segment interior node */
98419	int nNode, /* Size of buffer at zNode */
98420	sqlite3_int64 piLeaf / Selected leaf node */
98421	){
98422	int rc = SQLITE_OK; /* Return code */
98423	const char zCsr = zNode; / Cursor to iterate through node */
98424	const char zEnd = &zCsr[nNode];/ End of interior node buffer */
98425	char zBuffer = 0; / Buffer to load terms into */
98426	int nAlloc = 0; /* Size of allocated buffer */
98427
98428	while( 1 ){
98429	int isFirstTerm = 1; /* True when processing first term on page */
98430	int iHeight; /* Height of this node in tree */
98431	sqlite3_int64 iChild; /* Block id of child node to descend to */
98432	int nBlock; /* Size of child node in bytes */
98433
98434	zCsr += sqlite3Fts3GetVarint32(zCsr, &iHeight);
98435	zCsr += sqlite3Fts3GetVarint(zCsr, &iChild);
98436
98437	while( zCsr<zEnd ){
98438	int cmp; /* memcmp() result */
98439	int nSuffix; /* Size of term suffix */
98440	int nPrefix = 0; /* Size of term prefix */
98441	int nBuffer; /* Total term size */
98442
98443	/* Load the next term on the node into zBuffer */
98444	if( !isFirstTerm ){
98445	zCsr += sqlite3Fts3GetVarint32(zCsr, &nPrefix);
98446	}
98447	isFirstTerm = 0;
98448	zCsr += sqlite3Fts3GetVarint32(zCsr, &nSuffix);
98449	if( nPrefix+nSuffix>nAlloc ){
98450	char *zNew;
98451	nAlloc = (nPrefix+nSuffix) * 2;
98452	zNew = (char *)sqlite3_realloc(zBuffer, nAlloc);
98453	if( !zNew ){
98454	sqlite3_free(zBuffer);
98455	return SQLITE_NOMEM;
98456	}
98457	zBuffer = zNew;
98458	}
98459	memcpy(&zBuffer[nPrefix], zCsr, nSuffix);
98460	nBuffer = nPrefix + nSuffix;
98461	zCsr += nSuffix;
98462
98463	/* Compare the term we are searching for with the term just loaded from
98464	** the interior node. If the specified term is greater than or equal
98465	** to the term from the interior node, then all terms on the sub-tree
98466	** headed by node iChild are smaller than zTerm. No need to search
98467	** iChild.
98468	**
98469	** If the interior node term is larger than the specified term, then
98470	** the tree headed by iChild may contain the specified term.
98471	*/
98472	cmp = memcmp(zTerm, zBuffer, (nBuffer>nTerm ? nTerm : nBuffer));
98473	if( cmp<0 \|\| (cmp==0 && nBuffer>nTerm) ) break;
98474	iChild++;
98475	};
98476
98477	/* If (iHeight==1), the children of this interior node are leaves. The
98478	** specified term may be present on leaf node iChild.
98479	*/
98480	if( iHeight==1 ){
98481	*piLeaf = iChild;
98482	break;
98483	}
98484
98485	/* Descend to interior node iChild. */
98486	rc = sqlite3Fts3ReadBlock(p, iChild, &zCsr, &nBlock);
98487	if( rc!=SQLITE_OK ) break;
98488	zEnd = &zCsr[nBlock];
98489	}
98490	sqlite3_free(zBuffer);
98491	return rc;
98492	}
98493
98494	static void fts3PutDeltaVarint(
98495	char **pp,
98496	sqlite3_int64 *piPrev,
98497	sqlite3_int64 iVal
98498	){
98499	assert( iVal-*piPrev > 0 );
98500	pp += sqlite3Fts3PutVarint(pp, iVal-*piPrev);
98501	*piPrev = iVal;
98502	}
98503
98504	static void fts3PoslistCopy(char pp, char ppPoslist){
98505	char pEnd = ppPoslist;
98506	char c = 0;
98507	while( pEnd \| c ) c = pEnd++ & 0x80;
98508	pEnd++;
98509	if( pp ){
98510	int n = pEnd - *ppPoslist;
98511	char p = pp;
98512	memcpy(p, *ppPoslist, n);
98513	p += n;
98514	*pp = p;
98515	}
98516	*ppPoslist = pEnd;
98517	}
98518
98519	static void fts3ColumnlistCopy(char pp, char ppPoslist){
98520	char pEnd = ppPoslist;
98521	char c = 0;
98522	while( 0xFE & (pEnd \| c) ) c = pEnd++ & 0x80;
98523	if( pp ){
98524	int n = pEnd - *ppPoslist;
98525	char p = pp;
98526	memcpy(p, *ppPoslist, n);
98527	p += n;
98528	*pp = p;
98529	}
98530	*ppPoslist = pEnd;
98531	}
98532
98533	/*
98534	**
98535	*/
98536	static void fts3PoslistMerge(
98537	char *pp, / Output buffer */
98538	char *pp1, / Left input list */
98539	char *pp2 / Right input list */
98540	){
98541	char p = pp;
98542	char p1 = pp1;
98543	char p2 = pp2;
98544
98545	while( p1 && p2 ){
98546	int iCol1 = 0;
98547	int iCol2 = 0;
98548	if( *p1==0x01 ) sqlite3Fts3GetVarint32(&p1[1], &iCol1);
98549	if( *p2==0x01 ) sqlite3Fts3GetVarint32(&p2[1], &iCol2);
98550
98551	if( iCol1==iCol2 ){
98552	sqlite3_int64 i1 = 0;
98553	sqlite3_int64 i2 = 0;
98554	sqlite3_int64 iPrev = 0;
98555	if( iCol1!=0 ){
98556	int n;
98557	*p++ = 0x01;
98558	n = sqlite3Fts3PutVarint(p, iCol1);
98559	p += n;
98560	p1 += 1 + n;
98561	p2 += 1 + n;
98562	}
98563	while( (p1&0xFE) \|\| (p2&0xFE) ){
98564	if( i1==i2 ){
98565	fts3GetDeltaVarint(&p1, &i1); i1 -= 2;
98566	fts3GetDeltaVarint(&p2, &i2); i2 -= 2;
98567	}else if( i1<i2 ){
98568	fts3GetDeltaVarint(&p1, &i1); i1 -= 2;
98569	}else{
98570	fts3GetDeltaVarint(&p2, &i2); i2 -= 2;
98571	}
98572	fts3PutDeltaVarint(&p, &iPrev, (i1<i2 ? i1 : i2) + 2); iPrev -= 2;
98573	if( 0==(*p1&0xFE) ) i1 = 0x7FFFFFFF;
98574	if( 0==(*p2&0xFE) ) i2 = 0x7FFFFFFF;
98575	}
98576	}else if( iCol1<iCol2 ){
98577	fts3ColumnlistCopy(&p, &p1);
98578	}else{
98579	fts3ColumnlistCopy(&p, &p2);
98580	}
98581	}
98582
98583	*p++ = '\0';
98584	*pp = p;
98585	*pp1 = p1 + 1;
98586	*pp2 = p2 + 1;
98587	}
98588
98589	/*
98590	** nToken==1 searches for adjacent positions.
98591	*/
98592	static int fts3PoslistPhraseMerge(
98593	char *pp, / Output buffer */
98594	int nToken, /* Maximum difference in token positions */
98595	int isSaveLeft, /* Save the left position */
98596	char *pp1, / Left input list */
98597	char *pp2 / Right input list */
98598	){
98599	char p = (pp ? pp : 0);
98600	char p1 = pp1;
98601	char p2 = pp2;
98602
98603	int iCol1 = 0;
98604	int iCol2 = 0;
98605	assert( p1!=0 && p2!=0 );
98606	if( *p1==0x01 ){
98607	p1++;
98608	p1 += sqlite3Fts3GetVarint32(p1, &iCol1);
98609	}
98610	if( *p2==0x01 ){
98611	p2++;
98612	p2 += sqlite3Fts3GetVarint32(p2, &iCol2);
98613	}
98614
98615	while( 1 ){
98616	if( iCol1==iCol2 ){
98617	char *pSave = p;
98618	sqlite3_int64 iPrev = 0;
98619	sqlite3_int64 iPos1 = 0;
98620	sqlite3_int64 iPos2 = 0;
98621
98622	if( pp && iCol1 ){
98623	*p++ = 0x01;
98624	p += sqlite3Fts3PutVarint(p, iCol1);
98625	}
98626
98627	assert( p1!=0x00 && p2!=0x00 && p1!=0x01 && p2!=0x01 );
98628	fts3GetDeltaVarint(&p1, &iPos1); iPos1 -= 2;
98629	fts3GetDeltaVarint(&p2, &iPos2); iPos2 -= 2;
98630
98631	while( 1 ){
98632	if( iPos2>iPos1 && iPos2<=iPos1+nToken ){
98633	sqlite3_int64 iSave;
98634	if( !pp ){
98635	fts3PoslistCopy(0, &p2);
98636	fts3PoslistCopy(0, &p1);
98637	*pp1 = p1;
98638	*pp2 = p2;
98639	return 1;
98640	}
98641	iSave = isSaveLeft ? iPos1 : iPos2;
98642	fts3PutDeltaVarint(&p, &iPrev, iSave+2); iPrev -= 2;
98643	pSave = 0;
98644	}
98645	if( iPos2<=iPos1 ){
98646	if( (*p2&0xFE)==0 ) break;
98647	fts3GetDeltaVarint(&p2, &iPos2); iPos2 -= 2;
98648	}else{
98649	if( (*p1&0xFE)==0 ) break;
98650	fts3GetDeltaVarint(&p1, &iPos1); iPos1 -= 2;
98651	}
98652	}
98653	if( pSave && pp ){
98654	p = pSave;
98655	}
98656
98657	fts3ColumnlistCopy(0, &p1);
98658	fts3ColumnlistCopy(0, &p2);
98659	assert( (p1&0xFE)==0 && (p2&0xFE)==0 );
98660	if( 0==p1 \|\| 0==p2 ) break;
98661
98662	p1++;
98663	p1 += sqlite3Fts3GetVarint32(p1, &iCol1);
98664	p2++;
98665	p2 += sqlite3Fts3GetVarint32(p2, &iCol2);
98666	}
98667
98668	/* Advance pointer p1 or p2 (whichever corresponds to the smaller of
98669	** iCol1 and iCol2) so that it points to either the 0x00 that marks the
98670	** end of the position list, or the 0x01 that precedes the next
98671	** column-number in the position list.
98672	*/
98673	else if( iCol1<iCol2 ){
98674	fts3ColumnlistCopy(0, &p1);
98675	if( 0==*p1 ) break;
98676	p1++;
98677	p1 += sqlite3Fts3GetVarint32(p1, &iCol1);
98678	}else{
98679	fts3ColumnlistCopy(0, &p2);
98680	if( 0==*p2 ) break;
98681	p2++;
98682	p2 += sqlite3Fts3GetVarint32(p2, &iCol2);
98683	}
98684	}
98685
98686	fts3PoslistCopy(0, &p2);
98687	fts3PoslistCopy(0, &p1);
98688	*pp1 = p1;
98689	*pp2 = p2;
98690	if( !pp \|\| *pp==p ){
98691	return 0;
98692	}
98693	*p++ = 0x00;
98694	*pp = p;
98695	return 1;
98696	}
98697
98698	static int fts3PoslistNearMerge(
98699	char *pp, / Output buffer */
98700	char aTmp, / Temporary buffer space */
98701	int nRight, /* Maximum difference in token positions */
98702	int nLeft, /* Maximum difference in token positions */
98703	char *pp1, / Left input list */
98704	char *pp2 / Right input list */
98705	){
98706	char p1 = pp1;
98707	char p2 = pp2;
98708
98709	if( !pp ){
98710	if( fts3PoslistPhraseMerge(0, nRight, 0, pp1, pp2) ) return 1;
98711	*pp1 = p1;
98712	*pp2 = p2;
98713	return fts3PoslistPhraseMerge(0, nLeft, 0, pp2, pp1);
98714	}else{
98715	char *pTmp1 = aTmp;
98716	char *pTmp2;
98717	char *aTmp2;
98718	int res = 1;
98719
98720	fts3PoslistPhraseMerge(&pTmp1, nRight, 0, pp1, pp2);
98721	aTmp2 = pTmp2 = pTmp1;
98722	*pp1 = p1;
98723	*pp2 = p2;
98724	fts3PoslistPhraseMerge(&pTmp2, nLeft, 1, pp2, pp1);
98725	if( pTmp1!=aTmp && pTmp2!=aTmp2 ){
98726	fts3PoslistMerge(pp, &aTmp, &aTmp2);
98727	}else if( pTmp1!=aTmp ){
98728	fts3PoslistCopy(pp, &aTmp);
98729	}else if( pTmp2!=aTmp2 ){
98730	fts3PoslistCopy(pp, &aTmp2);
98731	}else{
98732	res = 0;
98733	}
98734
98735	return res;
98736	}
98737	}
98738
98739	/*
98740	** Values that may be used as the first parameter to fts3DoclistMerge().
98741	*/
98742	#define MERGE_NOT 2 /* D + D -> D */
98743	#define MERGE_AND 3 /* D + D -> D */
98744	#define MERGE_OR 4 /* D + D -> D */
98745	#define MERGE_POS_OR 5 /* P + P -> P */
98746	#define MERGE_PHRASE 6 /* P + P -> D */
98747	#define MERGE_POS_PHRASE 7 /* P + P -> P */
98748	#define MERGE_NEAR 8 /* P + P -> D */
98749	#define MERGE_POS_NEAR 9 /* P + P -> P */
98750
98751	/*
98752	** Merge the two doclists passed in buffer a1 (size n1 bytes) and a2
98753	** (size n2 bytes). The output is written to pre-allocated buffer aBuffer,
98754	** which is guaranteed to be large enough to hold the results. The number
98755	** of bytes written to aBuffer is stored in *pnBuffer before returning.
98756	**
98757	** If successful, SQLITE_OK is returned. Otherwise, if a malloc error
98758	** occurs while allocating a temporary buffer as part of the merge operation,
98759	** SQLITE_NOMEM is returned.
98760	*/
98761	static int fts3DoclistMerge(
98762	int mergetype, /* One of the MERGE_XXX constants */
98763	int nParam1, /* Used by MERGE_NEAR and MERGE_POS_NEAR */
98764	int nParam2, /* Used by MERGE_NEAR and MERGE_POS_NEAR */
98765	char aBuffer, / Pre-allocated output buffer */
98766	int pnBuffer, / OUT: Bytes written to aBuffer */
98767	char a1, / Buffer containing first doclist */
98768	int n1, /* Size of buffer a1 */
98769	char a2, / Buffer containing second doclist */
98770	int n2 /* Size of buffer a2 */
98771	){
98772	sqlite3_int64 i1 = 0;
98773	sqlite3_int64 i2 = 0;
98774	sqlite3_int64 iPrev = 0;
98775
98776	char *p = aBuffer;
98777	char *p1 = a1;
98778	char *p2 = a2;
98779	char *pEnd1 = &a1[n1];
98780	char *pEnd2 = &a2[n2];
98781
98782	assert( mergetype==MERGE_OR \|\| mergetype==MERGE_POS_OR
98783	\|\| mergetype==MERGE_AND \|\| mergetype==MERGE_NOT
98784	\|\| mergetype==MERGE_PHRASE \|\| mergetype==MERGE_POS_PHRASE
98785	\|\| mergetype==MERGE_NEAR \|\| mergetype==MERGE_POS_NEAR
98786	);
98787
98788	if( !aBuffer ){
98789	return SQLITE_NOMEM;
98790	}
98791
98792	/* Read the first docid from each doclist */
98793	fts3GetDeltaVarint2(&p1, pEnd1, &i1);
98794	fts3GetDeltaVarint2(&p2, pEnd2, &i2);
98795
98796	switch( mergetype ){
98797	case MERGE_OR:
98798	case MERGE_POS_OR:
98799	while( p1 \|\| p2 ){
98800	if( p2 && p1 && i1==i2 ){
98801	fts3PutDeltaVarint(&p, &iPrev, i1);
98802	if( mergetype==MERGE_POS_OR ) fts3PoslistMerge(&p, &p1, &p2);
98803	fts3GetDeltaVarint2(&p1, pEnd1, &i1);
98804	fts3GetDeltaVarint2(&p2, pEnd2, &i2);
98805	}else if( !p2 \|\| (p1 && i1<i2) ){
98806	fts3PutDeltaVarint(&p, &iPrev, i1);
98807	if( mergetype==MERGE_POS_OR ) fts3PoslistCopy(&p, &p1);
98808	fts3GetDeltaVarint2(&p1, pEnd1, &i1);
98809	}else{
98810	fts3PutDeltaVarint(&p, &iPrev, i2);
98811	if( mergetype==MERGE_POS_OR ) fts3PoslistCopy(&p, &p2);
98812	fts3GetDeltaVarint2(&p2, pEnd2, &i2);
98813	}
98814	}
98815	break;
98816
98817	case MERGE_AND:
98818	while( p1 && p2 ){
98819	if( i1==i2 ){
98820	fts3PutDeltaVarint(&p, &iPrev, i1);
98821	fts3GetDeltaVarint2(&p1, pEnd1, &i1);
98822	fts3GetDeltaVarint2(&p2, pEnd2, &i2);
98823	}else if( i1<i2 ){
98824	fts3GetDeltaVarint2(&p1, pEnd1, &i1);
98825	}else{
98826	fts3GetDeltaVarint2(&p2, pEnd2, &i2);
98827	}
98828	}
98829	break;
98830
98831	case MERGE_NOT:
98832	while( p1 ){
98833	if( p2 && i1==i2 ){
98834	fts3GetDeltaVarint2(&p1, pEnd1, &i1);
98835	fts3GetDeltaVarint2(&p2, pEnd2, &i2);
98836	}else if( !p2 \|\| i1<i2 ){
98837	fts3PutDeltaVarint(&p, &iPrev, i1);
98838	fts3GetDeltaVarint2(&p1, pEnd1, &i1);
98839	}else{
98840	fts3GetDeltaVarint2(&p2, pEnd2, &i2);
98841	}
98842	}
98843	break;
98844
98845	case MERGE_POS_PHRASE:
98846	case MERGE_PHRASE: {
98847	char **ppPos = (mergetype==MERGE_PHRASE ? 0 : &p);
98848	while( p1 && p2 ){
98849	if( i1==i2 ){
98850	char *pSave = p;
98851	sqlite3_int64 iPrevSave = iPrev;
98852	fts3PutDeltaVarint(&p, &iPrev, i1);
98853	if( 0==fts3PoslistPhraseMerge(ppPos, 1, 0, &p1, &p2) ){
98854	p = pSave;
98855	iPrev = iPrevSave;
98856	}
98857	fts3GetDeltaVarint2(&p1, pEnd1, &i1);
98858	fts3GetDeltaVarint2(&p2, pEnd2, &i2);
98859	}else if( i1<i2 ){
98860	fts3PoslistCopy(0, &p1);
98861	fts3GetDeltaVarint2(&p1, pEnd1, &i1);
98862	}else{
98863	fts3PoslistCopy(0, &p2);
98864	fts3GetDeltaVarint2(&p2, pEnd2, &i2);
98865	}
98866	}
98867	break;
98868	}
98869
98870	case MERGE_POS_NEAR:
98871	case MERGE_NEAR: {
98872	char *aTmp = 0;
98873	char **ppPos = 0;
98874	if( mergetype==MERGE_POS_NEAR ){
98875	ppPos = &p;
98876	aTmp = sqlite3_malloc(2*(n1+n2));
98877	if( !aTmp ){
98878	return SQLITE_NOMEM;
98879	}
98880	}
98881
98882	while( p1 && p2 ){
98883	if( i1==i2 ){
98884	char *pSave = p;
98885	sqlite3_int64 iPrevSave = iPrev;
98886	fts3PutDeltaVarint(&p, &iPrev, i1);
98887
98888	if( !fts3PoslistNearMerge(ppPos, aTmp, nParam1, nParam2, &p1, &p2) ){
98889	iPrev = iPrevSave;
98890	p = pSave;
98891	}
98892
98893	fts3GetDeltaVarint2(&p1, pEnd1, &i1);
98894	fts3GetDeltaVarint2(&p2, pEnd2, &i2);
98895	}else if( i1<i2 ){
98896	fts3PoslistCopy(0, &p1);
98897	fts3GetDeltaVarint2(&p1, pEnd1, &i1);
98898	}else{
98899	fts3PoslistCopy(0, &p2);
98900	fts3GetDeltaVarint2(&p2, pEnd2, &i2);
98901	}
98902	}
98903	sqlite3_free(aTmp);
98904	break;
98905	}
98906
98907	default:
98908	assert(!"Invalid mergetype value passed to fts3DoclistMerge()");
98909	}
98910
98911	*pnBuffer = (p-aBuffer);
98912	return SQLITE_OK;
98913	}
98914
98915	/*
98916	** A pointer to an instance of this structure is used as the context
98917	** argument to sqlite3Fts3SegReaderIterate()
98918	*/
98919	typedef struct TermSelect TermSelect;
98920	struct TermSelect {
98921	int isReqPos;
98922	char aOutput; / Malloc'd output buffer */
98923	int nOutput; /* Size of output in bytes */
98924	};
98925
98926	/*
98927	** This function is used as the sqlite3Fts3SegReaderIterate() callback when
98928	** querying the full-text index for a doclist associated with a term or
98929	** term-prefix.
98930	*/
98931	static int fts3TermSelectCb(
98932	Fts3Table p, / Virtual table object */
98933	void pContext, / Pointer to TermSelect structure */
98934	char *zTerm,
98935	int nTerm,
98936	char *aDoclist,
98937	int nDoclist
98938	){
98939	TermSelect pTS = (TermSelect )pContext;
98940	int nNew = pTS->nOutput + nDoclist;
98941	char *aNew = sqlite3_malloc(nNew);
98942
98943	if( !aNew ){
98944	return SQLITE_NOMEM;
98945	}
98946
98947	if( pTS->nOutput==0 ){
98948	/* If this is the first term selected, copy the doclist to the output
98949	** buffer using memcpy(). TODO: Add a way to transfer control of the
98950	** aDoclist buffer from the caller so as to avoid the memcpy().
98951	*/
98952	memcpy(aNew, aDoclist, nDoclist);
98953	}else{
98954	/* The output buffer is not empty. Merge doclist aDoclist with the
98955	** existing output. This can only happen with prefix-searches (as
98956	** searches for exact terms return exactly one doclist).
98957	*/
98958	int mergetype = (pTS->isReqPos ? MERGE_POS_OR : MERGE_OR);
98959	fts3DoclistMerge(mergetype, 0, 0,
98960	aNew, &nNew, pTS->aOutput, pTS->nOutput, aDoclist, nDoclist
98961	);
98962	}
98963
98964	sqlite3_free(pTS->aOutput);
98965	pTS->aOutput = aNew;
98966	pTS->nOutput = nNew;
98967
98968	return SQLITE_OK;
98969	}
98970
98971	/*
98972	** This function retreives the doclist for the specified term (or term
98973	** prefix) from the database.
98974	**
98975	** The returned doclist may be in one of two formats, depending on the
98976	** value of parameter isReqPos. If isReqPos is zero, then the doclist is
98977	** a sorted list of delta-compressed docids. If isReqPos is non-zero,
98978	** then the returned list is in the same format as is stored in the
98979	** database without the found length specifier at the start of on-disk
98980	** doclists.
98981	*/
98982	static int fts3TermSelect(
98983	Fts3Table p, / Virtual table handle */
98984	int iColumn, /* Column to query (or -ve for all columns) */
98985	const char zTerm, / Term to query for */
98986	int nTerm, /* Size of zTerm in bytes */
98987	int isPrefix, /* True for a prefix search */
98988	int isReqPos, /* True to include position lists in output */
98989	int pnOut, / OUT: Size of buffer at ppOut /
98990	char *ppOut / OUT: Malloced result buffer */
98991	){
98992	int i;
98993	TermSelect tsc;
98994	Fts3SegFilter filter; /* Segment term filter configuration */
98995	Fts3SegReader *apSegment = 0; / Array of segments to read data from */
98996	int nSegment = 0; /* Size of apSegment array */
98997	int nAlloc = 0; /* Allocated size of segment array */
98998	int rc; /* Return code */
98999	sqlite3_stmt pStmt; / SQL statement to scan %_segdir table */
99000	int iAge = 0; /* Used to assign ages to segments */
99001
99002	/* Loop through the entire %_segdir table. For each segment, create a
99003	** Fts3SegReader to iterate through the subset of the segment leaves
99004	** that may contain a term that matches zTerm/nTerm. For non-prefix
99005	** searches, this is always a single leaf. For prefix searches, this
99006	** may be a contiguous block of leaves.
99007	**
99008	** The code in this loop does not actually load any leaves into memory
99009	** (unless the root node happens to be a leaf). It simply examines the
99010	** b-tree structure to determine which leaves need to be inspected.
99011	*/
99012	rc = sqlite3Fts3AllSegdirs(p, &pStmt);
99013	while( rc==SQLITE_OK && SQLITE_ROW==(rc = sqlite3_step(pStmt)) ){
99014	Fts3SegReader *pNew = 0;
99015	int nRoot = sqlite3_column_bytes(pStmt, 4);
99016	char const *zRoot = sqlite3_column_blob(pStmt, 4);
99017	if( sqlite3_column_int64(pStmt, 1)==0 ){
99018	/* The entire segment is stored on the root node (which must be a
99019	** leaf). Do not bother inspecting any data in this case, just
99020	** create a Fts3SegReader to scan the single leaf.
99021	*/
99022	rc = sqlite3Fts3SegReaderNew(p, iAge, 0, 0, 0, zRoot, nRoot, &pNew);
99023	}else{
99024	sqlite3_int64 i1;
99025	rc = fts3SelectLeaf(p, zTerm, nTerm, zRoot, nRoot, &i1);
99026	if( rc==SQLITE_OK ){
99027	sqlite3_int64 i2 = sqlite3_column_int64(pStmt, 2);
99028	rc = sqlite3Fts3SegReaderNew(p, iAge, i1, i2, 0, 0, 0, &pNew);
99029	}
99030	}
99031	iAge++;
99032
99033	/* If a new Fts3SegReader was allocated, add it to the apSegment array. */
99034	assert( (rc==SQLITE_OK)==(pNew!=0) );
99035	if( pNew ){
99036	if( nSegment==nAlloc ){
99037	nAlloc += 16;
99038	Fts3SegReader pArray = (Fts3SegReader )sqlite3_realloc(
99039	apSegment, nAllocsizeof(Fts3SegReader )
99040	);
99041	if( !pArray ){
99042	sqlite3Fts3SegReaderFree(p, pNew);
99043	rc = SQLITE_NOMEM;
99044	goto finished;
99045	}
99046	apSegment = pArray;
99047	}
99048	apSegment[nSegment++] = pNew;
99049	}
99050	}
99051	if( rc!=SQLITE_DONE ){
99052	assert( rc!=SQLITE_OK );
99053	goto finished;
99054	}
99055
99056	memset(&tsc, 0, sizeof(TermSelect));
99057	tsc.isReqPos = isReqPos;
99058
99059	filter.flags = FTS3_SEGMENT_IGNORE_EMPTY
99060	\| (isPrefix ? FTS3_SEGMENT_PREFIX : 0)
99061	\| (isReqPos ? FTS3_SEGMENT_REQUIRE_POS : 0)
99062	\| (iColumn<p->nColumn ? FTS3_SEGMENT_COLUMN_FILTER : 0);
99063	filter.iCol = iColumn;
99064	filter.zTerm = zTerm;
99065	filter.nTerm = nTerm;
99066
99067	rc = sqlite3Fts3SegReaderIterate(p, apSegment, nSegment, &filter,
99068	fts3TermSelectCb, (void *)&tsc
99069	);
99070
99071	if( rc==SQLITE_OK ){
99072	*ppOut = tsc.aOutput;
99073	*pnOut = tsc.nOutput;
99074	}else{
99075	sqlite3_free(tsc.aOutput);
99076	}
99077
99078	finished:
99079	sqlite3_reset(pStmt);
99080	for(i=0; i<nSegment; i++){
99081	sqlite3Fts3SegReaderFree(p, apSegment[i]);
99082	}
99083	sqlite3_free(apSegment);
99084	return rc;
99085	}
99086
99087
99088	/*
99089	** Return a DocList corresponding to the phrase *pPhrase.



99090	*/
99091	static int fts3PhraseSelect(
99092	Fts3Table p, / Virtual table handle */
99093	Fts3Phrase pPhrase, / Phrase to return a doclist for */
99094	int isReqPos, /* True if output should contain positions */
99095	char *paOut, / OUT: Pointer to malloc'd result buffer */
99096	int pnOut / OUT: Size of buffer at paOut /
99097	){
99098	char *pOut = 0;
99099	int nOut = 0;
99100	int rc = SQLITE_OK;
99101	int ii;
99102	int iCol = pPhrase->iColumn;
99103	int isTermPos = (pPhrase->nToken>1 \|\| isReqPos);
99104
99105	assert( p->nPendingData==0 );
99106
99107	for(ii=0; ii<pPhrase->nToken; ii++){
99108	struct PhraseToken *pTok = &pPhrase->aToken[ii];
99109	char z = pTok->z; / Next token of the phrase */
99110	int n = pTok->n; /* Size of z in bytes */
99111	int isPrefix = pTok->isPrefix;/* True if token is a prefix */
99112	char pList; / Pointer to token doclist */
99113	int nList; /* Size of buffer at pList */
99114
99115	rc = fts3TermSelect(p, iCol, z, n, isPrefix, isTermPos, &nList, &pList);
99116	if( rc!=SQLITE_OK ) break;
99117
99118	if( ii==0 ){
99119	pOut = pList;
99120	nOut = nList;
99121	}else{
99122	/* Merge the new term list and the current output. If this is the
99123	** last term in the phrase, and positions are not required in the
99124	** output of this function, the positions can be dropped as part
99125	** of this merge. Either way, the result of this merge will be
99126	** smaller than nList bytes. The code in fts3DoclistMerge() is written
99127	** so that it is safe to use pList as the output as well as an input
99128	** in this case.
99129	*/
99130	int mergetype = MERGE_POS_PHRASE;
99131	if( ii==pPhrase->nToken-1 && !isReqPos ){
99132	mergetype = MERGE_PHRASE;
99133	}
99134	fts3DoclistMerge(mergetype, 0, 0, pList, &nOut, pOut, nOut, pList, nList);
99135	sqlite3_free(pOut);
99136	pOut = pList;
99137	}
99138	}
99139
99140	if( rc==SQLITE_OK ){
99141	*paOut = pOut;
99142	*pnOut = nOut;
99143	}else{
99144	sqlite3_free(pOut);
99145	}
99146	return rc;
99147	}
99148
99149	/*
99150	** Evaluate the full-text expression pExpr against fts3 table pTab. Store
99151	** the resulting doclist in paOut and pnOut.
99152	*/
99153	static int evalFts3Expr(
99154	Fts3Table p, / Virtual table handle */
99155	Fts3Expr pExpr, / Parsed fts3 expression */
99156	char *paOut, / OUT: Pointer to malloc'd result buffer */
99157	int pnOut / OUT: Size of buffer at paOut /
99158	){
99159	int rc = SQLITE_OK; /* Return code */
99160
99161	/* Zero the output parameters. */
99162	*paOut = 0;
99163	*pnOut = 0;


99164
99165	if( pExpr ){
99166	if( pExpr->eType==FTSQUERY_PHRASE ){
99167	int isReqPos = (pExpr->pParent && pExpr->pParent->eType==FTSQUERY_NEAR);
99168	rc = fts3PhraseSelect(p, pExpr->pPhrase, isReqPos, paOut, pnOut);



99169	}else{
99170	char *aLeft;
99171	char *aRight;
99172	int nLeft;
99173	int nRight;
99174
99175	if( SQLITE_OK==(rc = evalFts3Expr(p, pExpr->pRight, &aRight, &nRight))
99176	&& SQLITE_OK==(rc = evalFts3Expr(p, pExpr->pLeft, &aLeft, &nLeft))

99177	){
99178	switch( pExpr->eType ){
99179	case FTSQUERY_NEAR: {

99180	Fts3Expr *pLeft;
99181	Fts3Expr *pRight;
99182	int mergetype = MERGE_NEAR;
99183	int nParam1;
99184	int nParam2;
99185	char *aBuffer;
99186
99187	if( pExpr->pParent && pExpr->pParent->eType==FTSQUERY_NEAR ){
99188	mergetype = MERGE_POS_NEAR;
99189	}
99190	pLeft = pExpr->pLeft;
99191	while( pLeft->eType==FTSQUERY_NEAR ){
99192	pLeft=pLeft->pRight;
99193	}
99194	pRight = pExpr->pRight;
99195	assert( pRight->eType==FTSQUERY_PHRASE );
99196	assert( pLeft->eType==FTSQUERY_PHRASE );
99197
99198	nParam1 = pExpr->nNear+1;
99199	nParam2 = nParam1+pLeft->pPhrase->nToken+pRight->pPhrase->nToken-2;
99200	aBuffer = sqlite3_malloc(nLeft+nRight+1);
99201	rc = fts3DoclistMerge(mergetype, nParam1, nParam2, aBuffer,
99202	pnOut, aLeft, nLeft, aRight, nRight
99203	);
99204	if( rc!=SQLITE_OK ){
99205	sqlite3_free(aBuffer);
99206	}else{
99207	*paOut = aBuffer;
99208	}
99209	sqlite3_free(aLeft);
99210	break;
99211	}
99212
99213	case FTSQUERY_OR: {
99214	/* Allocate a buffer for the output. The maximum size is the
99215	** sum of the sizes of the two input buffers. The +1 term is
99216	** so that a buffer of zero bytes is never allocated - this can
99217	** cause fts3DoclistMerge() to incorrectly return SQLITE_NOMEM.
99218	*/
99219	char *aBuffer = sqlite3_malloc(nRight+nLeft+1);
99220	rc = fts3DoclistMerge(MERGE_OR, 0, 0, aBuffer, pnOut,
99221	aLeft, nLeft, aRight, nRight
99222	);
99223	*paOut = aBuffer;
99224	sqlite3_free(aLeft);
99225	break;
99226	}
99227
99228	case FTSQUERY_AND:
99229	case FTSQUERY_NOT: {
99230	assert( FTSQUERY_NOT==MERGE_NOT && FTSQUERY_AND==MERGE_AND );
99231	fts3DoclistMerge(pExpr->eType, 0, 0, aLeft, pnOut,
99232	aLeft, nLeft, aRight, nRight
99233	);
99234	*paOut = aLeft;
99235	break;
99236	}
99237	}
99238	}
99239	sqlite3_free(aRight);
99240	}
99241	}
99242
99243	return rc;
























99244	}
99245
99246	/*
99247	** This is the xFilter interface for the virtual table. See
99248	** the virtual table xFilter method documentation for additional
99249	** information.
99250	**
99251	** If idxNum==FTS3_FULLSCAN_SEARCH then do a full table scan against
99252	** the %_content table.
99253	**
99254	** If idxNum==FTS3_DOCID_SEARCH then do a docid lookup for a single entry
99255	** in the %_content table.
99256	**
99257	** If idxNum>=FTS3_FULLTEXT_SEARCH then use the full text index. The
99258	** column on the left-hand side of the MATCH operator is column
99259	** number idxNum-FTS3_FULLTEXT_SEARCH, 0 indexed. argv[0] is the right-hand
99260	** side of the MATCH operator.
99261	*/
99262	/* TODO(shess) Upgrade the cursor initialization and destruction to
99263	** account for fts3FilterMethod() being called multiple times on the
99264	** same cursor. The current solution is very fragile. Apply fix to
99265	** fts3 as appropriate.
99266	*/
99267	static int fts3FilterMethod(
99268	sqlite3_vtab_cursor pCursor, / The cursor used for this query */
99269	int idxNum, /* Strategy index */
99270	const char idxStr, / Unused */
99271	int nVal, /* Number of elements in apVal */
99272	sqlite3_value *apVal / Arguments for the indexing scheme */
99273	){
99274	const char *azSql[] = {
99275	"SELECT * FROM %Q.'%q_content' WHERE docid = ?", /* non-full-table-scan */
99276	"SELECT * FROM %Q.'%q_content'", /* full-table-scan */
99277	};
99278	int rc; /* Return code */
99279	char zSql; / SQL statement used to access %_content */
99280	Fts3Table p = (Fts3Table )pCursor->pVtab;
99281	Fts3Cursor pCsr = (Fts3Cursor )pCursor;
99282
99283	assert( idxNum>=0 && idxNum<=(FTS3_FULLTEXT_SEARCH+p->nColumn) );
99284	assert( nVal==0 \|\| nVal==1 );
99285	assert( (nVal==0)==(idxNum==FTS3_FULLSCAN_SEARCH) );
99286
99287	/* In case the cursor has been used before, clear it now. */
99288	sqlite3_finalize(pCsr->pStmt);
99289	sqlite3_free(pCsr->aDoclist);
99290	memset(&pCursor[1], 0, sizeof(Fts3Cursor)-sizeof(sqlite3_vtab_cursor));
99291
99292	/* Compile a SELECT statement for this cursor. For a full-table-scan, the
99293	** statement loops through all rows of the %_content table. For a
99294	** full-text query or docid lookup, the statement retrieves a single
99295	** row by docid.
99296	*/
99297	zSql = sqlite3_mprintf(azSql[idxNum==FTS3_FULLSCAN_SEARCH], p->zDb, p->zName);
99298	if( !zSql ){
99299	rc = SQLITE_NOMEM;
99300	}else{
99301	rc = sqlite3_prepare_v2(p->db, zSql, -1, &pCsr->pStmt, 0);
99302	sqlite3_free(zSql);
99303	}
99304	if( rc!=SQLITE_OK ) return rc;
99305	pCsr->eType = idxNum;
99306
99307	if( idxNum==FTS3_DOCID_SEARCH ){
99308	rc = sqlite3_bind_value(pCsr->pStmt, 1, apVal[0]);
99309	}else if( idxNum!=FTS3_FULLSCAN_SEARCH ){
99310	int iCol = idxNum-FTS3_FULLTEXT_SEARCH;
99311	const char zQuery = (const char )sqlite3_value_text(apVal[0]);
99312
99313	rc = sqlite3Fts3PendingTermsFlush(p);
99314	if( rc!=SQLITE_OK ) return rc;
99315
99316	rc = sqlite3Fts3ExprParse(p->pTokenizer, p->azColumn, p->nColumn,
99317	iCol, zQuery, -1, &pCsr->pExpr
99318	);
99319	if( rc!=SQLITE_OK ) return rc;
99320
99321	rc = evalFts3Expr(p, pCsr->pExpr, &pCsr->aDoclist, &pCsr->nDoclist);
99322	pCsr->pNextId = pCsr->aDoclist;
99323	pCsr->iPrevId = 0;
99324	}
99325
99326	if( rc!=SQLITE_OK ) return rc;
99327	return fts3NextMethod(pCursor);
99328	}
99329
99330	/*
99331	** This is the xEof method of the virtual table. SQLite calls this
99332	** routine to find out if it has reached the end of a result set.
99333	*/
99334	static int fts3EofMethod(sqlite3_vtab_cursor *pCursor){
99335	return ((Fts3Cursor *)pCursor)->isEof;
99336	}
99337
99338	/*
99339	** This is the xColumn method of the virtual table. The SQLite
















99340	** core calls this method during a query when it needs the value
99341	** of a column from the virtual table. This method needs to use
99342	** one of the sqlite3_result_*() routines to store the requested
99343	** value back in the pContext.
99344	*/
99345	static int fts3ColumnMethod(sqlite3_vtab_cursor *pCursor,
99346	sqlite3_context *pContext, int idxCol){
99347	Fts3Cursor c = (Fts3Cursor ) pCursor;
99348	Fts3Table *v = cursor_vtab(c);
99349	int rc = fts3CursorSeek(c);
99350	if( rc!=SQLITE_OK ){
99351	return rc;
99352	}
99353
99354	if( idxCol<v->nColumn ){
99355	sqlite3_value *pVal = sqlite3_column_value(c->pStmt, idxCol+1);
99356	sqlite3_result_value(pContext, pVal);
99357	}else if( idxCol==v->nColumn ){
	@@ -101700,2884 +99365,227 @@
99365	sqlite3_result_value(pContext, pVal);
99366	}
99367	return SQLITE_OK;
99368	}
99369
99370	/*
99371	** This is the xRowid method. The SQLite core calls this routine to
99372	** retrieve the rowid for the current row of the result set. fts3
99373	** exposes %_content.docid as the rowid for the virtual table. The
99374	** rowid should be written to *pRowid.
99375	*/
99376	static int fts3RowidMethod(sqlite3_vtab_cursor pCursor, sqlite_int64 pRowid){
99377	Fts3Cursor pCsr = (Fts3Cursor ) pCursor;
99378	if( pCsr->aDoclist ){
99379	*pRowid = pCsr->iPrevId;
99380	}else{
99381	*pRowid = sqlite3_column_int64(pCsr->pStmt, 0);
99382	}
99383	return SQLITE_OK;
99384	}
99385
99386	/*
99387	** This function is the implementation of the xUpdate callback used by
99388	** FTS3 virtual tables. It is invoked by SQLite each time a row is to be
99389	** inserted, updated or deleted.
99390	*/
99391	static int fts3UpdateMethod(
99392	sqlite3_vtab pVtab, / Virtual table handle */
99393	int nArg, /* Size of argument array */
99394	sqlite3_value *apVal, / Array of arguments */
99395	sqlite_int64 pRowid / OUT: The affected (or effected) rowid */
99396	){
99397	return sqlite3Fts3UpdateMethod(pVtab, nArg, apVal, pRowid);
99398	}
99399
99400	/*
99401	** Implementation of xSync() method. Flush the contents of the pending-terms
99402	** hash-table to the database.
99403	*/
99404	static int fts3SyncMethod(sqlite3_vtab *pVtab){
99405	return sqlite3Fts3PendingTermsFlush((Fts3Table *)pVtab);
99406	}
99407
99408	/*
99409	** Implementation of xBegin() method. This is a no-op.
99410	*/
99411	static int fts3BeginMethod(sqlite3_vtab *pVtab){
99412	assert( ((Fts3Table *)pVtab)->nPendingData==0 );
99413	return SQLITE_OK;
99414	}
99415
99416	/*
99417	** Implementation of xCommit() method. This is a no-op. The contents of
99418	** the pending-terms hash-table have already been flushed into the database
99419	** by fts3SyncMethod().
99420	*/
99421	static int fts3CommitMethod(sqlite3_vtab *pVtab){
99422	assert( ((Fts3Table *)pVtab)->nPendingData==0 );
99423	return SQLITE_OK;
99424	}
99425
99426	/*
99427	** Implementation of xRollback(). Discard the contents of the pending-terms
99428	** hash-table. Any changes made to the database are reverted by SQLite.
99429	*/
99430	static int fts3RollbackMethod(sqlite3_vtab *pVtab){
99431	sqlite3Fts3PendingTermsClear((Fts3Table *)pVtab);
99432	return SQLITE_OK;
99433	}
99434
99435	/*
99436	** Helper function used by the implementation of the overloaded snippet(),
99437	** offsets() and optimize() SQL functions.
99438	**
99439	** If the value passed as the third argument is a blob of size
99440	** sizeof(Fts3Cursor*), then the blob contents are copied to the
99441	** output variable *ppCsr and SQLITE_OK is returned. Otherwise, an error
99442	** message is written to context pContext and SQLITE_ERROR returned. The
99443	** string passed via zFunc is used as part of the error message.
99444	*/
99445	static int fts3FunctionArg(
99446	sqlite3_context pContext, / SQL function call context */
99447	const char zFunc, / Function name */
99448	sqlite3_value pVal, / argv[0] passed to function */
99449	Fts3Cursor *ppCsr / OUT: Store cursor handle here */
99450	){
99451	Fts3Cursor *pRet;
99452	if( sqlite3_value_type(pVal)!=SQLITE_BLOB
99453	&& sqlite3_value_bytes(pVal)!=sizeof(Fts3Cursor *)
99454	){
99455	char *zErr = sqlite3_mprintf("illegal first argument to %s", zFunc);
99456	sqlite3_result_error(pContext, zErr, -1);
99457	sqlite3_free(zErr);
99458	return SQLITE_ERROR;
99459	}
99460	memcpy(&pRet, sqlite3_value_blob(pVal), sizeof(Fts3Cursor *));
99461	*ppCsr = pRet;
99462	return SQLITE_OK;
































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































99463	}
99464
99465	/*
99466	** Implementation of the snippet() function for FTS3
99467	*/
99468	static void fts3SnippetFunc(
99469	sqlite3_context *pContext,
99470	int argc,
99471	sqlite3_value **argv
99472	){
99473	Fts3Cursor pCsr; / Cursor handle passed through apVal[0] */
99474	const char *zStart = "<b>";
99475	const char *zEnd = "</b>";
99476	const char *zEllipsis = "<b>...</b>";
99477
99478	if( argc<1 \|\| argc>4 ) return;
99479	if( fts3FunctionArg(pContext, "snippet", argv[0], &pCsr) ) return;
99480
99481	switch( argc ){
99482	case 4: zEllipsis = (const char*)sqlite3_value_text(argv[3]);
99483	case 3: zEnd = (const char*)sqlite3_value_text(argv[2]);
99484	case 2: zStart = (const char*)sqlite3_value_text(argv[1]);
99485	}
99486
99487	sqlite3Fts3Snippet(pContext, pCsr, zStart, zEnd, zEllipsis);









99488	}
99489
99490	/*
99491	** Implementation of the offsets() function for FTS3
99492	*/
99493	static void fts3OffsetsFunc(
99494	sqlite3_context pContext, / SQLite function call context */
99495	int nVal, /* Size of argument array */
99496	sqlite3_value *apVal / Array of arguments */
99497	){
99498	Fts3Cursor pCsr; / Cursor handle passed through apVal[0] */
99499
99500	assert( nVal==1 );
99501	if( fts3FunctionArg(pContext, "offsets", apVal[0], &pCsr) ) return;
99502	assert( pCsr );
99503	sqlite3Fts3Offsets(pContext, pCsr);
99504	}
99505
99506	/*
99507	** Implementation of the special optimize() function for FTS3. This
99508	** function merges all segments in the database to a single segment.
99509	** Example usage is:
99510	**
99511	** SELECT optimize(t) FROM t LIMIT 1;
99512	**
99513	** where 't' is the name of an FTS3 table.
99514	*/
99515	static void fts3OptimizeFunc(
99516	sqlite3_context pContext, / SQLite function call context */
99517	int nVal, /* Size of argument array */
99518	sqlite3_value *apVal / Array of arguments */
99519	){
99520	int rc; /* Return code */
99521	Fts3Table p; / Virtual table handle */
99522	Fts3Cursor pCursor; / Cursor handle passed through apVal[0] */
99523
99524	assert( nVal==1 );
99525	if( fts3FunctionArg(pContext, "optimize", apVal[0], &pCursor) ) return;
99526	p = (Fts3Table *)pCursor->base.pVtab;
99527	assert( p );
99528
99529	rc = sqlite3Fts3Optimize(p);
99530
99531	switch( rc ){
99532	case SQLITE_OK:
99533	sqlite3_result_text(pContext, "Index optimized", -1, SQLITE_STATIC);
99534	break;
99535	case SQLITE_DONE:
99536	sqlite3_result_text(pContext, "Index already optimal", -1, SQLITE_STATIC);
99537	break;
99538	default:
99539	sqlite3_result_error_code(pContext, rc);
99540	break;
99541	}
99542	}




















































































































































































































































































































































































































































































































































































































































99543
99544	/*
99545	** This routine implements the xFindFunction method for the FTS3
99546	** virtual table.
99547	*/
99548	static int fts3FindFunctionMethod(
99549	sqlite3_vtab pVtab, / Virtual table handle */
99550	int nArg, /* Number of SQL function arguments */
99551	const char zName, / Name of SQL function */
99552	void (*pxFunc)(sqlite3_context,int,sqlite3_value*), / OUT: Result */
99553	void *ppArg / Unused */
99554	){
99555	struct Overloaded {
99556	const char *zName;
99557	void (xFunc)(sqlite3_context,int,sqlite3_value**);
99558	} aOverload[] = {
99559	{ "snippet", fts3SnippetFunc },
99560	{ "offsets", fts3OffsetsFunc },
99561	{ "optimize", fts3OptimizeFunc },
99562	};
99563	int i; /* Iterator variable */
99564	for(i=0; i<SizeofArray(aOverload); i++){
99565	if( strcmp(zName, aOverload[i].zName)==0 ){
99566	*pxFunc = aOverload[i].xFunc;
99567	return 1;
99568	}
99569	}
99570
99571	/* No function of the specified name was found. Return 0. */







99572	return 0;
99573	}
99574
99575	/*
99576	** Implementation of FTS3 xRename method. Rename an fts3 table.
99577	*/
99578	static int fts3RenameMethod(
99579	sqlite3_vtab pVtab, / Virtual table handle */
99580	const char zName / New name of table */
99581	){
99582	Fts3Table p = (Fts3Table )pVtab;
99583	int rc = SQLITE_NOMEM; /* Return Code */
99584	char zSql; / SQL script to run to rename tables */
99585
99586	zSql = sqlite3_mprintf(
99587	"ALTER TABLE %Q.'%q_content' RENAME TO '%q_content';"
99588	"ALTER TABLE %Q.'%q_segments' RENAME TO '%q_segments';"
99589	"ALTER TABLE %Q.'%q_segdir' RENAME TO '%q_segdir';"
99590	, p->zDb, p->zName, zName
99591	, p->zDb, p->zName, zName
	@@ -104590,33 +99598,38 @@
99598	return rc;
99599	}
99600
99601	static const sqlite3_module fts3Module = {
99602	/* iVersion */ 0,
99603	/* xCreate */ fts3CreateMethod,
99604	/* xConnect */ fts3ConnectMethod,
99605	/* xBestIndex */ fts3BestIndexMethod,
99606	/* xDisconnect */ fts3DisconnectMethod,
99607	/* xDestroy */ fts3DestroyMethod,
99608	/* xOpen */ fts3OpenMethod,
99609	/* xClose */ fulltextClose,
99610	/* xFilter */ fts3FilterMethod,
99611	/* xNext */ fts3NextMethod,
99612	/* xEof */ fts3EofMethod,
99613	/* xColumn */ fts3ColumnMethod,
99614	/* xRowid */ fts3RowidMethod,
99615	/* xUpdate */ fts3UpdateMethod,
99616	/* xBegin */ fts3BeginMethod,
99617	/* xSync */ fts3SyncMethod,
99618	/* xCommit */ fts3CommitMethod,
99619	/* xRollback */ fts3RollbackMethod,
99620	/* xFindFunction */ fts3FindFunctionMethod,
99621	/* xRename */ fts3RenameMethod,
99622	};
99623
99624	/*
99625	** This function is registered as the module destructor (called when an
99626	** FTS3 enabled database connection is closed). It frees the memory
99627	** allocated for the tokenizer hash table.
99628	*/
99629	static void hashDestroy(void *p){
99630	Fts3Hash pHash = (Fts3Hash )p;
99631	sqlite3Fts3HashClear(pHash);
99632	sqlite3_free(pHash);
99633	}
99634
99635	/*
	@@ -104632,21 +99645,19 @@
99645	*/
99646	SQLITE_PRIVATE void sqlite3Fts3SimpleTokenizerModule(sqlite3_tokenizer_module const**ppModule);
99647	SQLITE_PRIVATE void sqlite3Fts3PorterTokenizerModule(sqlite3_tokenizer_module const**ppModule);
99648	SQLITE_PRIVATE void sqlite3Fts3IcuTokenizerModule(sqlite3_tokenizer_module const**ppModule);
99649


99650	/*
99651	** Initialise the fts3 extension. If this extension is built as part
99652	** of the sqlite library, then this function is called directly by
99653	** SQLite. If fts3 is built as a dynamically loadable extension, this
99654	** function is called by the sqlite3_extension_init() entry point.
99655	*/
99656	SQLITE_PRIVATE int sqlite3Fts3Init(sqlite3 *db){
99657	int rc = SQLITE_OK;
99658	Fts3Hash *pHash = 0;
99659	const sqlite3_tokenizer_module *pSimple = 0;
99660	const sqlite3_tokenizer_module *pPorter = 0;
99661	const sqlite3_tokenizer_module *pIcu = 0;
99662
99663	sqlite3Fts3SimpleTokenizerModule(&pSimple);
	@@ -104654,11 +99665,11 @@
99665	#ifdef SQLITE_ENABLE_ICU
99666	sqlite3Fts3IcuTokenizerModule(&pIcu);
99667	#endif
99668
99669	/* Allocate and initialise the hash-table used to store tokenizers. */
99670	pHash = sqlite3_malloc(sizeof(Fts3Hash));
99671	if( !pHash ){
99672	rc = SQLITE_NOMEM;
99673	}else{
99674	sqlite3Fts3HashInit(pHash, FTS3_HASH_STRING, 1);
99675	}
	@@ -104682,16 +99693,12 @@
99693	** module with sqlite.
99694	*/
99695	if( SQLITE_OK==rc
99696	&& SQLITE_OK==(rc = sqlite3Fts3InitHashTable(db, pHash, "fts3_tokenizer"))
99697	&& SQLITE_OK==(rc = sqlite3_overload_function(db, "snippet", -1))
99698	&& SQLITE_OK==(rc = sqlite3_overload_function(db, "offsets", 1))
99699	&& SQLITE_OK==(rc = sqlite3_overload_function(db, "optimize", 1))




99700	){
99701	return sqlite3_create_module_v2(
99702	db, "fts3", &fts3Module, (void *)pHash, hashDestroy
99703	);
99704	}
	@@ -104714,11 +99721,11 @@
99721	SQLITE_EXTENSION_INIT2(pApi)
99722	return sqlite3Fts3Init(db);
99723	}
99724	#endif
99725
99726	#endif
99727
99728	/************ End of fts3.c **********************************************/
99729	/************ Begin file fts3_expr.c *************************************/
99730	/*
99731	** 2008 Nov 28
	@@ -104733,12 +99740,11 @@
99740	******************************************************************************
99741	**
99742	** This module contains code that implements a parser for fts3 query strings
99743	** (the right-hand argument to the MATCH operator). Because the supported
99744	** syntax is relatively simple, the whole tokenizer/parser system is
99745	** hand-coded.

99746	*/
99747	#if !defined(SQLITE_CORE) \|\| defined(SQLITE_ENABLE_FTS3)
99748
99749	/*
99750	** By default, this module parses the legacy syntax that has been
	@@ -104760,11 +99766,33 @@
99766	** If compiled with SQLITE_TEST defined, then this module exports the
99767	** symbol "int sqlite3_fts3_enable_parentheses". Setting this variable
99768	** to zero causes the module to use the old syntax. If it is set to
99769	** non-zero the new syntax is activated. This is so both syntaxes can
99770	** be tested using a single build of testfixture.
99771	**
99772	** The following describes the syntax supported by the fts3 MATCH
99773	** operator in a similar format to that used by the lemon parser
99774	** generator. This module does not use actually lemon, it uses a
99775	** custom parser.
99776	**
99777	** query ::= andexpr (OR andexpr)*.
99778	**
99779	** andexpr ::= notexpr (AND? notexpr)*.
99780	**
99781	** notexpr ::= nearexpr (NOT nearexpr\|-TOKEN)*.
99782	** notexpr ::= LP query RP.
99783	**
99784	** nearexpr ::= phrase (NEAR distance_opt nearexpr)*.
99785	**
99786	** distance_opt ::= .
99787	** distance_opt ::= / INTEGER.
99788	**
99789	** phrase ::= TOKEN.
99790	** phrase ::= COLUMN:TOKEN.
99791	** phrase ::= "TOKEN TOKEN TOKEN...".
99792	*/
99793
99794	#ifdef SQLITE_TEST
99795	SQLITE_API int sqlite3_fts3_enable_parentheses = 0;
99796	#else
99797	# ifdef SQLITE_ENABLE_FTS3_PARENTHESIS
99798	# define sqlite3_fts3_enable_parentheses 1
	@@ -105069,10 +100097,13 @@
100097	cNext = zInput[nKey];
100098	if( fts3isspace(cNext)
100099	\|\| cNext=='"' \|\| cNext=='(' \|\| cNext==')' \|\| cNext==0
100100	){
100101	pRet = (Fts3Expr *)sqlite3_malloc(sizeof(Fts3Expr));
100102	if( !pRet ){
100103	return SQLITE_NOMEM;
100104	}
100105	memset(pRet, 0, sizeof(Fts3Expr));
100106	pRet->eType = pKey->eType;
100107	pRet->nNear = nNear;
100108	*ppExpr = pRet;
100109	*pnConsumed = (zInput - z) + nKey;
	@@ -105643,10 +100674,125 @@
100674	** SQLite (in which case SQLITE_ENABLE_FTS3 is defined).
100675	*/
100676	#if !defined(SQLITE_CORE) \|\| defined(SQLITE_ENABLE_FTS3)
100677
100678
100679	/************ Include fts3_hash.h in the middle of fts3_hash.c ***********/
100680	/************ Begin file fts3_hash.h *************************************/
100681	/*
100682	** 2001 September 22
100683	**
100684	** The author disclaims copyright to this source code. In place of
100685	** a legal notice, here is a blessing:
100686	**
100687	** May you do good and not evil.
100688	** May you find forgiveness for yourself and forgive others.
100689	** May you share freely, never taking more than you give.
100690	**
100691	*************************************************************************
100692	** This is the header file for the generic hash-table implemenation
100693	** used in SQLite. We've modified it slightly to serve as a standalone
100694	** hash table implementation for the full-text indexing module.
100695	**
100696	*/
100697	#ifndef _FTS3_HASH_H_
100698	#define _FTS3_HASH_H_
100699
100700	/* Forward declarations of structures. */
100701	typedef struct Fts3Hash Fts3Hash;
100702	typedef struct Fts3HashElem Fts3HashElem;
100703
100704	/* A complete hash table is an instance of the following structure.
100705	** The internals of this structure are intended to be opaque -- client
100706	** code should not attempt to access or modify the fields of this structure
100707	** directly. Change this structure only by using the routines below.
100708	** However, many of the "procedures" and "functions" for modifying and
100709	** accessing this structure are really macros, so we can't really make
100710	** this structure opaque.
100711	*/
100712	struct Fts3Hash {
100713	char keyClass; /* HASH_INT, _POINTER, _STRING, _BINARY */
100714	char copyKey; /* True if copy of key made on insert */
100715	int count; /* Number of entries in this table */
100716	Fts3HashElem first; / The first element of the array */
100717	int htsize; /* Number of buckets in the hash table */
100718	struct _fts3ht { /* the hash table */
100719	int count; /* Number of entries with this hash */
100720	Fts3HashElem chain; / Pointer to first entry with this hash */
100721	} *ht;
100722	};
100723
100724	/* Each element in the hash table is an instance of the following
100725	** structure. All elements are stored on a single doubly-linked list.
100726	**
100727	** Again, this structure is intended to be opaque, but it can't really
100728	** be opaque because it is used by macros.
100729	*/
100730	struct Fts3HashElem {
100731	Fts3HashElem next, prev; /* Next and previous elements in the table */
100732	void data; / Data associated with this element */
100733	void pKey; int nKey; / Key associated with this element */
100734	};
100735
100736	/*
100737	** There are 2 different modes of operation for a hash table:
100738	**
100739	** FTS3_HASH_STRING pKey points to a string that is nKey bytes long
100740	** (including the null-terminator, if any). Case
100741	** is respected in comparisons.
100742	**
100743	** FTS3_HASH_BINARY pKey points to binary data nKey bytes long.
100744	** memcmp() is used to compare keys.
100745	**
100746	** A copy of the key is made if the copyKey parameter to fts3HashInit is 1.
100747	*/
100748	#define FTS3_HASH_STRING 1
100749	#define FTS3_HASH_BINARY 2
100750
100751	/*
100752	** Access routines. To delete, insert a NULL pointer.
100753	*/
100754	SQLITE_PRIVATE void sqlite3Fts3HashInit(Fts3Hash*, int keytype, int copyKey);
100755	SQLITE_PRIVATE void sqlite3Fts3HashInsert(Fts3Hash, const void pKey, int nKey, void pData);
100756	SQLITE_PRIVATE void sqlite3Fts3HashFind(const Fts3Hash, const void *pKey, int nKey);
100757	SQLITE_PRIVATE void sqlite3Fts3HashClear(Fts3Hash*);
100758
100759	/*
100760	** Shorthand for the functions above
100761	*/
100762	#define fts3HashInit sqlite3Fts3HashInit
100763	#define fts3HashInsert sqlite3Fts3HashInsert
100764	#define fts3HashFind sqlite3Fts3HashFind
100765	#define fts3HashClear sqlite3Fts3HashClear
100766
100767	/*
100768	** Macros for looping over all elements of a hash table. The idiom is
100769	** like this:
100770	**
100771	** Fts3Hash h;
100772	** Fts3HashElem *p;
100773	** ...
100774	** for(p=fts3HashFirst(&h); p; p=fts3HashNext(p)){
100775	** SomeStructure *pData = fts3HashData(p);
100776	** // do something with pData
100777	** }
100778	*/
100779	#define fts3HashFirst(H) ((H)->first)
100780	#define fts3HashNext(E) ((E)->next)
100781	#define fts3HashData(E) ((E)->data)
100782	#define fts3HashKey(E) ((E)->pKey)
100783	#define fts3HashKeysize(E) ((E)->nKey)
100784
100785	/*
100786	** Number of entries in a hash table
100787	*/
100788	#define fts3HashCount(H) ((H)->count)
100789
100790	#endif /* _FTS3_HASH_H_ */
100791
100792	/************ End of fts3_hash.h *****************************************/
100793	/************ Continuing where we left off in fts3_hash.c ****************/
100794
100795	/*
100796	** Malloc and Free functions
100797	*/
100798	static void *fts3HashMalloc(int n){
	@@ -105668,11 +100814,11 @@
100814	** FTS3_HASH_BINARY or FTS3_HASH_STRING. The value of keyClass
100815	** determines what kind of key the hash table will use. "copyKey" is
100816	** true if the hash table should make its own private copy of keys and
100817	** false if it should just use the supplied pointer.
100818	*/
100819	SQLITE_PRIVATE void sqlite3Fts3HashInit(Fts3Hash *pNew, int keyClass, int copyKey){
100820	assert( pNew!=0 );
100821	assert( keyClass>=FTS3_HASH_STRING && keyClass<=FTS3_HASH_BINARY );
100822	pNew->keyClass = keyClass;
100823	pNew->copyKey = copyKey;
100824	pNew->first = 0;
	@@ -105683,21 +100829,21 @@
100829
100830	/* Remove all entries from a hash table. Reclaim all memory.
100831	** Call this routine to delete a hash table or to reset a hash table
100832	** to the empty state.
100833	*/
100834	SQLITE_PRIVATE void sqlite3Fts3HashClear(Fts3Hash *pH){
100835	Fts3HashElem elem; / For looping over all elements of the table */
100836
100837	assert( pH!=0 );
100838	elem = pH->first;
100839	pH->first = 0;
100840	fts3HashFree(pH->ht);
100841	pH->ht = 0;
100842	pH->htsize = 0;
100843	while( elem ){
100844	Fts3HashElem *next_elem = elem->next;
100845	if( pH->copyKey && elem->pKey ){
100846	fts3HashFree(elem->pKey);
100847	}
100848	fts3HashFree(elem);
100849	elem = next_elem;
	@@ -105776,15 +100922,15 @@
100922	}
100923
100924	/* Link an element into the hash table
100925	*/
100926	static void fts3HashInsertElement(
100927	Fts3Hash pH, / The complete hash table */
100928	struct _fts3ht pEntry, / The entry into which pNew is inserted */
100929	Fts3HashElem pNew / The element to be inserted */
100930	){
100931	Fts3HashElem pHead; / First element already in pEntry */
100932	pHead = pEntry->chain;
100933	if( pHead ){
100934	pNew->next = pHead;
100935	pNew->prev = pHead->prev;
100936	if( pHead->prev ){ pHead->prev->next = pNew; }
	@@ -105803,13 +100949,13 @@
100949
100950	/* Resize the hash table so that it cantains "new_size" buckets.
100951	** "new_size" must be a power of 2. The hash table might fail
100952	** to resize if sqliteMalloc() fails.
100953	*/
100954	static void fts3Rehash(Fts3Hash *pH, int new_size){
100955	struct _fts3ht new_ht; / The new hash table */
100956	Fts3HashElem elem, next_elem; /* For looping over existing elements */
100957	int (xHash)(const void,int); /* The hash function */
100958
100959	assert( (new_size & (new_size-1))==0 );
100960	new_ht = (struct _fts3ht )fts3HashMalloc( new_sizesizeof(struct _fts3ht) );
100961	if( new_ht==0 ) return;
	@@ -105826,17 +100972,17 @@
100972
100973	/* This function (for internal use only) locates an element in an
100974	** hash table that matches the given key. The hash for this key has
100975	** already been computed and is passed as the 4th parameter.
100976	*/
100977	static Fts3HashElem *fts3FindElementByHash(
100978	const Fts3Hash pH, / The pH to be searched */
100979	const void pKey, / The key we are searching for */
100980	int nKey,
100981	int h /* The hash for this key. */
100982	){
100983	Fts3HashElem elem; / Used to loop thru the element list */
100984	int count; /* Number of elements left to test */
100985	int (xCompare)(const void,int,const void,int); / comparison function */
100986
100987	if( pH->ht ){
100988	struct _fts3ht *pEntry = &pH->ht[h];
	@@ -105855,12 +101001,12 @@
101001
101002	/* Remove a single entry from the hash table given a pointer to that
101003	** element and a hash on the element's key.
101004	*/
101005	static void fts3RemoveElementByHash(
101006	Fts3Hash pH, / The pH containing "elem" */
101007	Fts3HashElem* elem, /* The element to be removed from the pH */
101008	int h /* Hash value for the element */
101009	){
101010	struct _fts3ht *pEntry;
101011	if( elem->prev ){
101012	elem->prev->next = elem->next;
	@@ -105892,13 +101038,13 @@
101038
101039	/* Attempt to locate an element of the hash table pH with a key
101040	** that matches pKey,nKey. Return the data for this element if it is
101041	** found, or NULL if there is no match.
101042	*/
101043	SQLITE_PRIVATE void sqlite3Fts3HashFind(const Fts3Hash pH, const void *pKey, int nKey){
101044	int h; /* A hash on key */
101045	Fts3HashElem elem; / The element that matches key */
101046	int (xHash)(const void,int); /* The hash function */
101047
101048	if( pH==0 \|\| pH->ht==0 ) return 0;
101049	xHash = ftsHashFunction(pH->keyClass);
101050	assert( xHash!=0 );
	@@ -105922,19 +101068,19 @@
101068	**
101069	** If the "data" parameter to this function is NULL, then the
101070	** element corresponding to "key" is removed from the hash table.
101071	*/
101072	SQLITE_PRIVATE void *sqlite3Fts3HashInsert(
101073	Fts3Hash pH, / The hash table to insert into */
101074	const void pKey, / The key */
101075	int nKey, /* Number of bytes in the key */
101076	void data / The data */
101077	){
101078	int hraw; /* Raw hash value of the key */
101079	int h; /* the hash of the key modulo hash table size */
101080	Fts3HashElem elem; / Used to loop thru the element list */
101081	Fts3HashElem new_elem; / New element added to the pH */
101082	int (xHash)(const void,int); /* The hash function */
101083
101084	assert( pH!=0 );
101085	xHash = ftsHashFunction(pH->keyClass);
101086	assert( xHash!=0 );
	@@ -105957,11 +101103,11 @@
101103	if( pH->htsize==0 ){
101104	pH->count = 0;
101105	return data;
101106	}
101107	}
101108	new_elem = (Fts3HashElem*)fts3HashMalloc( sizeof(Fts3HashElem) );
101109	if( new_elem==0 ) return data;
101110	if( pH->copyKey && pKey!=0 ){
101111	new_elem->pKey = fts3HashMalloc( nKey );
101112	if( new_elem->pKey==0 ){
101113	fts3HashFree(new_elem);
	@@ -106014,10 +101160,162 @@
101160	*/
101161	#if !defined(SQLITE_CORE) \|\| defined(SQLITE_ENABLE_FTS3)
101162
101163
101164
101165	/************ Include fts3_tokenizer.h in the middle of fts3_porter.c ****/
101166	/************ Begin file fts3_tokenizer.h ********************************/
101167	/*
101168	** 2006 July 10
101169	**
101170	** The author disclaims copyright to this source code.
101171	**
101172	*************************************************************************
101173	** Defines the interface to tokenizers used by fulltext-search. There
101174	** are three basic components:
101175	**
101176	** sqlite3_tokenizer_module is a singleton defining the tokenizer
101177	** interface functions. This is essentially the class structure for
101178	** tokenizers.
101179	**
101180	** sqlite3_tokenizer is used to define a particular tokenizer, perhaps
101181	** including customization information defined at creation time.
101182	**
101183	** sqlite3_tokenizer_cursor is generated by a tokenizer to generate
101184	** tokens from a particular input.
101185	*/
101186	#ifndef _FTS3_TOKENIZER_H_
101187	#define _FTS3_TOKENIZER_H_
101188
101189	/* TODO(shess) Only used for SQLITE_OK and SQLITE_DONE at this time.
101190	** If tokenizers are to be allowed to call sqlite3_*() functions, then
101191	** we will need a way to register the API consistently.
101192	*/
101193
101194	/*
101195	** Structures used by the tokenizer interface. When a new tokenizer
101196	** implementation is registered, the caller provides a pointer to
101197	** an sqlite3_tokenizer_module containing pointers to the callback
101198	** functions that make up an implementation.
101199	**
101200	** When an fts3 table is created, it passes any arguments passed to
101201	** the tokenizer clause of the CREATE VIRTUAL TABLE statement to the
101202	** sqlite3_tokenizer_module.xCreate() function of the requested tokenizer
101203	** implementation. The xCreate() function in turn returns an
101204	** sqlite3_tokenizer structure representing the specific tokenizer to
101205	** be used for the fts3 table (customized by the tokenizer clause arguments).
101206	**
101207	** To tokenize an input buffer, the sqlite3_tokenizer_module.xOpen()
101208	** method is called. It returns an sqlite3_tokenizer_cursor object
101209	** that may be used to tokenize a specific input buffer based on
101210	** the tokenization rules supplied by a specific sqlite3_tokenizer
101211	** object.
101212	*/
101213	typedef struct sqlite3_tokenizer_module sqlite3_tokenizer_module;
101214	typedef struct sqlite3_tokenizer sqlite3_tokenizer;
101215	typedef struct sqlite3_tokenizer_cursor sqlite3_tokenizer_cursor;
101216
101217	struct sqlite3_tokenizer_module {
101218
101219	/*
101220	** Structure version. Should always be set to 0.
101221	*/
101222	int iVersion;
101223
101224	/*
101225	** Create a new tokenizer. The values in the argv[] array are the
101226	** arguments passed to the "tokenizer" clause of the CREATE VIRTUAL
101227	** TABLE statement that created the fts3 table. For example, if
101228	** the following SQL is executed:
101229	**
101230	** CREATE .. USING fts3( ... , tokenizer <tokenizer-name> arg1 arg2)
101231	**
101232	** then argc is set to 2, and the argv[] array contains pointers
101233	** to the strings "arg1" and "arg2".
101234	**
101235	** This method should return either SQLITE_OK (0), or an SQLite error
101236	** code. If SQLITE_OK is returned, then *ppTokenizer should be set
101237	** to point at the newly created tokenizer structure. The generic
101238	** sqlite3_tokenizer.pModule variable should not be initialised by
101239	** this callback. The caller will do so.
101240	*/
101241	int (*xCreate)(
101242	int argc, /* Size of argv array */
101243	const char constargv, /* Tokenizer argument strings */
101244	sqlite3_tokenizer *ppTokenizer / OUT: Created tokenizer */
101245	);
101246
101247	/*
101248	** Destroy an existing tokenizer. The fts3 module calls this method
101249	** exactly once for each successful call to xCreate().
101250	*/
101251	int (xDestroy)(sqlite3_tokenizer pTokenizer);
101252
101253	/*
101254	** Create a tokenizer cursor to tokenize an input buffer. The caller
101255	** is responsible for ensuring that the input buffer remains valid
101256	** until the cursor is closed (using the xClose() method).
101257	*/
101258	int (*xOpen)(
101259	sqlite3_tokenizer pTokenizer, / Tokenizer object */
101260	const char pInput, int nBytes, / Input buffer */
101261	sqlite3_tokenizer_cursor *ppCursor / OUT: Created tokenizer cursor */
101262	);
101263
101264	/*
101265	** Destroy an existing tokenizer cursor. The fts3 module calls this
101266	** method exactly once for each successful call to xOpen().
101267	*/
101268	int (xClose)(sqlite3_tokenizer_cursor pCursor);
101269
101270	/*
101271	** Retrieve the next token from the tokenizer cursor pCursor. This
101272	** method should either return SQLITE_OK and set the values of the
101273	** "OUT" variables identified below, or SQLITE_DONE to indicate that
101274	** the end of the buffer has been reached, or an SQLite error code.
101275	**
101276	** *ppToken should be set to point at a buffer containing the
101277	** normalized version of the token (i.e. after any case-folding and/or
101278	** stemming has been performed). *pnBytes should be set to the length
101279	** of this buffer in bytes. The input text that generated the token is
101280	** identified by the byte offsets returned in *piStartOffset and
101281	** piEndOffset. piStartOffset should be set to the index of the first
101282	** byte of the token in the input buffer. *piEndOffset should be set
101283	** to the index of the first byte just past the end of the token in
101284	** the input buffer.
101285	**
101286	** The buffer *ppToken is set to point at is managed by the tokenizer
101287	** implementation. It is only required to be valid until the next call
101288	** to xNext() or xClose().
101289	*/
101290	/* TODO(shess) current implementation requires pInput to be
101291	** nul-terminated. This should either be fixed, or pInput/nBytes
101292	** should be converted to zInput.
101293	*/
101294	int (*xNext)(
101295	sqlite3_tokenizer_cursor pCursor, / Tokenizer cursor */
101296	const char *ppToken, int pnBytes, /* OUT: Normalized text for token */
101297	int piStartOffset, / OUT: Byte offset of token in input buffer */
101298	int piEndOffset, / OUT: Byte offset of end of token in input buffer */
101299	int piPosition / OUT: Number of tokens returned before this one */
101300	);
101301	};
101302
101303	struct sqlite3_tokenizer {
101304	const sqlite3_tokenizer_module pModule; / The module for this tokenizer */
101305	/* Tokenizer implementations will typically add additional fields */
101306	};
101307
101308	struct sqlite3_tokenizer_cursor {
101309	sqlite3_tokenizer pTokenizer; / Tokenizer for this cursor. */
101310	/* Tokenizer implementations will typically add additional fields */
101311	};
101312
101313	#endif /* _FTS3_TOKENIZER_H_ */
101314
101315	/************ End of fts3_tokenizer.h ************************************/
101316	/************ Continuing where we left off in fts3_porter.c **************/
101317
101318	/*
101319	** Class derived from sqlite3_tokenizer
101320	*/
101321	typedef struct porter_tokenizer {
	@@ -106682,18 +101980,18 @@
101980	static void scalarFunc(
101981	sqlite3_context *context,
101982	int argc,
101983	sqlite3_value **argv
101984	){
101985	Fts3Hash *pHash;
101986	void *pPtr = 0;
101987	const unsigned char *zName;
101988	int nName;
101989
101990	assert( argc==1 \|\| argc==2 );
101991
101992	pHash = (Fts3Hash *)sqlite3_user_data(context);
101993
101994	zName = sqlite3_value_text(argv[0]);
101995	nName = sqlite3_value_bytes(argv[0])+1;
101996
101997	if( argc==2 ){
	@@ -106719,10 +102017,132 @@
102017	}
102018	}
102019
102020	sqlite3_result_blob(context, (void *)&pPtr, sizeof(pPtr), SQLITE_TRANSIENT);
102021	}
102022
102023	static int fts3IsIdChar(char c){
102024	static const char isFtsIdChar[] = {
102025	0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 0x */
102026	0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 1x */
102027	0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 2x */
102028	1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, /* 3x */
102029	0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* 4x */
102030	1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 1, /* 5x */
102031	0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* 6x */
102032	1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, /* 7x */
102033	};
102034	return (c&0x80 \|\| isFtsIdChar[(int)(c)]);
102035	}
102036
102037	SQLITE_PRIVATE const char sqlite3Fts3NextToken(const char zStr, int *pn){
102038	const char *z1;
102039	const char *z2 = 0;
102040
102041	/* Find the start of the next token. */
102042	z1 = zStr;
102043	while( z2==0 ){
102044	switch( *z1 ){
102045	case '\0': return 0; /* No more tokens here */
102046	case '\'':
102047	case '"':
102048	case '`': {
102049	z2 = &z1[1];
102050	while( z2 && (z2[0]!=z1 \|\| z2[1]==*z1) ) z2++;
102051	if( *z2 ) z2++;
102052	break;
102053	}
102054	case '[':
102055	z2 = &z1[1];
102056	while( *z2 && z2[0]!=']' ) z2++;
102057	if( *z2 ) z2++;
102058	break;
102059
102060	default:
102061	if( fts3IsIdChar(*z1) ){
102062	z2 = &z1[1];
102063	while( fts3IsIdChar(*z2) ) z2++;
102064	}else{
102065	z1++;
102066	}
102067	}
102068	}
102069
102070	*pn = (z2-z1);
102071	return z1;
102072	}
102073
102074	SQLITE_PRIVATE int sqlite3Fts3InitTokenizer(
102075	Fts3Hash pHash, / Tokenizer hash table */
102076	const char zArg, / Possible tokenizer specification */
102077	sqlite3_tokenizer *ppTok, / OUT: Tokenizer (if applicable) */
102078	const char *pzTokenizer, / OUT: Set to zArg if is tokenizer */
102079	char *pzErr / OUT: Set to malloced error message */
102080	){
102081	int rc;
102082	char z = (char )zArg;
102083	int n;
102084	char *zCopy;
102085	char zEnd; / Pointer to nul-term of zCopy */
102086	sqlite3_tokenizer_module *m;
102087
102088	if( !z ){
102089	zCopy = sqlite3_mprintf("simple");
102090	}else{
102091	while( (z&0x80) && isspace(z) ) z++;
102092	if( sqlite3_strnicmp(z, "tokenize", 8) \|\| fts3IsIdChar(z[8])){
102093	return SQLITE_OK;
102094	}
102095	zCopy = sqlite3_mprintf("%s", &z[8]);
102096	*pzTokenizer = zArg;
102097	}
102098	if( !zCopy ){
102099	return SQLITE_NOMEM;
102100	}
102101
102102	zEnd = &zCopy[strlen(zCopy)];
102103
102104	z = (char *)sqlite3Fts3NextToken(zCopy, &n);
102105	z[n] = '\0';
102106	sqlite3Fts3Dequote(z);
102107
102108	m = (sqlite3_tokenizer_module *)sqlite3Fts3HashFind(pHash, z, strlen(z)+1);
102109	if( !m ){
102110	*pzErr = sqlite3_mprintf("unknown tokenizer: %s", z);
102111	rc = SQLITE_ERROR;
102112	}else{
102113	char const **aArg = 0;
102114	int iArg = 0;
102115	z = &z[n+1];
102116	while( z<zEnd && (z = (char *)sqlite3Fts3NextToken(z, &n)) ){
102117	int nNew = sizeof(char )(iArg+1);
102118	char const aNew = (const char )sqlite3_realloc(aArg, nNew);
102119	if( !aNew ){
102120	sqlite3_free(zCopy);
102121	sqlite3_free(aArg);
102122	return SQLITE_NOMEM;
102123	}
102124	aArg = aNew;
102125	aArg[iArg++] = z;
102126	z[n] = '\0';
102127	sqlite3Fts3Dequote(z);
102128	z = &z[n+1];
102129	}
102130	rc = m->xCreate(iArg, aArg, ppTok);
102131	assert( rc!=SQLITE_OK \|\| *ppTok );
102132	if( rc!=SQLITE_OK ){
102133	*pzErr = sqlite3_mprintf("unknown tokenizer: %s", z);
102134	}else{
102135	(*ppTok)->pModule = m;
102136	}
102137	sqlite3_free(aArg);
102138	}
102139
102140	sqlite3_free(zCopy);
102141	return rc;
102142	}
102143
102144
102145	#ifdef SQLITE_TEST
102146
102147
102148	/*
	@@ -106754,11 +102174,11 @@
102174	static void testFunc(
102175	sqlite3_context *context,
102176	int argc,
102177	sqlite3_value **argv
102178	){
102179	Fts3Hash *pHash;
102180	sqlite3_tokenizer_module *p;
102181	sqlite3_tokenizer *pTokenizer = 0;
102182	sqlite3_tokenizer_cursor *pCsr = 0;
102183
102184	const char *zErr = 0;
	@@ -106787,11 +102207,11 @@
102207
102208	if( argc==3 ){
102209	zArg = (const char *)sqlite3_value_text(argv[1]);
102210	}
102211
102212	pHash = (Fts3Hash *)sqlite3_user_data(context);
102213	p = (sqlite3_tokenizer_module *)sqlite3Fts3HashFind(pHash, zName, nName+1);
102214
102215	if( !p ){
102216	char *zErr = sqlite3_mprintf("unknown tokenizer: %s", zName);
102217	sqlite3_result_error(context, zErr, -1);
	@@ -106956,11 +102376,11 @@
102376	** The third argument to this function, zName, is used as the name
102377	** of both the scalar and, if created, the virtual table.
102378	*/
102379	SQLITE_PRIVATE int sqlite3Fts3InitHashTable(
102380	sqlite3 *db,
102381	Fts3Hash *pHash,
102382	const char *zName
102383	){
102384	int rc = SQLITE_OK;
102385	void p = (void )pHash;
102386	const int any = SQLITE_ANY;
	@@ -107219,10 +102639,2911 @@
102639	}
102640
102641	#endif /* !defined(SQLITE_CORE) \|\| defined(SQLITE_ENABLE_FTS3) */
102642
102643	/************ End of fts3_tokenizer1.c ***********************************/
102644	/************ Begin file fts3_write.c ************************************/
102645	/*
102646	** 2009 Oct 23
102647	**
102648	** The author disclaims copyright to this source code. In place of
102649	** a legal notice, here is a blessing:
102650	**
102651	** May you do good and not evil.
102652	** May you find forgiveness for yourself and forgive others.
102653	** May you share freely, never taking more than you give.
102654	**
102655	******************************************************************************
102656	**
102657	** This file is part of the SQLite FTS3 extension module. Specifically,
102658	** this file contains code to insert, update and delete rows from FTS3
102659	** tables. It also contains code to merge FTS3 b-tree segments. Some
102660	** of the sub-routines used to merge segments are also used by the query
102661	** code in fts3.c.
102662	*/
102663
102664	#if !defined(SQLITE_CORE) \|\| defined(SQLITE_ENABLE_FTS3)
102665
102666
102667	#define INTERIOR_MAX 2048 /* Soft limit for segment node size */
102668	#define LEAF_MAX 2048 /* Soft limit for segment leaf size */
102669
102670	typedef struct PendingList PendingList;
102671	typedef struct SegmentNode SegmentNode;
102672	typedef struct SegmentWriter SegmentWriter;
102673
102674	/*
102675	** Data structure used while accumulating terms in the pending-terms hash
102676	** table. The hash table entry maps from term (a string) to a malloced
102677	** instance of this structure.
102678	*/
102679	struct PendingList {
102680	int nData;
102681	char *aData;
102682	int nSpace;
102683	sqlite3_int64 iLastDocid;
102684	sqlite3_int64 iLastCol;
102685	sqlite3_int64 iLastPos;
102686	};
102687
102688	/*
102689	** An instance of this structure is used to iterate through the terms on
102690	** a contiguous set of segment b-tree leaf nodes. Although the details of
102691	** this structure are only manipulated by code in this file, opaque handles
102692	** of type Fts3SegReader* are also used by code in fts3.c to iterate through
102693	** terms when querying the full-text index. See functions:
102694	**
102695	** sqlite3Fts3SegReaderNew()
102696	** sqlite3Fts3SegReaderFree()
102697	** sqlite3Fts3SegReaderIterate()
102698	*/
102699	struct Fts3SegReader {
102700	int iIdx; /* Index within level */
102701	sqlite3_int64 iStartBlock;
102702	sqlite3_int64 iEndBlock;
102703	sqlite3_stmt pStmt; / SQL Statement to access leaf nodes */
102704	char aNode; / Pointer to node data (or NULL) */
102705	int nNode; /* Size of buffer at aNode (or 0) */
102706	int nTermAlloc; /* Allocated size of zTerm buffer */
102707
102708	/* Variables set by fts3SegReaderNext(). These may be read directly
102709	** by the caller. They are valid from the time SegmentReaderNew() returns
102710	** until SegmentReaderNext() returns something other than SQLITE_OK
102711	** (i.e. SQLITE_DONE).
102712	*/
102713	int nTerm; /* Number of bytes in current term */
102714	char zTerm; / Pointer to current term */
102715	char aDoclist; / Pointer to doclist of current entry */
102716	int nDoclist; /* Size of doclist in current entry */
102717
102718	/* The following variables are used to iterate through the current doclist */
102719	char *pOffsetList;
102720	sqlite3_int64 iDocid;
102721	};
102722
102723	/*
102724	** An instance of this structure is used to create a segment b-tree in the
102725	** database. The internal details of this type are only accessed by the
102726	** following functions:
102727	**
102728	** fts3SegWriterAdd()
102729	** fts3SegWriterFlush()
102730	** fts3SegWriterFree()
102731	*/
102732	struct SegmentWriter {
102733	SegmentNode pTree; / Pointer to interior tree structure */
102734	sqlite3_int64 iFirst; /* First slot in %_segments written */
102735	sqlite3_int64 iFree; /* Next free slot in %_segments */
102736	char zTerm; / Pointer to previous term buffer */
102737	int nTerm; /* Number of bytes in zTerm */
102738	int nMalloc; /* Size of malloc'd buffer at zMalloc */
102739	char zMalloc; / Malloc'd space (possibly) used for zTerm */
102740	int nSize; /* Size of allocation at aData */
102741	int nData; /* Bytes of data in aData */
102742	char aData; / Pointer to block from malloc() */
102743	};
102744
102745	/*
102746	** Type SegmentNode is used by the following three functions to create
102747	** the interior part of the segment b+-tree structures (everything except
102748	** the leaf nodes). These functions and type are only ever used by code
102749	** within the fts3SegWriterXXX() family of functions described above.
102750	**
102751	** fts3NodeAddTerm()
102752	** fts3NodeWrite()
102753	** fts3NodeFree()
102754	*/
102755	struct SegmentNode {
102756	SegmentNode pParent; / Parent node (or NULL for root node) */
102757	SegmentNode pRight; / Pointer to right-sibling */
102758	SegmentNode pLeftmost; / Pointer to left-most node of this depth */
102759	int nEntry; /* Number of terms written to node so far */
102760	char zTerm; / Pointer to previous term buffer */
102761	int nTerm; /* Number of bytes in zTerm */
102762	int nMalloc; /* Size of malloc'd buffer at zMalloc */
102763	char zMalloc; / Malloc'd space (possibly) used for zTerm */
102764	int nData; /* Bytes of valid data so far */
102765	char aData; / Node data */
102766	};
102767
102768	/*
102769	** Valid values for the second argument to fts3SqlStmt().
102770	*/
102771	#define SQL_DELETE_CONTENT 0
102772	#define SQL_IS_EMPTY 1
102773	#define SQL_DELETE_ALL_CONTENT 2
102774	#define SQL_DELETE_ALL_SEGMENTS 3
102775	#define SQL_DELETE_ALL_SEGDIR 4
102776	#define SQL_SELECT_CONTENT_BY_ROWID 5
102777	#define SQL_NEXT_SEGMENT_INDEX 6
102778	#define SQL_INSERT_SEGMENTS 7
102779	#define SQL_NEXT_SEGMENTS_ID 8
102780	#define SQL_INSERT_SEGDIR 9
102781	#define SQL_SELECT_LEVEL 10
102782	#define SQL_SELECT_ALL_LEVEL 11
102783	#define SQL_SELECT_LEVEL_COUNT 12
102784	#define SQL_SELECT_SEGDIR_COUNT_MAX 13
102785	#define SQL_DELETE_SEGDIR_BY_LEVEL 14
102786	#define SQL_DELETE_SEGMENTS_RANGE 15
102787	#define SQL_CONTENT_INSERT 16
102788	#define SQL_GET_BLOCK 17
102789
102790	/*
102791	** This function is used to obtain an SQLite prepared statement handle
102792	** for the statement identified by the second argument. If successful,
102793	** *pp is set to the requested statement handle and SQLITE_OK returned.
102794	** Otherwise, an SQLite error code is returned and *pp is set to 0.
102795	**
102796	** If argument apVal is not NULL, then it must point to an array with
102797	** at least as many entries as the requested statement has bound
102798	** parameters. The values are bound to the statements parameters before
102799	** returning.
102800	*/
102801	static int fts3SqlStmt(
102802	Fts3Table p, / Virtual table handle */
102803	int eStmt, /* One of the SQL_XXX constants above */
102804	sqlite3_stmt *pp, / OUT: Statement handle */
102805	sqlite3_value *apVal / Values to bind to statement */
102806	){
102807	const char *azSql[] = {
102808	/* 0 */ "DELETE FROM %Q.'%q_content' WHERE rowid = ?",
102809	/* 1 */ "SELECT NOT EXISTS(SELECT docid FROM %Q.'%q_content' WHERE rowid!=?)",
102810	/* 2 */ "DELETE FROM %Q.'%q_content'",
102811	/* 3 */ "DELETE FROM %Q.'%q_segments'",
102812	/* 4 */ "DELETE FROM %Q.'%q_segdir'",
102813	/* 5 / "SELECT FROM %Q.'%q_content' WHERE rowid=?",
102814	/* 6 */ "SELECT coalesce(max(idx)+1, 0) FROM %Q.'%q_segdir' WHERE level=?",
102815	/* 7 */ "INSERT INTO %Q.'%q_segments'(blockid, block) VALUES(?, ?)",
102816	/* 8 */ "SELECT coalesce(max(blockid)+1, 1) FROM %Q.'%q_segments'",
102817	/* 9 */ "INSERT INTO %Q.'%q_segdir' VALUES(?,?,?,?,?,?)",
102818
102819	/* Return segments in order from oldest to newest.*/
102820	/* 10 */ "SELECT idx, start_block, leaves_end_block, end_block, root "
102821	"FROM %Q.'%q_segdir' WHERE level = ? ORDER BY idx ASC",
102822	/* 11 */ "SELECT idx, start_block, leaves_end_block, end_block, root "
102823	"FROM %Q.'%q_segdir' ORDER BY level DESC, idx ASC",
102824
102825	/* 12 / "SELECT count() FROM %Q.'%q_segdir' WHERE level = ?",
102826	/* 13 / "SELECT count(), max(level) FROM %Q.'%q_segdir'",
102827
102828	/* 14 */ "DELETE FROM %Q.'%q_segdir' WHERE level = ?",
102829	/* 15 */ "DELETE FROM %Q.'%q_segments' WHERE blockid BETWEEN ? AND ?",
102830	/* 16 */ "INSERT INTO %Q.'%q_content' VALUES(%z)",
102831	/* 17 */ "SELECT block FROM %Q.'%q_segments' WHERE blockid = ?",
102832	};
102833	int rc = SQLITE_OK;
102834	sqlite3_stmt *pStmt;
102835
102836	assert( SizeofArray(azSql)==SizeofArray(p->aStmt) );
102837	assert( eStmt<SizeofArray(azSql) && eStmt>=0 );
102838
102839	pStmt = p->aStmt[eStmt];
102840	if( !pStmt ){
102841	char *zSql;
102842	if( eStmt==SQL_CONTENT_INSERT ){
102843	int i; /* Iterator variable */
102844	char zVarlist; / The "?, ?, ..." string */
102845	zVarlist = (char )sqlite3_malloc(2p->nColumn+2);
102846	if( !zVarlist ){
102847	*pp = 0;
102848	return SQLITE_NOMEM;
102849	}
102850	zVarlist[0] = '?';
102851	zVarlist[p->nColumn*2+1] = '\0';
102852	for(i=1; i<=p->nColumn; i++){
102853	zVarlist[i*2-1] = ',';
102854	zVarlist[i*2] = '?';
102855	}
102856	zSql = sqlite3_mprintf(azSql[eStmt], p->zDb, p->zName, zVarlist);
102857	}else{
102858	zSql = sqlite3_mprintf(azSql[eStmt], p->zDb, p->zName);
102859	}
102860	if( !zSql ){
102861	rc = SQLITE_NOMEM;
102862	}else{
102863	rc = sqlite3_prepare_v2(p->db, zSql, -1, &pStmt, NULL);
102864	sqlite3_free(zSql);
102865	assert( rc==SQLITE_OK \|\| pStmt==0 );
102866	p->aStmt[eStmt] = pStmt;
102867	}
102868	}
102869	if( apVal ){
102870	int i;
102871	int nParam = sqlite3_bind_parameter_count(pStmt);
102872	for(i=0; rc==SQLITE_OK && i<nParam; i++){
102873	rc = sqlite3_bind_value(pStmt, i+1, apVal[i]);
102874	}
102875	}
102876	*pp = pStmt;
102877	return rc;
102878	}
102879
102880	/*
102881	** Similar to fts3SqlStmt(). Except, after binding the parameters in
102882	** array apVal[] to the SQL statement identified by eStmt, the statement
102883	** is executed.
102884	**
102885	** Returns SQLITE_OK if the statement is successfully executed, or an
102886	** SQLite error code otherwise.
102887	*/
102888	static int fts3SqlExec(Fts3Table p, int eStmt, sqlite3_value *apVal){
102889	sqlite3_stmt *pStmt;
102890	int rc = fts3SqlStmt(p, eStmt, &pStmt, apVal);
102891	if( rc==SQLITE_OK ){
102892	sqlite3_step(pStmt);
102893	rc = sqlite3_reset(pStmt);
102894	}
102895	return rc;
102896	}
102897
102898
102899	/*
102900	** Read a single block from the %_segments table. If the specified block
102901	** does not exist, return SQLITE_CORRUPT. If some other error (malloc, IO
102902	** etc.) occurs, return the appropriate SQLite error code.
102903	**
102904	** Otherwise, if successful, set *pzBlock to point to a buffer containing
102905	** the block read from the database, and *pnBlock to the size of the read
102906	** block in bytes.
102907	**
102908	** WARNING: The returned buffer is only valid until the next call to
102909	** sqlite3Fts3ReadBlock().
102910	*/
102911	SQLITE_PRIVATE int sqlite3Fts3ReadBlock(
102912	Fts3Table *p,
102913	sqlite3_int64 iBlock,
102914	char const **pzBlock,
102915	int *pnBlock
102916	){
102917	sqlite3_stmt *pStmt;
102918	int rc = fts3SqlStmt(p, SQL_GET_BLOCK, &pStmt, 0);
102919	if( rc!=SQLITE_OK ) return rc;
102920	sqlite3_reset(pStmt);
102921
102922	sqlite3_bind_int64(pStmt, 1, iBlock);
102923	rc = sqlite3_step(pStmt);
102924	if( rc!=SQLITE_ROW ){
102925	return SQLITE_CORRUPT;
102926	}
102927
102928	*pnBlock = sqlite3_column_bytes(pStmt, 0);
102929	pzBlock = (char )sqlite3_column_blob(pStmt, 0);
102930	if( !*pzBlock ){
102931	return SQLITE_NOMEM;
102932	}
102933	return SQLITE_OK;
102934	}
102935
102936	/*
102937	** Set *ppStmt to a statement handle that may be used to iterate through
102938	** all rows in the %_segdir table, from oldest to newest. If successful,
102939	** return SQLITE_OK. If an error occurs while preparing the statement,
102940	** return an SQLite error code.
102941	**
102942	** There is only ever one instance of this SQL statement compiled for
102943	** each FTS3 table.
102944	**
102945	** The statement returns the following columns from the %_segdir table:
102946	**
102947	** 0: idx
102948	** 1: start_block
102949	** 2: leaves_end_block
102950	** 3: end_block
102951	** 4: root
102952	*/
102953	SQLITE_PRIVATE int sqlite3Fts3AllSegdirs(Fts3Table p, sqlite3_stmt *ppStmt){
102954	return fts3SqlStmt(p, SQL_SELECT_ALL_LEVEL, ppStmt, 0);
102955	}
102956
102957
102958	/*
102959	** Append a single varint to a PendingList buffer. SQLITE_OK is returned
102960	** if successful, or an SQLite error code otherwise.
102961	**
102962	** This function also serves to allocate the PendingList structure itself.
102963	** For example, to create a new PendingList structure containing two
102964	** varints:
102965	**
102966	** PendingList *p = 0;
102967	** fts3PendingListAppendVarint(&p, 1);
102968	** fts3PendingListAppendVarint(&p, 2);
102969	*/
102970	static int fts3PendingListAppendVarint(
102971	PendingList *pp, / IN/OUT: Pointer to PendingList struct */
102972	sqlite3_int64 i /* Value to append to data */
102973	){
102974	PendingList p = pp;
102975
102976	/* Allocate or grow the PendingList as required. */
102977	if( !p ){
102978	p = sqlite3_malloc(sizeof(*p) + 100);
102979	if( !p ){
102980	return SQLITE_NOMEM;
102981	}
102982	p->nSpace = 100;
102983	p->aData = (char *)&p[1];
102984	p->nData = 0;
102985	}
102986	else if( p->nData+FTS3_VARINT_MAX+1>p->nSpace ){
102987	int nNew = p->nSpace * 2;
102988	p = sqlite3_realloc(p, sizeof(*p) + nNew);
102989	if( !p ){
102990	sqlite3_free(*pp);
102991	*pp = 0;
102992	return SQLITE_NOMEM;
102993	}
102994	p->nSpace = nNew;
102995	p->aData = (char *)&p[1];
102996	}
102997
102998	/* Append the new serialized varint to the end of the list. */
102999	p->nData += sqlite3Fts3PutVarint(&p->aData[p->nData], i);
103000	p->aData[p->nData] = '\0';
103001	*pp = p;
103002	return SQLITE_OK;
103003	}
103004
103005	/*
103006	** Add a docid/column/position entry to a PendingList structure. Non-zero
103007	** is returned if the structure is sqlite3_realloced as part of adding
103008	** the entry. Otherwise, zero.
103009	**
103010	** If an OOM error occurs, *pRc is set to SQLITE_NOMEM before returning.
103011	** Zero is always returned in this case. Otherwise, if no OOM error occurs,
103012	** it is set to SQLITE_OK.
103013	*/
103014	static int fts3PendingListAppend(
103015	PendingList *pp, / IN/OUT: PendingList structure */
103016	sqlite3_int64 iDocid, /* Docid for entry to add */
103017	sqlite3_int64 iCol, /* Column for entry to add */
103018	sqlite3_int64 iPos, /* Position of term for entry to add */
103019	int pRc / OUT: Return code */
103020	){
103021	PendingList p = pp;
103022	int rc = SQLITE_OK;
103023
103024	assert( !p \|\| p->iLastDocid<=iDocid );
103025
103026	if( !p \|\| p->iLastDocid!=iDocid ){
103027	sqlite3_int64 iDelta = iDocid - (p ? p->iLastDocid : 0);
103028	if( p ){
103029	assert( p->nData<p->nSpace );
103030	assert( p->aData[p->nData]==0 );
103031	p->nData++;
103032	}
103033	if( SQLITE_OK!=(rc = fts3PendingListAppendVarint(&p, iDelta)) ){
103034	goto pendinglistappend_out;
103035	}
103036	p->iLastCol = -1;
103037	p->iLastPos = 0;
103038	p->iLastDocid = iDocid;
103039	}
103040	if( iCol>0 && p->iLastCol!=iCol ){
103041	if( SQLITE_OK!=(rc = fts3PendingListAppendVarint(&p, 1))
103042	\|\| SQLITE_OK!=(rc = fts3PendingListAppendVarint(&p, iCol))
103043	){
103044	goto pendinglistappend_out;
103045	}
103046	p->iLastCol = iCol;
103047	p->iLastPos = 0;
103048	}
103049	if( iCol>=0 ){
103050	assert( iPos>p->iLastPos \|\| (iPos==0 && p->iLastPos==0) );
103051	rc = fts3PendingListAppendVarint(&p, 2+iPos-p->iLastPos);
103052	p->iLastPos = iPos;
103053	}
103054
103055	pendinglistappend_out:
103056	*pRc = rc;
103057	if( p!=*pp ){
103058	*pp = p;
103059	return 1;
103060	}
103061	return 0;
103062	}
103063
103064	/*
103065	** Tokenize the nul-terminated string zText and add all tokens to the
103066	** pending-terms hash-table. The docid used is that currently stored in
103067	** p->iPrevDocid, and the column is specified by argument iCol.
103068	**
103069	** If successful, SQLITE_OK is returned. Otherwise, an SQLite error code.
103070	*/
103071	static int fts3PendingTermsAdd(Fts3Table p, const char zText, int iCol){
103072	int rc;
103073	int iStart;
103074	int iEnd;
103075	int iPos;
103076
103077	char const *zToken;
103078	int nToken;
103079
103080	sqlite3_tokenizer *pTokenizer = p->pTokenizer;
103081	sqlite3_tokenizer_module const *pModule = pTokenizer->pModule;
103082	sqlite3_tokenizer_cursor *pCsr;
103083	int (xNext)(sqlite3_tokenizer_cursor pCursor,
103084	const char*,int,int,int,int*);
103085
103086	assert( pTokenizer && pModule );
103087
103088	rc = pModule->xOpen(pTokenizer, zText, -1, &pCsr);
103089	if( rc!=SQLITE_OK ){
103090	return rc;
103091	}
103092	pCsr->pTokenizer = pTokenizer;
103093
103094	xNext = pModule->xNext;
103095	while( SQLITE_OK==rc
103096	&& SQLITE_OK==(rc = xNext(pCsr, &zToken, &nToken, &iStart, &iEnd, &iPos))
103097	){
103098	PendingList *pList;
103099
103100	/* Positions cannot be negative; we use -1 as a terminator internally.
103101	** Tokens must have a non-zero length.
103102	*/
103103	if( iPos<0 \|\| !zToken \|\| nToken<=0 ){
103104	rc = SQLITE_ERROR;
103105	break;
103106	}
103107
103108	pList = (PendingList *)fts3HashFind(&p->pendingTerms, zToken, nToken);
103109	if( pList ){
103110	p->nPendingData -= (pList->nData + nToken + sizeof(Fts3HashElem));
103111	}
103112	if( fts3PendingListAppend(&pList, p->iPrevDocid, iCol, iPos, &rc) ){
103113	if( pList==fts3HashInsert(&p->pendingTerms, zToken, nToken, pList) ){
103114	/* Malloc failed while inserting the new entry. This can only
103115	** happen if there was no previous entry for this token.
103116	*/
103117	assert( 0==fts3HashFind(&p->pendingTerms, zToken, nToken) );
103118	sqlite3_free(pList);
103119	rc = SQLITE_NOMEM;
103120	}
103121	}
103122	if( rc==SQLITE_OK ){
103123	p->nPendingData += (pList->nData + nToken + sizeof(Fts3HashElem));
103124	}
103125	}
103126
103127	pModule->xClose(pCsr);
103128	return (rc==SQLITE_DONE ? SQLITE_OK : rc);
103129	}
103130
103131	/*
103132	** Calling this function indicates that subsequent calls to
103133	** fts3PendingTermsAdd() are to add term/position-list pairs for the
103134	** contents of the document with docid iDocid.
103135	*/
103136	static int fts3PendingTermsDocid(Fts3Table *p, sqlite_int64 iDocid){
103137	/* TODO(shess) Explore whether partially flushing the buffer on
103138	** forced-flush would provide better performance. I suspect that if
103139	** we ordered the doclists by size and flushed the largest until the
103140	** buffer was half empty, that would let the less frequent terms
103141	** generate longer doclists.
103142	*/
103143	if( iDocid<=p->iPrevDocid \|\| p->nPendingData>FTS3_MAX_PENDING_DATA ){
103144	int rc = sqlite3Fts3PendingTermsFlush(p);
103145	if( rc!=SQLITE_OK ) return rc;
103146	}
103147	p->iPrevDocid = iDocid;
103148	return SQLITE_OK;
103149	}
103150
103151	SQLITE_PRIVATE void sqlite3Fts3PendingTermsClear(Fts3Table *p){
103152	Fts3HashElem *pElem;
103153	for(pElem=fts3HashFirst(&p->pendingTerms); pElem; pElem=fts3HashNext(pElem)){
103154	sqlite3_free(fts3HashData(pElem));
103155	}
103156	fts3HashClear(&p->pendingTerms);
103157	p->nPendingData = 0;
103158	}
103159
103160	/*
103161	** This function is called by the xUpdate() method as part of an INSERT
103162	** operation. It adds entries for each term in the new record to the
103163	** pendingTerms hash table.
103164	**
103165	** Argument apVal is the same as the similarly named argument passed to
103166	** fts3InsertData(). Parameter iDocid is the docid of the new row.
103167	*/
103168	static int fts3InsertTerms(Fts3Table p, sqlite3_value *apVal){
103169	int i; /* Iterator variable */
103170	for(i=2; i<p->nColumn+2; i++){
103171	const char zText = (const char )sqlite3_value_text(apVal[i]);
103172	if( zText ){
103173	int rc = fts3PendingTermsAdd(p, zText, i-2);
103174	if( rc!=SQLITE_OK ){
103175	return rc;
103176	}
103177	}
103178	}
103179	return SQLITE_OK;
103180	}
103181
103182	/*
103183	** This function is called by the xUpdate() method for an INSERT operation.
103184	** The apVal parameter is passed a copy of the apVal argument passed by
103185	** SQLite to the xUpdate() method. i.e:
103186	**
103187	** apVal[0] Not used for INSERT.
103188	** apVal[1] rowid
103189	** apVal[2] Left-most user-defined column
103190	** ...
103191	** apVal[p->nColumn+1] Right-most user-defined column
103192	** apVal[p->nColumn+2] Hidden column with same name as table
103193	** apVal[p->nColumn+3] Hidden "docid" column (alias for rowid)
103194	*/
103195	static int fts3InsertData(
103196	Fts3Table p, / Full-text table */
103197	sqlite3_value *apVal, / Array of values to insert */
103198	sqlite3_int64 piDocid / OUT: Docid for row just inserted */
103199	){
103200	int rc; /* Return code */
103201	int i; /* Iterator variable */
103202	sqlite3_stmt pContentInsert; / INSERT INTO %_content VALUES(...) */
103203
103204	/* Locate the statement handle used to insert data into the %_content
103205	** table. The SQL for this statement is:
103206	**
103207	** INSERT INTO %_content VALUES(?, ?, ?, ...)
103208	**
103209	** The statement features N '?' variables, where N is the number of user
103210	** defined columns in the FTS3 table, plus one for the docid field.
103211	*/
103212	rc = fts3SqlStmt(p, SQL_CONTENT_INSERT, &pContentInsert, &apVal[1]);
103213	if( rc!=SQLITE_OK ){
103214	return rc;
103215	}
103216
103217	/* There is a quirk here. The users INSERT statement may have specified
103218	** a value for the "rowid" field, for the "docid" field, or for both.
103219	** Which is a problem, since "rowid" and "docid" are aliases for the
103220	** same value. For example:
103221	**
103222	** INSERT INTO fts3tbl(rowid, docid) VALUES(1, 2);
103223	**
103224	** In FTS3, if a non-NULL docid value is specified, it is the value
103225	** inserted. Otherwise, the rowid value is used.
103226	*/
103227	if( SQLITE_NULL!=sqlite3_value_type(apVal[3+p->nColumn]) ){
103228	rc = sqlite3_bind_value(pContentInsert, 1, apVal[3+p->nColumn]);
103229	if( rc!=SQLITE_OK ) return rc;
103230	}
103231
103232	/* Execute the statement to insert the record. Set *piDocid to the
103233	** new docid value.
103234	*/
103235	sqlite3_step(pContentInsert);
103236	rc = sqlite3_reset(pContentInsert);
103237
103238	*piDocid = sqlite3_last_insert_rowid(p->db);
103239	return rc;
103240	}
103241
103242
103243
103244	/*
103245	** Remove all data from the FTS3 table. Clear the hash table containing
103246	** pending terms.
103247	*/
103248	static int fts3DeleteAll(Fts3Table *p){
103249	int rc; /* Return code */
103250
103251	/* Discard the contents of the pending-terms hash table. */
103252	sqlite3Fts3PendingTermsClear(p);
103253
103254	/* Delete everything from the %_content, %_segments and %_segdir tables. */
103255	rc = fts3SqlExec(p, SQL_DELETE_ALL_CONTENT, 0);
103256	if( rc==SQLITE_OK ){
103257	rc = fts3SqlExec(p, SQL_DELETE_ALL_SEGMENTS, 0);
103258	}
103259	if( rc==SQLITE_OK ){
103260	rc = fts3SqlExec(p, SQL_DELETE_ALL_SEGDIR, 0);
103261	}
103262	return rc;
103263	}
103264
103265	/*
103266	** The first element in the apVal[] array is assumed to contain the docid
103267	** (an integer) of a row about to be deleted. Remove all terms from the
103268	** full-text index.
103269	*/
103270	static int fts3DeleteTerms(Fts3Table p, sqlite3_value *apVal){
103271	int rc;
103272	sqlite3_stmt *pSelect;
103273
103274	rc = fts3SqlStmt(p, SQL_SELECT_CONTENT_BY_ROWID, &pSelect, apVal);
103275	if( rc==SQLITE_OK ){
103276	if( SQLITE_ROW==sqlite3_step(pSelect) ){
103277	int i;
103278	for(i=1; i<=p->nColumn; i++){
103279	const char zText = (const char )sqlite3_column_text(pSelect, i);
103280	rc = fts3PendingTermsAdd(p, zText, -1);
103281	if( rc!=SQLITE_OK ){
103282	sqlite3_reset(pSelect);
103283	return rc;
103284	}
103285	}
103286	}
103287	}
103288
103289	return sqlite3_reset(pSelect);
103290	}
103291
103292	/*
103293	** Forward declaration to account for the circular dependency between
103294	** functions fts3SegmentMerge() and fts3AllocateSegdirIdx().
103295	*/
103296	static int fts3SegmentMerge(Fts3Table *, int);
103297
103298	/*
103299	** This function allocates a new level iLevel index in the segdir table.
103300	** Usually, indexes are allocated within a level sequentially starting
103301	** with 0, so the allocated index is one greater than the value returned
103302	** by:
103303	**
103304	** SELECT max(idx) FROM %_segdir WHERE level = :iLevel
103305	**
103306	** However, if there are already FTS3_MERGE_COUNT indexes at the requested
103307	** level, they are merged into a single level (iLevel+1) segment and the
103308	** allocated index is 0.
103309	**
103310	** If successful, *piIdx is set to the allocated index slot and SQLITE_OK
103311	** returned. Otherwise, an SQLite error code is returned.
103312	*/
103313	static int fts3AllocateSegdirIdx(Fts3Table p, int iLevel, int piIdx){
103314	int rc; /* Return Code */
103315	sqlite3_stmt pNextIdx; / Query for next idx at level iLevel */
103316	int iNext; /* Result of query pNextIdx */
103317
103318	/* Set variable iNext to the next available segdir index at level iLevel. */
103319	rc = fts3SqlStmt(p, SQL_NEXT_SEGMENT_INDEX, &pNextIdx, 0);
103320	if( rc==SQLITE_OK ){
103321	sqlite3_bind_int(pNextIdx, 1, iLevel);
103322	if( SQLITE_ROW==sqlite3_step(pNextIdx) ){
103323	iNext = sqlite3_column_int64(pNextIdx, 0);
103324	}
103325	rc = sqlite3_reset(pNextIdx);
103326	}
103327
103328	if( rc==SQLITE_OK ){
103329	/* If iNext is FTS3_MERGE_COUNT, indicating that level iLevel is already
103330	** full, merge all segments in level iLevel into a single iLevel+1
103331	** segment and allocate (newly freed) index 0 at level iLevel. Otherwise,
103332	** if iNext is less than FTS3_MERGE_COUNT, allocate index iNext.
103333	*/
103334	if( iNext>=FTS3_MERGE_COUNT ){
103335	rc = fts3SegmentMerge(p, iLevel);
103336	*piIdx = 0;
103337	}else{
103338	*piIdx = iNext;
103339	}
103340	}
103341
103342	return rc;
103343	}
103344
103345	/*
103346	** Move the iterator passed as the first argument to the next term in the
103347	** segment. If successful, SQLITE_OK is returned. If there is no next term,
103348	** SQLITE_DONE. Otherwise, an SQLite error code.
103349	*/
103350	static int fts3SegReaderNext(Fts3SegReader *pReader){
103351	char pNext; / Cursor variable */
103352	int nPrefix; /* Number of bytes in term prefix */
103353	int nSuffix; /* Number of bytes in term suffix */
103354
103355	if( !pReader->aDoclist ){
103356	pNext = pReader->aNode;
103357	}else{
103358	pNext = &pReader->aDoclist[pReader->nDoclist];
103359	}
103360
103361	if( !pNext \|\| pNext>=&pReader->aNode[pReader->nNode] ){
103362	int rc;
103363	if( !pReader->pStmt ){
103364	pReader->aNode = 0;
103365	return SQLITE_OK;
103366	}
103367	rc = sqlite3_step(pReader->pStmt);
103368	if( rc!=SQLITE_ROW ){
103369	pReader->aNode = 0;
103370	return (rc==SQLITE_DONE ? SQLITE_OK : rc);
103371	}
103372	pReader->nNode = sqlite3_column_bytes(pReader->pStmt, 0);
103373	pReader->aNode = (char *)sqlite3_column_blob(pReader->pStmt, 0);
103374	pNext = pReader->aNode;
103375	}
103376
103377	pNext += sqlite3Fts3GetVarint32(pNext, &nPrefix);
103378	pNext += sqlite3Fts3GetVarint32(pNext, &nSuffix);
103379
103380	if( nPrefix+nSuffix>pReader->nTermAlloc ){
103381	int nNew = (nPrefix+nSuffix)*2;
103382	char *zNew = sqlite3_realloc(pReader->zTerm, nNew);
103383	if( !zNew ){
103384	return SQLITE_NOMEM;
103385	}
103386	pReader->zTerm = zNew;
103387	pReader->nTermAlloc = nNew;
103388	}
103389	memcpy(&pReader->zTerm[nPrefix], pNext, nSuffix);
103390	pReader->nTerm = nPrefix+nSuffix;
103391	pNext += nSuffix;
103392	pNext += sqlite3Fts3GetVarint32(pNext, &pReader->nDoclist);
103393	assert( pNext<&pReader->aNode[pReader->nNode] );
103394	pReader->aDoclist = pNext;
103395	pReader->pOffsetList = 0;
103396	return SQLITE_OK;
103397	}
103398
103399	/*
103400	** Set the SegReader to point to the first docid in the doclist associated
103401	** with the current term.
103402	*/
103403	static void fts3SegReaderFirstDocid(Fts3SegReader *pReader){
103404	int n;
103405	assert( pReader->aDoclist );
103406	assert( !pReader->pOffsetList );
103407	n = sqlite3Fts3GetVarint(pReader->aDoclist, &pReader->iDocid);
103408	pReader->pOffsetList = &pReader->aDoclist[n];
103409	}
103410
103411	/*
103412	** Advance the SegReader to point to the next docid in the doclist
103413	** associated with the current term.
103414	**
103415	** If arguments ppOffsetList and pnOffsetList are not NULL, then
103416	** *ppOffsetList is set to point to the first column-offset list
103417	** in the doclist entry (i.e. immediately past the docid varint).
103418	** *pnOffsetList is set to the length of the set of column-offset
103419	** lists, not including the nul-terminator byte. For example:
103420	*/
103421	static void fts3SegReaderNextDocid(
103422	Fts3SegReader *pReader,
103423	char **ppOffsetList,
103424	int *pnOffsetList
103425	){
103426	char *p = pReader->pOffsetList;
103427	char c = 0;
103428
103429	/* Pointer p currently points at the first byte of an offset list. The
103430	** following two lines advance it to point one byte past the end of
103431	** the same offset list.
103432	*/
103433	while( p \| c ) c = p++ & 0x80;
103434	p++;
103435
103436	/* If required, populate the output variables with a pointer to and the
103437	** size of the previous offset-list.
103438	*/
103439	if( ppOffsetList ){
103440	*ppOffsetList = pReader->pOffsetList;
103441	*pnOffsetList = p - pReader->pOffsetList - 1;
103442	}
103443
103444	/* If there are no more entries in the doclist, set pOffsetList to
103445	** NULL. Otherwise, set Fts3SegReader.iDocid to the next docid and
103446	** Fts3SegReader.pOffsetList to point to the next offset list before
103447	** returning.
103448	*/
103449	if( p>=&pReader->aDoclist[pReader->nDoclist] ){
103450	pReader->pOffsetList = 0;
103451	}else{
103452	sqlite3_int64 iDelta;
103453	pReader->pOffsetList = p + sqlite3Fts3GetVarint(p, &iDelta);
103454	pReader->iDocid += iDelta;
103455	}
103456	}
103457
103458	/*
103459	** Free all allocations associated with the iterator passed as the
103460	** second argument.
103461	*/
103462	SQLITE_PRIVATE void sqlite3Fts3SegReaderFree(Fts3Table p, Fts3SegReader pReader){
103463	if( pReader ){
103464	if( pReader->pStmt ){
103465	/* Move the leaf-range SELECT statement to the aLeavesStmt[] array,
103466	** so that it can be reused when required by another query.
103467	*/
103468	assert( p->nLeavesStmt<p->nLeavesTotal );
103469	sqlite3_reset(pReader->pStmt);
103470	p->aLeavesStmt[p->nLeavesStmt++] = pReader->pStmt;
103471	}
103472	sqlite3_free(pReader->zTerm);
103473	sqlite3_free(pReader);
103474	}
103475	}
103476
103477	/*
103478	** Allocate a new SegReader object.
103479	*/
103480	SQLITE_PRIVATE int sqlite3Fts3SegReaderNew(
103481	Fts3Table p, / Virtual table handle */
103482	int iAge, /* Segment "age". */
103483	sqlite3_int64 iStartLeaf, /* First leaf to traverse */
103484	sqlite3_int64 iEndLeaf, /* Final leaf to traverse */
103485	sqlite3_int64 iEndBlock, /* Final block of segment */
103486	const char zRoot, / Buffer containing root node */
103487	int nRoot, /* Size of buffer containing root node */
103488	Fts3SegReader *ppReader / OUT: Allocated Fts3SegReader */
103489	){
103490	int rc = SQLITE_OK; /* Return code */
103491	Fts3SegReader pReader; / Newly allocated SegReader object */
103492	int nExtra = 0; /* Bytes to allocate segment root node */
103493
103494	if( iStartLeaf==0 ){
103495	nExtra = nRoot;
103496	}
103497
103498	pReader = (Fts3SegReader *)sqlite3_malloc(sizeof(Fts3SegReader) + nExtra);
103499	if( !pReader ){
103500	return SQLITE_NOMEM;
103501	}
103502	memset(pReader, 0, sizeof(Fts3SegReader));
103503	pReader->iStartBlock = iStartLeaf;
103504	pReader->iIdx = iAge;
103505	pReader->iEndBlock = iEndBlock;
103506
103507	if( nExtra ){
103508	/* The entire segment is stored in the root node. */
103509	pReader->aNode = (char *)&pReader[1];
103510	pReader->nNode = nRoot;
103511	memcpy(pReader->aNode, zRoot, nRoot);
103512	}else{
103513	sqlite3_stmt *pStmt;
103514
103515	/* If the text of the SQL statement to iterate through a contiguous
103516	** set of entries in the %_segments table has not yet been composed,
103517	** compose it now.
103518	*/
103519	if( !p->zSelectLeaves ){
103520	p->zSelectLeaves = sqlite3_mprintf(
103521	"SELECT block FROM %Q.'%q_segments' WHERE blockid BETWEEN ? AND ? "
103522	"ORDER BY blockid", p->zDb, p->zName
103523	);
103524	if( !p->zSelectLeaves ){
103525	rc = SQLITE_NOMEM;
103526	goto finished;
103527	}
103528	}
103529
103530	/* If there are no free statements in the aLeavesStmt[] array, prepare
103531	** a new statement now. Otherwise, reuse a prepared statement from
103532	** aLeavesStmt[].
103533	*/
103534	if( p->nLeavesStmt==0 ){
103535	if( p->nLeavesTotal==p->nLeavesAlloc ){
103536	int nNew = p->nLeavesAlloc + 16;
103537	sqlite3_stmt aNew = (sqlite3_stmt )sqlite3_realloc(
103538	p->aLeavesStmt, nNewsizeof(sqlite3_stmt )
103539	);
103540	if( !aNew ){
103541	rc = SQLITE_NOMEM;
103542	goto finished;
103543	}
103544	p->nLeavesAlloc = nNew;
103545	p->aLeavesStmt = aNew;
103546	}
103547	rc = sqlite3_prepare_v2(p->db, p->zSelectLeaves, -1, &pReader->pStmt, 0);
103548	if( rc!=SQLITE_OK ){
103549	goto finished;
103550	}
103551	p->nLeavesTotal++;
103552	}else{
103553	pReader->pStmt = p->aLeavesStmt[--p->nLeavesStmt];
103554	}
103555
103556	/* Bind the start and end leaf blockids to the prepared SQL statement. */
103557	sqlite3_bind_int64(pReader->pStmt, 1, iStartLeaf);
103558	sqlite3_bind_int64(pReader->pStmt, 2, iEndLeaf);
103559	}
103560	rc = fts3SegReaderNext(pReader);
103561
103562	finished:
103563	if( rc==SQLITE_OK ){
103564	*ppReader = pReader;
103565	}else{
103566	sqlite3Fts3SegReaderFree(p, pReader);
103567	}
103568	return rc;
103569	}
103570
103571
103572	/*
103573	** The second argument to this function is expected to be a statement of
103574	** the form:
103575	**
103576	** SELECT
103577	** idx, -- col 0
103578	** start_block, -- col 1
103579	** leaves_end_block, -- col 2
103580	** end_block, -- col 3
103581	** root -- col 4
103582	** FROM %_segdir ...
103583	**
103584	** This function allocates and initializes a Fts3SegReader structure to
103585	** iterate through the terms stored in the segment identified by the
103586	** current row that pStmt is pointing to.
103587	**
103588	** If successful, the Fts3SegReader is left pointing to the first term
103589	** in the segment and SQLITE_OK is returned. Otherwise, an SQLite error
103590	** code is returned.
103591	*/
103592	static int fts3SegReaderNew(
103593	Fts3Table p, / Virtual table handle */
103594	sqlite3_stmt pStmt, / See above */
103595	int iAge, /* Segment "age". */
103596	Fts3SegReader *ppReader / OUT: Allocated Fts3SegReader */
103597	){
103598	return sqlite3Fts3SegReaderNew(p, iAge,
103599	sqlite3_column_int64(pStmt, 1),
103600	sqlite3_column_int64(pStmt, 2),
103601	sqlite3_column_int64(pStmt, 3),
103602	sqlite3_column_blob(pStmt, 4),
103603	sqlite3_column_bytes(pStmt, 4),
103604	ppReader
103605	);
103606	}
103607
103608	/*
103609	** Compare the entries pointed to by two Fts3SegReader structures.
103610	** Comparison is as follows:
103611	**
103612	** 1) EOF is greater than not EOF.
103613	**
103614	** 2) The current terms (if any) are compared with memcmp(). If one
103615	** term is a prefix of another, the longer term is considered the
103616	** larger.
103617	**
103618	** 3) By segment age. An older segment is considered larger.
103619	*/
103620	static int fts3SegReaderCmp(Fts3SegReader pLhs, Fts3SegReader pRhs){
103621	int rc;
103622	if( pLhs->aNode && pRhs->aNode ){
103623	int rc2 = pLhs->nTerm - pRhs->nTerm;
103624	if( rc2<0 ){
103625	rc = memcmp(pLhs->zTerm, pRhs->zTerm, pLhs->nTerm);
103626	}else{
103627	rc = memcmp(pLhs->zTerm, pRhs->zTerm, pRhs->nTerm);
103628	}
103629	if( rc==0 ){
103630	rc = rc2;
103631	}
103632	}else{
103633	rc = (pLhs->aNode==0) - (pRhs->aNode==0);
103634	}
103635	if( rc==0 ){
103636	rc = pRhs->iIdx - pLhs->iIdx;
103637	}
103638	assert( rc!=0 );
103639	return rc;
103640	}
103641
103642	/*
103643	** A different comparison function for SegReader structures. In this
103644	** version, it is assumed that each SegReader points to an entry in
103645	** a doclist for identical terms. Comparison is made as follows:
103646	**
103647	** 1) EOF (end of doclist in this case) is greater than not EOF.
103648	**
103649	** 2) By current docid.
103650	**
103651	** 3) By segment age. An older segment is considered larger.
103652	*/
103653	static int fts3SegReaderDoclistCmp(Fts3SegReader pLhs, Fts3SegReader pRhs){
103654	int rc = (pLhs->pOffsetList==0)-(pRhs->pOffsetList==0);
103655	if( rc==0 ){
103656	if( pLhs->iDocid==pRhs->iDocid ){
103657	rc = pRhs->iIdx - pLhs->iIdx;
103658	}else{
103659	rc = (pLhs->iDocid > pRhs->iDocid) ? 1 : -1;
103660	}
103661	}
103662	assert( pLhs->aNode && pRhs->aNode );
103663	return rc;
103664	}
103665
103666	/*
103667	** Compare the term that the Fts3SegReader object passed as the first argument
103668	** points to with the term specified by arguments zTerm and nTerm.
103669	**
103670	** If the pSeg iterator is already at EOF, return 0. Otherwise, return
103671	** -ve if the pSeg term is less than zTerm/nTerm, 0 if the two terms are
103672	** equal, or +ve if the pSeg term is greater than zTerm/nTerm.
103673	*/
103674	static int fts3SegReaderTermCmp(
103675	Fts3SegReader pSeg, / Segment reader object */
103676	const char zTerm, / Term to compare to */
103677	int nTerm /* Size of term zTerm in bytes */
103678	){
103679	int res = 0;
103680	if( pSeg->aNode ){
103681	if( pSeg->nTerm>nTerm ){
103682	res = memcmp(pSeg->zTerm, zTerm, nTerm);
103683	}else{
103684	res = memcmp(pSeg->zTerm, zTerm, pSeg->nTerm);
103685	}
103686	if( res==0 ){
103687	res = pSeg->nTerm-nTerm;
103688	}
103689	}
103690	return res;
103691	}
103692
103693	/*
103694	** Argument apSegment is an array of nSegment elements. It is known that
103695	** the final (nSegment-nSuspect) members are already in sorted order
103696	** (according to the comparison function provided). This function shuffles
103697	** the array around until all entries are in sorted order.
103698	*/
103699	static void fts3SegReaderSort(
103700	Fts3SegReader *apSegment, / Array to sort entries of */
103701	int nSegment, /* Size of apSegment array */
103702	int nSuspect, /* Unsorted entry count */
103703	int (xCmp)(Fts3SegReader , Fts3SegReader ) / Comparison function */
103704	){
103705	int i; /* Iterator variable */
103706
103707	assert( nSuspect<=nSegment );
103708
103709	if( nSuspect==nSegment ) nSuspect--;
103710	for(i=nSuspect-1; i>=0; i--){
103711	int j;
103712	for(j=i; j<(nSegment-1); j++){
103713	Fts3SegReader *pTmp;
103714	if( xCmp(apSegment[j], apSegment[j+1])<0 ) break;
103715	pTmp = apSegment[j+1];
103716	apSegment[j+1] = apSegment[j];
103717	apSegment[j] = pTmp;
103718	}
103719	}
103720
103721	#ifndef NDEBUG
103722	/* Check that the list really is sorted now. */
103723	for(i=0; i<(nSuspect-1); i++){
103724	assert( xCmp(apSegment[i], apSegment[i+1])<0 );
103725	}
103726	#endif
103727	}
103728
103729	/*
103730	** Insert a record into the %_segments table.
103731	*/
103732	static int fts3WriteSegment(
103733	Fts3Table p, / Virtual table handle */
103734	sqlite3_int64 iBlock, /* Block id for new block */
103735	char z, / Pointer to buffer containing block data */
103736	int n /* Size of buffer z in bytes */
103737	){
103738	sqlite3_stmt *pStmt;
103739	int rc = fts3SqlStmt(p, SQL_INSERT_SEGMENTS, &pStmt, 0);
103740	if( rc==SQLITE_OK ){
103741	sqlite3_bind_int64(pStmt, 1, iBlock);
103742	rc = sqlite3_bind_blob(pStmt, 2, z, n, SQLITE_STATIC);
103743	if( rc==SQLITE_OK ){
103744	sqlite3_step(pStmt);
103745	rc = sqlite3_reset(pStmt);
103746	}
103747	}
103748	return rc;
103749	}
103750
103751	/*
103752	** Insert a record into the %_segdir table.
103753	*/
103754	static int fts3WriteSegdir(
103755	Fts3Table p, / Virtual table handle */
103756	int iLevel, /* Value for "level" field */
103757	int iIdx, /* Value for "idx" field */
103758	sqlite3_int64 iStartBlock, /* Value for "start_block" field */
103759	sqlite3_int64 iLeafEndBlock, /* Value for "leaves_end_block" field */
103760	sqlite3_int64 iEndBlock, /* Value for "end_block" field */
103761	char zRoot, / Blob value for "root" field */
103762	int nRoot /* Number of bytes in buffer zRoot */
103763	){
103764	sqlite3_stmt *pStmt;
103765	int rc = fts3SqlStmt(p, SQL_INSERT_SEGDIR, &pStmt, 0);
103766	if( rc==SQLITE_OK ){
103767	sqlite3_bind_int(pStmt, 1, iLevel);
103768	sqlite3_bind_int(pStmt, 2, iIdx);
103769	sqlite3_bind_int64(pStmt, 3, iStartBlock);
103770	sqlite3_bind_int64(pStmt, 4, iLeafEndBlock);
103771	sqlite3_bind_int64(pStmt, 5, iEndBlock);
103772	rc = sqlite3_bind_blob(pStmt, 6, zRoot, nRoot, SQLITE_STATIC);
103773	if( rc==SQLITE_OK ){
103774	sqlite3_step(pStmt);
103775	rc = sqlite3_reset(pStmt);
103776	}
103777	}
103778	return rc;
103779	}
103780
103781	/*
103782	** Return the size of the common prefix (if any) shared by zPrev and
103783	** zNext, in bytes. For example,
103784	**
103785	** fts3PrefixCompress("abc", 3, "abcdef", 6) // returns 3
103786	** fts3PrefixCompress("abX", 3, "abcdef", 6) // returns 2
103787	** fts3PrefixCompress("abX", 3, "Xbcdef", 6) // returns 0
103788	*/
103789	static int fts3PrefixCompress(
103790	const char zPrev, / Buffer containing previous term */
103791	int nPrev, /* Size of buffer zPrev in bytes */
103792	const char zNext, / Buffer containing next term */
103793	int nNext /* Size of buffer zNext in bytes */
103794	){
103795	int n;
103796	for(n=0; n<nPrev && zPrev[n]==zNext[n]; n++);
103797	return n;
103798	}
103799
103800	/*
103801	** Add term zTerm to the SegmentNode. It is guaranteed that zTerm is larger
103802	** (according to memcmp) than the previous term.
103803	*/
103804	static int fts3NodeAddTerm(
103805	Fts3Table p, / Virtual table handle */
103806	SegmentNode *ppTree, / IN/OUT: SegmentNode handle */
103807	int isCopyTerm, /* True if zTerm/nTerm is transient */
103808	const char zTerm, / Pointer to buffer containing term */
103809	int nTerm /* Size of term in bytes */
103810	){
103811	SegmentNode pTree = ppTree;
103812	int rc;
103813	SegmentNode *pNew;
103814
103815	/* First try to append the term to the current node. Return early if
103816	** this is possible.
103817	*/
103818	if( pTree ){
103819	int nData = pTree->nData; /* Current size of node in bytes */
103820	int nReq = nData; /* Required space after adding zTerm */
103821	int nPrefix; /* Number of bytes of prefix compression */
103822	int nSuffix; /* Suffix length */
103823
103824	nPrefix = fts3PrefixCompress(pTree->zTerm, pTree->nTerm, zTerm, nTerm);
103825	nSuffix = nTerm-nPrefix;
103826
103827	nReq += sqlite3Fts3VarintLen(nPrefix)+sqlite3Fts3VarintLen(nSuffix)+nSuffix;
103828	if( nReq<=INTERIOR_MAX \|\| !pTree->zTerm ){
103829
103830	if( nReq>INTERIOR_MAX ){
103831	/* An unusual case: this is the first term to be added to the node
103832	** and the static node buffer (INTERIOR_MAX bytes) is not large
103833	** enough. Use a separately malloced buffer instead This wastes
103834	** INTERIOR_MAX bytes, but since this scenario only comes about when
103835	** the database contain two terms that share a prefix of almost 2KB,
103836	** this is not expected to be a serious problem.
103837	*/
103838	assert( pTree->aData==(char *)&pTree[1] );
103839	pTree->aData = (char *)sqlite3_malloc(nReq);
103840	if( !pTree->aData ){
103841	return SQLITE_NOMEM;
103842	}
103843	}
103844
103845	if( pTree->zTerm ){
103846	/* There is no prefix-length field for first term in a node */
103847	nData += sqlite3Fts3PutVarint(&pTree->aData[nData], nPrefix);
103848	}
103849
103850	nData += sqlite3Fts3PutVarint(&pTree->aData[nData], nSuffix);
103851	memcpy(&pTree->aData[nData], &zTerm[nPrefix], nSuffix);
103852	pTree->nData = nData + nSuffix;
103853	pTree->nEntry++;
103854
103855	if( isCopyTerm ){
103856	if( pTree->nMalloc<nTerm ){
103857	char zNew = sqlite3_realloc(pTree->zMalloc, nTerm2);
103858	if( !zNew ){
103859	return SQLITE_NOMEM;
103860	}
103861	pTree->nMalloc = nTerm*2;
103862	pTree->zMalloc = zNew;
103863	}
103864	pTree->zTerm = pTree->zMalloc;
103865	memcpy(pTree->zTerm, zTerm, nTerm);
103866	pTree->nTerm = nTerm;
103867	}else{
103868	pTree->zTerm = (char *)zTerm;
103869	pTree->nTerm = nTerm;
103870	}
103871	return SQLITE_OK;
103872	}
103873	}
103874
103875	/* If control flows to here, it was not possible to append zTerm to the
103876	** current node. Create a new node (a right-sibling of the current node).
103877	** If this is the first node in the tree, the term is added to it.
103878	**
103879	** Otherwise, the term is not added to the new node, it is left empty for
103880	** now. Instead, the term is inserted into the parent of pTree. If pTree
103881	** has no parent, one is created here.
103882	*/
103883	pNew = (SegmentNode *)sqlite3_malloc(sizeof(SegmentNode) + INTERIOR_MAX);
103884	if( !pNew ){
103885	return SQLITE_NOMEM;
103886	}
103887	memset(pNew, 0, sizeof(SegmentNode));
103888	pNew->nData = 1 + FTS3_VARINT_MAX;
103889	pNew->aData = (char *)&pNew[1];
103890
103891	if( pTree ){
103892	SegmentNode *pParent = pTree->pParent;
103893	rc = fts3NodeAddTerm(p, &pParent, isCopyTerm, zTerm, nTerm);
103894	if( pTree->pParent==0 ){
103895	pTree->pParent = pParent;
103896	}
103897	pTree->pRight = pNew;
103898	pNew->pLeftmost = pTree->pLeftmost;
103899	pNew->pParent = pParent;
103900	pNew->zMalloc = pTree->zMalloc;
103901	pNew->nMalloc = pTree->nMalloc;
103902	pTree->zMalloc = 0;
103903	}else{
103904	pNew->pLeftmost = pNew;
103905	rc = fts3NodeAddTerm(p, &pNew, isCopyTerm, zTerm, nTerm);
103906	}
103907
103908	*ppTree = pNew;
103909	return rc;
103910	}
103911
103912	/*
103913	** Helper function for fts3NodeWrite().
103914	*/
103915	static int fts3TreeFinishNode(
103916	SegmentNode *pTree,
103917	int iHeight,
103918	sqlite3_int64 iLeftChild
103919	){
103920	int nStart;
103921	assert( iHeight>=1 && iHeight<128 );
103922	nStart = FTS3_VARINT_MAX - sqlite3Fts3VarintLen(iLeftChild);
103923	pTree->aData[nStart] = (char)iHeight;
103924	sqlite3Fts3PutVarint(&pTree->aData[nStart+1], iLeftChild);
103925	return nStart;
103926	}
103927
103928	/*
103929	** Write the buffer for the segment node pTree and all of its peers to the
103930	** database. Then call this function recursively to write the parent of
103931	** pTree and its peers to the database.
103932	**
103933	** Except, if pTree is a root node, do not write it to the database. Instead,
103934	** set output variables paRoot and pnRoot to contain the root node.
103935	**
103936	** If successful, SQLITE_OK is returned and output variable *piLast is
103937	** set to the largest blockid written to the database (or zero if no
103938	** blocks were written to the db). Otherwise, an SQLite error code is
103939	** returned.
103940	*/
103941	static int fts3NodeWrite(
103942	Fts3Table p, / Virtual table handle */
103943	SegmentNode pTree, / SegmentNode handle */
103944	int iHeight, /* Height of this node in tree */
103945	sqlite3_int64 iLeaf, /* Block id of first leaf node */
103946	sqlite3_int64 iFree, /* Block id of next free slot in %_segments */
103947	sqlite3_int64 piLast, / OUT: Block id of last entry written */
103948	char *paRoot, / OUT: Data for root node */
103949	int pnRoot / OUT: Size of root node in bytes */
103950	){
103951	int rc = SQLITE_OK;
103952
103953	if( !pTree->pParent ){
103954	/* Root node of the tree. */
103955	int nStart = fts3TreeFinishNode(pTree, iHeight, iLeaf);
103956	*piLast = iFree-1;
103957	*pnRoot = pTree->nData - nStart;
103958	*paRoot = &pTree->aData[nStart];
103959	}else{
103960	SegmentNode *pIter;
103961	sqlite3_int64 iNextFree = iFree;
103962	sqlite3_int64 iNextLeaf = iLeaf;
103963	for(pIter=pTree->pLeftmost; pIter && rc==SQLITE_OK; pIter=pIter->pRight){
103964	int nStart = fts3TreeFinishNode(pIter, iHeight, iNextLeaf);
103965	int nWrite = pIter->nData - nStart;
103966
103967	rc = fts3WriteSegment(p, iNextFree, &pIter->aData[nStart], nWrite);
103968	iNextFree++;
103969	iNextLeaf += (pIter->nEntry+1);
103970	}
103971	if( rc==SQLITE_OK ){
103972	assert( iNextLeaf==iFree );
103973	rc = fts3NodeWrite(
103974	p, pTree->pParent, iHeight+1, iFree, iNextFree, piLast, paRoot, pnRoot
103975	);
103976	}
103977	}
103978
103979	return rc;
103980	}
103981
103982	/*
103983	** Free all memory allocations associated with the tree pTree.
103984	*/
103985	static void fts3NodeFree(SegmentNode *pTree){
103986	if( pTree ){
103987	SegmentNode *p = pTree->pLeftmost;
103988	fts3NodeFree(p->pParent);
103989	while( p ){
103990	SegmentNode *pRight = p->pRight;
103991	if( p->aData!=(char *)&p[1] ){
103992	sqlite3_free(p->aData);
103993	}
103994	assert( pRight==0 \|\| p->zMalloc==0 );
103995	sqlite3_free(p->zMalloc);
103996	sqlite3_free(p);
103997	p = pRight;
103998	}
103999	}
104000	}
104001
104002	/*
104003	** Add a term to the segment being constructed by the SegmentWriter object
104004	** ppWriter. When adding the first term to a segment, ppWriter should
104005	** be passed NULL. This function will allocate a new SegmentWriter object
104006	** and return it via the input/output variable *ppWriter in this case.
104007	**
104008	** If successful, SQLITE_OK is returned. Otherwise, an SQLite error code.
104009	*/
104010	static int fts3SegWriterAdd(
104011	Fts3Table p, / Virtual table handle */
104012	SegmentWriter *ppWriter, / IN/OUT: SegmentWriter handle */
104013	int isCopyTerm, /* True if buffer zTerm must be copied */
104014	const char zTerm, / Pointer to buffer containing term */
104015	int nTerm, /* Size of term in bytes */
104016	const char aDoclist, / Pointer to buffer containing doclist */
104017	int nDoclist /* Size of doclist in bytes */
104018	){
104019	int nPrefix; /* Size of term prefix in bytes */
104020	int nSuffix; /* Size of term suffix in bytes */
104021	int nReq; /* Number of bytes required on leaf page */
104022	int nData;
104023	SegmentWriter pWriter = ppWriter;
104024
104025	if( !pWriter ){
104026	int rc;
104027	sqlite3_stmt *pStmt;
104028
104029	/* Allocate the SegmentWriter structure */
104030	pWriter = (SegmentWriter *)sqlite3_malloc(sizeof(SegmentWriter));
104031	if( !pWriter ) return SQLITE_NOMEM;
104032	memset(pWriter, 0, sizeof(SegmentWriter));
104033	*ppWriter = pWriter;
104034
104035	/* Allocate a buffer in which to accumulate data */
104036	pWriter->aData = (char *)sqlite3_malloc(LEAF_MAX);
104037	if( !pWriter->aData ) return SQLITE_NOMEM;
104038	pWriter->nSize = LEAF_MAX;
104039
104040	/* Find the next free blockid in the %_segments table */
104041	rc = fts3SqlStmt(p, SQL_NEXT_SEGMENTS_ID, &pStmt, 0);
104042	if( rc!=SQLITE_OK ) return rc;
104043	if( SQLITE_ROW==sqlite3_step(pStmt) ){
104044	pWriter->iFree = sqlite3_column_int64(pStmt, 0);
104045	pWriter->iFirst = pWriter->iFree;
104046	}
104047	rc = sqlite3_reset(pStmt);
104048	if( rc!=SQLITE_OK ) return rc;
104049	}
104050	nData = pWriter->nData;
104051
104052	nPrefix = fts3PrefixCompress(pWriter->zTerm, pWriter->nTerm, zTerm, nTerm);
104053	nSuffix = nTerm-nPrefix;
104054
104055	/* Figure out how many bytes are required by this new entry */
104056	nReq = sqlite3Fts3VarintLen(nPrefix) + /* varint containing prefix size */
104057	sqlite3Fts3VarintLen(nSuffix) + /* varint containing suffix size */
104058	nSuffix + /* Term suffix */
104059	sqlite3Fts3VarintLen(nDoclist) + /* Size of doclist */
104060	nDoclist; /* Doclist data */
104061
104062	if( nData>0 && nData+nReq>LEAF_MAX ){
104063	int rc;
104064
104065	/* The current leaf node is full. Write it out to the database. */
104066	rc = fts3WriteSegment(p, pWriter->iFree++, pWriter->aData, nData);
104067	if( rc!=SQLITE_OK ) return rc;
104068
104069	/* Add the current term to the interior node tree. The term added to
104070	** the interior tree must:
104071	**
104072	** a) be greater than the largest term on the leaf node just written
104073	** to the database (still available in pWriter->zTerm), and
104074	**
104075	** b) be less than or equal to the term about to be added to the new
104076	** leaf node (zTerm/nTerm).
104077	**
104078	** In other words, it must be the prefix of zTerm 1 byte longer than
104079	** the common prefix (if any) of zTerm and pWriter->zTerm.
104080	*/
104081	assert( nPrefix<nTerm );
104082	rc = fts3NodeAddTerm(p, &pWriter->pTree, isCopyTerm, zTerm, nPrefix+1);
104083	if( rc!=SQLITE_OK ) return rc;
104084
104085	nData = 0;
104086	pWriter->nTerm = 0;
104087
104088	nPrefix = 0;
104089	nSuffix = nTerm;
104090	nReq = 1 + /* varint containing prefix size */
104091	sqlite3Fts3VarintLen(nTerm) + /* varint containing suffix size */
104092	nTerm + /* Term suffix */
104093	sqlite3Fts3VarintLen(nDoclist) + /* Size of doclist */
104094	nDoclist; /* Doclist data */
104095	}
104096
104097	/* If the buffer currently allocated is too small for this entry, realloc
104098	** the buffer to make it large enough.
104099	*/
104100	if( nReq>pWriter->nSize ){
104101	char *aNew = sqlite3_realloc(pWriter->aData, nReq);
104102	if( !aNew ) return SQLITE_NOMEM;
104103	pWriter->aData = aNew;
104104	pWriter->nSize = nReq;
104105	}
104106	assert( nData+nReq<=pWriter->nSize );
104107
104108	/* Append the prefix-compressed term and doclist to the buffer. */
104109	nData += sqlite3Fts3PutVarint(&pWriter->aData[nData], nPrefix);
104110	nData += sqlite3Fts3PutVarint(&pWriter->aData[nData], nSuffix);
104111	memcpy(&pWriter->aData[nData], &zTerm[nPrefix], nSuffix);
104112	nData += nSuffix;
104113	nData += sqlite3Fts3PutVarint(&pWriter->aData[nData], nDoclist);
104114	memcpy(&pWriter->aData[nData], aDoclist, nDoclist);
104115	pWriter->nData = nData + nDoclist;
104116
104117	/* Save the current term so that it can be used to prefix-compress the next.
104118	** If the isCopyTerm parameter is true, then the buffer pointed to by
104119	** zTerm is transient, so take a copy of the term data. Otherwise, just
104120	** store a copy of the pointer.
104121	*/
104122	if( isCopyTerm ){
104123	if( nTerm>pWriter->nMalloc ){
104124	char zNew = sqlite3_realloc(pWriter->zMalloc, nTerm2);
104125	if( !zNew ){
104126	return SQLITE_NOMEM;
104127	}
104128	pWriter->nMalloc = nTerm*2;
104129	pWriter->zMalloc = zNew;
104130	pWriter->zTerm = zNew;
104131	}
104132	assert( pWriter->zTerm==pWriter->zMalloc );
104133	memcpy(pWriter->zTerm, zTerm, nTerm);
104134	}else{
104135	pWriter->zTerm = (char *)zTerm;
104136	}
104137	pWriter->nTerm = nTerm;
104138
104139	return SQLITE_OK;
104140	}
104141
104142	/*
104143	** Flush all data associated with the SegmentWriter object pWriter to the
104144	** database. This function must be called after all terms have been added
104145	** to the segment using fts3SegWriterAdd(). If successful, SQLITE_OK is
104146	** returned. Otherwise, an SQLite error code.
104147	*/
104148	static int fts3SegWriterFlush(
104149	Fts3Table p, / Virtual table handle */
104150	SegmentWriter pWriter, / SegmentWriter to flush to the db */
104151	int iLevel, /* Value for 'level' column of %_segdir */
104152	int iIdx /* Value for 'idx' column of %_segdir */
104153	){
104154	int rc; /* Return code */
104155	if( pWriter->pTree ){
104156	sqlite3_int64 iLast; /* Largest block id written to database */
104157	sqlite3_int64 iLastLeaf; /* Largest leaf block id written to db */
104158	char zRoot; / Pointer to buffer containing root node */
104159	int nRoot; /* Size of buffer zRoot */
104160
104161	iLastLeaf = pWriter->iFree;
104162	rc = fts3WriteSegment(p, pWriter->iFree++, pWriter->aData, pWriter->nData);
104163	if( rc==SQLITE_OK ){
104164	rc = fts3NodeWrite(p, pWriter->pTree, 1,
104165	pWriter->iFirst, pWriter->iFree, &iLast, &zRoot, &nRoot);
104166	}
104167	if( rc==SQLITE_OK ){
104168	rc = fts3WriteSegdir(
104169	p, iLevel, iIdx, pWriter->iFirst, iLastLeaf, iLast, zRoot, nRoot);
104170	}
104171	}else{
104172	/* The entire tree fits on the root node. Write it to the segdir table. */
104173	rc = fts3WriteSegdir(
104174	p, iLevel, iIdx, 0, 0, 0, pWriter->aData, pWriter->nData);
104175	}
104176	return rc;
104177	}
104178
104179	/*
104180	** Release all memory held by the SegmentWriter object passed as the
104181	** first argument.
104182	*/
104183	static void fts3SegWriterFree(SegmentWriter *pWriter){
104184	if( pWriter ){
104185	sqlite3_free(pWriter->aData);
104186	sqlite3_free(pWriter->zMalloc);
104187	fts3NodeFree(pWriter->pTree);
104188	sqlite3_free(pWriter);
104189	}
104190	}
104191
104192	/*
104193	** The first value in the apVal[] array is assumed to contain an integer.
104194	** This function tests if there exist any documents with docid values that
104195	** are different from that integer. i.e. if deleting the document with docid
104196	** apVal[0] would mean the FTS3 table were empty.
104197	**
104198	** If successful, *pisEmpty is set to true if the table is empty except for
104199	** document apVal[0], or false otherwise, and SQLITE_OK is returned. If an
104200	** error occurs, an SQLite error code is returned.
104201	*/
104202	static int fts3IsEmpty(Fts3Table p, sqlite3_value apVal, int pisEmpty){
104203	sqlite3_stmt *pStmt;
104204	int rc;
104205	rc = fts3SqlStmt(p, SQL_IS_EMPTY, &pStmt, apVal);
104206	if( rc==SQLITE_OK ){
104207	if( SQLITE_ROW==sqlite3_step(pStmt) ){
104208	*pisEmpty = sqlite3_column_int(pStmt, 0);
104209	}
104210	rc = sqlite3_reset(pStmt);
104211	}
104212	return rc;
104213	}
104214
104215	/*
104216	** Set *pnSegment to the number of segments of level iLevel in the database.
104217	**
104218	** Return SQLITE_OK if successful, or an SQLite error code if not.
104219	*/
104220	static int fts3SegmentCount(Fts3Table p, int iLevel, int pnSegment){
104221	sqlite3_stmt *pStmt;
104222	int rc;
104223
104224	assert( iLevel>=0 );
104225	rc = fts3SqlStmt(p, SQL_SELECT_LEVEL_COUNT, &pStmt, 0);
104226	if( rc!=SQLITE_OK ) return rc;
104227	sqlite3_bind_int(pStmt, 1, iLevel);
104228	if( SQLITE_ROW==sqlite3_step(pStmt) ){
104229	*pnSegment = sqlite3_column_int(pStmt, 0);
104230	}
104231	return sqlite3_reset(pStmt);
104232	}
104233
104234	/*
104235	** Set *pnSegment to the total number of segments in the database. Set
104236	** *pnMax to the largest segment level in the database (segment levels
104237	** are stored in the 'level' column of the %_segdir table).
104238	**
104239	** Return SQLITE_OK if successful, or an SQLite error code if not.
104240	*/
104241	static int fts3SegmentCountMax(Fts3Table p, int pnSegment, int *pnMax){
104242	sqlite3_stmt *pStmt;
104243	int rc;
104244
104245	rc = fts3SqlStmt(p, SQL_SELECT_SEGDIR_COUNT_MAX, &pStmt, 0);
104246	if( rc!=SQLITE_OK ) return rc;
104247	if( SQLITE_ROW==sqlite3_step(pStmt) ){
104248	*pnSegment = sqlite3_column_int(pStmt, 0);
104249	*pnMax = sqlite3_column_int(pStmt, 1);
104250	}
104251	return sqlite3_reset(pStmt);
104252	}
104253
104254	/*
104255	** This function is used after merging multiple segments into a single large
104256	** segment to delete the old, now redundant, segment b-trees. Specifically,
104257	** it:
104258	**
104259	** 1) Deletes all %_segments entries for the segments associated with
104260	** each of the SegReader objects in the array passed as the third
104261	** argument, and
104262	**
104263	** 2) deletes all %_segdir entries with level iLevel, or all %_segdir
104264	** entries regardless of level if (iLevel<0).
104265	**
104266	** SQLITE_OK is returned if successful, otherwise an SQLite error code.
104267	*/
104268	static int fts3DeleteSegdir(
104269	Fts3Table p, / Virtual table handle */
104270	int iLevel, /* Level of %_segdir entries to delete */
104271	Fts3SegReader *apSegment, / Array of SegReader objects */
104272	int nReader /* Size of array apSegment */
104273	){
104274	int rc; /* Return Code */
104275	int i; /* Iterator variable */
104276	sqlite3_stmt pDelete; / SQL statement to delete rows */
104277
104278	rc = fts3SqlStmt(p, SQL_DELETE_SEGMENTS_RANGE, &pDelete, 0);
104279	for(i=0; rc==SQLITE_OK && i<nReader; i++){
104280	Fts3SegReader *pSegment = apSegment[i];
104281	if( pSegment->iStartBlock ){
104282	sqlite3_bind_int64(pDelete, 1, pSegment->iStartBlock);
104283	sqlite3_bind_int64(pDelete, 2, pSegment->iEndBlock);
104284	sqlite3_step(pDelete);
104285	rc = sqlite3_reset(pDelete);
104286	}
104287	}
104288	if( rc!=SQLITE_OK ){
104289	return rc;
104290	}
104291
104292	if( iLevel>=0 ){
104293	rc = fts3SqlStmt(p, SQL_DELETE_SEGDIR_BY_LEVEL, &pDelete, 0);
104294	if( rc==SQLITE_OK ){
104295	sqlite3_bind_int(pDelete, 1, iLevel);
104296	sqlite3_step(pDelete);
104297	rc = sqlite3_reset(pDelete);
104298	}
104299	}else{
104300	rc = fts3SqlExec(p, SQL_DELETE_ALL_SEGDIR, 0);
104301	}
104302
104303	return rc;
104304	}
104305
104306	/*
104307	** When this function is called, buffer ppList (size pnList bytes) contains
104308	** a position list that may (or may not) feature multiple columns. This
104309	** function adjusts the pointer ppList and the length pnList so that they
104310	** identify the subset of the position list that corresponds to column iCol.
104311	**
104312	** If there are no entries in the input position list for column iCol, then
104313	** *pnList is set to zero before returning.
104314	*/
104315	static void fts3ColumnFilter(
104316	int iCol, /* Column to filter on */
104317	char *ppList, / IN/OUT: Pointer to position list */
104318	int pnList / IN/OUT: Size of buffer ppList in bytes /
104319	){
104320	char pList = ppList;
104321	int nList = *pnList;
104322	char *pEnd = &pList[nList];
104323	int iCurrent = 0;
104324	char *p = pList;
104325
104326	assert( iCol>=0 );
104327	while( 1 ){
104328	char c = 0;
104329	while( p<pEnd && (c \| p)&0xFE ) c = p++ & 0x80;
104330
104331	if( iCol==iCurrent ){
104332	nList = (p - pList);
104333	break;
104334	}
104335
104336	nList -= (p - pList);
104337	pList = p;
104338	if( nList==0 ){
104339	break;
104340	}
104341	p = &pList[1];
104342	p += sqlite3Fts3GetVarint32(p, &iCurrent);
104343	}
104344
104345	*ppList = pList;
104346	*pnList = nList;
104347	}
104348
104349	/*
104350	** sqlite3Fts3SegReaderIterate() callback used when merging multiple
104351	** segments to create a single, larger segment.
104352	*/
104353	static int fts3MergeCallback(
104354	Fts3Table *p,
104355	void *pContext,
104356	char *zTerm,
104357	int nTerm,
104358	char *aDoclist,
104359	int nDoclist
104360	){
104361	SegmentWriter ppW = (SegmentWriter )pContext;
104362	return fts3SegWriterAdd(p, ppW, 1, zTerm, nTerm, aDoclist, nDoclist);
104363	}
104364
104365	/*
104366	** This function is used to iterate through a contiguous set of terms
104367	** stored in the full-text index. It merges data contained in one or
104368	** more segments to support this.
104369	**
104370	** The second argument is passed an array of pointers to SegReader objects
104371	** allocated with sqlite3Fts3SegReaderNew(). This function merges the range
104372	** of terms selected by each SegReader. If a single term is present in
104373	** more than one segment, the associated doclists are merged. For each
104374	** term and (possibly merged) doclist in the merged range, the callback
104375	** function xFunc is invoked with its arguments set as follows.
104376	**
104377	** arg 0: Copy of 'p' parameter passed to this function
104378	** arg 1: Copy of 'pContext' parameter passed to this function
104379	** arg 2: Pointer to buffer containing term
104380	** arg 3: Size of arg 2 buffer in bytes
104381	** arg 4: Pointer to buffer containing doclist
104382	** arg 5: Size of arg 2 buffer in bytes
104383	**
104384	** The 4th argument to this function is a pointer to a structure of type
104385	** Fts3SegFilter, defined in fts3Int.h. The contents of this structure
104386	** further restrict the range of terms that callbacks are made for and
104387	** modify the behaviour of this function. See comments above structure
104388	** definition for details.
104389	*/
104390	SQLITE_PRIVATE int sqlite3Fts3SegReaderIterate(
104391	Fts3Table p, / Virtual table handle */
104392	Fts3SegReader *apSegment, / Array of Fts3SegReader objects */
104393	int nSegment, /* Size of apSegment array */
104394	Fts3SegFilter pFilter, / Restrictions on range of iteration */
104395	int (xFunc)(Fts3Table , void , char , int, char , int), / Callback */
104396	void pContext / Callback context (2nd argument) */
104397	){
104398	int i; /* Iterator variable */
104399	char aBuffer = 0; / Buffer to merge doclists in */
104400	int nAlloc = 0; /* Allocated size of aBuffer buffer */
104401	int rc = SQLITE_OK; /* Return code */
104402
104403	int isIgnoreEmpty = (pFilter->flags & FTS3_SEGMENT_IGNORE_EMPTY);
104404	int isRequirePos = (pFilter->flags & FTS3_SEGMENT_REQUIRE_POS);
104405	int isColFilter = (pFilter->flags & FTS3_SEGMENT_COLUMN_FILTER);
104406	int isPrefix = (pFilter->flags & FTS3_SEGMENT_PREFIX);
104407
104408	/* If the Fts3SegFilter defines a specific term (or term prefix) to search
104409	** for, then advance each segment iterator until it points to a term of
104410	** equal or greater value than the specified term. This prevents many
104411	** unnecessary merge/sort operations for the case where single segment
104412	** b-tree leaf nodes contain more than one term.
104413	*/
104414	if( pFilter->zTerm ){
104415	int nTerm = pFilter->nTerm;
104416	const char *zTerm = pFilter->zTerm;
104417	for(i=0; i<nSegment; i++){
104418	Fts3SegReader *pSeg = apSegment[i];
104419	while( fts3SegReaderTermCmp(pSeg, zTerm, nTerm)<0 ){
104420	rc = fts3SegReaderNext(pSeg);
104421	if( rc!=SQLITE_OK ) goto finished;
104422	}
104423	}
104424	}
104425
104426	fts3SegReaderSort(apSegment, nSegment, nSegment, fts3SegReaderCmp);
104427	while( apSegment[0]->aNode ){
104428	int nTerm = apSegment[0]->nTerm;
104429	char *zTerm = apSegment[0]->zTerm;
104430	int nMerge = 1;
104431
104432	/* If this is a prefix-search, and if the term that apSegment[0] points
104433	** to does not share a suffix with pFilter->zTerm/nTerm, then all
104434	** required callbacks have been made. In this case exit early.
104435	**
104436	** Similarly, if this is a search for an exact match, and the first term
104437	** of segment apSegment[0] is not a match, exit early.
104438	*/
104439	if( pFilter->zTerm ){
104440	if( nTerm<pFilter->nTerm
104441	\|\| (!isPrefix && nTerm>pFilter->nTerm)
104442	\|\| memcmp(zTerm, pFilter->zTerm, pFilter->nTerm)
104443	){
104444	goto finished;
104445	}
104446	}
104447
104448	while( nMerge<nSegment
104449	&& apSegment[nMerge]->aNode
104450	&& apSegment[nMerge]->nTerm==nTerm
104451	&& 0==memcmp(zTerm, apSegment[nMerge]->zTerm, nTerm)
104452	){
104453	nMerge++;
104454	}
104455
104456	if( nMerge==1 && !isIgnoreEmpty && !isColFilter && isRequirePos ){
104457	Fts3SegReader *p0 = apSegment[0];
104458	rc = xFunc(p, pContext, zTerm, nTerm, p0->aDoclist, p0->nDoclist);
104459	if( rc!=SQLITE_OK ) goto finished;
104460	}else{
104461	int nDoclist = 0; /* Size of doclist */
104462	sqlite3_int64 iPrev = 0; /* Previous docid stored in doclist */
104463
104464	/* The current term of the first nMerge entries in the array
104465	** of Fts3SegReader objects is the same. The doclists must be merged
104466	** and a single term added to the new segment.
104467	*/
104468	for(i=0; i<nMerge; i++){
104469	fts3SegReaderFirstDocid(apSegment[i]);
104470	}
104471	fts3SegReaderSort(apSegment, nMerge, nMerge, fts3SegReaderDoclistCmp);
104472	while( apSegment[0]->pOffsetList ){
104473	int j; /* Number of segments that share a docid */
104474	char *pList;
104475	int nList;
104476	int nByte;
104477	sqlite3_int64 iDocid = apSegment[0]->iDocid;
104478	fts3SegReaderNextDocid(apSegment[0], &pList, &nList);
104479	j = 1;
104480	while( j<nMerge
104481	&& apSegment[j]->pOffsetList
104482	&& apSegment[j]->iDocid==iDocid
104483	){
104484	fts3SegReaderNextDocid(apSegment[j], 0, 0);
104485	j++;
104486	}
104487
104488	if( isColFilter ){
104489	fts3ColumnFilter(pFilter->iCol, &pList, &nList);
104490	}
104491
104492	if( !isIgnoreEmpty \|\| nList>0 ){
104493	nByte = sqlite3Fts3VarintLen(iDocid-iPrev) + (isRequirePos?nList+1:0);
104494	if( nDoclist+nByte>nAlloc ){
104495	char *aNew;
104496	nAlloc = nDoclist+nByte*2;
104497	aNew = sqlite3_realloc(aBuffer, nAlloc);
104498	if( !aNew ){
104499	rc = SQLITE_NOMEM;
104500	goto finished;
104501	}
104502	aBuffer = aNew;
104503	}
104504	nDoclist += sqlite3Fts3PutVarint(&aBuffer[nDoclist], iDocid-iPrev);
104505	iPrev = iDocid;
104506	if( isRequirePos ){
104507	memcpy(&aBuffer[nDoclist], pList, nList);
104508	nDoclist += nList;
104509	aBuffer[nDoclist++] = '\0';
104510	}
104511	}
104512
104513	fts3SegReaderSort(apSegment, nMerge, j, fts3SegReaderDoclistCmp);
104514	}
104515
104516	if( nDoclist>0 ){
104517	rc = xFunc(p, pContext, zTerm, nTerm, aBuffer, nDoclist);
104518	if( rc!=SQLITE_OK ) goto finished;
104519	}
104520	}
104521
104522	/* If there is a term specified to filter on, and this is not a prefix
104523	** search, return now. The callback that corresponds to the required
104524	** term (if such a term exists in the index) has already been made.
104525	*/
104526	if( pFilter->zTerm && !isPrefix ){
104527	goto finished;
104528	}
104529
104530	for(i=0; i<nMerge; i++){
104531	rc = fts3SegReaderNext(apSegment[i]);
104532	if( rc!=SQLITE_OK ) goto finished;
104533	}
104534	fts3SegReaderSort(apSegment, nSegment, nMerge, fts3SegReaderCmp);
104535	}
104536
104537	finished:
104538	sqlite3_free(aBuffer);
104539	return rc;
104540	}
104541
104542	/*
104543	** Merge all level iLevel segments in the database into a single
104544	** iLevel+1 segment. Or, if iLevel<0, merge all segments into a
104545	** single segment with a level equal to the numerically largest level
104546	** currently present in the database.
104547	**
104548	** If this function is called with iLevel<0, but there is only one
104549	** segment in the database, SQLITE_DONE is returned immediately.
104550	** Otherwise, if successful, SQLITE_OK is returned. If an error occurs,
104551	** an SQLite error code is returned.
104552	*/
104553	static int fts3SegmentMerge(Fts3Table *p, int iLevel){
104554	int i; /* Iterator variable */
104555	int rc; /* Return code */
104556	int iIdx; /* Index of new segment */
104557	int iNewLevel; /* Level to create new segment at */
104558	sqlite3_stmt *pStmt;
104559	SegmentWriter *pWriter = 0;
104560	int nSegment = 0; /* Number of segments being merged */
104561	Fts3SegReader *apSegment = 0; / Array of Segment iterators */
104562	Fts3SegFilter filter; /* Segment term filter condition */
104563
104564	if( iLevel<0 ){
104565	/* This call is to merge all segments in the database to a single
104566	** segment. The level of the new segment is equal to the the numerically
104567	** greatest segment level currently present in the database. The index
104568	** of the new segment is always 0.
104569	*/
104570	iIdx = 0;
104571	rc = fts3SegmentCountMax(p, &nSegment, &iNewLevel);
104572	if( nSegment==1 ){
104573	return SQLITE_DONE;
104574	}
104575	}else{
104576	/* This call is to merge all segments at level iLevel. Find the next
104577	** available segment index at level iLevel+1. The call to
104578	** fts3AllocateSegdirIdx() will merge the segments at level iLevel+1 to
104579	** a single iLevel+2 segment if necessary.
104580	*/
104581	iNewLevel = iLevel+1;
104582	rc = fts3AllocateSegdirIdx(p, iNewLevel, &iIdx);
104583	if( rc!=SQLITE_OK ) return rc;
104584	rc = fts3SegmentCount(p, iLevel, &nSegment);
104585	}
104586	if( rc!=SQLITE_OK ) return rc;
104587	assert( nSegment>0 );
104588	assert( iNewLevel>=0 );
104589
104590	/* Allocate space for an array of pointers to segment iterators. */
104591	apSegment = (Fts3SegReader*)sqlite3_malloc(sizeof(Fts3SegReader )*nSegment);
104592	if( !apSegment ){
104593	return SQLITE_NOMEM;
104594	}
104595	memset(apSegment, 0, sizeof(Fts3SegReader )nSegment);
104596
104597	/* Allocate a Fts3SegReader structure for each segment being merged. A
104598	** Fts3SegReader stores the state data required to iterate through all
104599	** entries on all leaves of a single segment.
104600	*/
104601	assert( SQL_SELECT_LEVEL+1==SQL_SELECT_ALL_LEVEL);
104602	rc = fts3SqlStmt(p, SQL_SELECT_LEVEL+(iLevel<0), &pStmt, 0);
104603	if( rc!=SQLITE_OK ) goto finished;
104604	sqlite3_bind_int(pStmt, 1, iLevel);
104605	for(i=0; SQLITE_ROW==(sqlite3_step(pStmt)); i++){
104606	rc = fts3SegReaderNew(p, pStmt, i, &apSegment[i]);
104607	if( rc!=SQLITE_OK ){
104608	goto finished;
104609	}
104610	}
104611	rc = sqlite3_reset(pStmt);
104612	pStmt = 0;
104613	if( rc!=SQLITE_OK ) goto finished;
104614
104615	memset(&filter, 0, sizeof(Fts3SegFilter));
104616	filter.flags = FTS3_SEGMENT_REQUIRE_POS;
104617	filter.flags \|= (iLevel<0 ? FTS3_SEGMENT_IGNORE_EMPTY : 0);
104618	rc = sqlite3Fts3SegReaderIterate(p, apSegment, nSegment,
104619	&filter, fts3MergeCallback, (void *)&pWriter
104620	);
104621	if( rc!=SQLITE_OK ) goto finished;
104622
104623	rc = fts3DeleteSegdir(p, iLevel, apSegment, nSegment);
104624	if( rc==SQLITE_OK ){
104625	rc = fts3SegWriterFlush(p, pWriter, iNewLevel, iIdx);
104626	}
104627
104628	finished:
104629	fts3SegWriterFree(pWriter);
104630	if( apSegment ){
104631	for(i=0; i<nSegment; i++){
104632	sqlite3Fts3SegReaderFree(p, apSegment[i]);
104633	}
104634	sqlite3_free(apSegment);
104635	}
104636	sqlite3_reset(pStmt);
104637	return rc;
104638	}
104639
104640	/*
104641	** This is a comparison function used as a qsort() callback when sorting
104642	** an array of pending terms by term. This occurs as part of flushing
104643	** the contents of the pending-terms hash table to the database.
104644	*/
104645	static int qsortCompare(const void lhs, const void rhs){
104646	char z1 = fts3HashKey((Fts3HashElem **)lhs);
104647	char z2 = fts3HashKey((Fts3HashElem **)rhs);
104648	int n1 = fts3HashKeysize((Fts3HashElem *)lhs);
104649	int n2 = fts3HashKeysize((Fts3HashElem *)rhs);
104650
104651	int n = (n1<n2 ? n1 : n2);
104652	int c = memcmp(z1, z2, n);
104653	if( c==0 ){
104654	c = n1 - n2;
104655	}
104656	return c;
104657	}
104658
104659
104660	/*
104661	** Flush the contents of pendingTerms to a level 0 segment.
104662	*/
104663	SQLITE_PRIVATE int sqlite3Fts3PendingTermsFlush(Fts3Table *p){
104664	Fts3HashElem *pElem;
104665	int idx, rc, i;
104666	Fts3HashElem *apElem; / Array of pointers to hash elements */
104667	int nElem; /* Number of terms in new segment */
104668	SegmentWriter pWriter = 0; / Used to write the segment */
104669
104670	/* Find the number of terms that will make up the new segment. If there
104671	** are no terms, return early (do not bother to write an empty segment).
104672	*/
104673	nElem = fts3HashCount(&p->pendingTerms);
104674	if( nElem==0 ){
104675	assert( p->nPendingData==0 );
104676	return SQLITE_OK;
104677	}
104678
104679	/* Determine the next index at level 0, merging as necessary. */
104680	rc = fts3AllocateSegdirIdx(p, 0, &idx);
104681	if( rc!=SQLITE_OK ){
104682	return rc;
104683	}
104684
104685	apElem = sqlite3_malloc(nElemsizeof(Fts3HashElem ));
104686	if( !apElem ){
104687	return SQLITE_NOMEM;
104688	}
104689
104690	i = 0;
104691	for(pElem=fts3HashFirst(&p->pendingTerms); pElem; pElem=fts3HashNext(pElem)){
104692	apElem[i++] = pElem;
104693	}
104694	assert( i==nElem );
104695
104696	/* TODO(shess) Should we allow user-defined collation sequences,
104697	** here? I think we only need that once we support prefix searches.
104698	** Also, should we be using qsort()?
104699	*/
104700	if( nElem>1 ){
104701	qsort(apElem, nElem, sizeof(Fts3HashElem *), qsortCompare);
104702	}
104703
104704
104705	/* Write the segment tree into the database. */
104706	for(i=0; rc==SQLITE_OK && i<nElem; i++){
104707	const char *z = fts3HashKey(apElem[i]);
104708	int n = fts3HashKeysize(apElem[i]);
104709	PendingList *pList = fts3HashData(apElem[i]);
104710	rc = fts3SegWriterAdd(p, &pWriter, 0, z, n, pList->aData, pList->nData+1);
104711	}
104712	if( rc==SQLITE_OK ){
104713	rc = fts3SegWriterFlush(p, pWriter, 0, idx);
104714	}
104715
104716	/* Free all allocated resources before returning */
104717	fts3SegWriterFree(pWriter);
104718	sqlite3_free(apElem);
104719	sqlite3Fts3PendingTermsClear(p);
104720	return rc;
104721	}
104722
104723	/*
104724	** This function does the work for the xUpdate method of FTS3 virtual
104725	** tables.
104726	*/
104727	SQLITE_PRIVATE int sqlite3Fts3UpdateMethod(
104728	sqlite3_vtab pVtab, / FTS3 vtab object */
104729	int nArg, /* Size of argument array */
104730	sqlite3_value *apVal, / Array of arguments */
104731	sqlite_int64 pRowid / OUT: The affected (or effected) rowid */
104732	){
104733	Fts3Table p = (Fts3Table )pVtab;
104734	int rc = SQLITE_OK; /* Return Code */
104735	int isRemove = 0; /* True for an UPDATE or DELETE */
104736	sqlite3_int64 iRemove; /* Rowid removed by UPDATE or DELETE */
104737
104738	/* If this is a DELETE or UPDATE operation, remove the old record. */
104739	if( sqlite3_value_type(apVal[0])!=SQLITE_NULL ){
104740	int isEmpty;
104741	rc = fts3IsEmpty(p, apVal, &isEmpty);
104742	if( rc==SQLITE_OK ){
104743	if( isEmpty ){
104744	/* Deleting this row means the whole table is empty. In this case
104745	** delete the contents of all three tables and throw away any
104746	** data in the pendingTerms hash table.
104747	*/
104748	rc = fts3DeleteAll(p);
104749	}else{
104750	isRemove = 1;
104751	iRemove = sqlite3_value_int64(apVal[0]);
104752	rc = fts3PendingTermsDocid(p, iRemove);
104753	if( rc==SQLITE_OK ){
104754	rc = fts3DeleteTerms(p, apVal);
104755	if( rc==SQLITE_OK ){
104756	rc = fts3SqlExec(p, SQL_DELETE_CONTENT, apVal);
104757	}
104758	}
104759	}
104760	}
104761	}
104762
104763	/* If this is an INSERT or UPDATE operation, insert the new record. */
104764	if( nArg>1 && rc==SQLITE_OK ){
104765	rc = fts3InsertData(p, apVal, pRowid);
104766	if( rc==SQLITE_OK && (!isRemove \|\| *pRowid!=iRemove) ){
104767	rc = fts3PendingTermsDocid(p, *pRowid);
104768	}
104769	if( rc==SQLITE_OK ){
104770	rc = fts3InsertTerms(p, apVal);
104771	}
104772	}
104773
104774	return rc;
104775	}
104776
104777	/*
104778	** Flush any data in the pending-terms hash table to disk. If successful,
104779	** merge all segments in the database (including the new segment, if
104780	** there was any data to flush) into a single segment.
104781	*/
104782	SQLITE_PRIVATE int sqlite3Fts3Optimize(Fts3Table *p){
104783	int rc = sqlite3Fts3PendingTermsFlush(p);
104784	if( rc==SQLITE_OK ){
104785	rc = fts3SegmentMerge(p, -1);
104786	}
104787	return rc;
104788	}
104789
104790	#endif
104791
104792	/************ End of fts3_write.c ****************************************/
104793	/************ Begin file fts3_snippet.c **********************************/
104794	/*
104795	** 2009 Oct 23
104796	**
104797	** The author disclaims copyright to this source code. In place of
104798	** a legal notice, here is a blessing:
104799	**
104800	** May you do good and not evil.
104801	** May you find forgiveness for yourself and forgive others.
104802	** May you share freely, never taking more than you give.
104803	**
104804	******************************************************************************
104805	*/
104806
104807	#if !defined(SQLITE_CORE) \|\| defined(SQLITE_ENABLE_FTS3)
104808
104809
104810	typedef struct Snippet Snippet;
104811
104812	/*
104813	** An instance of the following structure keeps track of generated
104814	** matching-word offset information and snippets.
104815	*/
104816	struct Snippet {
104817	int nMatch; /* Total number of matches */
104818	int nAlloc; /* Space allocated for aMatch[] */
104819	struct snippetMatch { /* One entry for each matching term */
104820	char snStatus; /* Status flag for use while constructing snippets */
104821	short int iCol; /* The column that contains the match */
104822	short int iTerm; /* The index in Query.pTerms[] of the matching term */
104823	int iToken; /* The index of the matching document token */
104824	short int nByte; /* Number of bytes in the term */
104825	int iStart; /* The offset to the first character of the term */
104826	} aMatch; / Points to space obtained from malloc */
104827	char zOffset; / Text rendering of aMatch[] */
104828	int nOffset; /* strlen(zOffset) */
104829	char zSnippet; / Snippet text */
104830	int nSnippet; /* strlen(zSnippet) */
104831	};
104832
104833
104834	/* It is not safe to call isspace(), tolower(), or isalnum() on
104835	** hi-bit-set characters. This is the same solution used in the
104836	** tokenizer.
104837	*/
104838	/* TODO(shess) The snippet-generation code should be using the
104839	** tokenizer-generated tokens rather than doing its own local
104840	** tokenization.
104841	*/
104842	/* TODO(shess) Is __isascii() a portable version of (c&0x80)==0? */
104843	static int safe_isspace(char c){
104844	return (c&0x80)==0 ? isspace(c) : 0;
104845	}
104846	static int safe_isalnum(char c){
104847	return (c&0x80)==0 ? isalnum(c) : 0;
104848	}
104849
104850	/*******************************************************************/
104851	/* DataBuffer is used to collect data into a buffer in piecemeal
104852	** fashion. It implements the usual distinction between amount of
104853	** data currently stored (nData) and buffer capacity (nCapacity).
104854	**
104855	** dataBufferInit - create a buffer with given initial capacity.
104856	** dataBufferReset - forget buffer's data, retaining capacity.
104857	** dataBufferSwap - swap contents of two buffers.
104858	** dataBufferExpand - expand capacity without adding data.
104859	** dataBufferAppend - append data.
104860	** dataBufferAppend2 - append two pieces of data at once.
104861	** dataBufferReplace - replace buffer's data.
104862	*/
104863	typedef struct DataBuffer {
104864	char pData; / Pointer to malloc'ed buffer. */
104865	int nCapacity; /* Size of pData buffer. */
104866	int nData; /* End of data loaded into pData. */
104867	} DataBuffer;
104868
104869	static void dataBufferInit(DataBuffer *pBuffer, int nCapacity){
104870	assert( nCapacity>=0 );
104871	pBuffer->nData = 0;
104872	pBuffer->nCapacity = nCapacity;
104873	pBuffer->pData = nCapacity==0 ? NULL : sqlite3_malloc(nCapacity);
104874	}
104875	static void dataBufferReset(DataBuffer *pBuffer){
104876	pBuffer->nData = 0;
104877	}
104878	static void dataBufferExpand(DataBuffer *pBuffer, int nAddCapacity){
104879	assert( nAddCapacity>0 );
104880	/* TODO(shess) Consider expanding more aggressively. Note that the
104881	** underlying malloc implementation may take care of such things for
104882	** us already.
104883	*/
104884	if( pBuffer->nData+nAddCapacity>pBuffer->nCapacity ){
104885	pBuffer->nCapacity = pBuffer->nData+nAddCapacity;
104886	pBuffer->pData = sqlite3_realloc(pBuffer->pData, pBuffer->nCapacity);
104887	}
104888	}
104889	static void dataBufferAppend(DataBuffer *pBuffer,
104890	const char *pSource, int nSource){
104891	assert( nSource>0 && pSource!=NULL );
104892	dataBufferExpand(pBuffer, nSource);
104893	memcpy(pBuffer->pData+pBuffer->nData, pSource, nSource);
104894	pBuffer->nData += nSource;
104895	}
104896	static void dataBufferAppend2(DataBuffer *pBuffer,
104897	const char *pSource1, int nSource1,
104898	const char *pSource2, int nSource2){
104899	assert( nSource1>0 && pSource1!=NULL );
104900	assert( nSource2>0 && pSource2!=NULL );
104901	dataBufferExpand(pBuffer, nSource1+nSource2);
104902	memcpy(pBuffer->pData+pBuffer->nData, pSource1, nSource1);
104903	memcpy(pBuffer->pData+pBuffer->nData+nSource1, pSource2, nSource2);
104904	pBuffer->nData += nSource1+nSource2;
104905	}
104906	static void dataBufferReplace(DataBuffer *pBuffer,
104907	const char *pSource, int nSource){
104908	dataBufferReset(pBuffer);
104909	dataBufferAppend(pBuffer, pSource, nSource);
104910	}
104911
104912
104913	/* StringBuffer is a null-terminated version of DataBuffer. */
104914	typedef struct StringBuffer {
104915	DataBuffer b; /* Includes null terminator. */
104916	} StringBuffer;
104917
104918	static void initStringBuffer(StringBuffer *sb){
104919	dataBufferInit(&sb->b, 100);
104920	dataBufferReplace(&sb->b, "", 1);
104921	}
104922	static int stringBufferLength(StringBuffer *sb){
104923	return sb->b.nData-1;
104924	}
104925	static char stringBufferData(StringBuffer sb){
104926	return sb->b.pData;
104927	}
104928
104929	static void nappend(StringBuffer sb, const char zFrom, int nFrom){
104930	assert( sb->b.nData>0 );
104931	if( nFrom>0 ){
104932	sb->b.nData--;
104933	dataBufferAppend2(&sb->b, zFrom, nFrom, "", 1);
104934	}
104935	}
104936	static void append(StringBuffer sb, const char zFrom){
104937	nappend(sb, zFrom, strlen(zFrom));
104938	}
104939
104940	static int endsInWhiteSpace(StringBuffer *p){
104941	return stringBufferLength(p)>0 &&
104942	safe_isspace(stringBufferData(p)[stringBufferLength(p)-1]);
104943	}
104944
104945	/* If the StringBuffer ends in something other than white space, add a
104946	** single space character to the end.
104947	*/
104948	static void appendWhiteSpace(StringBuffer *p){
104949	if( stringBufferLength(p)==0 ) return;
104950	if( !endsInWhiteSpace(p) ) append(p, " ");
104951	}
104952
104953	/* Remove white space from the end of the StringBuffer */
104954	static void trimWhiteSpace(StringBuffer *p){
104955	while( endsInWhiteSpace(p) ){
104956	p->b.pData[--p->b.nData-1] = '\0';
104957	}
104958	}
104959
104960
104961	/*
104962	** Release all memory associated with the Snippet structure passed as
104963	** an argument.
104964	*/
104965	static void fts3SnippetFree(Snippet *p){
104966	sqlite3_free(p->aMatch);
104967	sqlite3_free(p->zOffset);
104968	sqlite3_free(p->zSnippet);
104969	sqlite3_free(p);
104970	}
104971
104972	/*
104973	** Append a single entry to the p->aMatch[] log.
104974	*/
104975	static void snippetAppendMatch(
104976	Snippet p, / Append the entry to this snippet */
104977	int iCol, int iTerm, /* The column and query term */
104978	int iToken, /* Matching token in document */
104979	int iStart, int nByte /* Offset and size of the match */
104980	){
104981	int i;
104982	struct snippetMatch *pMatch;
104983	if( p->nMatch+1>=p->nAlloc ){
104984	p->nAlloc = p->nAlloc*2 + 10;
104985	p->aMatch = sqlite3_realloc(p->aMatch, p->nAlloc*sizeof(p->aMatch[0]) );
104986	if( p->aMatch==0 ){
104987	p->nMatch = 0;
104988	p->nAlloc = 0;
104989	return;
104990	}
104991	}
104992	i = p->nMatch++;
104993	pMatch = &p->aMatch[i];
104994	pMatch->iCol = iCol;
104995	pMatch->iTerm = iTerm;
104996	pMatch->iToken = iToken;
104997	pMatch->iStart = iStart;
104998	pMatch->nByte = nByte;
104999	}
105000
105001	/*
105002	** Sizing information for the circular buffer used in snippetOffsetsOfColumn()
105003	*/
105004	#define FTS3_ROTOR_SZ (32)
105005	#define FTS3_ROTOR_MASK (FTS3_ROTOR_SZ-1)
105006
105007	/*
105008	** Function to iterate through the tokens of a compiled expression.
105009	**
105010	** Except, skip all tokens on the right-hand side of a NOT operator.
105011	** This function is used to find tokens as part of snippet and offset
105012	** generation and we do nt want snippets and offsets to report matches
105013	** for tokens on the RHS of a NOT.
105014	*/
105015	static int fts3NextExprToken(Fts3Expr *ppExpr, int piToken){
105016	Fts3Expr p = ppExpr;
105017	int iToken = *piToken;
105018	if( iToken<0 ){
105019	/* In this case the expression p is the root of an expression tree.
105020	** Move to the first token in the expression tree.
105021	*/
105022	while( p->pLeft ){
105023	p = p->pLeft;
105024	}
105025	iToken = 0;
105026	}else{
105027	assert(p && p->eType==FTSQUERY_PHRASE );
105028	if( iToken<(p->pPhrase->nToken-1) ){
105029	iToken++;
105030	}else{
105031	iToken = 0;
105032	while( p->pParent && p->pParent->pLeft!=p ){
105033	assert( p->pParent->pRight==p );
105034	p = p->pParent;
105035	}
105036	p = p->pParent;
105037	if( p ){
105038	assert( p->pRight!=0 );
105039	p = p->pRight;
105040	while( p->pLeft ){
105041	p = p->pLeft;
105042	}
105043	}
105044	}
105045	}
105046
105047	*ppExpr = p;
105048	*piToken = iToken;
105049	return p?1:0;
105050	}
105051
105052	/*
105053	** Return TRUE if the expression node pExpr is located beneath the
105054	** RHS of a NOT operator.
105055	*/
105056	static int fts3ExprBeneathNot(Fts3Expr *p){
105057	Fts3Expr *pParent;
105058	while( p ){
105059	pParent = p->pParent;
105060	if( pParent && pParent->eType==FTSQUERY_NOT && pParent->pRight==p ){
105061	return 1;
105062	}
105063	p = pParent;
105064	}
105065	return 0;
105066	}
105067
105068	/*
105069	** Add entries to pSnippet->aMatch[] for every match that occurs against
105070	** document zDoc[0..nDoc-1] which is stored in column iColumn.
105071	*/
105072	static void snippetOffsetsOfColumn(
105073	Fts3Cursor pCur, / The fulltest search cursor */
105074	Snippet pSnippet, / The Snippet object to be filled in */
105075	int iColumn, /* Index of fulltext table column */
105076	const char zDoc, / Text of the fulltext table column */
105077	int nDoc /* Length of zDoc in bytes */
105078	){
105079	const sqlite3_tokenizer_module pTModule; / The tokenizer module */
105080	sqlite3_tokenizer pTokenizer; / The specific tokenizer */
105081	sqlite3_tokenizer_cursor pTCursor; / Tokenizer cursor */
105082	Fts3Table pVtab; / The full text index */
105083	int nColumn; /* Number of columns in the index */
105084	int i, j; /* Loop counters */
105085	int rc; /* Return code */
105086	unsigned int match, prevMatch; /* Phrase search bitmasks */
105087	const char zToken; / Next token from the tokenizer */
105088	int nToken; /* Size of zToken */
105089	int iBegin, iEnd, iPos; /* Offsets of beginning and end */
105090
105091	/* The following variables keep a circular buffer of the last
105092	** few tokens */
105093	unsigned int iRotor = 0; /* Index of current token */
105094	int iRotorBegin[FTS3_ROTOR_SZ]; /* Beginning offset of token */
105095	int iRotorLen[FTS3_ROTOR_SZ]; /* Length of token */
105096
105097	pVtab = (Fts3Table *)pCur->base.pVtab;
105098	nColumn = pVtab->nColumn;
105099	pTokenizer = pVtab->pTokenizer;
105100	pTModule = pTokenizer->pModule;
105101	rc = pTModule->xOpen(pTokenizer, zDoc, nDoc, &pTCursor);
105102	if( rc ) return;
105103	pTCursor->pTokenizer = pTokenizer;
105104
105105	prevMatch = 0;
105106	while( !pTModule->xNext(pTCursor, &zToken, &nToken, &iBegin, &iEnd, &iPos) ){
105107	Fts3Expr *pIter = pCur->pExpr;
105108	int iIter = -1;
105109	iRotorBegin[iRotor&FTS3_ROTOR_MASK] = iBegin;
105110	iRotorLen[iRotor&FTS3_ROTOR_MASK] = iEnd-iBegin;
105111	match = 0;
105112	for(i=0; i<(FTS3_ROTOR_SZ-1) && fts3NextExprToken(&pIter, &iIter); i++){
105113	int nPhrase; /* Number of tokens in current phrase */
105114	struct PhraseToken pToken; / Current token */
105115	int iCol; /* Column index */
105116
105117	if( fts3ExprBeneathNot(pIter) ) continue;
105118	nPhrase = pIter->pPhrase->nToken;
105119	pToken = &pIter->pPhrase->aToken[iIter];
105120	iCol = pIter->pPhrase->iColumn;
105121	if( iCol>=0 && iCol<nColumn && iCol!=iColumn ) continue;
105122	if( pToken->n>nToken ) continue;
105123	if( !pToken->isPrefix && pToken->n<nToken ) continue;
105124	assert( pToken->n<=nToken );
105125	if( memcmp(pToken->z, zToken, pToken->n) ) continue;
105126	if( iIter>0 && (prevMatch & (1<<i))==0 ) continue;
105127	match \|= 1<<i;
105128	if( i==(FTS3_ROTOR_SZ-2) \|\| nPhrase==iIter+1 ){
105129	for(j=nPhrase-1; j>=0; j--){
105130	int k = (iRotor-j) & FTS3_ROTOR_MASK;
105131	snippetAppendMatch(pSnippet, iColumn, i-j, iPos-j,
105132	iRotorBegin[k], iRotorLen[k]);
105133	}
105134	}
105135	}
105136	prevMatch = match<<1;
105137	iRotor++;
105138	}
105139	pTModule->xClose(pTCursor);
105140	}
105141
105142	/*
105143	** Remove entries from the pSnippet structure to account for the NEAR
105144	** operator. When this is called, pSnippet contains the list of token
105145	** offsets produced by treating all NEAR operators as AND operators.
105146	** This function removes any entries that should not be present after
105147	** accounting for the NEAR restriction. For example, if the queried
105148	** document is:
105149	**
105150	** "A B C D E A"
105151	**
105152	** and the query is:
105153	**
105154	** A NEAR/0 E
105155	**
105156	** then when this function is called the Snippet contains token offsets
105157	** 0, 4 and 5. This function removes the "0" entry (because the first A
105158	** is not near enough to an E).
105159	**
105160	** When this function is called, the value pointed to by parameter piLeft is
105161	** the integer id of the left-most token in the expression tree headed by
105162	** pExpr. This function increments *piLeft by the total number of tokens
105163	** in the expression tree headed by pExpr.
105164	**
105165	** Return 1 if any trimming occurs. Return 0 if no trimming is required.
105166	*/
105167	static int trimSnippetOffsets(
105168	Fts3Expr pExpr, / The search expression */
105169	Snippet pSnippet, / The set of snippet offsets to be trimmed */
105170	int piLeft / Index of left-most token in pExpr */
105171	){
105172	if( pExpr ){
105173	if( trimSnippetOffsets(pExpr->pLeft, pSnippet, piLeft) ){
105174	return 1;
105175	}
105176
105177	switch( pExpr->eType ){
105178	case FTSQUERY_PHRASE:
105179	*piLeft += pExpr->pPhrase->nToken;
105180	break;
105181	case FTSQUERY_NEAR: {
105182	/* The right-hand-side of a NEAR operator is always a phrase. The
105183	** left-hand-side is either a phrase or an expression tree that is
105184	** itself headed by a NEAR operator. The following initializations
105185	** set local variable iLeft to the token number of the left-most
105186	** token in the right-hand phrase, and iRight to the right most
105187	** token in the same phrase. For example, if we had:
105188	**
105189	** <col> MATCH '"abc def" NEAR/2 "ghi jkl"'
105190	**
105191	** then iLeft will be set to 2 (token number of ghi) and nToken will
105192	** be set to 4.
105193	*/
105194	Fts3Expr *pLeft = pExpr->pLeft;
105195	Fts3Expr *pRight = pExpr->pRight;
105196	int iLeft = *piLeft;
105197	int nNear = pExpr->nNear;
105198	int nToken = pRight->pPhrase->nToken;
105199	int jj, ii;
105200	if( pLeft->eType==FTSQUERY_NEAR ){
105201	pLeft = pLeft->pRight;
105202	}
105203	assert( pRight->eType==FTSQUERY_PHRASE );
105204	assert( pLeft->eType==FTSQUERY_PHRASE );
105205	nToken += pLeft->pPhrase->nToken;
105206
105207	for(ii=0; ii<pSnippet->nMatch; ii++){
105208	struct snippetMatch *p = &pSnippet->aMatch[ii];
105209	if( p->iTerm==iLeft ){
105210	int isOk = 0;
105211	/* Snippet ii is an occurence of query term iLeft in the document.
105212	** It occurs at position (p->iToken) of the document. We now
105213	** search for an instance of token (iLeft-1) somewhere in the
105214	** range (p->iToken - nNear)...(p->iToken + nNear + nToken) within
105215	** the set of snippetMatch structures. If one is found, proceed.
105216	** If one cannot be found, then remove snippets ii..(ii+N-1)
105217	** from the matching snippets, where N is the number of tokens
105218	** in phrase pRight->pPhrase.
105219	*/
105220	for(jj=0; isOk==0 && jj<pSnippet->nMatch; jj++){
105221	struct snippetMatch *p2 = &pSnippet->aMatch[jj];
105222	if( p2->iTerm==(iLeft-1) ){
105223	if( p2->iToken>=(p->iToken-nNear-1)
105224	&& p2->iToken<(p->iToken+nNear+nToken)
105225	){
105226	isOk = 1;
105227	}
105228	}
105229	}
105230	if( !isOk ){
105231	int kk;
105232	for(kk=0; kk<pRight->pPhrase->nToken; kk++){
105233	pSnippet->aMatch[kk+ii].iTerm = -2;
105234	}
105235	return 1;
105236	}
105237	}
105238	if( p->iTerm==(iLeft-1) ){
105239	int isOk = 0;
105240	for(jj=0; isOk==0 && jj<pSnippet->nMatch; jj++){
105241	struct snippetMatch *p2 = &pSnippet->aMatch[jj];
105242	if( p2->iTerm==iLeft ){
105243	if( p2->iToken<=(p->iToken+nNear+1)
105244	&& p2->iToken>(p->iToken-nNear-nToken)
105245	){
105246	isOk = 1;
105247	}
105248	}
105249	}
105250	if( !isOk ){
105251	int kk;
105252	for(kk=0; kk<pLeft->pPhrase->nToken; kk++){
105253	pSnippet->aMatch[ii-kk].iTerm = -2;
105254	}
105255	return 1;
105256	}
105257	}
105258	}
105259	break;
105260	}
105261	}
105262
105263	if( trimSnippetOffsets(pExpr->pRight, pSnippet, piLeft) ){
105264	return 1;
105265	}
105266	}
105267	return 0;
105268	}
105269
105270	/*
105271	** Compute all offsets for the current row of the query.
105272	** If the offsets have already been computed, this routine is a no-op.
105273	*/
105274	static int snippetAllOffsets(Fts3Cursor pCsr, Snippet *ppSnippet){
105275	Fts3Table p = (Fts3Table )pCsr->base.pVtab;
105276	int nColumn;
105277	int iColumn, i;
105278	int iFirst, iLast;
105279	int iTerm = 0;
105280	Snippet *pSnippet;
105281
105282	if( pCsr->pExpr==0 ){
105283	return SQLITE_OK;
105284	}
105285
105286	pSnippet = (Snippet *)sqlite3_malloc(sizeof(Snippet));
105287	*ppSnippet = pSnippet;
105288	if( !pSnippet ){
105289	return SQLITE_NOMEM;
105290	}
105291	memset(pSnippet, 0, sizeof(Snippet));
105292
105293	nColumn = p->nColumn;
105294	iColumn = (pCsr->eType - 2);
105295	if( iColumn<0 \|\| iColumn>=nColumn ){
105296	/* Look for matches over all columns of the full-text index */
105297	iFirst = 0;
105298	iLast = nColumn-1;
105299	}else{
105300	/* Look for matches in the iColumn-th column of the index only */
105301	iFirst = iColumn;
105302	iLast = iColumn;
105303	}
105304	for(i=iFirst; i<=iLast; i++){
105305	const char *zDoc;
105306	int nDoc;
105307	zDoc = (const char*)sqlite3_column_text(pCsr->pStmt, i+1);
105308	nDoc = sqlite3_column_bytes(pCsr->pStmt, i+1);
105309	snippetOffsetsOfColumn(pCsr, pSnippet, i, zDoc, nDoc);
105310	}
105311
105312	while( trimSnippetOffsets(pCsr->pExpr, pSnippet, &iTerm) ){
105313	iTerm = 0;
105314	}
105315
105316	return SQLITE_OK;
105317	}
105318
105319	/*
105320	** Convert the information in the aMatch[] array of the snippet
105321	** into the string zOffset[0..nOffset-1]. This string is used as
105322	** the return of the SQL offsets() function.
105323	*/
105324	static void snippetOffsetText(Snippet *p){
105325	int i;
105326	int cnt = 0;
105327	StringBuffer sb;
105328	char zBuf[200];
105329	if( p->zOffset ) return;
105330	initStringBuffer(&sb);
105331	for(i=0; i<p->nMatch; i++){
105332	struct snippetMatch *pMatch = &p->aMatch[i];
105333	if( pMatch->iTerm>=0 ){
105334	/* If snippetMatch.iTerm is less than 0, then the match was
105335	** discarded as part of processing the NEAR operator (see the
105336	** trimSnippetOffsetsForNear() function for details). Ignore
105337	** it in this case
105338	*/
105339	zBuf[0] = ' ';
105340	sqlite3_snprintf(sizeof(zBuf)-1, &zBuf[cnt>0], "%d %d %d %d",
105341	pMatch->iCol, pMatch->iTerm, pMatch->iStart, pMatch->nByte);
105342	append(&sb, zBuf);
105343	cnt++;
105344	}
105345	}
105346	p->zOffset = stringBufferData(&sb);
105347	p->nOffset = stringBufferLength(&sb);
105348	}
105349
105350	/*
105351	** zDoc[0..nDoc-1] is phrase of text. aMatch[0..nMatch-1] are a set
105352	** of matching words some of which might be in zDoc. zDoc is column
105353	** number iCol.
105354	**
105355	** iBreak is suggested spot in zDoc where we could begin or end an
105356	** excerpt. Return a value similar to iBreak but possibly adjusted
105357	** to be a little left or right so that the break point is better.
105358	*/
105359	static int wordBoundary(
105360	int iBreak, /* The suggested break point */
105361	const char zDoc, / Document text */
105362	int nDoc, /* Number of bytes in zDoc[] */
105363	struct snippetMatch aMatch, / Matching words */
105364	int nMatch, /* Number of entries in aMatch[] */
105365	int iCol /* The column number for zDoc[] */
105366	){
105367	int i;
105368	if( iBreak<=10 ){
105369	return 0;
105370	}
105371	if( iBreak>=nDoc-10 ){
105372	return nDoc;
105373	}
105374	for(i=0; i<nMatch && aMatch[i].iCol<iCol; i++){}
105375	while( i<nMatch && aMatch[i].iStart+aMatch[i].nByte<iBreak ){ i++; }
105376	if( i<nMatch ){
105377	if( aMatch[i].iStart<iBreak+10 ){
105378	return aMatch[i].iStart;
105379	}
105380	if( i>0 && aMatch[i-1].iStart+aMatch[i-1].nByte>=iBreak ){
105381	return aMatch[i-1].iStart;
105382	}
105383	}
105384	for(i=1; i<=10; i++){
105385	if( safe_isspace(zDoc[iBreak-i]) ){
105386	return iBreak - i + 1;
105387	}
105388	if( safe_isspace(zDoc[iBreak+i]) ){
105389	return iBreak + i + 1;
105390	}
105391	}
105392	return iBreak;
105393	}
105394
105395
105396
105397	/*
105398	** Allowed values for Snippet.aMatch[].snStatus
105399	*/
105400	#define SNIPPET_IGNORE 0 /* It is ok to omit this match from the snippet */
105401	#define SNIPPET_DESIRED 1 /* We want to include this match in the snippet */
105402
105403	/*
105404	** Generate the text of a snippet.
105405	*/
105406	static void snippetText(
105407	Fts3Cursor pCursor, / The cursor we need the snippet for */
105408	Snippet *pSnippet,
105409	const char zStartMark, / Markup to appear before each match */
105410	const char zEndMark, / Markup to appear after each match */
105411	const char zEllipsis / Ellipsis mark */
105412	){
105413	int i, j;
105414	struct snippetMatch *aMatch;
105415	int nMatch;
105416	int nDesired;
105417	StringBuffer sb;
105418	int tailCol;
105419	int tailOffset;
105420	int iCol;
105421	int nDoc;
105422	const char *zDoc;
105423	int iStart, iEnd;
105424	int tailEllipsis = 0;
105425	int iMatch;
105426
105427
105428	sqlite3_free(pSnippet->zSnippet);
105429	pSnippet->zSnippet = 0;
105430	aMatch = pSnippet->aMatch;
105431	nMatch = pSnippet->nMatch;
105432	initStringBuffer(&sb);
105433
105434	for(i=0; i<nMatch; i++){
105435	aMatch[i].snStatus = SNIPPET_IGNORE;
105436	}
105437	nDesired = 0;
105438	for(i=0; i<FTS3_ROTOR_SZ; i++){
105439	for(j=0; j<nMatch; j++){
105440	if( aMatch[j].iTerm==i ){
105441	aMatch[j].snStatus = SNIPPET_DESIRED;
105442	nDesired++;
105443	break;
105444	}
105445	}
105446	}
105447
105448	iMatch = 0;
105449	tailCol = -1;
105450	tailOffset = 0;
105451	for(i=0; i<nMatch && nDesired>0; i++){
105452	if( aMatch[i].snStatus!=SNIPPET_DESIRED ) continue;
105453	nDesired--;
105454	iCol = aMatch[i].iCol;
105455	zDoc = (const char*)sqlite3_column_text(pCursor->pStmt, iCol+1);
105456	nDoc = sqlite3_column_bytes(pCursor->pStmt, iCol+1);
105457	iStart = aMatch[i].iStart - 40;
105458	iStart = wordBoundary(iStart, zDoc, nDoc, aMatch, nMatch, iCol);
105459	if( iStart<=10 ){
105460	iStart = 0;
105461	}
105462	if( iCol==tailCol && iStart<=tailOffset+20 ){
105463	iStart = tailOffset;
105464	}
105465	if( (iCol!=tailCol && tailCol>=0) \|\| iStart!=tailOffset ){
105466	trimWhiteSpace(&sb);
105467	appendWhiteSpace(&sb);
105468	append(&sb, zEllipsis);
105469	appendWhiteSpace(&sb);
105470	}
105471	iEnd = aMatch[i].iStart + aMatch[i].nByte + 40;
105472	iEnd = wordBoundary(iEnd, zDoc, nDoc, aMatch, nMatch, iCol);
105473	if( iEnd>=nDoc-10 ){
105474	iEnd = nDoc;
105475	tailEllipsis = 0;
105476	}else{
105477	tailEllipsis = 1;
105478	}
105479	while( iMatch<nMatch && aMatch[iMatch].iCol<iCol ){ iMatch++; }
105480	while( iStart<iEnd ){
105481	while( iMatch<nMatch && aMatch[iMatch].iStart<iStart
105482	&& aMatch[iMatch].iCol<=iCol ){
105483	iMatch++;
105484	}
105485	if( iMatch<nMatch && aMatch[iMatch].iStart<iEnd
105486	&& aMatch[iMatch].iCol==iCol ){
105487	nappend(&sb, &zDoc[iStart], aMatch[iMatch].iStart - iStart);
105488	iStart = aMatch[iMatch].iStart;
105489	append(&sb, zStartMark);
105490	nappend(&sb, &zDoc[iStart], aMatch[iMatch].nByte);
105491	append(&sb, zEndMark);
105492	iStart += aMatch[iMatch].nByte;
105493	for(j=iMatch+1; j<nMatch; j++){
105494	if( aMatch[j].iTerm==aMatch[iMatch].iTerm
105495	&& aMatch[j].snStatus==SNIPPET_DESIRED ){
105496	nDesired--;
105497	aMatch[j].snStatus = SNIPPET_IGNORE;
105498	}
105499	}
105500	}else{
105501	nappend(&sb, &zDoc[iStart], iEnd - iStart);
105502	iStart = iEnd;
105503	}
105504	}
105505	tailCol = iCol;
105506	tailOffset = iEnd;
105507	}
105508	trimWhiteSpace(&sb);
105509	if( tailEllipsis ){
105510	appendWhiteSpace(&sb);
105511	append(&sb, zEllipsis);
105512	}
105513	pSnippet->zSnippet = stringBufferData(&sb);
105514	pSnippet->nSnippet = stringBufferLength(&sb);
105515	}
105516
105517	SQLITE_PRIVATE void sqlite3Fts3Offsets(
105518	sqlite3_context pCtx, / SQLite function call context */
105519	Fts3Cursor pCsr / Cursor object */
105520	){
105521	Snippet p; / Snippet structure */
105522	int rc = snippetAllOffsets(pCsr, &p);
105523	snippetOffsetText(p);
105524	sqlite3_result_text(pCtx, p->zOffset, p->nOffset, SQLITE_TRANSIENT);
105525	fts3SnippetFree(p);
105526	}
105527
105528	SQLITE_PRIVATE void sqlite3Fts3Snippet(
105529	sqlite3_context pCtx, / SQLite function call context */
105530	Fts3Cursor pCsr, / Cursor object */
105531	const char zStart, / Snippet start text - "<b>" */
105532	const char zEnd, / Snippet end text - "</b>" */
105533	const char zEllipsis / Snippet ellipsis text - "<b>...</b>" */
105534	){
105535	Snippet p; / Snippet structure */
105536	int rc = snippetAllOffsets(pCsr, &p);
105537	snippetText(pCsr, p, zStart, zEnd, zEllipsis);
105538	sqlite3_result_text(pCtx, p->zSnippet, p->nSnippet, SQLITE_TRANSIENT);
105539	fts3SnippetFree(p);
105540	}
105541
105542	#endif
105543
105544	/************ End of fts3_snippet.c **************************************/
105545	/************ Begin file rtree.c *****************************************/
105546	/*
105547	** 2001 September 15
105548	**
105549	** The author disclaims copyright to this source code. In place of
	@@ -107233,12 +105554,10 @@
105554	** May you share freely, never taking more than you give.
105555	**
105556	*************************************************************************
105557	** This file contains code for implementations of the r-tree and r*-tree
105558	** algorithms packaged as an SQLite virtual table module.


105559	*/
105560
105561	#if !defined(SQLITE_CORE) \|\| defined(SQLITE_ENABLE_RTREE)
105562
105563	/*
105564

M src/sqlite3.h

+3 -3

		--- src/sqlite3.h
		+++ src/sqlite3.h
		@@ -117,13 +117,13 @@
117	117	** [sqlite3_libversion_number()], [sqlite3_sourceid()],
118	118	** [sqlite_version()] and [sqlite_source_id()].
119	119	**
120	120	** Requirements: [H10011] [H10014]
121	121	*/
122		-#define SQLITE_VERSION "3.6.20"
123		-#define SQLITE_VERSION_NUMBER 3006020
124		-#define SQLITE_SOURCE_ID "2009-11-04 23:02:52 f1c09acaca3e205acf5b077c9b2d0fe35f035c1e"
	122	+#define SQLITE_VERSION "3.6.21"
	123	+#define SQLITE_VERSION_NUMBER 3006021
	124	+#define SQLITE_SOURCE_ID "2009-11-23 13:17:27 39214aee6553db76309851e7aa74fcc02d4f59b7"
125	125
126	126	/*
127	127	** CAPI3REF: Run-Time Library Version Numbers {H10020} <S60100>
128	128	** KEYWORDS: sqlite3_version
129	129	**
130	130

	--- src/sqlite3.h
	+++ src/sqlite3.h
	@@ -117,13 +117,13 @@
117	** [sqlite3_libversion_number()], [sqlite3_sourceid()],
118	** [sqlite_version()] and [sqlite_source_id()].
119	**
120	** Requirements: [H10011] [H10014]
121	*/
122	#define SQLITE_VERSION "3.6.20"
123	#define SQLITE_VERSION_NUMBER 3006020
124	#define SQLITE_SOURCE_ID "2009-11-04 23:02:52 f1c09acaca3e205acf5b077c9b2d0fe35f035c1e"
125
126	/*
127	** CAPI3REF: Run-Time Library Version Numbers {H10020} <S60100>
128	** KEYWORDS: sqlite3_version
129	**
130

	--- src/sqlite3.h
	+++ src/sqlite3.h
	@@ -117,13 +117,13 @@
117	** [sqlite3_libversion_number()], [sqlite3_sourceid()],
118	** [sqlite_version()] and [sqlite_source_id()].
119	**
120	** Requirements: [H10011] [H10014]
121	*/
122	#define SQLITE_VERSION "3.6.21"
123	#define SQLITE_VERSION_NUMBER 3006021
124	#define SQLITE_SOURCE_ID "2009-11-23 13:17:27 39214aee6553db76309851e7aa74fcc02d4f59b7"
125
126	/*
127	** CAPI3REF: Run-Time Library Version Numbers {H10020} <S60100>
128	** KEYWORDS: sqlite3_version
129	**
130

Fossil SCM

Keyboard Shortcuts