Fossil SCM

Update the built-in SQLite to the latest 3.7.8 alpha, for the purpose of testing SQLite.

drh 2011-07-19 23:44 trunk

Commit 4adc11edb756c53452e0527c6f8ded128c66beba

Parent b317471d8c8567e…

2 files changed +1212 -471 +21 -4

M src/sqlite3.c

+1212 -471

		--- src/sqlite3.c
		+++ src/sqlite3.c
		@@ -1,8 +1,8 @@
1	1	/******************************************************************************
2	2	** This file is an amalgamation of many separate C source files from SQLite
3		-** version 3.7.7. By combining all the individual C code files into this
	3	+** version 3.7.8. By combining all the individual C code files into this
4	4	** single large file, the entire code can be compiled as a single translation
5	5	** unit. This allows many compilers to do optimizations that would not be
6	6	** possible if the files were compiled separately. Performance improvements
7	7	** of 5% or more are commonly seen when SQLite is compiled as a single
8	8	** translation unit.
		@@ -648,13 +648,13 @@
648	648	**
649	649	** See also: [sqlite3_libversion()],
650	650	** [sqlite3_libversion_number()], [sqlite3_sourceid()],
651	651	** [sqlite_version()] and [sqlite_source_id()].
652	652	*/
653		-#define SQLITE_VERSION "3.7.7"
654		-#define SQLITE_VERSION_NUMBER 3007007
655		-#define SQLITE_SOURCE_ID "2011-06-24 11:29:51 9b191bb4c7c1e1b12b188c0b3eee1f8f587887c8"
	653	+#define SQLITE_VERSION "3.7.8"
	654	+#define SQLITE_VERSION_NUMBER 3007008
	655	+#define SQLITE_SOURCE_ID "2011-07-19 18:29:00 ed5f0aad6b21066bacd01521e82c22e96991f400"
656	656
657	657	/*
658	658	** CAPI3REF: Run-Time Library Version Numbers
659	659	** KEYWORDS: sqlite3_version, sqlite3_sourceid
660	660	**
		@@ -1282,20 +1282,37 @@
1282	1282	** when [PRAGMA synchronous \| PRAGMA synchronous=OFF] is set, but most
1283	1283	** VFSes do not need this signal and should silently ignore this opcode.
1284	1284	** Applications should not call [sqlite3_file_control()] with this
1285	1285	** opcode as doing so may disrupt the operation of the specialized VFSes
1286	1286	** that do require it.
	1287	+**
	1288	+** ^The [SQLITE_FCNTL_WIN32_AV_RETRY] opcode is used to configure automatic
	1289	+** retry counts and intervals for certain disk I/O operations for the
	1290	+** windows [VFS] in order to work to provide robustness against
	1291	+** anti-virus programs. By default, the windows VFS will retry file read,
	1292	+** file write, and file delete opertions up to 10 times, with a delay
	1293	+** of 25 milliseconds before the first retry and with the delay increasing
	1294	+** by an additional 25 milliseconds with each subsequent retry. This
	1295	+** opcode allows those to values (10 retries and 25 milliseconds of delay)
	1296	+** to be adjusted. The values are changed for all database connections
	1297	+** within the same process. The argument is a pointer to an array of two
	1298	+** integers where the first integer i the new retry count and the second
	1299	+** integer is the delay. If either integer is negative, then the setting
	1300	+** is not changed but instead the prior value of that setting is written
	1301	+** into the array entry, allowing the current retry settings to be
	1302	+** interrogated. The zDbName parameter is ignored.
	1303	+**
1287	1304	*/
1288	1305	#define SQLITE_FCNTL_LOCKSTATE 1
1289	1306	#define SQLITE_GET_LOCKPROXYFILE 2
1290	1307	#define SQLITE_SET_LOCKPROXYFILE 3
1291	1308	#define SQLITE_LAST_ERRNO 4
1292	1309	#define SQLITE_FCNTL_SIZE_HINT 5
1293	1310	#define SQLITE_FCNTL_CHUNK_SIZE 6
1294	1311	#define SQLITE_FCNTL_FILE_POINTER 7
1295	1312	#define SQLITE_FCNTL_SYNC_OMITTED 8
1296		-
	1313	+#define SQLITE_FCNTL_WIN32_AV_RETRY 9
1297	1314
1298	1315	/*
1299	1316	** CAPI3REF: Mutex Handle
1300	1317	**
1301	1318	** The mutex module within SQLite defines [sqlite3_mutex] to be an
		@@ -9558,10 +9575,11 @@
9558	9575	#define SQLITE_IndexSearch 0x08 /* Disable indexes for searching */
9559	9576	#define SQLITE_IndexCover 0x10 /* Disable index covering table */
9560	9577	#define SQLITE_GroupByOrder 0x20 /* Disable GROUPBY cover of ORDERBY */
9561	9578	#define SQLITE_FactorOutConst 0x40 /* Disable factoring out constants */
9562	9579	#define SQLITE_IdxRealAsInt 0x80 /* Store REAL as INT in indices */
	9580	+#define SQLITE_DistinctOpt 0x80 /* DISTINCT using indexes */
9563	9581	#define SQLITE_OptMask 0xff /* Mask of all disablable opts */
9564	9582
9565	9583	/*
9566	9584	** Possible values for the sqlite.magic field.
9567	9585	** The numbers are obtained at random and have no special meaning, other
		@@ -10449,10 +10467,11 @@
10449	10467	Table pTab; / An SQL table corresponding to zName */
10450	10468	Select pSelect; / A SELECT statement used in place of a table name */
10451	10469	u8 isPopulated; /* Temporary table associated with SELECT is populated */
10452	10470	u8 jointype; /* Type of join between this able and the previous */
10453	10471	u8 notIndexed; /* True if there is a NOT INDEXED clause */
	10472	+ u8 isCorrelated; /* True if sub-query is correlated */
10454	10473	#ifndef SQLITE_OMIT_EXPLAIN
10455	10474	u8 iSelectId; /* If pSelect!=0, the id of the sub-select in EQP */
10456	10475	#endif
10457	10476	int iCursor; /* The VDBE cursor number used to access this table */
10458	10477	Expr pOn; / The ON clause of a join */
		@@ -10568,10 +10587,11 @@
10568	10587	struct WhereInfo {
10569	10588	Parse pParse; / Parsing and code generating context */
10570	10589	u16 wctrlFlags; /* Flags originally passed to sqlite3WhereBegin() */
10571	10590	u8 okOnePass; /* Ok to use one-pass algorithm for UPDATE or DELETE */
10572	10591	u8 untestedTerms; /* Not all WHERE terms resolved by outer loop */
	10592	+ u8 eDistinct;
10573	10593	SrcList pTabList; / List of tables in the join */
10574	10594	int iTop; /* The very beginning of the WHERE loop */
10575	10595	int iContinue; /* Jump here to continue with next record */
10576	10596	int iBreak; /* Jump here to break out of the loop */
10577	10597	int nLevel; /* Number of nested loop */
		@@ -10579,10 +10599,13 @@
10579	10599	double savedNQueryLoop; /* pParse->nQueryLoop outside the WHERE loop */
10580	10600	double nRowOut; /* Estimated number of output rows */
10581	10601	WhereLevel a[1]; /* Information about each nest loop in WHERE */
10582	10602	};
10583	10603
	10604	+#define WHERE_DISTINCT_UNIQUE 1
	10605	+#define WHERE_DISTINCT_ORDERED 2
	10606	+
10584	10607	/*
10585	10608	** A NameContext defines a context in which to resolve table and column
10586	10609	** names. The context consists of a list of tables (the pSrcList) field and
10587	10610	** a list of named expression (pEList). The named expression list may
10588	10611	** be NULL. The pSrc corresponds to the FROM clause of a SELECT or
		@@ -11340,11 +11363,11 @@
11340	11363	#if defined(SQLITE_ENABLE_UPDATE_DELETE_LIMIT) && !defined(SQLITE_OMIT_SUBQUERY)
11341	11364	SQLITE_PRIVATE Expr sqlite3LimitWhere(Parse , SrcList , Expr , ExprList , Expr , Expr , char );
11342	11365	#endif
11343	11366	SQLITE_PRIVATE void sqlite3DeleteFrom(Parse, SrcList, Expr*);
11344	11367	SQLITE_PRIVATE void sqlite3Update(Parse, SrcList, ExprList, Expr, int);
11345		-SQLITE_PRIVATE WhereInfo sqlite3WhereBegin(Parse, SrcList, Expr, ExprList**, u16);
	11368	+SQLITE_PRIVATE WhereInfo sqlite3WhereBegin(Parse, SrcList, Expr, ExprList*,ExprList,u16);
11346	11369	SQLITE_PRIVATE void sqlite3WhereEnd(WhereInfo*);
11347	11370	SQLITE_PRIVATE int sqlite3ExprCodeGetColumn(Parse, Table, int, int, int);
11348	11371	SQLITE_PRIVATE void sqlite3ExprCodeGetColumnOfTable(Vdbe, Table, int, int, int);
11349	11372	SQLITE_PRIVATE void sqlite3ExprCodeMove(Parse*, int, int, int);
11350	11373	SQLITE_PRIVATE void sqlite3ExprCodeCopy(Parse*, int, int, int);
		@@ -15841,11 +15864,11 @@
15841	15864	** Free an outstanding memory allocation.
15842	15865	**
15843	15866	** This function assumes that the necessary mutexes, if any, are
15844	15867	** already held by the caller. Hence "Unsafe".
15845	15868	*/
15846		-void memsys3FreeUnsafe(void *pOld){
	15869	+static void memsys3FreeUnsafe(void *pOld){
15847	15870	Mem3Block p = (Mem3Block)pOld;
15848	15871	int i;
15849	15872	u32 size, x;
15850	15873	assert( sqlite3_mutex_held(mem3.mutex) );
15851	15874	assert( p>mem3.aPool && p<&mem3.aPool[mem3.nPool] );
		@@ -15916,21 +15939,21 @@
15916	15939	}
15917	15940
15918	15941	/*
15919	15942	** Free memory.
15920	15943	*/
15921		-void memsys3Free(void *pPrior){
	15944	+static void memsys3Free(void *pPrior){
15922	15945	assert( pPrior );
15923	15946	memsys3Enter();
15924	15947	memsys3FreeUnsafe(pPrior);
15925	15948	memsys3Leave();
15926	15949	}
15927	15950
15928	15951	/*
15929	15952	** Change the size of an existing memory allocation
15930	15953	*/
15931		-void memsys3Realloc(void pPrior, int nBytes){
	15954	+static void memsys3Realloc(void pPrior, int nBytes){
15932	15955	int nOld;
15933	15956	void *p;
15934	15957	if( pPrior==0 ){
15935	15958	return sqlite3_malloc(nBytes);
15936	15959	}
		@@ -25143,11 +25166,13 @@
25143	25166	case EINVAL:
25144	25167	case ENOTCONN:
25145	25168	case ENODEV:
25146	25169	case ENXIO:
25147	25170	case ENOENT:
	25171	+#ifdef ESTALE /* ESTALE is not defined on Interix systems */
25148	25172	case ESTALE:
	25173	+#endif
25149	25174	case ENOSYS:
25150	25175	/* these should force the client to close the file and reconnect */
25151	25176
25152	25177	default:
25153	25178	return sqliteIOErr;
		@@ -31820,10 +31845,58 @@
31820	31845	iLine, iErrno, zFunc, zPath, zMsg
31821	31846	);
31822	31847
31823	31848	return errcode;
31824	31849	}
	31850	+
	31851	+/*
	31852	+** The number of times that a ReadFile(), WriteFile(), and DeleteFile()
	31853	+** will be retried following a locking error - probably caused by
	31854	+** antivirus software. Also the initial delay before the first retry.
	31855	+** The delay increases linearly with each retry.
	31856	+*/
	31857	+#ifndef SQLITE_WIN32_IOERR_RETRY
	31858	+# define SQLITE_WIN32_IOERR_RETRY 10
	31859	+#endif
	31860	+#ifndef SQLITE_WIN32_IOERR_RETRY_DELAY
	31861	+# define SQLITE_WIN32_IOERR_RETRY_DELAY 25
	31862	+#endif
	31863	+static int win32IoerrRetry = SQLITE_WIN32_IOERR_RETRY;
	31864	+static int win32IoerrRetryDelay = SQLITE_WIN32_IOERR_RETRY_DELAY;
	31865	+
	31866	+/*
	31867	+** If a ReadFile() or WriteFile() error occurs, invoke this routine
	31868	+** to see if it should be retried. Return TRUE to retry. Return FALSE
	31869	+** to give up with an error.
	31870	+*/
	31871	+static int retryIoerr(int *pnRetry){
	31872	+ DWORD e;
	31873	+ if( *pnRetry>=win32IoerrRetry ){
	31874	+ return 0;
	31875	+ }
	31876	+ e = GetLastError();
	31877	+ if( e==ERROR_ACCESS_DENIED \|\|
	31878	+ e==ERROR_LOCK_VIOLATION \|\|
	31879	+ e==ERROR_SHARING_VIOLATION ){
	31880	+ Sleep(win32IoerrRetryDelay(1+pnRetry));
	31881	+ ++*pnRetry;
	31882	+ return 1;
	31883	+ }
	31884	+ return 0;
	31885	+}
	31886	+
	31887	+/*
	31888	+** Log a I/O error retry episode.
	31889	+*/
	31890	+static void logIoerr(int nRetry){
	31891	+ if( nRetry ){
	31892	+ sqlite3_log(SQLITE_IOERR,
	31893	+ "delayed %dms for lock/sharing conflict",
	31894	+ win32IoerrRetryDelaynRetry(nRetry+1)/2
	31895	+ );
	31896	+ }
	31897	+}
31825	31898
31826	31899	#if SQLITE_OS_WINCE
31827	31900	/*************************************************************************
31828	31901	** This section contains code for WinCE only.
31829	31902	*/
		@@ -32238,22 +32311,25 @@
32238	32311	int amt, /* Number of bytes to read */
32239	32312	sqlite3_int64 offset /* Begin reading at this offset */
32240	32313	){
32241	32314	winFile pFile = (winFile)id; /* file handle */
32242	32315	DWORD nRead; /* Number of bytes actually read from file */
	32316	+ int nRetry = 0; /* Number of retrys */
32243	32317
32244	32318	assert( id!=0 );
32245	32319	SimulateIOError(return SQLITE_IOERR_READ);
32246	32320	OSTRACE(("READ %d lock=%d\n", pFile->h, pFile->locktype));
32247	32321
32248	32322	if( seekWinFile(pFile, offset) ){
32249	32323	return SQLITE_FULL;
32250	32324	}
32251		- if( !ReadFile(pFile->h, pBuf, amt, &nRead, 0) ){
	32325	+ while( !ReadFile(pFile->h, pBuf, amt, &nRead, 0) ){
	32326	+ if( retryIoerr(&nRetry) ) continue;
32252	32327	pFile->lastErrno = GetLastError();
32253	32328	return winLogError(SQLITE_IOERR_READ, "winRead", pFile->zPath);
32254	32329	}
	32330	+ logIoerr(nRetry);
32255	32331	if( nRead<(DWORD)amt ){
32256	32332	/* Unread parts of the buffer must be zero-filled */
32257	32333	memset(&((char*)pBuf)[nRead], 0, amt-nRead);
32258	32334	return SQLITE_IOERR_SHORT_READ;
32259	32335	}
		@@ -32271,10 +32347,11 @@
32271	32347	int amt, /* Number of bytes to write */
32272	32348	sqlite3_int64 offset /* Offset into the file to begin writing at */
32273	32349	){
32274	32350	int rc; /* True if error has occured, else false */
32275	32351	winFile pFile = (winFile)id; /* File handle */
	32352	+ int nRetry = 0; /* Number of retries */
32276	32353
32277	32354	assert( amt>0 );
32278	32355	assert( pFile );
32279	32356	SimulateIOError(return SQLITE_IOERR_WRITE);
32280	32357	SimulateDiskfullError(return SQLITE_FULL);
		@@ -32285,11 +32362,16 @@
32285	32362	if( rc==0 ){
32286	32363	u8 aRem = (u8 )pBuf; /* Data yet to be written */
32287	32364	int nRem = amt; /* Number of bytes yet to be written */
32288	32365	DWORD nWrite; /* Bytes written by each WriteFile() call */
32289	32366
32290		- while( nRem>0 && WriteFile(pFile->h, aRem, nRem, &nWrite, 0) && nWrite>0 ){
	32367	+ while( nRem>0 ){
	32368	+ if( !WriteFile(pFile->h, aRem, nRem, &nWrite, 0) ){
	32369	+ if( retryIoerr(&nRetry) ) continue;
	32370	+ break;
	32371	+ }
	32372	+ if( nWrite<=0 ) break;
32291	32373	aRem += nWrite;
32292	32374	nRem -= nWrite;
32293	32375	}
32294	32376	if( nRem>0 ){
32295	32377	pFile->lastErrno = GetLastError();
		@@ -32301,10 +32383,12 @@
32301	32383	if( ( pFile->lastErrno==ERROR_HANDLE_DISK_FULL )
32302	32384	\|\| ( pFile->lastErrno==ERROR_DISK_FULL )){
32303	32385	return SQLITE_FULL;
32304	32386	}
32305	32387	return winLogError(SQLITE_IOERR_WRITE, "winWrite", pFile->zPath);
	32388	+ }else{
	32389	+ logIoerr(nRetry);
32306	32390	}
32307	32391	return SQLITE_OK;
32308	32392	}
32309	32393
32310	32394	/*
		@@ -32716,10 +32800,24 @@
32716	32800	SimulateIOErrorBenign(0);
32717	32801	return SQLITE_OK;
32718	32802	}
32719	32803	case SQLITE_FCNTL_SYNC_OMITTED: {
32720	32804	return SQLITE_OK;
	32805	+ }
	32806	+ case SQLITE_FCNTL_WIN32_AV_RETRY: {
	32807	+ int a = (int)pArg;
	32808	+ if( a[0]>0 ){
	32809	+ win32IoerrRetry = a[0];
	32810	+ }else{
	32811	+ a[0] = win32IoerrRetry;
	32812	+ }
	32813	+ if( a[1]>0 ){
	32814	+ win32IoerrRetryDelay = a[1];
	32815	+ }else{
	32816	+ a[1] = win32IoerrRetryDelay;
	32817	+ }
	32818	+ return SQLITE_OK;
32721	32819	}
32722	32820	}
32723	32821	return SQLITE_NOTFOUND;
32724	32822	}
32725	32823
		@@ -33734,19 +33832,17 @@
33734	33832	** file open, we will be unable to delete it. To work around this
33735	33833	** problem, we delay 100 milliseconds and try to delete again. Up
33736	33834	** to MX_DELETION_ATTEMPTs deletion attempts are run before giving
33737	33835	** up and returning an error.
33738	33836	*/
33739		-#define MX_DELETION_ATTEMPTS 5
33740	33837	static int winDelete(
33741	33838	sqlite3_vfs pVfs, / Not used on win32 */
33742	33839	const char zFilename, / Name of file to delete */
33743	33840	int syncDir /* Not used on win32 */
33744	33841	){
33745	33842	int cnt = 0;
33746		- DWORD rc;
33747		- DWORD error = 0;
	33843	+ int rc;
33748	33844	void *zConverted;
33749	33845	UNUSED_PARAMETER(pVfs);
33750	33846	UNUSED_PARAMETER(syncDir);
33751	33847
33752	33848	SimulateIOError(return SQLITE_IOERR_DELETE);
		@@ -33753,38 +33849,34 @@
33753	33849	zConverted = convertUtf8Filename(zFilename);
33754	33850	if( zConverted==0 ){
33755	33851	return SQLITE_NOMEM;
33756	33852	}
33757	33853	if( isNT() ){
33758		- do{
33759		- DeleteFileW(zConverted);
33760		- }while( ( ((rc = GetFileAttributesW(zConverted)) != INVALID_FILE_ATTRIBUTES)
33761		- \|\| ((error = GetLastError()) == ERROR_ACCESS_DENIED))
33762		- && (++cnt < MX_DELETION_ATTEMPTS)
33763		- && (Sleep(100), 1) );
	33854	+ rc = 1;
	33855	+ while( GetFileAttributesW(zConverted)!=INVALID_FILE_ATTRIBUTES &&
	33856	+ (rc = DeleteFileW(zConverted))==0 && retryIoerr(&cnt) ){}
	33857	+ rc = rc ? SQLITE_OK : SQLITE_ERROR;
33764	33858	/* isNT() is 1 if SQLITE_OS_WINCE==1, so this else is never executed.
33765	33859	** Since the ASCII version of these Windows API do not exist for WINCE,
33766	33860	** it's important to not reference them for WINCE builds.
33767	33861	*/
33768	33862	#if SQLITE_OS_WINCE==0
33769	33863	}else{
33770		- do{
33771		- DeleteFileA(zConverted);
33772		- }while( ( ((rc = GetFileAttributesA(zConverted)) != INVALID_FILE_ATTRIBUTES)
33773		- \|\| ((error = GetLastError()) == ERROR_ACCESS_DENIED))
33774		- && (++cnt < MX_DELETION_ATTEMPTS)
33775		- && (Sleep(100), 1) );
	33864	+ rc = 1;
	33865	+ while( GetFileAttributesA(zConverted)!=INVALID_FILE_ATTRIBUTES &&
	33866	+ (rc = DeleteFileA(zConverted))==0 && retryIoerr(&cnt) ){}
	33867	+ rc = rc ? SQLITE_OK : SQLITE_ERROR;
33776	33868	#endif
33777	33869	}
	33870	+ if( rc ){
	33871	+ rc = winLogError(SQLITE_IOERR_DELETE, "winDelete", zFilename);
	33872	+ }else{
	33873	+ logIoerr(cnt);
	33874	+ }
33778	33875	free(zConverted);
33779		- OSTRACE(("DELETE \"%s\" %s\n", zFilename,
33780		- ( (rc==INVALID_FILE_ATTRIBUTES) && (error==ERROR_FILE_NOT_FOUND)) ?
33781		- "ok" : "failed" ));
33782		-
33783		- return ( (rc == INVALID_FILE_ATTRIBUTES)
33784		- && (error == ERROR_FILE_NOT_FOUND)) ? SQLITE_OK :
33785		- winLogError(SQLITE_IOERR_DELETE, "winDelete", zFilename);
	33876	+ OSTRACE(("DELETE \"%s\" %s\n", zFilename, (rc ? "failed" : "ok" )));
	33877	+ return rc;
33786	33878	}
33787	33879
33788	33880	/*
33789	33881	** Check the existance and status of a file.
33790	33882	*/
		@@ -59049,10 +59141,11 @@
59049	59141	nMem = 10;
59050	59142	}
59051	59143	memset(zCsr, 0, zEnd-zCsr);
59052	59144	zCsr += (zCsr - (u8*)0)&7;
59053	59145	assert( EIGHT_BYTE_ALIGNMENT(zCsr) );
	59146	+ p->expired = 0;
59054	59147
59055	59148	/* Memory for registers, parameters, cursor, etc, is allocated in two
59056	59149	** passes. On the first pass, we try to reuse unused space at the
59057	59150	** end of the opcode array. If we are unable to satisfy all memory
59058	59151	** requirements by reusing the opcode array tail, then the second
		@@ -61277,11 +61370,11 @@
61277	61370	sqlite3_mutex_enter(db->mutex);
61278	61371	while( (rc = sqlite3Step(v))==SQLITE_SCHEMA
61279	61372	&& cnt++ < SQLITE_MAX_SCHEMA_RETRY
61280	61373	&& (rc2 = rc = sqlite3Reprepare(v))==SQLITE_OK ){
61281	61374	sqlite3_reset(pStmt);
61282		- v->expired = 0;
	61375	+ assert( v->expired==0 );
61283	61376	}
61284	61377	if( rc2!=SQLITE_OK && ALWAYS(v->isPrepareV2) && ALWAYS(db->pErr) ){
61285	61378	/* This case occurs after failing to recompile an sql statement.
61286	61379	** The error message from the SQL compiler has already been loaded
61287	61380	** into the database handle. This block copies the error message
		@@ -65968,11 +66061,11 @@
65968	66061	** automatically created table with root-page 1 (an BLOB_INTKEY table).
65969	66062	*/
65970	66063	if( pOp->p4.pKeyInfo ){
65971	66064	int pgno;
65972	66065	assert( pOp->p4type==P4_KEYINFO );
65973		- rc = sqlite3BtreeCreateTable(u.ax.pCx->pBt, &pgno, BTREE_BLOBKEY);
	66066	+ rc = sqlite3BtreeCreateTable(u.ax.pCx->pBt, &pgno, BTREE_BLOBKEY \| pOp->p5);
65974	66067	if( rc==SQLITE_OK ){
65975	66068	assert( pgno==MASTER_ROOT+1 );
65976	66069	rc = sqlite3BtreeCursor(u.ax.pCx->pBt, pgno, 1,
65977	66070	(KeyInfo*)pOp->p4.z, u.ax.pCx->pCursor);
65978	66071	u.ax.pCx->pKeyInfo = pOp->p4.pKeyInfo;
		@@ -71007,15 +71100,29 @@
71007	71100	/* Recursively resolve names in all subqueries
71008	71101	*/
71009	71102	for(i=0; i<p->pSrc->nSrc; i++){
71010	71103	struct SrcList_item *pItem = &p->pSrc->a[i];
71011	71104	if( pItem->pSelect ){
	71105	+ NameContext pNC; / Used to iterate name contexts */
	71106	+ int nRef = 0; /* Refcount for pOuterNC and outer contexts */
71012	71107	const char *zSavedContext = pParse->zAuthContext;
	71108	+
	71109	+ /* Count the total number of references to pOuterNC and all of its
	71110	+ ** parent contexts. After resolving references to expressions in
	71111	+ ** pItem->pSelect, check if this value has changed. If so, then
	71112	+ ** SELECT statement pItem->pSelect must be correlated. Set the
	71113	+ ** pItem->isCorrelated flag if this is the case. */
	71114	+ for(pNC=pOuterNC; pNC; pNC=pNC->pNext) nRef += pNC->nRef;
	71115	+
71013	71116	if( pItem->zName ) pParse->zAuthContext = pItem->zName;
71014	71117	sqlite3ResolveSelectNames(pParse, pItem->pSelect, pOuterNC);
71015	71118	pParse->zAuthContext = zSavedContext;
71016	71119	if( pParse->nErr \|\| db->mallocFailed ) return WRC_Abort;
	71120	+
	71121	+ for(pNC=pOuterNC; pNC; pNC=pNC->pNext) nRef -= pNC->nRef;
	71122	+ assert( pItem->isCorrelated==0 && nRef<=0 );
	71123	+ pItem->isCorrelated = (nRef!=0);
71017	71124	}
71018	71125	}
71019	71126
71020	71127	/* If there are no aggregate functions in the result-set, and no GROUP BY
71021	71128	** expression, do not allow aggregates in any of the other expressions.
		@@ -72119,10 +72226,11 @@
72119	72226	pNewItem->zName = sqlite3DbStrDup(db, pOldItem->zName);
72120	72227	pNewItem->zAlias = sqlite3DbStrDup(db, pOldItem->zAlias);
72121	72228	pNewItem->jointype = pOldItem->jointype;
72122	72229	pNewItem->iCursor = pOldItem->iCursor;
72123	72230	pNewItem->isPopulated = pOldItem->isPopulated;
	72231	+ pNewItem->isCorrelated = pOldItem->isCorrelated;
72124	72232	pNewItem->zIndex = sqlite3DbStrDup(db, pOldItem->zIndex);
72125	72233	pNewItem->notIndexed = pOldItem->notIndexed;
72126	72234	pNewItem->pIndex = pOldItem->pIndex;
72127	72235	pTab = pNewItem->pTab = pOldItem->pTab;
72128	72236	if( pTab ){
		@@ -80126,11 +80234,11 @@
80126	80234	if( pStart ){
80127	80235	assert( pEnd!=0 );
80128	80236	/* A named index with an explicit CREATE INDEX statement */
80129	80237	zStmt = sqlite3MPrintf(db, "CREATE%s INDEX %.*s",
80130	80238	onError==OE_None ? "" : " UNIQUE",
80131		- pEnd->z - pName->z + 1,
	80239	+ (int)(pEnd->z - pName->z) + 1,
80132	80240	pName->z);
80133	80241	}else{
80134	80242	/* An automatic index created by a PRIMARY KEY or UNIQUE constraint */
80135	80243	/* zStmt = sqlite3MPrintf(""); */
80136	80244	zStmt = 0;
		@@ -81921,11 +82029,13 @@
81921	82029	int regRowid; /* Actual register containing rowids */
81922	82030
81923	82031	/* Collect rowids of every row to be deleted.
81924	82032	*/
81925	82033	sqlite3VdbeAddOp2(v, OP_Null, 0, iRowSet);
81926		- pWInfo = sqlite3WhereBegin(pParse, pTabList, pWhere,0,WHERE_DUPLICATES_OK);
	82034	+ pWInfo = sqlite3WhereBegin(
	82035	+ pParse, pTabList, pWhere, 0, 0, WHERE_DUPLICATES_OK
	82036	+ );
81927	82037	if( pWInfo==0 ) goto delete_from_cleanup;
81928	82038	regRowid = sqlite3ExprCodeGetColumn(pParse, pTab, -1, iCur, iRowid);
81929	82039	sqlite3VdbeAddOp2(v, OP_RowSetAdd, iRowSet, regRowid);
81930	82040	if( db->flags & SQLITE_CountRows ){
81931	82041	sqlite3VdbeAddOp2(v, OP_AddImm, memCnt, 1);
		@@ -84368,11 +84478,11 @@
84368	84478
84369	84479	/* Create VDBE to loop through the entries in pSrc that match the WHERE
84370	84480	** clause. If the constraint is not deferred, throw an exception for
84371	84481	** each row found. Otherwise, for deferred constraints, increment the
84372	84482	** deferred constraint counter by nIncr for each row selected. */
84373		- pWInfo = sqlite3WhereBegin(pParse, pSrc, pWhere, 0, 0);
	84483	+ pWInfo = sqlite3WhereBegin(pParse, pSrc, pWhere, 0, 0, 0);
84374	84484	if( nIncr>0 && pFKey->isDeferred==0 ){
84375	84485	sqlite3ParseToplevel(pParse)->mayAbort = 1;
84376	84486	}
84377	84487	sqlite3VdbeAddOp2(v, OP_FkCounter, pFKey->isDeferred, nIncr);
84378	84488	if( pWInfo ){
		@@ -87235,10 +87345,13 @@
87235	87345	int (strnicmp)(const char,const char*,int);
87236	87346	int (unlock_notify)(sqlite3,void()(void,int),void);
87237	87347	int (wal_autocheckpoint)(sqlite3,int);
87238	87348	int (wal_checkpoint)(sqlite3,const char*);
87239	87349	void (wal_hook)(sqlite3,int()(void,sqlite3,const char,int),void);
	87350	+ int (blob_reopen)(sqlite3_blob,sqlite3_int64);
	87351	+ int (vtab_config)(sqlite3,int op,...);
	87352	+ int (vtab_on_conflict)(sqlite3);
87240	87353	};
87241	87354
87242	87355	/*
87243	87356	** The following macros redefine the API routines so that they are
87244	87357	** redirected throught the global sqlite3_api structure.
		@@ -87435,10 +87548,13 @@
87435	87548	#define sqlite3_strnicmp sqlite3_api->strnicmp
87436	87549	#define sqlite3_unlock_notify sqlite3_api->unlock_notify
87437	87550	#define sqlite3_wal_autocheckpoint sqlite3_api->wal_autocheckpoint
87438	87551	#define sqlite3_wal_checkpoint sqlite3_api->wal_checkpoint
87439	87552	#define sqlite3_wal_hook sqlite3_api->wal_hook
	87553	+#define sqlite3_blob_reopen sqlite3_api->blob_reopen
	87554	+#define sqlite3_vtab_config sqlite3_api->vtab_config
	87555	+#define sqlite3_vtab_on_conflict sqlite3_api->vtab_on_conflict
87440	87556	#endif /* SQLITE_CORE */
87441	87557
87442	87558	#define SQLITE_EXTENSION_INIT1 const sqlite3_api_routines *sqlite3_api = 0;
87443	87559	#define SQLITE_EXTENSION_INIT2(v) sqlite3_api = v;
87444	87560
		@@ -87509,10 +87625,12 @@
87509	87625
87510	87626	#ifdef SQLITE_OMIT_VIRTUALTABLE
87511	87627	# define sqlite3_create_module 0
87512	87628	# define sqlite3_create_module_v2 0
87513	87629	# define sqlite3_declare_vtab 0
	87630	+# define sqlite3_vtab_config 0
	87631	+# define sqlite3_vtab_on_conflict 0
87514	87632	#endif
87515	87633
87516	87634	#ifdef SQLITE_OMIT_SHARED_CACHE
87517	87635	# define sqlite3_enable_shared_cache 0
87518	87636	#endif
		@@ -87532,10 +87650,11 @@
87532	87650	#define sqlite3_blob_bytes 0
87533	87651	#define sqlite3_blob_close 0
87534	87652	#define sqlite3_blob_open 0
87535	87653	#define sqlite3_blob_read 0
87536	87654	#define sqlite3_blob_write 0
	87655	+#define sqlite3_blob_reopen 0
87537	87656	#endif
87538	87657
87539	87658	/*
87540	87659	** The following structure contains pointers to all SQLite API routines.
87541	87660	** A pointer to this structure is passed into extensions when they are
		@@ -87797,10 +87916,13 @@
87797	87916	#else
87798	87917	0,
87799	87918	0,
87800	87919	0,
87801	87920	#endif
	87921	+ sqlite3_blob_reopen,
	87922	+ sqlite3_vtab_config,
	87923	+ sqlite3_vtab_on_conflict,
87802	87924	};
87803	87925
87804	87926	/*
87805	87927	** Attempt to load an SQLite extension library contained in the file
87806	87928	** zFile. The entry point is zProc. zProc may be 0 in which case a
		@@ -94186,10 +94308,11 @@
94186	94308	Expr pHaving; / The HAVING clause. May be NULL */
94187	94309	int isDistinct; /* True if the DISTINCT keyword is present */
94188	94310	int distinct; /* Table to use for the distinct set */
94189	94311	int rc = 1; /* Value to return from this function */
94190	94312	int addrSortIndex; /* Address of an OP_OpenEphemeral instruction */
	94313	+ int addrDistinctIndex; /* Address of an OP_OpenEphemeral instruction */
94191	94314	AggInfo sAggInfo; /* Information used by aggregate queries */
94192	94315	int iEnd; /* Address of the end of the query */
94193	94316	sqlite3 db; / The database connection */
94194	94317
94195	94318	#ifndef SQLITE_OMIT_EXPLAIN
		@@ -94312,20 +94435,10 @@
94312	94435	explainSetInteger(pParse->iSelectId, iRestoreSelectId);
94313	94436	return rc;
94314	94437	}
94315	94438	#endif
94316	94439
94317		- /* If possible, rewrite the query to use GROUP BY instead of DISTINCT.
94318		- ** GROUP BY might use an index, DISTINCT never does.
94319		- */
94320		- assert( p->pGroupBy==0 \|\| (p->selFlags & SF_Aggregate)!=0 );
94321		- if( (p->selFlags & (SF_Distinct\|SF_Aggregate))==SF_Distinct ){
94322		- p->pGroupBy = sqlite3ExprListDup(db, p->pEList, 0);
94323		- pGroupBy = p->pGroupBy;
94324		- p->selFlags &= ~SF_Distinct;
94325		- }
94326		-
94327	94440	/* If there is both a GROUP BY and an ORDER BY clause and they are
94328	94441	** identical, then disable the ORDER BY clause since the GROUP BY
94329	94442	** will cause elements to come out in the correct order. This is
94330	94443	** an optimization - the correct answer should result regardless.
94331	94444	** Use the SQLITE_GroupByOrder flag with SQLITE_TESTCTRL_OPTIMIZER
		@@ -94333,10 +94446,34 @@
94333	94446	*/
94334	94447	if( sqlite3ExprListCompare(p->pGroupBy, pOrderBy)==0
94335	94448	&& (db->flags & SQLITE_GroupByOrder)==0 ){
94336	94449	pOrderBy = 0;
94337	94450	}
	94451	+
	94452	+ /* If the query is DISTINCT with an ORDER BY but is not an aggregate, and
	94453	+ ** if the select-list is the same as the ORDER BY list, then this query
	94454	+ ** can be rewritten as a GROUP BY. In other words, this:
	94455	+ **
	94456	+ ** SELECT DISTINCT xyz FROM ... ORDER BY xyz
	94457	+ **
	94458	+ ** is transformed to:
	94459	+ **
	94460	+ ** SELECT xyz FROM ... GROUP BY xyz
	94461	+ **
	94462	+ ** The second form is preferred as a single index (or temp-table) may be
	94463	+ ** used for both the ORDER BY and DISTINCT processing. As originally
	94464	+ ** written the query must use a temp-table for at least one of the ORDER
	94465	+ ** BY and DISTINCT, and an index or separate temp-table for the other.
	94466	+ */
	94467	+ if( (p->selFlags & (SF_Distinct\|SF_Aggregate))==SF_Distinct
	94468	+ && sqlite3ExprListCompare(pOrderBy, p->pEList)==0
	94469	+ ){
	94470	+ p->selFlags &= ~SF_Distinct;
	94471	+ p->pGroupBy = sqlite3ExprListDup(db, p->pEList, 0);
	94472	+ pGroupBy = p->pGroupBy;
	94473	+ pOrderBy = 0;
	94474	+ }
94338	94475
94339	94476	/* If there is an ORDER BY clause, then this sorting
94340	94477	** index might end up being unused if the data can be
94341	94478	** extracted in pre-sorted order. If that is the case, then the
94342	94479	** OP_OpenEphemeral instruction will be changed to an OP_Noop once
		@@ -94369,26 +94506,25 @@
94369	94506
94370	94507	/* Open a virtual index to use for the distinct set.
94371	94508	*/
94372	94509	if( p->selFlags & SF_Distinct ){
94373	94510	KeyInfo *pKeyInfo;
94374		- assert( isAgg \|\| pGroupBy );
94375	94511	distinct = pParse->nTab++;
94376	94512	pKeyInfo = keyInfoFromExprList(pParse, p->pEList);
94377		- sqlite3VdbeAddOp4(v, OP_OpenEphemeral, distinct, 0, 0,
94378		- (char*)pKeyInfo, P4_KEYINFO_HANDOFF);
	94513	+ addrDistinctIndex = sqlite3VdbeAddOp4(v, OP_OpenEphemeral, distinct, 0, 0,
	94514	+ (char*)pKeyInfo, P4_KEYINFO_HANDOFF);
94379	94515	sqlite3VdbeChangeP5(v, BTREE_UNORDERED);
94380	94516	}else{
94381		- distinct = -1;
	94517	+ distinct = addrDistinctIndex = -1;
94382	94518	}
94383	94519
94384	94520	/* Aggregate and non-aggregate queries are handled differently */
94385	94521	if( !isAgg && pGroupBy==0 ){
94386		- /* This case is for non-aggregate queries
94387		- ** Begin the database scan
94388		- */
94389		- pWInfo = sqlite3WhereBegin(pParse, pTabList, pWhere, &pOrderBy, 0);
	94522	+ ExprList *pDist = (isDistinct ? p->pEList : 0);
	94523	+
	94524	+ /* Begin the database scan. */
	94525	+ pWInfo = sqlite3WhereBegin(pParse, pTabList, pWhere, &pOrderBy, pDist, 0);
94390	94526	if( pWInfo==0 ) goto select_end;
94391	94527	if( pWInfo->nRowOut < p->nSelectRow ) p->nSelectRow = pWInfo->nRowOut;
94392	94528
94393	94529	/* If sorting index that was created by a prior OP_OpenEphemeral
94394	94530	** instruction ended up not being needed, then change the OP_OpenEphemeral
		@@ -94397,14 +94533,56 @@
94397	94533	if( addrSortIndex>=0 && pOrderBy==0 ){
94398	94534	sqlite3VdbeChangeToNoop(v, addrSortIndex, 1);
94399	94535	p->addrOpenEphm[2] = -1;
94400	94536	}
94401	94537
94402		- /* Use the standard inner loop
94403		- */
94404		- assert(!isDistinct);
94405		- selectInnerLoop(pParse, p, pEList, 0, 0, pOrderBy, -1, pDest,
	94538	+ if( pWInfo->eDistinct ){
	94539	+ VdbeOp pOp; / No longer required OpenEphemeral instr. */
	94540	+
	94541	+ assert( addrDistinctIndex>0 );
	94542	+ pOp = sqlite3VdbeGetOp(v, addrDistinctIndex);
	94543	+
	94544	+ assert( isDistinct );
	94545	+ assert( pWInfo->eDistinct==WHERE_DISTINCT_ORDERED
	94546	+ \|\| pWInfo->eDistinct==WHERE_DISTINCT_UNIQUE
	94547	+ );
	94548	+ distinct = -1;
	94549	+ if( pWInfo->eDistinct==WHERE_DISTINCT_ORDERED ){
	94550	+ int iJump;
	94551	+ int iExpr;
	94552	+ int iFlag = ++pParse->nMem;
	94553	+ int iBase = pParse->nMem+1;
	94554	+ int iBase2 = iBase + pEList->nExpr;
	94555	+ pParse->nMem += (pEList->nExpr*2);
	94556	+
	94557	+ /* Change the OP_OpenEphemeral coded earlier to an OP_Integer. The
	94558	+ ** OP_Integer initializes the "first row" flag. */
	94559	+ pOp->opcode = OP_Integer;
	94560	+ pOp->p1 = 1;
	94561	+ pOp->p2 = iFlag;
	94562	+
	94563	+ sqlite3ExprCodeExprList(pParse, pEList, iBase, 1);
	94564	+ iJump = sqlite3VdbeCurrentAddr(v) + 1 + pEList->nExpr + 1 + 1;
	94565	+ sqlite3VdbeAddOp2(v, OP_If, iFlag, iJump-1);
	94566	+ for(iExpr=0; iExpr<pEList->nExpr; iExpr++){
	94567	+ CollSeq *pColl = sqlite3ExprCollSeq(pParse, pEList->a[iExpr].pExpr);
	94568	+ sqlite3VdbeAddOp3(v, OP_Ne, iBase+iExpr, iJump, iBase2+iExpr);
	94569	+ sqlite3VdbeChangeP4(v, -1, (const char *)pColl, P4_COLLSEQ);
	94570	+ sqlite3VdbeChangeP5(v, SQLITE_NULLEQ);
	94571	+ }
	94572	+ sqlite3VdbeAddOp2(v, OP_Goto, 0, pWInfo->iContinue);
	94573	+
	94574	+ sqlite3VdbeAddOp2(v, OP_Integer, 0, iFlag);
	94575	+ assert( sqlite3VdbeCurrentAddr(v)==iJump );
	94576	+ sqlite3VdbeAddOp3(v, OP_Move, iBase, iBase2, pEList->nExpr);
	94577	+ }else{
	94578	+ pOp->opcode = OP_Noop;
	94579	+ }
	94580	+ }
	94581	+
	94582	+ /* Use the standard inner loop. */
	94583	+ selectInnerLoop(pParse, p, pEList, 0, 0, pOrderBy, distinct, pDest,
94406	94584	pWInfo->iContinue, pWInfo->iBreak);
94407	94585
94408	94586	/* End the database scan loop.
94409	94587	*/
94410	94588	sqlite3WhereEnd(pWInfo);
		@@ -94510,11 +94688,11 @@
94510	94688	** This might involve two separate loops with an OP_Sort in between, or
94511	94689	** it might be a single loop that uses an index to extract information
94512	94690	** in the right order to begin with.
94513	94691	*/
94514	94692	sqlite3VdbeAddOp2(v, OP_Gosub, regReset, addrReset);
94515		- pWInfo = sqlite3WhereBegin(pParse, pTabList, pWhere, &pGroupBy, 0);
	94693	+ pWInfo = sqlite3WhereBegin(pParse, pTabList, pWhere, &pGroupBy, 0, 0);
94516	94694	if( pWInfo==0 ) goto select_end;
94517	94695	if( pGroupBy==0 ){
94518	94696	/* The optimizer is able to deliver rows in group by order so
94519	94697	** we do not have to sort. The OP_OpenEphemeral table will be
94520	94698	** cancelled later because we still need to use the pKeyInfo
		@@ -94772,11 +94950,11 @@
94772	94950	/* This case runs if the aggregate has no GROUP BY clause. The
94773	94951	** processing is much simpler since there is only a single row
94774	94952	** of output.
94775	94953	*/
94776	94954	resetAccumulator(pParse, &sAggInfo);
94777		- pWInfo = sqlite3WhereBegin(pParse, pTabList, pWhere, &pMinMax, flag);
	94955	+ pWInfo = sqlite3WhereBegin(pParse, pTabList, pWhere, &pMinMax, 0, flag);
94778	94956	if( pWInfo==0 ){
94779	94957	sqlite3ExprListDelete(db, pDel);
94780	94958	goto select_end;
94781	94959	}
94782	94960	updateAccumulator(pParse, &sAggInfo);
		@@ -95248,19 +95426,32 @@
95248	95426	iDb = sqlite3TwoPartName(pParse, pName1, pName2, &pName);
95249	95427	if( iDb<0 ){
95250	95428	goto trigger_cleanup;
95251	95429	}
95252	95430	}
	95431	+ if( !pTableName \|\| db->mallocFailed ){
	95432	+ goto trigger_cleanup;
	95433	+ }
	95434	+
	95435	+ /* A long-standing parser bug is that this syntax was allowed:
	95436	+ **
	95437	+ ** CREATE TRIGGER attached.demo AFTER INSERT ON attached.tab ....
	95438	+ ** ^^^^^^^^
	95439	+ **
	95440	+ ** To maintain backwards compatibility, ignore the database
	95441	+ ** name on pTableName if we are reparsing our of SQLITE_MASTER.
	95442	+ */
	95443	+ if( db->init.busy && iDb!=1 ){
	95444	+ sqlite3DbFree(db, pTableName->a[0].zDatabase);
	95445	+ pTableName->a[0].zDatabase = 0;
	95446	+ }
95253	95447
95254	95448	/* If the trigger name was unqualified, and the table is a temp table,
95255	95449	** then set iDb to 1 to create the trigger in the temporary database.
95256	95450	** If sqlite3SrcListLookup() returns 0, indicating the table does not
95257	95451	** exist, the error is caught by the block below.
95258	95452	*/
95259		- if( !pTableName \|\| db->mallocFailed ){
95260		- goto trigger_cleanup;
95261		- }
95262	95453	pTab = sqlite3SrcListLookup(pParse, pTableName);
95263	95454	if( db->init.busy==0 && pName2->n==0 && pTab
95264	95455	&& pTab->pSchema==db->aDb[1].pSchema ){
95265	95456	iDb = 1;
95266	95457	}
		@@ -96554,11 +96745,13 @@
96554	96745	}
96555	96746
96556	96747	/* Begin the database scan
96557	96748	*/
96558	96749	sqlite3VdbeAddOp2(v, OP_Null, 0, regOldRowid);
96559		- pWInfo = sqlite3WhereBegin(pParse, pTabList, pWhere,0, WHERE_ONEPASS_DESIRED);
	96750	+ pWInfo = sqlite3WhereBegin(
	96751	+ pParse, pTabList, pWhere, 0, 0, WHERE_ONEPASS_DESIRED
	96752	+ );
96560	96753	if( pWInfo==0 ) goto update_cleanup;
96561	96754	okOnePass = pWInfo->okOnePass;
96562	96755
96563	96756	/* Remember the rowid of every item to be updated.
96564	96757	*/
		@@ -98580,10 +98773,11 @@
98580	98773	#define WHERE_REVERSE 0x02000000 /* Scan in reverse order */
98581	98774	#define WHERE_UNIQUE 0x04000000 /* Selects no more than one row */
98582	98775	#define WHERE_VIRTUALTABLE 0x08000000 /* Use virtual-table processing */
98583	98776	#define WHERE_MULTI_OR 0x10000000 /* OR using multiple indices */
98584	98777	#define WHERE_TEMP_INDEX 0x20000000 /* Uses an ephemeral index */
	98778	+#define WHERE_DISTINCT 0x40000000 /* Correct order for DISTINCT */
98585	98779
98586	98780	/*
98587	98781	** Initialize a preallocated WhereClause structure.
98588	98782	*/
98589	98783	static void whereClauseInit(
		@@ -99724,10 +99918,166 @@
99724	99918	}
99725	99919	}
99726	99920	return 0;
99727	99921	}
99728	99922
	99923	+/*
	99924	+** This function searches the expression list passed as the second argument
	99925	+** for an expression of type TK_COLUMN that refers to the same column and
	99926	+** uses the same collation sequence as the iCol'th column of index pIdx.
	99927	+** Argument iBase is the cursor number used for the table that pIdx refers
	99928	+** to.
	99929	+**
	99930	+** If such an expression is found, its index in pList->a[] is returned. If
	99931	+** no expression is found, -1 is returned.
	99932	+*/
	99933	+static int findIndexCol(
	99934	+ Parse pParse, / Parse context */
	99935	+ ExprList pList, / Expression list to search */
	99936	+ int iBase, /* Cursor for table associated with pIdx */
	99937	+ Index pIdx, / Index to match column of */
	99938	+ int iCol /* Column of index to match */
	99939	+){
	99940	+ int i;
	99941	+ const char *zColl = pIdx->azColl[iCol];
	99942	+
	99943	+ for(i=0; i<pList->nExpr; i++){
	99944	+ Expr *p = pList->a[i].pExpr;
	99945	+ if( p->op==TK_COLUMN
	99946	+ && p->iColumn==pIdx->aiColumn[iCol]
	99947	+ && p->iTable==iBase
	99948	+ ){
	99949	+ CollSeq *pColl = sqlite3ExprCollSeq(pParse, p);
	99950	+ if( ALWAYS(pColl) && 0==sqlite3StrICmp(pColl->zName, zColl) ){
	99951	+ return i;
	99952	+ }
	99953	+ }
	99954	+ }
	99955	+
	99956	+ return -1;
	99957	+}
	99958	+
	99959	+/*
	99960	+** This routine determines if pIdx can be used to assist in processing a
	99961	+** DISTINCT qualifier. In other words, it tests whether or not using this
	99962	+** index for the outer loop guarantees that rows with equal values for
	99963	+** all expressions in the pDistinct list are delivered grouped together.
	99964	+**
	99965	+** For example, the query
	99966	+**
	99967	+** SELECT DISTINCT a, b, c FROM tbl WHERE a = ?
	99968	+**
	99969	+** can benefit from any index on columns "b" and "c".
	99970	+*/
	99971	+static int isDistinctIndex(
	99972	+ Parse pParse, / Parsing context */
	99973	+ WhereClause pWC, / The WHERE clause */
	99974	+ Index pIdx, / The index being considered */
	99975	+ int base, /* Cursor number for the table pIdx is on */
	99976	+ ExprList pDistinct, / The DISTINCT expressions */
	99977	+ int nEqCol /* Number of index columns with == */
	99978	+){
	99979	+ Bitmask mask = 0; /* Mask of unaccounted for pDistinct exprs */
	99980	+ int i; /* Iterator variable */
	99981	+
	99982	+ if( pIdx->zName==0 \|\| pDistinct==0 \|\| pDistinct->nExpr>=BMS ) return 0;
	99983	+ testcase( pDistinct->nExpr==BMS-1 );
	99984	+
	99985	+ /* Loop through all the expressions in the distinct list. If any of them
	99986	+ ** are not simple column references, return early. Otherwise, test if the
	99987	+ ** WHERE clause contains a "col=X" clause. If it does, the expression
	99988	+ ** can be ignored. If it does not, and the column does not belong to the
	99989	+ ** same table as index pIdx, return early. Finally, if there is no
	99990	+ ** matching "col=X" expression and the column is on the same table as pIdx,
	99991	+ ** set the corresponding bit in variable mask.
	99992	+ */
	99993	+ for(i=0; i<pDistinct->nExpr; i++){
	99994	+ WhereTerm *pTerm;
	99995	+ Expr *p = pDistinct->a[i].pExpr;
	99996	+ if( p->op!=TK_COLUMN ) return 0;
	99997	+ pTerm = findTerm(pWC, p->iTable, p->iColumn, ~(Bitmask)0, WO_EQ, 0);
	99998	+ if( pTerm ){
	99999	+ Expr *pX = pTerm->pExpr;
	100000	+ CollSeq *p1 = sqlite3BinaryCompareCollSeq(pParse, pX->pLeft, pX->pRight);
	100001	+ CollSeq *p2 = sqlite3ExprCollSeq(pParse, p);
	100002	+ if( p1==p2 ) continue;
	100003	+ }
	100004	+ if( p->iTable!=base ) return 0;
	100005	+ mask \|= (((Bitmask)1) << i);
	100006	+ }
	100007	+
	100008	+ for(i=nEqCol; mask && i<pIdx->nColumn; i++){
	100009	+ int iExpr = findIndexCol(pParse, pDistinct, base, pIdx, i);
	100010	+ if( iExpr<0 ) break;
	100011	+ mask &= ~(((Bitmask)1) << iExpr);
	100012	+ }
	100013	+
	100014	+ return (mask==0);
	100015	+}
	100016	+
	100017	+
	100018	+/*
	100019	+** Return true if the DISTINCT expression-list passed as the third argument
	100020	+** is redundant. A DISTINCT list is redundant if the database contains a
	100021	+** UNIQUE index that guarantees that the result of the query will be distinct
	100022	+** anyway.
	100023	+*/
	100024	+static int isDistinctRedundant(
	100025	+ Parse *pParse,
	100026	+ SrcList *pTabList,
	100027	+ WhereClause *pWC,
	100028	+ ExprList *pDistinct
	100029	+){
	100030	+ Table *pTab;
	100031	+ Index *pIdx;
	100032	+ int i;
	100033	+ int iBase;
	100034	+
	100035	+ /* If there is more than one table or sub-select in the FROM clause of
	100036	+ ** this query, then it will not be possible to show that the DISTINCT
	100037	+ ** clause is redundant. */
	100038	+ if( pTabList->nSrc!=1 ) return 0;
	100039	+ iBase = pTabList->a[0].iCursor;
	100040	+ pTab = pTabList->a[0].pTab;
	100041	+
	100042	+ /* If any of the expressions is an IPK column on table iBase, then return
	100043	+ ** true. Note: The (p->iTable==iBase) part of this test may be false if the
	100044	+ ** current SELECT is a correlated sub-query.
	100045	+ */
	100046	+ for(i=0; i<pDistinct->nExpr; i++){
	100047	+ Expr *p = pDistinct->a[i].pExpr;
	100048	+ if( p->op==TK_COLUMN && p->iTable==iBase && p->iColumn<0 ) return 1;
	100049	+ }
	100050	+
	100051	+ /* Loop through all indices on the table, checking each to see if it makes
	100052	+ ** the DISTINCT qualifier redundant. It does so if:
	100053	+ **
	100054	+ ** 1. The index is itself UNIQUE, and
	100055	+ **
	100056	+ ** 2. All of the columns in the index are either part of the pDistinct
	100057	+ ** list, or else the WHERE clause contains a term of the form "col=X",
	100058	+ ** where X is a constant value. The collation sequences of the
	100059	+ ** comparison and select-list expressions must match those of the index.
	100060	+ */
	100061	+ for(pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext){
	100062	+ if( pIdx->onError==OE_None ) continue;
	100063	+ for(i=0; i<pIdx->nColumn; i++){
	100064	+ int iCol = pIdx->aiColumn[i];
	100065	+ if( 0==findTerm(pWC, iBase, iCol, ~(Bitmask)0, WO_EQ, pIdx)
	100066	+ && 0>findIndexCol(pParse, pDistinct, iBase, pIdx, i)
	100067	+ ){
	100068	+ break;
	100069	+ }
	100070	+ }
	100071	+ if( i==pIdx->nColumn ){
	100072	+ /* This index implies that the DISTINCT qualifier is redundant. */
	100073	+ return 1;
	100074	+ }
	100075	+ }
	100076	+
	100077	+ return 0;
	100078	+}
99729	100079
99730	100080	/*
99731	100081	** This routine decides if pIdx can be used to satisfy the ORDER BY
99732	100082	** clause. If it can, it returns 1. If pIdx cannot satisfy the
99733	100083	** ORDER BY clause, this routine returns 0.
		@@ -99760,11 +100110,14 @@
99760	100110	int sortOrder = 0; /* XOR of index and ORDER BY sort direction */
99761	100111	int nTerm; /* Number of ORDER BY terms */
99762	100112	struct ExprList_item pTerm; / A term of the ORDER BY clause */
99763	100113	sqlite3 *db = pParse->db;
99764	100114
99765		- assert( pOrderBy!=0 );
	100115	+ if( !pOrderBy ) return 0;
	100116	+ if( wsFlags & WHERE_COLUMN_IN ) return 0;
	100117	+ if( pIdx->bUnordered ) return 0;
	100118	+
99766	100119	nTerm = pOrderBy->nExpr;
99767	100120	assert( nTerm>0 );
99768	100121
99769	100122	/* Argument pIdx must either point to a 'real' named index structure,
99770	100123	** or an index structure allocated on the stack by bestBtreeIndex() to
		@@ -100073,10 +100426,14 @@
100073	100426	double costTempIdx; /* per-query cost of the transient index */
100074	100427	WhereTerm pTerm; / A single term of the WHERE clause */
100075	100428	WhereTerm pWCEnd; / End of pWC->a[] */
100076	100429	Table pTable; / Table tht might be indexed */
100077	100430
	100431	+ if( pParse->nQueryLoop<=(double)1 ){
	100432	+ /* There is no point in building an automatic index for a single scan */
	100433	+ return;
	100434	+ }
100078	100435	if( (pParse->db->flags & SQLITE_AutoIndex)==0 ){
100079	100436	/* Automatic indices are disabled at run-time */
100080	100437	return;
100081	100438	}
100082	100439	if( (pCost->plan.wsFlags & WHERE_NOT_FULLSCAN)!=0 ){
		@@ -100084,10 +100441,14 @@
100084	100441	return;
100085	100442	}
100086	100443	if( pSrc->notIndexed ){
100087	100444	/* The NOT INDEXED clause appears in the SQL. */
100088	100445	return;
	100446	+ }
	100447	+ if( pSrc->isCorrelated ){
	100448	+ /* The source is a correlated sub-query. No point in indexing it. */
	100449	+ return;
100089	100450	}
100090	100451
100091	100452	assert( pParse->nQueryLoop >= (double)1 );
100092	100453	pTable = pSrc->pTab;
100093	100454	nTableRow = pTable->nRowEst;
		@@ -101016,10 +101377,11 @@
101016	101377	WhereClause pWC, / The WHERE clause */
101017	101378	struct SrcList_item pSrc, / The FROM clause term to search */
101018	101379	Bitmask notReady, /* Mask of cursors not available for indexing */
101019	101380	Bitmask notValid, /* Cursors not available for any purpose */
101020	101381	ExprList pOrderBy, / The ORDER BY clause */
	101382	+ ExprList pDistinct, / The select-list if query is DISTINCT */
101021	101383	WhereCost pCost / Lowest cost query plan */
101022	101384	){
101023	101385	int iCur = pSrc->iCursor; /* The cursor of the table to be accessed */
101024	101386	Index pProbe; / An index we are evaluating */
101025	101387	Index pIdx; / Copy of pProbe, or zero for IPK index */
		@@ -101156,11 +101518,12 @@
101156	101518	int nEq; /* Number of == or IN terms matching index */
101157	101519	int bInEst = 0; /* True if "x IN (SELECT...)" seen */
101158	101520	int nInMul = 1; /* Number of distinct equalities to lookup */
101159	101521	int estBound = 100; /* Estimated reduction in search space */
101160	101522	int nBound = 0; /* Number of range constraints seen */
101161		- int bSort = 0; /* True if external sort required */
	101523	+ int bSort = !!pOrderBy; /* True if external sort required */
	101524	+ int bDist = !!pDistinct; /* True if index cannot help with DISTINCT */
101162	101525	int bLookup = 0; /* True if not a covering index */
101163	101526	WhereTerm pTerm; / A single term of the WHERE clause */
101164	101527	#ifdef SQLITE_ENABLE_STAT2
101165	101528	WhereTerm pFirstTerm = 0; / First term matching the index */
101166	101529	#endif
		@@ -101220,21 +101583,24 @@
101220	101583
101221	101584	/* If there is an ORDER BY clause and the index being considered will
101222	101585	** naturally scan rows in the required order, set the appropriate flags
101223	101586	** in wsFlags. Otherwise, if there is an ORDER BY clause but the index
101224	101587	** will scan rows in a different order, set the bSort variable. */
101225		- if( pOrderBy ){
101226		- if( (wsFlags & WHERE_COLUMN_IN)==0
101227		- && pProbe->bUnordered==0
101228		- && isSortingIndex(pParse, pWC->pMaskSet, pProbe, iCur, pOrderBy,
101229		- nEq, wsFlags, &rev)
101230		- ){
101231		- wsFlags \|= WHERE_ROWID_RANGE\|WHERE_COLUMN_RANGE\|WHERE_ORDERBY;
101232		- wsFlags \|= (rev ? WHERE_REVERSE : 0);
101233		- }else{
101234		- bSort = 1;
101235		- }
	101588	+ if( isSortingIndex(
	101589	+ pParse, pWC->pMaskSet, pProbe, iCur, pOrderBy, nEq, wsFlags, &rev)
	101590	+ ){
	101591	+ bSort = 0;
	101592	+ wsFlags \|= WHERE_ROWID_RANGE\|WHERE_COLUMN_RANGE\|WHERE_ORDERBY;
	101593	+ wsFlags \|= (rev ? WHERE_REVERSE : 0);
	101594	+ }
	101595	+
	101596	+ /* If there is a DISTINCT qualifier and this index will scan rows in
	101597	+ ** order of the DISTINCT expressions, clear bDist and set the appropriate
	101598	+ ** flags in wsFlags. */
	101599	+ if( isDistinctIndex(pParse, pWC, pProbe, iCur, pDistinct, nEq) ){
	101600	+ bDist = 0;
	101601	+ wsFlags \|= WHERE_ROWID_RANGE\|WHERE_COLUMN_RANGE\|WHERE_DISTINCT;
101236	101602	}
101237	101603
101238	101604	/* If currently calculating the cost of using an index (not the IPK
101239	101605	** index), determine if all required column data may be obtained without
101240	101606	** using the main table (i.e. if the index is a covering
		@@ -101265,16 +101631,17 @@
101265	101631	nRow = aiRowEst[0]/2;
101266	101632	nInMul = (int)(nRow / aiRowEst[nEq]);
101267	101633	}
101268	101634
101269	101635	#ifdef SQLITE_ENABLE_STAT2
101270		- /* If the constraint is of the form x=VALUE and histogram
	101636	+ /* If the constraint is of the form x=VALUE or x IN (E1,E2,...)
	101637	+ ** and we do not think that values of x are unique and if histogram
101271	101638	** data is available for column x, then it might be possible
101272	101639	** to get a better estimate on the number of rows based on
101273	101640	** VALUE and how common that value is according to the histogram.
101274	101641	*/
101275		- if( nRow>(double)1 && nEq==1 && pFirstTerm!=0 ){
	101642	+ if( nRow>(double)1 && nEq==1 && pFirstTerm!=0 && aiRowEst[1]>1 ){
101276	101643	if( pFirstTerm->eOperator & (WO_EQ\|WO_ISNULL) ){
101277	101644	testcase( pFirstTerm->eOperator==WO_EQ );
101278	101645	testcase( pFirstTerm->eOperator==WO_ISNULL );
101279	101646	whereEqualScanEst(pParse, pProbe, pFirstTerm->pExpr->pRight, &nRow);
101280	101647	}else if( pFirstTerm->eOperator==WO_IN && bInEst==0 ){
		@@ -101347,10 +101714,13 @@
101347	101714	** difference and select C of 3.0.
101348	101715	*/
101349	101716	if( bSort ){
101350	101717	cost += nRowestLog(nRow)3;
101351	101718	}
	101719	+ if( bDist ){
	101720	+ cost += nRowestLog(nRow)3;
	101721	+ }
101352	101722
101353	101723	/** Cost of using this index has now been computed **/
101354	101724
101355	101725	/* If there are additional constraints on this table that cannot
101356	101726	** be used with the current index, but which might lower the number
		@@ -101492,11 +101862,11 @@
101492	101862	}
101493	101863	sqlite3DbFree(pParse->db, p);
101494	101864	}else
101495	101865	#endif
101496	101866	{
101497		- bestBtreeIndex(pParse, pWC, pSrc, notReady, notValid, pOrderBy, pCost);
	101867	+ bestBtreeIndex(pParse, pWC, pSrc, notReady, notValid, pOrderBy, 0, pCost);
101498	101868	}
101499	101869	}
101500	101870
101501	101871	/*
101502	101872	** Disable a term in the WHERE clause. Except, do not disable the term
		@@ -102454,11 +102824,11 @@
102454	102824	for(ii=0; ii<pOrWc->nTerm; ii++){
102455	102825	WhereTerm *pOrTerm = &pOrWc->a[ii];
102456	102826	if( pOrTerm->leftCursor==iCur \|\| pOrTerm->eOperator==WO_AND ){
102457	102827	WhereInfo pSubWInfo; / Info for single OR-term scan */
102458	102828	/* Loop through table entries that match term pOrTerm. */
102459		- pSubWInfo = sqlite3WhereBegin(pParse, pOrTab, pOrTerm->pExpr, 0,
	102829	+ pSubWInfo = sqlite3WhereBegin(pParse, pOrTab, pOrTerm->pExpr, 0, 0,
102460	102830	WHERE_OMIT_OPEN \| WHERE_OMIT_CLOSE \|
102461	102831	WHERE_FORCE_TABLE \| WHERE_ONETABLE_ONLY);
102462	102832	if( pSubWInfo ){
102463	102833	explainOneScan(
102464	102834	pParse, pOrTab, &pSubWInfo->a[0], iLevel, pLevel->iFrom, 0
		@@ -102695,10 +103065,11 @@
102695	103065	SQLITE_PRIVATE WhereInfo *sqlite3WhereBegin(
102696	103066	Parse pParse, / The parser context */
102697	103067	SrcList pTabList, / A list of all tables to be scanned */
102698	103068	Expr pWhere, / The WHERE clause */
102699	103069	ExprList *ppOrderBy, / An ORDER BY clause, or NULL */
	103070	+ ExprList pDistinct, / The select-list for DISTINCT queries - or NULL */
102700	103071	u16 wctrlFlags /* One of the WHERE_* flags defined in sqliteInt.h */
102701	103072	){
102702	103073	int i; /* Loop counter */
102703	103074	int nByteWInfo; /* Num. bytes allocated for WhereInfo struct */
102704	103075	int nTabList; /* Number of elements in pTabList */
		@@ -102754,10 +103125,14 @@
102754	103125	pWInfo->iBreak = sqlite3VdbeMakeLabel(v);
102755	103126	pWInfo->pWC = pWC = (WhereClause )&((u8 )pWInfo)[nByteWInfo];
102756	103127	pWInfo->wctrlFlags = wctrlFlags;
102757	103128	pWInfo->savedNQueryLoop = pParse->nQueryLoop;
102758	103129	pMaskSet = (WhereMaskSet*)&pWC[1];
	103130	+
	103131	+ /* Disable the DISTINCT optimization if SQLITE_DistinctOpt is set via
	103132	+ ** sqlite3_test_ctrl(SQLITE_TESTCTRL_OPTIMIZATIONS,...) */
	103133	+ if( db->flags & SQLITE_DistinctOpt ) pDistinct = 0;
102759	103134
102760	103135	/* Split the WHERE clause into separate subexpressions where each
102761	103136	** subexpression is separated by an AND operator.
102762	103137	*/
102763	103138	initMaskSet(pMaskSet);
		@@ -102821,10 +103196,19 @@
102821	103196	*/
102822	103197	exprAnalyzeAll(pTabList, pWC);
102823	103198	if( db->mallocFailed ){
102824	103199	goto whereBeginError;
102825	103200	}
	103201	+
	103202	+ /* Check if the DISTINCT qualifier, if there is one, is redundant.
	103203	+ ** If it is, then set pDistinct to NULL and WhereInfo.eDistinct to
	103204	+ ** WHERE_DISTINCT_UNIQUE to tell the caller to ignore the DISTINCT.
	103205	+ */
	103206	+ if( pDistinct && isDistinctRedundant(pParse, pTabList, pWC, pDistinct) ){
	103207	+ pDistinct = 0;
	103208	+ pWInfo->eDistinct = WHERE_DISTINCT_UNIQUE;
	103209	+ }
102826	103210
102827	103211	/* Chose the best index to use for each table in the FROM clause.
102828	103212	**
102829	103213	** This loop fills in the following fields:
102830	103214	**
		@@ -102905,10 +103289,11 @@
102905	103289	Bitmask mask; /* Mask of tables not yet ready */
102906	103290	for(j=iFrom, pTabItem=&pTabList->a[j]; j<nTabList; j++, pTabItem++){
102907	103291	int doNotReorder; /* True if this table should not be reordered */
102908	103292	WhereCost sCost; /* Cost information from best[Virtual]Index() */
102909	103293	ExprList pOrderBy; / ORDER BY clause for index to optimize */
	103294	+ ExprList pDist; / DISTINCT clause for index to optimize */
102910	103295
102911	103296	doNotReorder = (pTabItem->jointype & (JT_LEFT\|JT_CROSS))!=0;
102912	103297	if( j!=iFrom && doNotReorder ) break;
102913	103298	m = getMask(pMaskSet, pTabItem->iCursor);
102914	103299	if( (m & notReady)==0 ){
		@@ -102915,10 +103300,11 @@
102915	103300	if( j==iFrom ) iFrom++;
102916	103301	continue;
102917	103302	}
102918	103303	mask = (isOptimal ? m : notReady);
102919	103304	pOrderBy = ((i==0 && ppOrderBy )?*ppOrderBy:0);
	103305	+ pDist = (i==0 ? pDistinct : 0);
102920	103306	if( pTabItem->pIndex==0 ) nUnconstrained++;
102921	103307
102922	103308	WHERETRACE(("=== trying table %d with isOptimal=%d ===\n",
102923	103309	j, isOptimal));
102924	103310	assert( pTabItem->pTab );
		@@ -102929,11 +103315,11 @@
102929	103315	&sCost, pp);
102930	103316	}else
102931	103317	#endif
102932	103318	{
102933	103319	bestBtreeIndex(pParse, pWC, pTabItem, mask, notReady, pOrderBy,
102934		- &sCost);
	103320	+ pDist, &sCost);
102935	103321	}
102936	103322	assert( isOptimal \|\| (sCost.used&notReady)==0 );
102937	103323
102938	103324	/* If an INDEXED BY clause is present, then the plan must use that
102939	103325	** index if it uses any index at all */
		@@ -102989,10 +103375,14 @@
102989	103375	WHERETRACE(("*** Optimizer selects table %d for loop %d"
102990	103376	" with cost=%g and nRow=%g\n",
102991	103377	bestJ, pLevel-pWInfo->a, bestPlan.rCost, bestPlan.plan.nRow));
102992	103378	if( (bestPlan.plan.wsFlags & WHERE_ORDERBY)!=0 ){
102993	103379	*ppOrderBy = 0;
	103380	+ }
	103381	+ if( (bestPlan.plan.wsFlags & WHERE_DISTINCT)!=0 ){
	103382	+ assert( pWInfo->eDistinct==0 );
	103383	+ pWInfo->eDistinct = WHERE_DISTINCT_ORDERED;
102994	103384	}
102995	103385	andFlags &= bestPlan.plan.wsFlags;
102996	103386	pLevel->plan = bestPlan.plan;
102997	103387	testcase( bestPlan.plan.wsFlags & WHERE_INDEXED );
102998	103388	testcase( bestPlan.plan.wsFlags & WHERE_TEMP_INDEX );
		@@ -111519,11 +111909,17 @@
111519	111909	*/
111520	111910	#if defined(SQLITE_ENABLE_FTS4) && !defined(SQLITE_ENABLE_FTS3)
111521	111911	# define SQLITE_ENABLE_FTS3
111522	111912	#endif
111523	111913
111524		-#ifdef SQLITE_ENABLE_FTS3
	111914	+#if !defined(SQLITE_CORE) \|\| defined(SQLITE_ENABLE_FTS3)
	111915	+
	111916	+/* If not building as part of the core, include sqlite3ext.h. */
	111917	+#ifndef SQLITE_CORE
	111918	+SQLITE_API extern const sqlite3_api_routines *sqlite3_api;
	111919	+#endif
	111920	+
111525	111921	/************ Include fts3_tokenizer.h in the middle of fts3Int.h ********/
111526	111922	/************ Begin file fts3_tokenizer.h ********************************/
111527	111923	/*
111528	111924	** 2006 July 10
111529	111925	**
		@@ -112052,11 +112448,11 @@
112052	112448
112053	112449	sqlite3_int64 iDocid; /* Current docid (if pList!=0) */
112054	112450	int bFreeList; /* True if pList should be sqlite3_free()d */
112055	112451	char pList; / Pointer to position list following iDocid */
112056	112452	int nList; /* Length of position list */
112057		-} doclist;
	112453	+};
112058	112454
112059	112455	/*
112060	112456	** A "phrase" is a sequence of one or more tokens that must match in
112061	112457	** sequence. A single token is the base case and the most common case.
112062	112458	** For a sequence of tokens contained in double-quotes (i.e. "one two three")
		@@ -112252,23 +112648,12 @@
112252	112648	#endif
112253	112649
112254	112650	/* fts3_aux.c */
112255	112651	SQLITE_PRIVATE int sqlite3Fts3InitAux(sqlite3 *db);
112256	112652
112257		-SQLITE_PRIVATE int sqlite3Fts3TermSegReaderCursor(
112258		- Fts3Cursor pCsr, / Virtual table cursor handle */
112259		- const char zTerm, / Term to query for */
112260		- int nTerm, /* Size of zTerm in bytes */
112261		- int isPrefix, /* True for a prefix search */
112262		- Fts3MultiSegReader *ppSegcsr / OUT: Allocated seg-reader cursor */
112263		-);
112264		-
112265	112653	SQLITE_PRIVATE void sqlite3Fts3EvalPhraseCleanup(Fts3Phrase *);
112266	112654
112267		-SQLITE_PRIVATE int sqlite3Fts3EvalStart(Fts3Cursor , Fts3Expr , int);
112268		-SQLITE_PRIVATE int sqlite3Fts3EvalNext(Fts3Cursor *pCsr);
112269		-
112270	112655	SQLITE_PRIVATE int sqlite3Fts3MsrIncrStart(
112271	112656	Fts3Table, Fts3MultiSegReader, int, const char*, int);
112272	112657	SQLITE_PRIVATE int sqlite3Fts3MsrIncrNext(
112273	112658	Fts3Table , Fts3MultiSegReader , sqlite3_int64 , char , int );
112274	112659	SQLITE_PRIVATE char sqlite3Fts3EvalPhrasePoslist(Fts3Cursor , Fts3Expr *, int iCol);
		@@ -112275,11 +112660,11 @@
112275	112660	SQLITE_PRIVATE int sqlite3Fts3MsrOvfl(Fts3Cursor , Fts3MultiSegReader , int *);
112276	112661	SQLITE_PRIVATE int sqlite3Fts3MsrIncrRestart(Fts3MultiSegReader *pCsr);
112277	112662
112278	112663	SQLITE_PRIVATE int sqlite3Fts3DeferredTokenList(Fts3DeferredToken , char , int );
112279	112664
112280		-#endif /* SQLITE_ENABLE_FTS3 */
	112665	+#endif /* !SQLITE_CORE \|\| SQLITE_ENABLE_FTS3 */
112281	112666	#endif /* _FTSINT_H */
112282	112667
112283	112668	/************ End of fts3Int.h *******************************************/
112284	112669	/************ Continuing where we left off in fts3.c *********************/
112285	112670	#if !defined(SQLITE_CORE) \|\| defined(SQLITE_ENABLE_FTS3)
		@@ -112291,10 +112676,15 @@
112291	112676
112292	112677	#ifndef SQLITE_CORE
112293	112678	SQLITE_EXTENSION_INIT1
112294	112679	#endif
112295	112680
	112681	+static int fts3EvalNext(Fts3Cursor *pCsr);
	112682	+static int fts3EvalStart(Fts3Cursor *pCsr);
	112683	+static int fts3TermSegReaderCursor(
	112684	+ Fts3Cursor , const char , int, int, Fts3MultiSegReader **);
	112685	+
112296	112686	/*
112297	112687	** Write a 64-bit variable-length integer to memory starting at p[0].
112298	112688	** The length of data written will be between 1 and FTS3_VARINT_MAX bytes.
112299	112689	** The number of bytes written is returned.
112300	112690	*/
		@@ -112799,31 +113189,60 @@
112799	113189	}
112800	113190	sqlite3_free(zFree);
112801	113191	return zRet;
112802	113192	}
112803	113193
	113194	+/*
	113195	+** This function interprets the string at (*pp) as a non-negative integer
	113196	+** value. It reads the integer and sets *pnOut to the value read, then
	113197	+** sets *pp to point to the byte immediately following the last byte of
	113198	+** the integer value.
	113199	+**
	113200	+** Only decimal digits ('0'..'9') may be part of an integer value.
	113201	+**
	113202	+** If *pp does not being with a decimal digit SQLITE_ERROR is returned and
	113203	+** the output value undefined. Otherwise SQLITE_OK is returned.
	113204	+**
	113205	+** This function is used when parsing the "prefix=" FTS4 parameter.
	113206	+*/
112804	113207	static int fts3GobbleInt(const char *pp, int pnOut){
112805		- const char p = pp;
112806		- int nInt = 0;
	113208	+ const char p = pp; /* Iterator pointer */
	113209	+ int nInt = 0; /* Output value */
	113210	+
112807	113211	for(p=*pp; p[0]>='0' && p[0]<='9'; p++){
112808	113212	nInt = nInt * 10 + (p[0] - '0');
112809	113213	}
112810	113214	if( p==*pp ) return SQLITE_ERROR;
112811	113215	*pnOut = nInt;
112812	113216	*pp = p;
112813	113217	return SQLITE_OK;
112814	113218	}
112815	113219
112816		-
	113220	+/*
	113221	+** This function is called to allocate an array of Fts3Index structures
	113222	+** representing the indexes maintained by the current FTS table. FTS tables
	113223	+** always maintain the main "terms" index, but may also maintain one or
	113224	+** more "prefix" indexes, depending on the value of the "prefix=" parameter
	113225	+** (if any) specified as part of the CREATE VIRTUAL TABLE statement.
	113226	+**
	113227	+** Argument zParam is passed the value of the "prefix=" option if one was
	113228	+** specified, or NULL otherwise.
	113229	+**
	113230	+** If no error occurs, SQLITE_OK is returned and *apIndex set to point to
	113231	+** the allocated array. *pnIndex is set to the number of elements in the
	113232	+** array. If an error does occur, an SQLite error code is returned.
	113233	+**
	113234	+** Regardless of whether or not an error is returned, it is the responsibility
	113235	+** of the caller to call sqlite3_free() on the output array to free it.
	113236	+*/
112817	113237	static int fts3PrefixParameter(
112818	113238	const char zParam, / ABC in prefix=ABC parameter to parse */
112819	113239	int pnIndex, / OUT: size of apIndex[] array /
112820		- struct Fts3Index *apIndex, / OUT: Array of indexes for this table */
112821		- struct Fts3Index *apFree / OUT: Free this with sqlite3_free() */
	113240	+ struct Fts3Index *apIndex / OUT: Array of indexes for this table */
112822	113241	){
112823		- struct Fts3Index *aIndex;
112824		- int nIndex = 1;
	113242	+ struct Fts3Index aIndex; / Allocated array */
	113243	+ int nIndex = 1; /* Number of entries in array */
112825	113244
112826	113245	if( zParam && zParam[0] ){
112827	113246	const char *p;
112828	113247	nIndex++;
112829	113248	for(p=zParam; *p; p++){
		@@ -112830,11 +113249,11 @@
112830	113249	if( *p==',' ) nIndex++;
112831	113250	}
112832	113251	}
112833	113252
112834	113253	aIndex = sqlite3_malloc(sizeof(struct Fts3Index) * nIndex);
112835		- apIndex = apFree = aIndex;
	113254	+ *apIndex = aIndex;
112836	113255	*pnIndex = nIndex;
112837	113256	if( !aIndex ){
112838	113257	return SQLITE_NOMEM;
112839	113258	}
112840	113259
		@@ -112887,12 +113306,11 @@
112887	113306	int isFts4 = (argv[0][3]=='4'); /* True for FTS4, false for FTS3 */
112888	113307	const char *aCol; / Array of column names */
112889	113308	sqlite3_tokenizer pTokenizer = 0; / Tokenizer for this table */
112890	113309
112891	113310	int nIndex; /* Size of aIndex[] array */
112892		- struct Fts3Index aIndex; / Array of indexes for this table */
112893		- struct Fts3Index aFree = 0; / Free this before returning */
	113311	+ struct Fts3Index aIndex = 0; / Array of indexes for this table */
112894	113312
112895	113313	/* The results of parsing supported FTS4 key=value options: */
112896	113314	int bNoDocsize = 0; /* True to omit %_docsize table */
112897	113315	int bDescIdx = 0; /* True to store descending indexes */
112898	113316	char zPrefix = 0; / Prefix parameter value (or NULL) */
		@@ -113025,11 +113443,11 @@
113025	113443	rc = sqlite3Fts3InitTokenizer(pHash, "simple", &pTokenizer, pzErr);
113026	113444	if( rc!=SQLITE_OK ) goto fts3_init_out;
113027	113445	}
113028	113446	assert( pTokenizer );
113029	113447
113030		- rc = fts3PrefixParameter(zPrefix, &nIndex, &aIndex, &aFree);
	113448	+ rc = fts3PrefixParameter(zPrefix, &nIndex, &aIndex);
113031	113449	if( rc==SQLITE_ERROR ){
113032	113450	assert( zPrefix );
113033	113451	*pzErr = sqlite3_mprintf("error parsing prefix parameter: %s", zPrefix);
113034	113452	}
113035	113453	if( rc!=SQLITE_OK ) goto fts3_init_out;
		@@ -113112,11 +113530,11 @@
113112	113530	/* Declare the table schema to SQLite. */
113113	113531	fts3DeclareVtab(&rc, p);
113114	113532
113115	113533	fts3_init_out:
113116	113534	sqlite3_free(zPrefix);
113117		- sqlite3_free(aFree);
	113535	+ sqlite3_free(aIndex);
113118	113536	sqlite3_free(zCompress);
113119	113537	sqlite3_free(zUncompress);
113120	113538	sqlite3_free((void *)aCol);
113121	113539	if( rc!=SQLITE_OK ){
113122	113540	if( p ){
		@@ -113703,12 +114121,10 @@
113703	114121	*pp1 = p1 + 1;
113704	114122	*pp2 = p2 + 1;
113705	114123	}
113706	114124
113707	114125	/*
113708		-** nToken==1 searches for adjacent positions.
113709		-**
113710	114126	** This function is used to merge two position lists into one. When it is
113711	114127	** called, pp1 and pp2 must both point to position lists. A position-list is
113712	114128	** the part of a doclist that follows each document id. For example, if a row
113713	114129	** contains:
113714	114130	**
		@@ -113724,10 +114140,12 @@
113724	114140	** If isSaveLeft is 0, an entry is added to the output position list for
113725	114141	** each position in *pp2 for which there exists one or more positions in
113726	114142	** pp1 so that (pos(pp2)>pos(pp1) && pos(pp2)-pos(*pp1)<=nToken). i.e.
113727	114143	** when the pp1 token appears before the pp2 token, but not more than nToken
113728	114144	** slots before it.
	114145	+**
	114146	+** e.g. nToken==1 searches for adjacent positions.
113729	114147	*/
113730	114148	static int fts3PoslistPhraseMerge(
113731	114149	char *pp, / IN/OUT: Preallocated output buffer */
113732	114150	int nToken, /* Maximum difference in token positions */
113733	114151	int isSaveLeft, /* Save the left position */
		@@ -113890,26 +114308,38 @@
113890	114308
113891	114309	return res;
113892	114310	}
113893	114311
113894	114312	/*
113895		-** A pointer to an instance of this structure is used as the context
113896		-** argument to sqlite3Fts3SegReaderIterate()
	114313	+** An instance of this function is used to merge together the (potentially
	114314	+** large number of) doclists for each term that matches a prefix query.
	114315	+** See function fts3TermSelectMerge() for details.
113897	114316	*/
113898	114317	typedef struct TermSelect TermSelect;
113899	114318	struct TermSelect {
113900		- int isReqPos;
113901		- char aaOutput[16]; / Malloc'd output buffer */
113902		- int anOutput[16]; /* Size of output in bytes */
	114319	+ char aaOutput[16]; / Malloc'd output buffers */
	114320	+ int anOutput[16]; /* Size each output buffer in bytes */
113903	114321	};
113904	114322
113905		-
	114323	+/*
	114324	+** This function is used to read a single varint from a buffer. Parameter
	114325	+** pEnd points 1 byte past the end of the buffer. When this function is
	114326	+** called, if *pp points to pEnd or greater, then the end of the buffer
	114327	+** has been reached. In this case *pp is set to 0 and the function returns.
	114328	+**
	114329	+** If *pp does not point to or past pEnd, then a single varint is read
	114330	+** from pp. pp is then set to point 1 byte past the end of the read varint.
	114331	+**
	114332	+** If bDescIdx is false, the value read is added to *pVal before returning.
	114333	+** If it is true, the value read is subtracted from *pVal before this
	114334	+** function returns.
	114335	+*/
113906	114336	static void fts3GetDeltaVarint3(
113907		- char **pp,
113908		- char *pEnd,
113909		- int bDescIdx,
113910		- sqlite3_int64 *pVal
	114337	+ char *pp, / IN/OUT: Point to read varint from */
	114338	+ char pEnd, / End of buffer */
	114339	+ int bDescIdx, /* True if docids are descending */
	114340	+ sqlite3_int64 pVal / IN/OUT: Integer value */
113911	114341	){
113912	114342	if( *pp>=pEnd ){
113913	114343	*pp = 0;
113914	114344	}else{
113915	114345	sqlite3_int64 iVal;
		@@ -113920,10 +114350,25 @@
113920	114350	*pVal += iVal;
113921	114351	}
113922	114352	}
113923	114353	}
113924	114354
	114355	+/*
	114356	+** This function is used to write a single varint to a buffer. The varint
	114357	+** is written to pp. Before returning, pp is set to point 1 byte past the
	114358	+** end of the value written.
	114359	+**
	114360	+** If *pbFirst is zero when this function is called, the value written to
	114361	+** the buffer is that of parameter iVal.
	114362	+**
	114363	+** If *pbFirst is non-zero when this function is called, then the value
	114364	+** written is either (iVal-piPrev) (if bDescIdx is zero) or (piPrev-iVal)
	114365	+** (if bDescIdx is non-zero).
	114366	+**
	114367	+** Before returning, this function always sets pbFirst to 1 and piPrev
	114368	+** to the value of parameter iVal.
	114369	+*/
113925	114370	static void fts3PutDeltaVarint3(
113926	114371	char *pp, / IN/OUT: Output pointer */
113927	114372	int bDescIdx, /* True for descending docids */
113928	114373	sqlite3_int64 piPrev, / IN/OUT: Previous value written to list */
113929	114374	int pbFirst, / IN/OUT: True after first int written */
		@@ -113940,14 +114385,38 @@
113940	114385	pp += sqlite3Fts3PutVarint(pp, iWrite);
113941	114386	*piPrev = iVal;
113942	114387	*pbFirst = 1;
113943	114388	}
113944	114389
113945		-#define COMPARE_DOCID(i1, i2) ((bDescIdx?-1:1) * (i1-i2))
113946	114390
	114391	+/*
	114392	+** This macro is used by various functions that merge doclists. The two
	114393	+** arguments are 64-bit docid values. If the value of the stack variable
	114394	+** bDescDoclist is 0 when this macro is invoked, then it returns (i1-i2).
	114395	+** Otherwise, (i2-i1).
	114396	+**
	114397	+** Using this makes it easier to write code that can merge doclists that are
	114398	+** sorted in either ascending or descending order.
	114399	+*/
	114400	+#define DOCID_CMP(i1, i2) ((bDescDoclist?-1:1) * (i1-i2))
	114401	+
	114402	+/*
	114403	+** This function does an "OR" merge of two doclists (output contains all
	114404	+** positions contained in either argument doclist). If the docids in the
	114405	+** input doclists are sorted in ascending order, parameter bDescDoclist
	114406	+** should be false. If they are sorted in ascending order, it should be
	114407	+** passed a non-zero value.
	114408	+**
	114409	+** If no error occurs, *paOut is set to point at an sqlite3_malloc'd buffer
	114410	+** containing the output doclist and SQLITE_OK is returned. In this case
	114411	+** *pnOut is set to the number of bytes in the output doclist.
	114412	+**
	114413	+** If an error occurs, an SQLite error code is returned. The output values
	114414	+** are undefined in this case.
	114415	+*/
113947	114416	static int fts3DoclistOrMerge(
113948		- int bDescIdx, /* True if arguments are desc */
	114417	+ int bDescDoclist, /* True if arguments are desc */
113949	114418	char a1, int n1, / First doclist */
113950	114419	char a2, int n2, / Second doclist */
113951	114420	char *paOut, int pnOut /* OUT: Malloc'd doclist */
113952	114421	){
113953	114422	sqlite3_int64 i1 = 0;
		@@ -113968,35 +114437,47 @@
113968	114437
113969	114438	p = aOut;
113970	114439	fts3GetDeltaVarint3(&p1, pEnd1, 0, &i1);
113971	114440	fts3GetDeltaVarint3(&p2, pEnd2, 0, &i2);
113972	114441	while( p1 \|\| p2 ){
113973		- sqlite3_int64 iDiff = COMPARE_DOCID(i1, i2);
	114442	+ sqlite3_int64 iDiff = DOCID_CMP(i1, i2);
113974	114443
113975	114444	if( p2 && p1 && iDiff==0 ){
113976		- fts3PutDeltaVarint3(&p, bDescIdx, &iPrev, &bFirstOut, i1);
	114445	+ fts3PutDeltaVarint3(&p, bDescDoclist, &iPrev, &bFirstOut, i1);
113977	114446	fts3PoslistMerge(&p, &p1, &p2);
113978		- fts3GetDeltaVarint3(&p1, pEnd1, bDescIdx, &i1);
113979		- fts3GetDeltaVarint3(&p2, pEnd2, bDescIdx, &i2);
	114447	+ fts3GetDeltaVarint3(&p1, pEnd1, bDescDoclist, &i1);
	114448	+ fts3GetDeltaVarint3(&p2, pEnd2, bDescDoclist, &i2);
113980	114449	}else if( !p2 \|\| (p1 && iDiff<0) ){
113981		- fts3PutDeltaVarint3(&p, bDescIdx, &iPrev, &bFirstOut, i1);
	114450	+ fts3PutDeltaVarint3(&p, bDescDoclist, &iPrev, &bFirstOut, i1);
113982	114451	fts3PoslistCopy(&p, &p1);
113983		- fts3GetDeltaVarint3(&p1, pEnd1, bDescIdx, &i1);
	114452	+ fts3GetDeltaVarint3(&p1, pEnd1, bDescDoclist, &i1);
113984	114453	}else{
113985		- fts3PutDeltaVarint3(&p, bDescIdx, &iPrev, &bFirstOut, i2);
	114454	+ fts3PutDeltaVarint3(&p, bDescDoclist, &iPrev, &bFirstOut, i2);
113986	114455	fts3PoslistCopy(&p, &p2);
113987		- fts3GetDeltaVarint3(&p2, pEnd2, bDescIdx, &i2);
	114456	+ fts3GetDeltaVarint3(&p2, pEnd2, bDescDoclist, &i2);
113988	114457	}
113989	114458	}
113990	114459
113991	114460	*paOut = aOut;
113992	114461	*pnOut = (p-aOut);
113993	114462	return SQLITE_OK;
113994	114463	}
113995	114464
	114465	+/*
	114466	+** This function does a "phrase" merge of two doclists. In a phrase merge,
	114467	+** the output contains a copy of each position from the right-hand input
	114468	+** doclist for which there is a position in the left-hand input doclist
	114469	+** exactly nDist tokens before it.
	114470	+**
	114471	+** If the docids in the input doclists are sorted in ascending order,
	114472	+** parameter bDescDoclist should be false. If they are sorted in ascending
	114473	+** order, it should be passed a non-zero value.
	114474	+**
	114475	+** The right-hand input doclist is overwritten by this function.
	114476	+*/
113996	114477	static void fts3DoclistPhraseMerge(
113997		- int bDescIdx, /* True if arguments are desc */
	114478	+ int bDescDoclist, /* True if arguments are desc */
113998	114479	int nDist, /* Distance from left to right (1=adjacent) */
113999	114480	char aLeft, int nLeft, / Left doclist */
114000	114481	char aRight, int pnRight /* IN/OUT: Right/output doclist */
114001	114482	){
114002	114483	sqlite3_int64 i1 = 0;
		@@ -114015,30 +114496,30 @@
114015	114496	p = aOut;
114016	114497	fts3GetDeltaVarint3(&p1, pEnd1, 0, &i1);
114017	114498	fts3GetDeltaVarint3(&p2, pEnd2, 0, &i2);
114018	114499
114019	114500	while( p1 && p2 ){
114020		- sqlite3_int64 iDiff = COMPARE_DOCID(i1, i2);
	114501	+ sqlite3_int64 iDiff = DOCID_CMP(i1, i2);
114021	114502	if( iDiff==0 ){
114022	114503	char *pSave = p;
114023	114504	sqlite3_int64 iPrevSave = iPrev;
114024	114505	int bFirstOutSave = bFirstOut;
114025	114506
114026		- fts3PutDeltaVarint3(&p, bDescIdx, &iPrev, &bFirstOut, i1);
	114507	+ fts3PutDeltaVarint3(&p, bDescDoclist, &iPrev, &bFirstOut, i1);
114027	114508	if( 0==fts3PoslistPhraseMerge(&p, nDist, 0, 1, &p1, &p2) ){
114028	114509	p = pSave;
114029	114510	iPrev = iPrevSave;
114030	114511	bFirstOut = bFirstOutSave;
114031	114512	}
114032		- fts3GetDeltaVarint3(&p1, pEnd1, bDescIdx, &i1);
114033		- fts3GetDeltaVarint3(&p2, pEnd2, bDescIdx, &i2);
	114513	+ fts3GetDeltaVarint3(&p1, pEnd1, bDescDoclist, &i1);
	114514	+ fts3GetDeltaVarint3(&p2, pEnd2, bDescDoclist, &i2);
114034	114515	}else if( iDiff<0 ){
114035	114516	fts3PoslistCopy(0, &p1);
114036		- fts3GetDeltaVarint3(&p1, pEnd1, bDescIdx, &i1);
	114517	+ fts3GetDeltaVarint3(&p1, pEnd1, bDescDoclist, &i1);
114037	114518	}else{
114038	114519	fts3PoslistCopy(0, &p2);
114039		- fts3GetDeltaVarint3(&p2, pEnd2, bDescIdx, &i2);
	114520	+ fts3GetDeltaVarint3(&p2, pEnd2, bDescDoclist, &i2);
114040	114521	}
114041	114522	}
114042	114523
114043	114524	*pnRight = p - aOut;
114044	114525	}
		@@ -114051,11 +114532,11 @@
114051	114532	**
114052	114533	** If an OOM error occurs, return SQLITE_NOMEM. In this case it is
114053	114534	** the responsibility of the caller to free any doclists left in the
114054	114535	** TermSelect.aaOutput[] array.
114055	114536	*/
114056		-static int fts3TermSelectMerge(Fts3Table p, TermSelect pTS){
	114537	+static int fts3TermSelectFinishMerge(Fts3Table p, TermSelect pTS){
114057	114538	char *aOut = 0;
114058	114539	int nOut = 0;
114059	114540	int i;
114060	114541
114061	114542	/* Loop through the doclists in the aaOutput[] array. Merge them all
		@@ -114092,28 +114573,29 @@
114092	114573	pTS->anOutput[0] = nOut;
114093	114574	return SQLITE_OK;
114094	114575	}
114095	114576
114096	114577	/*
114097		-** This function is used as the sqlite3Fts3SegReaderIterate() callback when
114098		-** querying the full-text index for a doclist associated with a term or
114099		-** term-prefix.
	114578	+** Merge the doclist aDoclist/nDoclist into the TermSelect object passed
	114579	+** as the first argument. The merge is an "OR" merge (see function
	114580	+** fts3DoclistOrMerge() for details).
	114581	+**
	114582	+** This function is called with the doclist for each term that matches
	114583	+** a queried prefix. It merges all these doclists into one, the doclist
	114584	+** for the specified prefix. Since there can be a very large number of
	114585	+** doclists to merge, the merging is done pair-wise using the TermSelect
	114586	+** object.
	114587	+**
	114588	+** This function returns SQLITE_OK if the merge is successful, or an
	114589	+** SQLite error code (SQLITE_NOMEM) if an error occurs.
114100	114590	*/
114101		-static int fts3TermSelectCb(
114102		- Fts3Table p, / Virtual table object */
114103		- void pContext, / Pointer to TermSelect structure */
114104		- char *zTerm,
114105		- int nTerm,
114106		- char *aDoclist,
114107		- int nDoclist
	114591	+static int fts3TermSelectMerge(
	114592	+ Fts3Table p, / FTS table handle */
	114593	+ TermSelect pTS, / TermSelect object to merge into */
	114594	+ char aDoclist, / Pointer to doclist */
	114595	+ int nDoclist /* Size of aDoclist in bytes */
114108	114596	){
114109		- TermSelect pTS = (TermSelect )pContext;
114110		-
114111		- UNUSED_PARAMETER(p);
114112		- UNUSED_PARAMETER(zTerm);
114113		- UNUSED_PARAMETER(nTerm);
114114		-
114115	114597	if( pTS->aaOutput[0]==0 ){
114116	114598	/* If this is the first term selected, copy the doclist to the output
114117	114599	** buffer using memcpy(). */
114118	114600	pTS->aaOutput[0] = sqlite3_malloc(nDoclist);
114119	114601	pTS->anOutput[0] = nDoclist;
		@@ -114180,23 +114662,30 @@
114180	114662	}
114181	114663	pCsr->apSegment[pCsr->nSegment++] = pNew;
114182	114664	return SQLITE_OK;
114183	114665	}
114184	114666
	114667	+/*
	114668	+** Add seg-reader objects to the Fts3MultiSegReader object passed as the
	114669	+** 8th argument.
	114670	+**
	114671	+** This function returns SQLITE_OK if successful, or an SQLite error code
	114672	+** otherwise.
	114673	+*/
114185	114674	static int fts3SegReaderCursor(
114186	114675	Fts3Table p, / FTS3 table handle */
114187	114676	int iIndex, /* Index to search (from 0 to p->nIndex-1) */
114188	114677	int iLevel, /* Level of segments to scan */
114189	114678	const char zTerm, / Term to query for */
114190	114679	int nTerm, /* Size of zTerm in bytes */
114191	114680	int isPrefix, /* True for a prefix search */
114192	114681	int isScan, /* True to scan from zTerm to EOF */
114193		- Fts3MultiSegReader pCsr / Cursor object to populate */
	114682	+ Fts3MultiSegReader pCsr / Cursor object to populate */
114194	114683	){
114195		- int rc = SQLITE_OK;
114196		- int rc2;
114197		- sqlite3_stmt *pStmt = 0;
	114684	+ int rc = SQLITE_OK; /* Error code */
	114685	+ sqlite3_stmt pStmt = 0; / Statement to iterate through segments */
	114686	+ int rc2; /* Result of sqlite3_reset() */
114198	114687
114199	114688	/* If iLevel is less than 0 and this is not a scan, include a seg-reader
114200	114689	** for the pending-terms. If this is a scan, then this call must be being
114201	114690	** made by an fts4aux module, not an FTS table. In this case calling
114202	114691	** Fts3SegReaderPending might segfault, as the data structures used by
		@@ -114281,28 +114770,46 @@
114281	114770	return fts3SegReaderCursor(
114282	114771	p, iIndex, iLevel, zTerm, nTerm, isPrefix, isScan, pCsr
114283	114772	);
114284	114773	}
114285	114774
	114775	+/*
	114776	+** In addition to its current configuration, have the Fts3MultiSegReader
	114777	+** passed as the 4th argument also scan the doclist for term zTerm/nTerm.
	114778	+**
	114779	+** SQLITE_OK is returned if no error occurs, otherwise an SQLite error code.
	114780	+*/
114286	114781	static int fts3SegReaderCursorAddZero(
114287		- Fts3Table *p,
114288		- const char *zTerm,
114289		- int nTerm,
114290		- Fts3MultiSegReader *pCsr
	114782	+ Fts3Table p, / FTS virtual table handle */
	114783	+ const char zTerm, / Term to scan doclist of */
	114784	+ int nTerm, /* Number of bytes in zTerm */
	114785	+ Fts3MultiSegReader pCsr / Fts3MultiSegReader to modify */
114291	114786	){
114292	114787	return fts3SegReaderCursor(p, 0, FTS3_SEGCURSOR_ALL, zTerm, nTerm, 0, 0,pCsr);
114293	114788	}
114294	114789
114295		-
114296		-SQLITE_PRIVATE int sqlite3Fts3TermSegReaderCursor(
	114790	+/*
	114791	+** Open an Fts3MultiSegReader to scan the doclist for term zTerm/nTerm. Or,
	114792	+** if isPrefix is true, to scan the doclist for all terms for which
	114793	+** zTerm/nTerm is a prefix. If successful, return SQLITE_OK and write
	114794	+** a pointer to the new Fts3MultiSegReader to *ppSegcsr. Otherwise, return
	114795	+** an SQLite error code.
	114796	+**
	114797	+** It is the responsibility of the caller to free this object by eventually
	114798	+** passing it to fts3SegReaderCursorFree()
	114799	+**
	114800	+** SQLITE_OK is returned if no error occurs, otherwise an SQLite error code.
	114801	+** Output parameter *ppSegcsr is set to 0 if an error occurs.
	114802	+*/
	114803	+static int fts3TermSegReaderCursor(
114297	114804	Fts3Cursor pCsr, / Virtual table cursor handle */
114298	114805	const char zTerm, / Term to query for */
114299	114806	int nTerm, /* Size of zTerm in bytes */
114300	114807	int isPrefix, /* True for a prefix search */
114301	114808	Fts3MultiSegReader *ppSegcsr / OUT: Allocated seg-reader cursor */
114302	114809	){
114303		- Fts3MultiSegReader pSegcsr; / Object to allocate and return */
	114810	+ Fts3MultiSegReader pSegcsr; / Object to allocate and return */
114304	114811	int rc = SQLITE_NOMEM; /* Return code */
114305	114812
114306	114813	pSegcsr = sqlite3_malloc(sizeof(Fts3MultiSegReader));
114307	114814	if( pSegcsr ){
114308	114815	int i;
		@@ -114342,62 +114849,53 @@
114342	114849
114343	114850	*ppSegcsr = pSegcsr;
114344	114851	return rc;
114345	114852	}
114346	114853
	114854	+/*
	114855	+** Free an Fts3MultiSegReader allocated by fts3TermSegReaderCursor().
	114856	+*/
114347	114857	static void fts3SegReaderCursorFree(Fts3MultiSegReader *pSegcsr){
114348	114858	sqlite3Fts3SegReaderFinish(pSegcsr);
114349	114859	sqlite3_free(pSegcsr);
114350	114860	}
114351	114861
114352	114862	/*
114353	114863	** This function retreives the doclist for the specified term (or term
114354		-** prefix) from the database.
114355		-**
114356		-** The returned doclist may be in one of two formats, depending on the
114357		-** value of parameter isReqPos. If isReqPos is zero, then the doclist is
114358		-** a sorted list of delta-compressed docids (a bare doclist). If isReqPos
114359		-** is non-zero, then the returned list is in the same format as is stored
114360		-** in the database without the found length specifier at the start of on-disk
114361		-** doclists.
	114864	+** prefix) from the database.
114362	114865	*/
114363	114866	static int fts3TermSelect(
114364	114867	Fts3Table p, / Virtual table handle */
114365	114868	Fts3PhraseToken pTok, / Token to query for */
114366	114869	int iColumn, /* Column to query (or -ve for all columns) */
114367		- int isReqPos, /* True to include position lists in output */
114368	114870	int pnOut, / OUT: Size of buffer at ppOut /
114369	114871	char *ppOut / OUT: Malloced result buffer */
114370	114872	){
114371	114873	int rc; /* Return code */
114372		- Fts3MultiSegReader pSegcsr; / Seg-reader cursor for this term */
114373		- TermSelect tsc; /* Context object for fts3TermSelectCb() */
	114874	+ Fts3MultiSegReader pSegcsr; / Seg-reader cursor for this term */
	114875	+ TermSelect tsc; /* Object for pair-wise doclist merging */
114374	114876	Fts3SegFilter filter; /* Segment term filter configuration */
114375	114877
114376	114878	pSegcsr = pTok->pSegcsr;
114377	114879	memset(&tsc, 0, sizeof(TermSelect));
114378		- tsc.isReqPos = isReqPos;
114379	114880
114380		- filter.flags = FTS3_SEGMENT_IGNORE_EMPTY
	114881	+ filter.flags = FTS3_SEGMENT_IGNORE_EMPTY \| FTS3_SEGMENT_REQUIRE_POS
114381	114882	\| (pTok->isPrefix ? FTS3_SEGMENT_PREFIX : 0)
114382		- \| (isReqPos ? FTS3_SEGMENT_REQUIRE_POS : 0)
114383	114883	\| (iColumn<p->nColumn ? FTS3_SEGMENT_COLUMN_FILTER : 0);
114384	114884	filter.iCol = iColumn;
114385	114885	filter.zTerm = pTok->z;
114386	114886	filter.nTerm = pTok->n;
114387	114887
114388	114888	rc = sqlite3Fts3SegReaderStart(p, pSegcsr, &filter);
114389	114889	while( SQLITE_OK==rc
114390	114890	&& SQLITE_ROW==(rc = sqlite3Fts3SegReaderStep(p, pSegcsr))
114391	114891	){
114392		- rc = fts3TermSelectCb(p, (void *)&tsc,
114393		- pSegcsr->zTerm, pSegcsr->nTerm, pSegcsr->aDoclist, pSegcsr->nDoclist
114394		- );
	114892	+ rc = fts3TermSelectMerge(p, &tsc, pSegcsr->aDoclist, pSegcsr->nDoclist);
114395	114893	}
114396	114894
114397	114895	if( rc==SQLITE_OK ){
114398		- rc = fts3TermSelectMerge(p, &tsc);
	114896	+ rc = fts3TermSelectFinishMerge(p, &tsc);
114399	114897	}
114400	114898	if( rc==SQLITE_OK ){
114401	114899	*ppOut = tsc.aaOutput[0];
114402	114900	*pnOut = tsc.anOutput[0];
114403	114901	}else{
		@@ -114419,28 +114917,19 @@
114419	114917	** If the isPoslist argument is true, then it is assumed that the doclist
114420	114918	** contains a position-list following each docid. Otherwise, it is assumed
114421	114919	** that the doclist is simply a list of docids stored as delta encoded
114422	114920	** varints.
114423	114921	*/
114424		-static int fts3DoclistCountDocids(int isPoslist, char *aList, int nList){
	114922	+static int fts3DoclistCountDocids(char *aList, int nList){
114425	114923	int nDoc = 0; /* Return value */
114426	114924	if( aList ){
114427	114925	char aEnd = &aList[nList]; / Pointer to one byte after EOF */
114428	114926	char p = aList; / Cursor */
114429		- if( !isPoslist ){
114430		- /* The number of docids in the list is the same as the number of
114431		- ** varints. In FTS3 a varint consists of a single byte with the 0x80
114432		- ** bit cleared and zero or more bytes with the 0x80 bit set. So to
114433		- ** count the varints in the buffer, just count the number of bytes
114434		- ** with the 0x80 bit clear. */
114435		- while( p<aEnd ) nDoc += (((*p++)&0x80)==0);
114436		- }else{
114437		- while( p<aEnd ){
114438		- nDoc++;
114439		- while( (p++)&0x80 ); / Skip docid varint */
114440		- fts3PoslistCopy(0, &p); /* Skip over position list */
114441		- }
	114927	+ while( p<aEnd ){
	114928	+ nDoc++;
	114929	+ while( (p++)&0x80 ); / Skip docid varint */
	114930	+ fts3PoslistCopy(0, &p); /* Skip over position list */
114442	114931	}
114443	114932	}
114444	114933
114445	114934	return nDoc;
114446	114935	}
		@@ -114466,11 +114955,11 @@
114466	114955	}else{
114467	114956	pCsr->iPrevId = sqlite3_column_int64(pCsr->pStmt, 0);
114468	114957	rc = SQLITE_OK;
114469	114958	}
114470	114959	}else{
114471		- rc = sqlite3Fts3EvalNext((Fts3Cursor *)pCursor);
	114960	+ rc = fts3EvalNext((Fts3Cursor *)pCursor);
114472	114961	}
114473	114962	assert( ((Fts3Table *)pCsr->base.pVtab)->pSegments==0 );
114474	114963	return rc;
114475	114964	}
114476	114965
		@@ -114543,11 +115032,11 @@
114543	115032	}
114544	115033
114545	115034	rc = sqlite3Fts3ReadLock(p);
114546	115035	if( rc!=SQLITE_OK ) return rc;
114547	115036
114548		- rc = sqlite3Fts3EvalStart(pCsr, pCsr->pExpr, 1);
	115037	+ rc = fts3EvalStart(pCsr);
114549	115038
114550	115039	sqlite3Fts3SegmentsClose(p);
114551	115040	if( rc!=SQLITE_OK ) return rc;
114552	115041	pCsr->pNextId = pCsr->aDoclist;
114553	115042	pCsr->iPrevId = 0;
		@@ -114950,26 +115439,43 @@
114950	115439	p->zDb, p->zName, zName
114951	115440	);
114952	115441	return rc;
114953	115442	}
114954	115443
	115444	+/*
	115445	+** The xSavepoint() method.
	115446	+**
	115447	+** Flush the contents of the pending-terms table to disk.
	115448	+*/
114955	115449	static int fts3SavepointMethod(sqlite3_vtab *pVtab, int iSavepoint){
114956	115450	UNUSED_PARAMETER(iSavepoint);
114957	115451	assert( ((Fts3Table *)pVtab)->inTransaction );
114958	115452	assert( ((Fts3Table *)pVtab)->mxSavepoint < iSavepoint );
114959	115453	TESTONLY( ((Fts3Table *)pVtab)->mxSavepoint = iSavepoint );
114960	115454	return fts3SyncMethod(pVtab);
114961	115455	}
	115456	+
	115457	+/*
	115458	+** The xRelease() method.
	115459	+**
	115460	+** This is a no-op.
	115461	+*/
114962	115462	static int fts3ReleaseMethod(sqlite3_vtab *pVtab, int iSavepoint){
114963	115463	TESTONLY( Fts3Table p = (Fts3Table)pVtab );
114964	115464	UNUSED_PARAMETER(iSavepoint);
114965	115465	UNUSED_PARAMETER(pVtab);
114966	115466	assert( p->inTransaction );
114967	115467	assert( p->mxSavepoint >= iSavepoint );
114968	115468	TESTONLY( p->mxSavepoint = iSavepoint-1 );
114969	115469	return SQLITE_OK;
114970	115470	}
	115471	+
	115472	+/*
	115473	+** The xRollbackTo() method.
	115474	+**
	115475	+** Discard the contents of the pending terms table.
	115476	+*/
114971	115477	static int fts3RollbackToMethod(sqlite3_vtab *pVtab, int iSavepoint){
114972	115478	Fts3Table p = (Fts3Table)pVtab;
114973	115479	UNUSED_PARAMETER(iSavepoint);
114974	115480	assert( p->inTransaction );
114975	115481	assert( p->mxSavepoint >= iSavepoint );
		@@ -115114,22 +115620,10 @@
115114	115620	sqlite3Fts3HashClear(pHash);
115115	115621	sqlite3_free(pHash);
115116	115622	}
115117	115623	return rc;
115118	115624	}
115119		-
115120		-#if !SQLITE_CORE
115121		-SQLITE_API int sqlite3_extension_init(
115122		- sqlite3 *db,
115123		- char **pzErrMsg,
115124		- const sqlite3_api_routines *pApi
115125		-){
115126		- SQLITE_EXTENSION_INIT2(pApi)
115127		- return sqlite3Fts3Init(db);
115128		-}
115129		-#endif
115130		-
115131	115625
115132	115626	/*
115133	115627	** Allocate an Fts3MultiSegReader for each token in the expression headed
115134	115628	** by pExpr.
115135	115629	**
		@@ -115143,24 +115637,24 @@
115143	115637	** there exists prefix b-tree of the right length) then it may be traversed
115144	115638	** and merged incrementally. Otherwise, it has to be merged into an in-memory
115145	115639	** doclist and then traversed.
115146	115640	*/
115147	115641	static void fts3EvalAllocateReaders(
115148		- Fts3Cursor *pCsr,
115149		- Fts3Expr *pExpr,
	115642	+ Fts3Cursor pCsr, / FTS cursor handle */
	115643	+ Fts3Expr pExpr, / Allocate readers for this expression */
115150	115644	int pnToken, / OUT: Total number of tokens in phrase. */
115151	115645	int pnOr, / OUT: Total number of OR nodes in expr. */
115152		- int *pRc
	115646	+ int pRc / IN/OUT: Error code */
115153	115647	){
115154	115648	if( pExpr && SQLITE_OK==*pRc ){
115155	115649	if( pExpr->eType==FTSQUERY_PHRASE ){
115156	115650	int i;
115157	115651	int nToken = pExpr->pPhrase->nToken;
115158	115652	*pnToken += nToken;
115159	115653	for(i=0; i<nToken; i++){
115160	115654	Fts3PhraseToken *pToken = &pExpr->pPhrase->aToken[i];
115161		- int rc = sqlite3Fts3TermSegReaderCursor(pCsr,
	115655	+ int rc = fts3TermSegReaderCursor(pCsr,
115162	115656	pToken->z, pToken->n, pToken->isPrefix, &pToken->pSegcsr
115163	115657	);
115164	115658	if( rc!=SQLITE_OK ){
115165	115659	*pRc = rc;
115166	115660	return;
		@@ -115174,16 +115668,24 @@
115174	115668	fts3EvalAllocateReaders(pCsr, pExpr->pRight, pnToken, pnOr, pRc);
115175	115669	}
115176	115670	}
115177	115671	}
115178	115672
	115673	+/*
	115674	+** Arguments pList/nList contain the doclist for token iToken of phrase p.
	115675	+** It is merged into the main doclist stored in p->doclist.aAll/nAll.
	115676	+**
	115677	+** This function assumes that pList points to a buffer allocated using
	115678	+** sqlite3_malloc(). This function takes responsibility for eventually
	115679	+** freeing the buffer.
	115680	+*/
115179	115681	static void fts3EvalPhraseMergeToken(
115180		- Fts3Table *pTab,
115181		- Fts3Phrase *p,
115182		- int iToken,
115183		- char *pList,
115184		- int nList
	115682	+ Fts3Table pTab, / FTS Table pointer */
	115683	+ Fts3Phrase p, / Phrase to merge pList/nList into */
	115684	+ int iToken, /* Token pList/nList corresponds to */
	115685	+ char pList, / Pointer to doclist */
	115686	+ int nList /* Number of bytes in pList */
115185	115687	){
115186	115688	assert( iToken!=p->iDoclistToken );
115187	115689
115188	115690	if( pList==0 ){
115189	115691	sqlite3_free(p->doclist.aAll);
		@@ -115228,13 +115730,19 @@
115228	115730	}
115229	115731
115230	115732	if( iToken>p->iDoclistToken ) p->iDoclistToken = iToken;
115231	115733	}
115232	115734
	115735	+/*
	115736	+** Load the doclist for phrase p into p->doclist.aAll/nAll. The loaded doclist
	115737	+** does not take deferred tokens into account.
	115738	+**
	115739	+** SQLITE_OK is returned if no error occurs, otherwise an SQLite error code.
	115740	+*/
115233	115741	static int fts3EvalPhraseLoad(
115234		- Fts3Cursor *pCsr,
115235		- Fts3Phrase *p
	115742	+ Fts3Cursor pCsr, / FTS Cursor handle */
	115743	+ Fts3Phrase p / Phrase object */
115236	115744	){
115237	115745	Fts3Table pTab = (Fts3Table )pCsr->base.pVtab;
115238	115746	int iToken;
115239	115747	int rc = SQLITE_OK;
115240	115748
		@@ -115243,11 +115751,11 @@
115243	115751	assert( pToken->pDeferred==0 \|\| pToken->pSegcsr==0 );
115244	115752
115245	115753	if( pToken->pSegcsr ){
115246	115754	int nThis = 0;
115247	115755	char *pThis = 0;
115248		- rc = fts3TermSelect(pTab, pToken, p->iColumn, 1, &nThis, &pThis);
	115756	+ rc = fts3TermSelect(pTab, pToken, p->iColumn, &nThis, &pThis);
115249	115757	if( rc==SQLITE_OK ){
115250	115758	fts3EvalPhraseMergeToken(pTab, p, iToken, pThis, nThis);
115251	115759	}
115252	115760	}
115253	115761	assert( pToken->pSegcsr==0 );
		@@ -115254,18 +115762,26 @@
115254	115762	}
115255	115763
115256	115764	return rc;
115257	115765	}
115258	115766
	115767	+/*
	115768	+** This function is called on each phrase after the position lists for
	115769	+** any deferred tokens have been loaded into memory. It updates the phrases
	115770	+** current position list to include only those positions that are really
	115771	+** instances of the phrase (after considering deferred tokens). If this
	115772	+** means that the phrase does not appear in the current row, doclist.pList
	115773	+** and doclist.nList are both zeroed.
	115774	+**
	115775	+** SQLITE_OK is returned if no error occurs, otherwise an SQLite error code.
	115776	+*/
115259	115777	static int fts3EvalDeferredPhrase(Fts3Cursor pCsr, Fts3Phrase pPhrase){
115260		- int iToken;
115261		- int rc = SQLITE_OK;
115262		-
115263		- int nMaxUndeferred = pPhrase->iDoclistToken;
115264		- char *aPoslist = 0;
115265		- int nPoslist = 0;
115266		- int iPrev = -1;
	115778	+ int iToken; /* Used to iterate through phrase tokens */
	115779	+ int rc = SQLITE_OK; /* Return code */
	115780	+ char aPoslist = 0; / Position list for deferred tokens */
	115781	+ int nPoslist = 0; /* Number of bytes in aPoslist */
	115782	+ int iPrev = -1; /* Token number of previous deferred token */
115267	115783
115268	115784	assert( pPhrase->doclist.bFreeList==0 );
115269	115785
115270	115786	for(iToken=0; rc==SQLITE_OK && iToken<pPhrase->nToken; iToken++){
115271	115787	Fts3PhraseToken *pToken = &pPhrase->aToken[iToken];
		@@ -115307,10 +115823,11 @@
115307	115823	iPrev = iToken;
115308	115824	}
115309	115825	}
115310	115826
115311	115827	if( iPrev>=0 ){
	115828	+ int nMaxUndeferred = pPhrase->iDoclistToken;
115312	115829	if( nMaxUndeferred<0 ){
115313	115830	pPhrase->doclist.pList = aPoslist;
115314	115831	pPhrase->doclist.nList = nPoslist;
115315	115832	pPhrase->doclist.iDocid = pCsr->iPrevId;
115316	115833	pPhrase->doclist.bFreeList = 1;
		@@ -115355,13 +115872,19 @@
115355	115872	/*
115356	115873	** This function is called for each Fts3Phrase in a full-text query
115357	115874	** expression to initialize the mechanism for returning rows. Once this
115358	115875	** function has been called successfully on an Fts3Phrase, it may be
115359	115876	** used with fts3EvalPhraseNext() to iterate through the matching docids.
	115877	+**
	115878	+** If parameter bOptOk is true, then the phrase may (or may not) use the
	115879	+** incremental loading strategy. Otherwise, the entire doclist is loaded into
	115880	+** memory within this call.
	115881	+**
	115882	+** SQLITE_OK is returned if no error occurs, otherwise an SQLite error code.
115360	115883	*/
115361	115884	static int fts3EvalPhraseStart(Fts3Cursor pCsr, int bOptOk, Fts3Phrase p){
115362		- int rc;
	115885	+ int rc; /* Error code */
115363	115886	Fts3PhraseToken *pFirst = &p->aToken[0];
115364	115887	Fts3Table pTab = (Fts3Table )pCsr->base.pVtab;
115365	115888
115366	115889	if( pCsr->bDesc==pTab->bDescIdx
115367	115890	&& bOptOk==1
		@@ -115385,11 +115908,17 @@
115385	115908	return rc;
115386	115909	}
115387	115910
115388	115911	/*
115389	115912	** This function is used to iterate backwards (from the end to start)
115390		-** through doclists.
	115913	+** through doclists. It is used by this module to iterate through phrase
	115914	+** doclists in reverse and by the fts3_write.c module to iterate through
	115915	+** pending-terms lists when writing to databases with "order=desc".
	115916	+**
	115917	+** The doclist may be sorted in ascending (parameter bDescIdx==0) or
	115918	+** descending (parameter bDescIdx==1) order of docid. Regardless, this
	115919	+** function iterates from the end of the doclist to the beginning.
115391	115920	*/
115392	115921	SQLITE_PRIVATE void sqlite3Fts3DoclistPrev(
115393	115922	int bDescIdx, /* True if the doclist is desc */
115394	115923	char aDoclist, / Pointer to entire doclist */
115395	115924	int nDoclist, /* Length of aDoclist in bytes */
		@@ -115450,13 +115979,13 @@
115450	115979	** If there is no "next" entry and no error occurs, then *pbEof is set to
115451	115980	** 1 before returning. Otherwise, if no error occurs and the iterator is
115452	115981	** successfully advanced, *pbEof is set to 0.
115453	115982	*/
115454	115983	static int fts3EvalPhraseNext(
115455		- Fts3Cursor *pCsr,
115456		- Fts3Phrase *p,
115457		- u8 *pbEof
	115984	+ Fts3Cursor pCsr, / FTS Cursor handle */
	115985	+ Fts3Phrase p, / Phrase object to advance to next docid */
	115986	+ u8 pbEof / OUT: Set to 1 if EOF */
115458	115987	){
115459	115988	int rc = SQLITE_OK;
115460	115989	Fts3Doclist *pDL = &p->doclist;
115461	115990	Fts3Table pTab = (Fts3Table )pCsr->base.pVtab;
115462	115991
		@@ -115498,14 +116027,14 @@
115498	116027	fts3PoslistCopy(0, &pIter);
115499	116028	pDL->nList = (pIter - pDL->pList);
115500	116029
115501	116030	/* pIter now points just past the 0x00 that terminates the position-
115502	116031	** list for document pDL->iDocid. However, if this position-list was
115503		- ** edited in place by fts3EvalNearTrim2(), then pIter may not actually
	116032	+ ** edited in place by fts3EvalNearTrim(), then pIter may not actually
115504	116033	** point to the start of the next docid value. The following line deals
115505	116034	** with this case by advancing pIter past the zero-padding added by
115506		- ** fts3EvalNearTrim2(). */
	116035	+ ** fts3EvalNearTrim(). */
115507	116036	while( pIter<pEnd && *pIter==0 ) pIter++;
115508	116037
115509	116038	pDL->pNextDocid = pIter;
115510	116039	assert( pIter>=&pDL->aAll[pDL->nAll] \|\| *pIter );
115511	116040	*pbEof = 0;
		@@ -115513,15 +116042,31 @@
115513	116042	}
115514	116043
115515	116044	return rc;
115516	116045	}
115517	116046
	116047	+/*
	116048	+**
	116049	+** If *pRc is not SQLITE_OK when this function is called, it is a no-op.
	116050	+** Otherwise, fts3EvalPhraseStart() is called on all phrases within the
	116051	+** expression. Also the Fts3Expr.bDeferred variable is set to true for any
	116052	+** expressions for which all descendent tokens are deferred.
	116053	+**
	116054	+** If parameter bOptOk is zero, then it is guaranteed that the
	116055	+** Fts3Phrase.doclist.aAll/nAll variables contain the entire doclist for
	116056	+** each phrase in the expression (subject to deferred token processing).
	116057	+** Or, if bOptOk is non-zero, then one or more tokens within the expression
	116058	+** may be loaded incrementally, meaning doclist.aAll/nAll is not available.
	116059	+**
	116060	+** If an error occurs within this function, *pRc is set to an SQLite error
	116061	+** code before returning.
	116062	+*/
115518	116063	static void fts3EvalStartReaders(
115519		- Fts3Cursor *pCsr,
115520		- Fts3Expr *pExpr,
115521		- int bOptOk,
115522		- int *pRc
	116064	+ Fts3Cursor pCsr, / FTS Cursor handle */
	116065	+ Fts3Expr pExpr, / Expression to initialize phrases in */
	116066	+ int bOptOk, /* True to enable incremental loading */
	116067	+ int pRc / IN/OUT: Error code */
115523	116068	){
115524	116069	if( pExpr && SQLITE_OK==*pRc ){
115525	116070	if( pExpr->eType==FTSQUERY_PHRASE ){
115526	116071	int i;
115527	116072	int nToken = pExpr->pPhrase->nToken;
		@@ -115536,27 +116081,46 @@
115536	116081	pExpr->bDeferred = (pExpr->pLeft->bDeferred && pExpr->pRight->bDeferred);
115537	116082	}
115538	116083	}
115539	116084	}
115540	116085
	116086	+/*
	116087	+** An array of the following structures is assembled as part of the process
	116088	+** of selecting tokens to defer before the query starts executing (as part
	116089	+** of the xFilter() method). There is one element in the array for each
	116090	+** token in the FTS expression.
	116091	+**
	116092	+** Tokens are divided into AND/NEAR clusters. All tokens in a cluster belong
	116093	+** to phrases that are connected only by AND and NEAR operators (not OR or
	116094	+** NOT). When determining tokens to defer, each AND/NEAR cluster is considered
	116095	+** separately. The root of a tokens AND/NEAR cluster is stored in
	116096	+** Fts3TokenAndCost.pRoot.
	116097	+*/
115541	116098	typedef struct Fts3TokenAndCost Fts3TokenAndCost;
115542	116099	struct Fts3TokenAndCost {
115543	116100	Fts3Phrase pPhrase; / The phrase the token belongs to */
115544	116101	int iToken; /* Position of token in phrase */
115545	116102	Fts3PhraseToken pToken; / The token itself */
115546		- Fts3Expr *pRoot;
115547		- int nOvfl;
	116103	+ Fts3Expr pRoot; / Root of NEAR/AND cluster */
	116104	+ int nOvfl; /* Number of overflow pages to load doclist */
115548	116105	int iCol; /* The column the token must match */
115549	116106	};
115550	116107
	116108	+/*
	116109	+** This function is used to populate an allocated Fts3TokenAndCost array.
	116110	+**
	116111	+** If *pRc is not SQLITE_OK when this function is called, it is a no-op.
	116112	+** Otherwise, if an error occurs during execution, *pRc is set to an
	116113	+** SQLite error code.
	116114	+*/
115551	116115	static void fts3EvalTokenCosts(
115552		- Fts3Cursor *pCsr,
115553		- Fts3Expr *pRoot,
115554		- Fts3Expr *pExpr,
115555		- Fts3TokenAndCost **ppTC,
115556		- Fts3Expr ***ppOr,
115557		- int *pRc
	116116	+ Fts3Cursor pCsr, / FTS Cursor handle */
	116117	+ Fts3Expr pRoot, / Root of current AND/NEAR cluster */
	116118	+ Fts3Expr pExpr, / Expression to consider */
	116119	+ Fts3TokenAndCost *ppTC, / Write new entries to (ppTC)++ */
	116120	+ Fts3Expr **ppOr, / Write new OR root to (ppOr)++ */
	116121	+ int pRc / IN/OUT: Error code */
115558	116122	){
115559	116123	if( *pRc==SQLITE_OK && pExpr ){
115560	116124	if( pExpr->eType==FTSQUERY_PHRASE ){
115561	116125	Fts3Phrase *pPhrase = pExpr->pPhrase;
115562	116126	int i;
		@@ -115584,23 +116148,34 @@
115584	116148	fts3EvalTokenCosts(pCsr, pRoot, pExpr->pRight, ppTC, ppOr, pRc);
115585	116149	}
115586	116150	}
115587	116151	}
115588	116152
	116153	+/*
	116154	+** Determine the average document (row) size in pages. If successful,
	116155	+** write this value to *pnPage and return SQLITE_OK. Otherwise, return
	116156	+** an SQLite error code.
	116157	+**
	116158	+** The average document size in pages is calculated by first calculating
	116159	+** determining the average size in bytes, B. If B is less than the amount
	116160	+** of data that will fit on a single leaf page of an intkey table in
	116161	+** this database, then the average docsize is 1. Otherwise, it is 1 plus
	116162	+** the number of overflow pages consumed by a record B bytes in size.
	116163	+*/
115589	116164	static int fts3EvalAverageDocsize(Fts3Cursor pCsr, int pnPage){
115590	116165	if( pCsr->nRowAvg==0 ){
115591	116166	/* The average document size, which is required to calculate the cost
115592		- ** of each doclist, has not yet been determined. Read the required
115593		- ** data from the %_stat table to calculate it.
115594		- **
115595		- ** Entry 0 of the %_stat table is a blob containing (nCol+1) FTS3
115596		- ** varints, where nCol is the number of columns in the FTS3 table.
115597		- ** The first varint is the number of documents currently stored in
115598		- ** the table. The following nCol varints contain the total amount of
115599		- ** data stored in all rows of each column of the table, from left
115600		- ** to right.
115601		- */
	116167	+ ** of each doclist, has not yet been determined. Read the required
	116168	+ ** data from the %_stat table to calculate it.
	116169	+ **
	116170	+ ** Entry 0 of the %_stat table is a blob containing (nCol+1) FTS3
	116171	+ ** varints, where nCol is the number of columns in the FTS3 table.
	116172	+ ** The first varint is the number of documents currently stored in
	116173	+ ** the table. The following nCol varints contain the total amount of
	116174	+ ** data stored in all rows of each column of the table, from left
	116175	+ ** to right.
	116176	+ */
115602	116177	int rc;
115603	116178	Fts3Table p = (Fts3Table)pCsr->base.pVtab;
115604	116179	sqlite3_stmt *pStmt;
115605	116180	sqlite3_int64 nDoc = 0;
115606	116181	sqlite3_int64 nByte = 0;
		@@ -115631,109 +116206,151 @@
115631	116206
115632	116207	*pnPage = pCsr->nRowAvg;
115633	116208	return SQLITE_OK;
115634	116209	}
115635	116210
	116211	+/*
	116212	+** This function is called to select the tokens (if any) that will be
	116213	+** deferred. The array aTC[] has already been populated when this is
	116214	+** called.
	116215	+**
	116216	+** This function is called once for each AND/NEAR cluster in the
	116217	+** expression. Each invocation determines which tokens to defer within
	116218	+** the cluster with root node pRoot. See comments above the definition
	116219	+** of struct Fts3TokenAndCost for more details.
	116220	+**
	116221	+** If no error occurs, SQLITE_OK is returned and sqlite3Fts3DeferToken()
	116222	+** called on each token to defer. Otherwise, an SQLite error code is
	116223	+** returned.
	116224	+*/
115636	116225	static int fts3EvalSelectDeferred(
115637		- Fts3Cursor *pCsr,
115638		- Fts3Expr *pRoot,
115639		- Fts3TokenAndCost *aTC,
115640		- int nTC
	116226	+ Fts3Cursor pCsr, / FTS Cursor handle */
	116227	+ Fts3Expr pRoot, / Consider tokens with this root node */
	116228	+ Fts3TokenAndCost aTC, / Array of expression tokens and costs */
	116229	+ int nTC /* Number of entries in aTC[] */
115641	116230	){
115642		- int nDocSize = 0;
115643		- int nDocEst = 0;
115644		- int rc = SQLITE_OK;
115645	116231	Fts3Table pTab = (Fts3Table )pCsr->base.pVtab;
115646		- int ii;
	116232	+ int nDocSize = 0; /* Number of pages per doc loaded */
	116233	+ int rc = SQLITE_OK; /* Return code */
	116234	+ int ii; /* Iterator variable for various purposes */
	116235	+ int nOvfl = 0; /* Total overflow pages used by doclists */
	116236	+ int nToken = 0; /* Total number of tokens in cluster */
115647	116237
115648		- int nOvfl = 0;
115649		- int nTerm = 0;
	116238	+ int nMinEst = 0; /* The minimum count for any phrase so far. */
	116239	+ int nLoad4 = 1; /* (Phrases that will be loaded)^4. */
115650	116240
	116241	+ /* Count the tokens in this AND/NEAR cluster. If none of the doclists
	116242	+ ** associated with the tokens spill onto overflow pages, or if there is
	116243	+ ** only 1 token, exit early. No tokens to defer in this case. */
115651	116244	for(ii=0; ii<nTC; ii++){
115652	116245	if( aTC[ii].pRoot==pRoot ){
115653	116246	nOvfl += aTC[ii].nOvfl;
115654		- nTerm++;
	116247	+ nToken++;
115655	116248	}
115656	116249	}
115657		- if( nOvfl==0 \|\| nTerm<2 ) return SQLITE_OK;
	116250	+ if( nOvfl==0 \|\| nToken<2 ) return SQLITE_OK;
115658	116251
	116252	+ /* Obtain the average docsize (in pages). */
115659	116253	rc = fts3EvalAverageDocsize(pCsr, &nDocSize);
115660		-
115661		- for(ii=0; ii<nTerm && rc==SQLITE_OK; ii++){
115662		- int jj;
115663		- Fts3TokenAndCost *pTC = 0;
115664		-
115665		- for(jj=0; jj<nTC; jj++){
115666		- if( aTC[jj].pToken && aTC[jj].pRoot==pRoot
115667		- && (!pTC \|\| aTC[jj].nOvfl<pTC->nOvfl)
	116254	+ assert( rc!=SQLITE_OK \|\| nDocSize>0 );
	116255	+
	116256	+
	116257	+ /* Iterate through all tokens in this AND/NEAR cluster, in ascending order
	116258	+ ** of the number of overflow pages that will be loaded by the pager layer
	116259	+ ** to retrieve the entire doclist for the token from the full-text index.
	116260	+ ** Load the doclists for tokens that are either:
	116261	+ **
	116262	+ ** a. The cheapest token in the entire query (i.e. the one visited by the
	116263	+ ** first iteration of this loop), or
	116264	+ **
	116265	+ ** b. Part of a multi-token phrase.
	116266	+ **
	116267	+ ** After each token doclist is loaded, merge it with the others from the
	116268	+ ** same phrase and count the number of documents that the merged doclist
	116269	+ ** contains. Set variable "nMinEst" to the smallest number of documents in
	116270	+ ** any phrase doclist for which 1 or more token doclists have been loaded.
	116271	+ ** Let nOther be the number of other phrases for which it is certain that
	116272	+ ** one or more tokens will not be deferred.
	116273	+ **
	116274	+ ** Then, for each token, defer it if loading the doclist would result in
	116275	+ ** loading N or more overflow pages into memory, where N is computed as:
	116276	+ **
	116277	+ ** (nMinEst + 4^nOther - 1) / (4^nOther)
	116278	+ */
	116279	+ for(ii=0; ii<nToken && rc==SQLITE_OK; ii++){
	116280	+ int iTC; /* Used to iterate through aTC[] array. */
	116281	+ Fts3TokenAndCost pTC = 0; / Set to cheapest remaining token. */
	116282	+
	116283	+ /* Set pTC to point to the cheapest remaining token. */
	116284	+ for(iTC=0; iTC<nTC; iTC++){
	116285	+ if( aTC[iTC].pToken && aTC[iTC].pRoot==pRoot
	116286	+ && (!pTC \|\| aTC[iTC].nOvfl<pTC->nOvfl)
115668	116287	){
115669		- pTC = &aTC[jj];
	116288	+ pTC = &aTC[iTC];
115670	116289	}
115671	116290	}
115672	116291	assert( pTC );
115673	116292
115674		- /* At this point pTC points to the cheapest remaining token. */
115675		- if( ii==0 ){
115676		- if( pTC->nOvfl ){
115677		- nDocEst = (pTC->nOvfl * pTab->nPgsz + pTab->nPgsz) / 10;
115678		- }else{
	116293	+ if( ii && pTC->nOvfl>=((nMinEst+(nLoad4/4)-1)/(nLoad4/4))*nDocSize ){
	116294	+ /* The number of overflow pages to load for this (and therefore all
	116295	+ ** subsequent) tokens is greater than the estimated number of pages
	116296	+ ** that will be loaded if all subsequent tokens are deferred.
	116297	+ */
	116298	+ Fts3PhraseToken *pToken = pTC->pToken;
	116299	+ rc = sqlite3Fts3DeferToken(pCsr, pToken, pTC->iCol);
	116300	+ fts3SegReaderCursorFree(pToken->pSegcsr);
	116301	+ pToken->pSegcsr = 0;
	116302	+ }else{
	116303	+ nLoad4 = nLoad4*4;
	116304	+ if( ii==0 \|\| pTC->pPhrase->nToken>1 ){
	116305	+ /* Either this is the cheapest token in the entire query, or it is
	116306	+ ** part of a multi-token phrase. Either way, the entire doclist will
	116307	+ ** (eventually) be loaded into memory. It may as well be now. */
115679	116308	Fts3PhraseToken *pToken = pTC->pToken;
115680	116309	int nList = 0;
115681	116310	char *pList = 0;
115682		- rc = fts3TermSelect(pTab, pToken, pTC->iCol, 1, &nList, &pList);
	116311	+ rc = fts3TermSelect(pTab, pToken, pTC->iCol, &nList, &pList);
115683	116312	assert( rc==SQLITE_OK \|\| pList==0 );
115684		-
115685	116313	if( rc==SQLITE_OK ){
115686		- nDocEst = fts3DoclistCountDocids(1, pList, nList);
	116314	+ int nCount;
115687	116315	fts3EvalPhraseMergeToken(pTab, pTC->pPhrase, pTC->iToken,pList,nList);
	116316	+ nCount = fts3DoclistCountDocids(
	116317	+ pTC->pPhrase->doclist.aAll, pTC->pPhrase->doclist.nAll
	116318	+ );
	116319	+ if( ii==0 \|\| nCount<nMinEst ) nMinEst = nCount;
115688	116320	}
115689	116321	}
115690		- }else{
115691		- if( pTC->nOvfl>=(nDocEst*nDocSize) ){
115692		- Fts3PhraseToken *pToken = pTC->pToken;
115693		- rc = sqlite3Fts3DeferToken(pCsr, pToken, pTC->iCol);
115694		- fts3SegReaderCursorFree(pToken->pSegcsr);
115695		- pToken->pSegcsr = 0;
115696		- }
115697		- nDocEst = 1 + (nDocEst/4);
115698	116322	}
115699	116323	pTC->pToken = 0;
115700	116324	}
115701	116325
115702	116326	return rc;
115703	116327	}
115704	116328
115705		-SQLITE_PRIVATE int sqlite3Fts3EvalStart(Fts3Cursor pCsr, Fts3Expr pExpr, int bOptOk){
	116329	+/*
	116330	+** This function is called from within the xFilter method. It initializes
	116331	+** the full-text query currently stored in pCsr->pExpr. To iterate through
	116332	+** the results of a query, the caller does:
	116333	+**
	116334	+** fts3EvalStart(pCsr);
	116335	+** while( 1 ){
	116336	+** fts3EvalNext(pCsr);
	116337	+** if( pCsr->bEof ) break;
	116338	+** ... return row pCsr->iPrevId to the caller ...
	116339	+** }
	116340	+*/
	116341	+static int fts3EvalStart(Fts3Cursor *pCsr){
115706	116342	Fts3Table pTab = (Fts3Table )pCsr->base.pVtab;
115707	116343	int rc = SQLITE_OK;
115708	116344	int nToken = 0;
115709	116345	int nOr = 0;
115710	116346
115711	116347	/* Allocate a MultiSegReader for each token in the expression. */
115712		- fts3EvalAllocateReaders(pCsr, pExpr, &nToken, &nOr, &rc);
115713		-
115714		- /* Call fts3EvalPhraseStart() on all phrases in the expression. TODO:
115715		- ** This call will eventually also be responsible for determining which
115716		- ** tokens are 'deferred' until the document text is loaded into memory.
115717		- **
115718		- ** Each token in each phrase is dealt with using one of the following
115719		- ** three strategies:
115720		- **
115721		- ** 1. Entire doclist loaded into memory as part of the
115722		- ** fts3EvalStartReaders() call.
115723		- **
115724		- ** 2. Doclist loaded into memory incrementally, as part of each
115725		- ** sqlite3Fts3EvalNext() call.
115726		- **
115727		- ** 3. Token doclist is never loaded. Instead, documents are loaded into
115728		- ** memory and scanned for the token as part of the sqlite3Fts3EvalNext()
115729		- ** call. This is known as a "deferred" token.
115730		- */
115731		-
115732		- /* If bOptOk is true, check if there are any tokens that should be deferred.
115733		- */
115734		- if( rc==SQLITE_OK && bOptOk && nToken>1 && pTab->bHasStat ){
	116348	+ fts3EvalAllocateReaders(pCsr, pCsr->pExpr, &nToken, &nOr, &rc);
	116349	+
	116350	+ /* Determine which, if any, tokens in the expression should be deferred. */
	116351	+ if( rc==SQLITE_OK && nToken>1 && pTab->bHasStat ){
115735	116352	Fts3TokenAndCost *aTC;
115736	116353	Fts3Expr **apOr;
115737	116354	aTC = (Fts3TokenAndCost *)sqlite3_malloc(
115738	116355	sizeof(Fts3TokenAndCost) * nToken
115739	116356	+ sizeof(Fts3Expr ) nOr * 2
		@@ -115745,11 +116362,11 @@
115745	116362	}else{
115746	116363	int ii;
115747	116364	Fts3TokenAndCost *pTC = aTC;
115748	116365	Fts3Expr **ppOr = apOr;
115749	116366
115750		- fts3EvalTokenCosts(pCsr, 0, pExpr, &pTC, &ppOr, &rc);
	116367	+ fts3EvalTokenCosts(pCsr, 0, pCsr->pExpr, &pTC, &ppOr, &rc);
115751	116368	nToken = pTC-aTC;
115752	116369	nOr = ppOr-apOr;
115753	116370
115754	116371	if( rc==SQLITE_OK ){
115755	116372	rc = fts3EvalSelectDeferred(pCsr, 0, aTC, nToken);
		@@ -115760,25 +116377,50 @@
115760	116377
115761	116378	sqlite3_free(aTC);
115762	116379	}
115763	116380	}
115764	116381
115765		- fts3EvalStartReaders(pCsr, pExpr, bOptOk, &rc);
	116382	+ fts3EvalStartReaders(pCsr, pCsr->pExpr, 1, &rc);
115766	116383	return rc;
115767	116384	}
115768	116385
115769		-static void fts3EvalZeroPoslist(Fts3Phrase *pPhrase){
	116386	+/*
	116387	+** Invalidate the current position list for phrase pPhrase.
	116388	+*/
	116389	+static void fts3EvalInvalidatePoslist(Fts3Phrase *pPhrase){
115770	116390	if( pPhrase->doclist.bFreeList ){
115771	116391	sqlite3_free(pPhrase->doclist.pList);
115772	116392	}
115773	116393	pPhrase->doclist.pList = 0;
115774	116394	pPhrase->doclist.nList = 0;
115775	116395	pPhrase->doclist.bFreeList = 0;
115776	116396	}
115777	116397
115778		-static int fts3EvalNearTrim2(
115779		- int nNear,
	116398	+/*
	116399	+** This function is called to edit the position list associated with
	116400	+** the phrase object passed as the fifth argument according to a NEAR
	116401	+** condition. For example:
	116402	+**
	116403	+** abc NEAR/5 "def ghi"
	116404	+**
	116405	+** Parameter nNear is passed the NEAR distance of the expression (5 in
	116406	+** the example above). When this function is called, *paPoslist points to
	116407	+** the position list, and *pnToken is the number of phrase tokens in, the
	116408	+** phrase on the other side of the NEAR operator to pPhrase. For example,
	116409	+** if pPhrase refers to the "def ghi" phrase, then *paPoslist points to
	116410	+** the position list associated with phrase "abc".
	116411	+**
	116412	+** All positions in the pPhrase position list that are not sufficiently
	116413	+** close to a position in the *paPoslist position list are removed. If this
	116414	+** leaves 0 positions, zero is returned. Otherwise, non-zero.
	116415	+**
	116416	+** Before returning, *paPoslist is set to point to the position lsit
	116417	+** associated with pPhrase. And *pnToken is set to the number of tokens in
	116418	+** pPhrase.
	116419	+*/
	116420	+static int fts3EvalNearTrim(
	116421	+ int nNear, /* NEAR distance. As in "NEAR/nNear". */
115780	116422	char aTmp, / Temporary space to use */
115781	116423	char *paPoslist, / IN/OUT: Position list */
115782	116424	int pnToken, / IN/OUT: Tokens in phrase of paPoslist /
115783	116425	Fts3Phrase pPhrase / The phrase object to trim the doclist of */
115784	116426	){
		@@ -115806,10 +116448,176 @@
115806	116448	}
115807	116449
115808	116450	return res;
115809	116451	}
115810	116452
	116453	+/*
	116454	+** This function is a no-op if *pRc is other than SQLITE_OK when it is called.
	116455	+** Otherwise, it advances the expression passed as the second argument to
	116456	+** point to the next matching row in the database. Expressions iterate through
	116457	+** matching rows in docid order. Ascending order if Fts3Cursor.bDesc is zero,
	116458	+** or descending if it is non-zero.
	116459	+**
	116460	+** If an error occurs, *pRc is set to an SQLite error code. Otherwise, if
	116461	+** successful, the following variables in pExpr are set:
	116462	+**
	116463	+** Fts3Expr.bEof (non-zero if EOF - there is no next row)
	116464	+** Fts3Expr.iDocid (valid if bEof==0. The docid of the next row)
	116465	+**
	116466	+** If the expression is of type FTSQUERY_PHRASE, and the expression is not
	116467	+** at EOF, then the following variables are populated with the position list
	116468	+** for the phrase for the visited row:
	116469	+**
	116470	+** FTs3Expr.pPhrase->doclist.nList (length of pList in bytes)
	116471	+** FTs3Expr.pPhrase->doclist.pList (pointer to position list)
	116472	+**
	116473	+** It says above that this function advances the expression to the next
	116474	+** matching row. This is usually true, but there are the following exceptions:
	116475	+**
	116476	+** 1. Deferred tokens are not taken into account. If a phrase consists
	116477	+** entirely of deferred tokens, it is assumed to match every row in
	116478	+** the db. In this case the position-list is not populated at all.
	116479	+**
	116480	+** Or, if a phrase contains one or more deferred tokens and one or
	116481	+** more non-deferred tokens, then the expression is advanced to the
	116482	+** next possible match, considering only non-deferred tokens. In other
	116483	+** words, if the phrase is "A B C", and "B" is deferred, the expression
	116484	+** is advanced to the next row that contains an instance of "A * C",
	116485	+** where "*" may match any single token. The position list in this case
	116486	+** is populated as for "A * C" before returning.
	116487	+**
	116488	+** 2. NEAR is treated as AND. If the expression is "x NEAR y", it is
	116489	+** advanced to point to the next row that matches "x AND y".
	116490	+**
	116491	+** See fts3EvalTestDeferredAndNear() for details on testing if a row is
	116492	+** really a match, taking into account deferred tokens and NEAR operators.
	116493	+*/
	116494	+static void fts3EvalNextRow(
	116495	+ Fts3Cursor pCsr, / FTS Cursor handle */
	116496	+ Fts3Expr pExpr, / Expr. to advance to next matching row */
	116497	+ int pRc / IN/OUT: Error code */
	116498	+){
	116499	+ if( *pRc==SQLITE_OK ){
	116500	+ int bDescDoclist = pCsr->bDesc; /* Used by DOCID_CMP() macro */
	116501	+ assert( pExpr->bEof==0 );
	116502	+ pExpr->bStart = 1;
	116503	+
	116504	+ switch( pExpr->eType ){
	116505	+ case FTSQUERY_NEAR:
	116506	+ case FTSQUERY_AND: {
	116507	+ Fts3Expr *pLeft = pExpr->pLeft;
	116508	+ Fts3Expr *pRight = pExpr->pRight;
	116509	+ assert( !pLeft->bDeferred \|\| !pRight->bDeferred );
	116510	+
	116511	+ if( pLeft->bDeferred ){
	116512	+ /* LHS is entirely deferred. So we assume it matches every row.
	116513	+ ** Advance the RHS iterator to find the next row visited. */
	116514	+ fts3EvalNextRow(pCsr, pRight, pRc);
	116515	+ pExpr->iDocid = pRight->iDocid;
	116516	+ pExpr->bEof = pRight->bEof;
	116517	+ }else if( pRight->bDeferred ){
	116518	+ /* RHS is entirely deferred. So we assume it matches every row.
	116519	+ ** Advance the LHS iterator to find the next row visited. */
	116520	+ fts3EvalNextRow(pCsr, pLeft, pRc);
	116521	+ pExpr->iDocid = pLeft->iDocid;
	116522	+ pExpr->bEof = pLeft->bEof;
	116523	+ }else{
	116524	+ /* Neither the RHS or LHS are deferred. */
	116525	+ fts3EvalNextRow(pCsr, pLeft, pRc);
	116526	+ fts3EvalNextRow(pCsr, pRight, pRc);
	116527	+ while( !pLeft->bEof && !pRight->bEof && *pRc==SQLITE_OK ){
	116528	+ sqlite3_int64 iDiff = DOCID_CMP(pLeft->iDocid, pRight->iDocid);
	116529	+ if( iDiff==0 ) break;
	116530	+ if( iDiff<0 ){
	116531	+ fts3EvalNextRow(pCsr, pLeft, pRc);
	116532	+ }else{
	116533	+ fts3EvalNextRow(pCsr, pRight, pRc);
	116534	+ }
	116535	+ }
	116536	+ pExpr->iDocid = pLeft->iDocid;
	116537	+ pExpr->bEof = (pLeft->bEof \|\| pRight->bEof);
	116538	+ }
	116539	+ break;
	116540	+ }
	116541	+
	116542	+ case FTSQUERY_OR: {
	116543	+ Fts3Expr *pLeft = pExpr->pLeft;
	116544	+ Fts3Expr *pRight = pExpr->pRight;
	116545	+ sqlite3_int64 iCmp = DOCID_CMP(pLeft->iDocid, pRight->iDocid);
	116546	+
	116547	+ assert( pLeft->bStart \|\| pLeft->iDocid==pRight->iDocid );
	116548	+ assert( pRight->bStart \|\| pLeft->iDocid==pRight->iDocid );
	116549	+
	116550	+ if( pRight->bEof \|\| (pLeft->bEof==0 && iCmp<0) ){
	116551	+ fts3EvalNextRow(pCsr, pLeft, pRc);
	116552	+ }else if( pLeft->bEof \|\| (pRight->bEof==0 && iCmp>0) ){
	116553	+ fts3EvalNextRow(pCsr, pRight, pRc);
	116554	+ }else{
	116555	+ fts3EvalNextRow(pCsr, pLeft, pRc);
	116556	+ fts3EvalNextRow(pCsr, pRight, pRc);
	116557	+ }
	116558	+
	116559	+ pExpr->bEof = (pLeft->bEof && pRight->bEof);
	116560	+ iCmp = DOCID_CMP(pLeft->iDocid, pRight->iDocid);
	116561	+ if( pRight->bEof \|\| (pLeft->bEof==0 && iCmp<0) ){
	116562	+ pExpr->iDocid = pLeft->iDocid;
	116563	+ }else{
	116564	+ pExpr->iDocid = pRight->iDocid;
	116565	+ }
	116566	+
	116567	+ break;
	116568	+ }
	116569	+
	116570	+ case FTSQUERY_NOT: {
	116571	+ Fts3Expr *pLeft = pExpr->pLeft;
	116572	+ Fts3Expr *pRight = pExpr->pRight;
	116573	+
	116574	+ if( pRight->bStart==0 ){
	116575	+ fts3EvalNextRow(pCsr, pRight, pRc);
	116576	+ assert( *pRc!=SQLITE_OK \|\| pRight->bStart );
	116577	+ }
	116578	+
	116579	+ fts3EvalNextRow(pCsr, pLeft, pRc);
	116580	+ if( pLeft->bEof==0 ){
	116581	+ while( !*pRc
	116582	+ && !pRight->bEof
	116583	+ && DOCID_CMP(pLeft->iDocid, pRight->iDocid)>0
	116584	+ ){
	116585	+ fts3EvalNextRow(pCsr, pRight, pRc);
	116586	+ }
	116587	+ }
	116588	+ pExpr->iDocid = pLeft->iDocid;
	116589	+ pExpr->bEof = pLeft->bEof;
	116590	+ break;
	116591	+ }
	116592	+
	116593	+ default: {
	116594	+ Fts3Phrase *pPhrase = pExpr->pPhrase;
	116595	+ fts3EvalInvalidatePoslist(pPhrase);
	116596	+ *pRc = fts3EvalPhraseNext(pCsr, pPhrase, &pExpr->bEof);
	116597	+ pExpr->iDocid = pPhrase->doclist.iDocid;
	116598	+ break;
	116599	+ }
	116600	+ }
	116601	+ }
	116602	+}
	116603	+
	116604	+/*
	116605	+** If *pRc is not SQLITE_OK, or if pExpr is not the root node of a NEAR
	116606	+** cluster, then this function returns 1 immediately.
	116607	+**
	116608	+** Otherwise, it checks if the current row really does match the NEAR
	116609	+** expression, using the data currently stored in the position lists
	116610	+** (Fts3Expr->pPhrase.doclist.pList/nList) for each phrase in the expression.
	116611	+**
	116612	+** If the current row is a match, the position list associated with each
	116613	+** phrase in the NEAR expression is edited in place to contain only those
	116614	+** phrase instances sufficiently close to their peers to satisfy all NEAR
	116615	+** constraints. In this case it returns 1. If the NEAR expression does not
	116616	+** match the current row, 0 is returned. The position lists may or may not
	116617	+** be edited if 0 is returned.
	116618	+*/
115811	116619	static int fts3EvalNearTest(Fts3Expr pExpr, int pRc){
115812	116620	int res = 1;
115813	116621
115814	116622	/* The following block runs if pExpr is the root of a NEAR query.
115815	116623	** For example, the query:
		@@ -115827,11 +116635,11 @@
115827	116635	** \| \|
115828	116636	** "w" "x"
115829	116637	**
115830	116638	** The right-hand child of a NEAR node is always a phrase. The
115831	116639	** left-hand child may be either a phrase or a NEAR node. There are
115832		- ** no exceptions to this.
	116640	+ ** no exceptions to this - it's the way the parser in fts3_expr.c works.
115833	116641	*/
115834	116642	if( *pRc==SQLITE_OK
115835	116643	&& pExpr->eType==FTSQUERY_NEAR
115836	116644	&& pExpr->bEof==0
115837	116645	&& (pExpr->pParent==0 \|\| pExpr->pParent->eType!=FTSQUERY_NEAR)
		@@ -115854,21 +116662,21 @@
115854	116662	int nToken = p->pPhrase->nToken;
115855	116663
115856	116664	for(p=p->pParent;res && p && p->eType==FTSQUERY_NEAR; p=p->pParent){
115857	116665	Fts3Phrase *pPhrase = p->pRight->pPhrase;
115858	116666	int nNear = p->nNear;
115859		- res = fts3EvalNearTrim2(nNear, aTmp, &aPoslist, &nToken, pPhrase);
	116667	+ res = fts3EvalNearTrim(nNear, aTmp, &aPoslist, &nToken, pPhrase);
115860	116668	}
115861	116669
115862	116670	aPoslist = pExpr->pRight->pPhrase->doclist.pList;
115863	116671	nToken = pExpr->pRight->pPhrase->nToken;
115864	116672	for(p=pExpr->pLeft; p && res; p=p->pLeft){
115865	116673	int nNear = p->pParent->nNear;
115866	116674	Fts3Phrase *pPhrase = (
115867	116675	p->eType==FTSQUERY_NEAR ? p->pRight->pPhrase : p->pPhrase
115868	116676	);
115869		- res = fts3EvalNearTrim2(nNear, aTmp, &aPoslist, &nToken, pPhrase);
	116677	+ res = fts3EvalNearTrim(nNear, aTmp, &aPoslist, &nToken, pPhrase);
115870	116678	}
115871	116679	}
115872	116680
115873	116681	sqlite3_free(aTmp);
115874	116682	}
		@@ -115875,132 +116683,33 @@
115875	116683
115876	116684	return res;
115877	116685	}
115878	116686
115879	116687	/*
115880		-** This macro is used by the fts3EvalNext() function. The two arguments are
115881		-** 64-bit docid values. If the current query is "ORDER BY docid ASC", then
115882		-** the macro returns (i1 - i2). Or if it is "ORDER BY docid DESC", then
115883		-** it returns (i2 - i1). This allows the same code to be used for merging
115884		-** doclists in ascending or descending order.
	116688	+** This function is a helper function for fts3EvalTestDeferredAndNear().
	116689	+** Assuming no error occurs or has occurred, It returns non-zero if the
	116690	+** expression passed as the second argument matches the row that pCsr
	116691	+** currently points to, or zero if it does not.
	116692	+**
	116693	+** If *pRc is not SQLITE_OK when this function is called, it is a no-op.
	116694	+** If an error occurs during execution of this function, *pRc is set to
	116695	+** the appropriate SQLite error code. In this case the returned value is
	116696	+** undefined.
115885	116697	*/
115886		-#define DOCID_CMP(i1, i2) ((pCsr->bDesc?-1:1) * (i1-i2))
115887		-
115888		-static void fts3EvalNext(
115889		- Fts3Cursor *pCsr,
115890		- Fts3Expr *pExpr,
115891		- int *pRc
115892		-){
115893		- if( *pRc==SQLITE_OK ){
115894		- assert( pExpr->bEof==0 );
115895		- pExpr->bStart = 1;
115896		-
115897		- switch( pExpr->eType ){
115898		- case FTSQUERY_NEAR:
115899		- case FTSQUERY_AND: {
115900		- Fts3Expr *pLeft = pExpr->pLeft;
115901		- Fts3Expr *pRight = pExpr->pRight;
115902		- assert( !pLeft->bDeferred \|\| !pRight->bDeferred );
115903		- if( pLeft->bDeferred ){
115904		- fts3EvalNext(pCsr, pRight, pRc);
115905		- pExpr->iDocid = pRight->iDocid;
115906		- pExpr->bEof = pRight->bEof;
115907		- }else if( pRight->bDeferred ){
115908		- fts3EvalNext(pCsr, pLeft, pRc);
115909		- pExpr->iDocid = pLeft->iDocid;
115910		- pExpr->bEof = pLeft->bEof;
115911		- }else{
115912		- fts3EvalNext(pCsr, pLeft, pRc);
115913		- fts3EvalNext(pCsr, pRight, pRc);
115914		-
115915		- while( !pLeft->bEof && !pRight->bEof && *pRc==SQLITE_OK ){
115916		- sqlite3_int64 iDiff = DOCID_CMP(pLeft->iDocid, pRight->iDocid);
115917		- if( iDiff==0 ) break;
115918		- if( iDiff<0 ){
115919		- fts3EvalNext(pCsr, pLeft, pRc);
115920		- }else{
115921		- fts3EvalNext(pCsr, pRight, pRc);
115922		- }
115923		- }
115924		-
115925		- pExpr->iDocid = pLeft->iDocid;
115926		- pExpr->bEof = (pLeft->bEof \|\| pRight->bEof);
115927		- }
115928		- break;
115929		- }
115930		-
115931		- case FTSQUERY_OR: {
115932		- Fts3Expr *pLeft = pExpr->pLeft;
115933		- Fts3Expr *pRight = pExpr->pRight;
115934		- sqlite3_int64 iCmp = DOCID_CMP(pLeft->iDocid, pRight->iDocid);
115935		-
115936		- assert( pLeft->bStart \|\| pLeft->iDocid==pRight->iDocid );
115937		- assert( pRight->bStart \|\| pLeft->iDocid==pRight->iDocid );
115938		-
115939		- if( pRight->bEof \|\| (pLeft->bEof==0 && iCmp<0) ){
115940		- fts3EvalNext(pCsr, pLeft, pRc);
115941		- }else if( pLeft->bEof \|\| (pRight->bEof==0 && iCmp>0) ){
115942		- fts3EvalNext(pCsr, pRight, pRc);
115943		- }else{
115944		- fts3EvalNext(pCsr, pLeft, pRc);
115945		- fts3EvalNext(pCsr, pRight, pRc);
115946		- }
115947		-
115948		- pExpr->bEof = (pLeft->bEof && pRight->bEof);
115949		- iCmp = DOCID_CMP(pLeft->iDocid, pRight->iDocid);
115950		- if( pRight->bEof \|\| (pLeft->bEof==0 && iCmp<0) ){
115951		- pExpr->iDocid = pLeft->iDocid;
115952		- }else{
115953		- pExpr->iDocid = pRight->iDocid;
115954		- }
115955		-
115956		- break;
115957		- }
115958		-
115959		- case FTSQUERY_NOT: {
115960		- Fts3Expr *pLeft = pExpr->pLeft;
115961		- Fts3Expr *pRight = pExpr->pRight;
115962		-
115963		- if( pRight->bStart==0 ){
115964		- fts3EvalNext(pCsr, pRight, pRc);
115965		- assert( *pRc!=SQLITE_OK \|\| pRight->bStart );
115966		- }
115967		-
115968		- fts3EvalNext(pCsr, pLeft, pRc);
115969		- if( pLeft->bEof==0 ){
115970		- while( !*pRc
115971		- && !pRight->bEof
115972		- && DOCID_CMP(pLeft->iDocid, pRight->iDocid)>0
115973		- ){
115974		- fts3EvalNext(pCsr, pRight, pRc);
115975		- }
115976		- }
115977		- pExpr->iDocid = pLeft->iDocid;
115978		- pExpr->bEof = pLeft->bEof;
115979		- break;
115980		- }
115981		-
115982		- default: {
115983		- Fts3Phrase *pPhrase = pExpr->pPhrase;
115984		- fts3EvalZeroPoslist(pPhrase);
115985		- *pRc = fts3EvalPhraseNext(pCsr, pPhrase, &pExpr->bEof);
115986		- pExpr->iDocid = pPhrase->doclist.iDocid;
115987		- break;
115988		- }
115989		- }
115990		- }
115991		-}
115992		-
115993		-static int fts3EvalDeferredTest(Fts3Cursor pCsr, Fts3Expr pExpr, int *pRc){
115994		- int bHit = 1;
	116698	+static int fts3EvalTestExpr(
	116699	+ Fts3Cursor pCsr, / FTS cursor handle */
	116700	+ Fts3Expr pExpr, / Expr to test. May or may not be root. */
	116701	+ int pRc / IN/OUT: Error code */
	116702	+){
	116703	+ int bHit = 1; /* Return value */
115995	116704	if( *pRc==SQLITE_OK ){
115996	116705	switch( pExpr->eType ){
115997	116706	case FTSQUERY_NEAR:
115998	116707	case FTSQUERY_AND:
115999	116708	bHit = (
116000		- fts3EvalDeferredTest(pCsr, pExpr->pLeft, pRc)
116001		- && fts3EvalDeferredTest(pCsr, pExpr->pRight, pRc)
	116709	+ fts3EvalTestExpr(pCsr, pExpr->pLeft, pRc)
	116710	+ && fts3EvalTestExpr(pCsr, pExpr->pRight, pRc)
116002	116711	&& fts3EvalNearTest(pExpr, pRc)
116003	116712	);
116004	116713
116005	116714	/* If the NEAR expression does not match any rows, zero the doclist for
116006	116715	** all phrases involved in the NEAR. This is because the snippet(),
		@@ -116022,31 +116731,31 @@
116022	116731	&& (pExpr->pParent==0 \|\| pExpr->pParent->eType!=FTSQUERY_NEAR)
116023	116732	){
116024	116733	Fts3Expr *p;
116025	116734	for(p=pExpr; p->pPhrase==0; p=p->pLeft){
116026	116735	if( p->pRight->iDocid==pCsr->iPrevId ){
116027		- fts3EvalZeroPoslist(p->pRight->pPhrase);
	116736	+ fts3EvalInvalidatePoslist(p->pRight->pPhrase);
116028	116737	}
116029	116738	}
116030	116739	if( p->iDocid==pCsr->iPrevId ){
116031		- fts3EvalZeroPoslist(p->pPhrase);
	116740	+ fts3EvalInvalidatePoslist(p->pPhrase);
116032	116741	}
116033	116742	}
116034	116743
116035	116744	break;
116036	116745
116037	116746	case FTSQUERY_OR: {
116038		- int bHit1 = fts3EvalDeferredTest(pCsr, pExpr->pLeft, pRc);
116039		- int bHit2 = fts3EvalDeferredTest(pCsr, pExpr->pRight, pRc);
	116747	+ int bHit1 = fts3EvalTestExpr(pCsr, pExpr->pLeft, pRc);
	116748	+ int bHit2 = fts3EvalTestExpr(pCsr, pExpr->pRight, pRc);
116040	116749	bHit = bHit1 \|\| bHit2;
116041	116750	break;
116042	116751	}
116043	116752
116044	116753	case FTSQUERY_NOT:
116045	116754	bHit = (
116046		- fts3EvalDeferredTest(pCsr, pExpr->pLeft, pRc)
116047		- && !fts3EvalDeferredTest(pCsr, pExpr->pRight, pRc)
	116755	+ fts3EvalTestExpr(pCsr, pExpr->pLeft, pRc)
	116756	+ && !fts3EvalTestExpr(pCsr, pExpr->pRight, pRc)
116048	116757	);
116049	116758	break;
116050	116759
116051	116760	default: {
116052	116761	if( pCsr->pDeferred
		@@ -116053,11 +116762,11 @@
116053	116762	&& (pExpr->iDocid==pCsr->iPrevId \|\| pExpr->bDeferred)
116054	116763	){
116055	116764	Fts3Phrase *pPhrase = pExpr->pPhrase;
116056	116765	assert( pExpr->bDeferred \|\| pPhrase->doclist.bFreeList==0 );
116057	116766	if( pExpr->bDeferred ){
116058		- fts3EvalZeroPoslist(pPhrase);
	116767	+ fts3EvalInvalidatePoslist(pPhrase);
116059	116768	}
116060	116769	*pRc = fts3EvalDeferredPhrase(pCsr, pPhrase);
116061	116770	bHit = (pPhrase->doclist.pList!=0);
116062	116771	pExpr->iDocid = pCsr->iPrevId;
116063	116772	}else{
		@@ -116069,31 +116778,53 @@
116069	116778	}
116070	116779	return bHit;
116071	116780	}
116072	116781
116073	116782	/*
116074		-** Return 1 if both of the following are true:
	116783	+** This function is called as the second part of each xNext operation when
	116784	+** iterating through the results of a full-text query. At this point the
	116785	+** cursor points to a row that matches the query expression, with the
	116786	+** following caveats:
	116787	+**
	116788	+** * Up until this point, "NEAR" operators in the expression have been
	116789	+** treated as "AND".
	116790	+**
	116791	+** * Deferred tokens have not yet been considered.
	116792	+**
	116793	+** If *pRc is not SQLITE_OK when this function is called, it immediately
	116794	+** returns 0. Otherwise, it tests whether or not after considering NEAR
	116795	+** operators and deferred tokens the current row is still a match for the
	116796	+** expression. It returns 1 if both of the following are true:
116075	116797	**
116076	116798	** 1. *pRc is SQLITE_OK when this function returns, and
116077	116799	**
116078	116800	** 2. After scanning the current FTS table row for the deferred tokens,
116079		-** it is determined that the row does not match the query.
	116801	+** it is determined that the row does not match the query.
116080	116802	**
116081	116803	** Or, if no error occurs and it seems the current row does match the FTS
116082	116804	** query, return 0.
116083	116805	*/
116084		-static int fts3EvalLoadDeferred(Fts3Cursor pCsr, int pRc){
	116806	+static int fts3EvalTestDeferredAndNear(Fts3Cursor pCsr, int pRc){
116085	116807	int rc = *pRc;
116086	116808	int bMiss = 0;
116087	116809	if( rc==SQLITE_OK ){
	116810	+
	116811	+ /* If there are one or more deferred tokens, load the current row into
	116812	+ ** memory and scan it to determine the position list for each deferred
	116813	+ ** token. Then, see if this row is really a match, considering deferred
	116814	+ ** tokens and NEAR operators (neither of which were taken into account
	116815	+ ** earlier, by fts3EvalNextRow()).
	116816	+ */
116088	116817	if( pCsr->pDeferred ){
116089	116818	rc = fts3CursorSeek(0, pCsr);
116090	116819	if( rc==SQLITE_OK ){
116091	116820	rc = sqlite3Fts3CacheDeferredDoclists(pCsr);
116092	116821	}
116093	116822	}
116094		- bMiss = (0==fts3EvalDeferredTest(pCsr, pCsr->pExpr, &rc));
	116823	+ bMiss = (0==fts3EvalTestExpr(pCsr, pCsr->pExpr, &rc));
	116824	+
	116825	+ /* Free the position-lists accumulated for each deferred token above. */
116095	116826	sqlite3Fts3FreeDeferredDoclists(pCsr);
116096	116827	*pRc = rc;
116097	116828	}
116098	116829	return (rc==SQLITE_OK && bMiss);
116099	116830	}
		@@ -116100,11 +116831,11 @@
116100	116831
116101	116832	/*
116102	116833	** Advance to the next document that matches the FTS expression in
116103	116834	** Fts3Cursor.pExpr.
116104	116835	*/
116105		-SQLITE_PRIVATE int sqlite3Fts3EvalNext(Fts3Cursor *pCsr){
	116836	+static int fts3EvalNext(Fts3Cursor *pCsr){
116106	116837	int rc = SQLITE_OK; /* Return Code */
116107	116838	Fts3Expr *pExpr = pCsr->pExpr;
116108	116839	assert( pCsr->isEof==0 );
116109	116840	if( pExpr==0 ){
116110	116841	pCsr->isEof = 1;
		@@ -116112,23 +116843,23 @@
116112	116843	do {
116113	116844	if( pCsr->isRequireSeek==0 ){
116114	116845	sqlite3_reset(pCsr->pStmt);
116115	116846	}
116116	116847	assert( sqlite3_data_count(pCsr->pStmt)==0 );
116117		- fts3EvalNext(pCsr, pExpr, &rc);
	116848	+ fts3EvalNextRow(pCsr, pExpr, &rc);
116118	116849	pCsr->isEof = pExpr->bEof;
116119	116850	pCsr->isRequireSeek = 1;
116120	116851	pCsr->isMatchinfoNeeded = 1;
116121	116852	pCsr->iPrevId = pExpr->iDocid;
116122		- }while( pCsr->isEof==0 && fts3EvalLoadDeferred(pCsr, &rc) );
	116853	+ }while( pCsr->isEof==0 && fts3EvalTestDeferredAndNear(pCsr, &rc) );
116123	116854	}
116124	116855	return rc;
116125	116856	}
116126	116857
116127	116858	/*
116128	116859	** Restart interation for expression pExpr so that the next call to
116129		-** sqlite3Fts3EvalNext() visits the first row. Do not allow incremental
	116860	+** fts3EvalNext() visits the first row. Do not allow incremental
116130	116861	** loading or merging of phrase doclists for this iteration.
116131	116862	**
116132	116863	** If *pRc is other than SQLITE_OK when this function is called, it is
116133	116864	** a no-op. If an error occurs within this function, *pRc is set to an
116134	116865	** SQLite error code before returning.
		@@ -116140,11 +116871,11 @@
116140	116871	){
116141	116872	if( pExpr && *pRc==SQLITE_OK ){
116142	116873	Fts3Phrase *pPhrase = pExpr->pPhrase;
116143	116874
116144	116875	if( pPhrase ){
116145		- fts3EvalZeroPoslist(pPhrase);
	116876	+ fts3EvalInvalidatePoslist(pPhrase);
116146	116877	if( pPhrase->bIncr ){
116147	116878	assert( pPhrase->nToken==1 );
116148	116879	assert( pPhrase->aToken[0].pSegcsr );
116149	116880	sqlite3Fts3MsrIncrRestart(pPhrase->aToken[0].pSegcsr);
116150	116881	*pRc = fts3EvalPhraseStart(pCsr, 0, pPhrase);
		@@ -116256,18 +116987,18 @@
116256	116987	/* Ensure the %_content statement is reset. */
116257	116988	if( pCsr->isRequireSeek==0 ) sqlite3_reset(pCsr->pStmt);
116258	116989	assert( sqlite3_data_count(pCsr->pStmt)==0 );
116259	116990
116260	116991	/* Advance to the next document */
116261		- fts3EvalNext(pCsr, pRoot, &rc);
	116992	+ fts3EvalNextRow(pCsr, pRoot, &rc);
116262	116993	pCsr->isEof = pRoot->bEof;
116263	116994	pCsr->isRequireSeek = 1;
116264	116995	pCsr->isMatchinfoNeeded = 1;
116265	116996	pCsr->iPrevId = pRoot->iDocid;
116266	116997	}while( pCsr->isEof==0
116267	116998	&& pRoot->eType==FTSQUERY_NEAR
116268		- && fts3EvalLoadDeferred(pCsr, &rc)
	116999	+ && fts3EvalTestDeferredAndNear(pCsr, &rc)
116269	117000	);
116270	117001
116271	117002	if( rc==SQLITE_OK && pCsr->isEof==0 ){
116272	117003	fts3EvalUpdateCounts(pRoot);
116273	117004	}
		@@ -116285,14 +117016,14 @@
116285	117016	**
116286	117017	** do {...} while( pRoot->iDocid<iDocid && rc==SQLITE_OK );
116287	117018	*/
116288	117019	fts3EvalRestart(pCsr, pRoot, &rc);
116289	117020	do {
116290		- fts3EvalNext(pCsr, pRoot, &rc);
	117021	+ fts3EvalNextRow(pCsr, pRoot, &rc);
116291	117022	assert( pRoot->bEof==0 );
116292	117023	}while( pRoot->iDocid!=iDocid && rc==SQLITE_OK );
116293		- fts3EvalLoadDeferred(pCsr, &rc);
	117024	+ fts3EvalTestDeferredAndNear(pCsr, &rc);
116294	117025	}
116295	117026	}
116296	117027	return rc;
116297	117028	}
116298	117029
		@@ -116419,18 +117150,32 @@
116419	117150	*/
116420	117151	SQLITE_PRIVATE void sqlite3Fts3EvalPhraseCleanup(Fts3Phrase *pPhrase){
116421	117152	if( pPhrase ){
116422	117153	int i;
116423	117154	sqlite3_free(pPhrase->doclist.aAll);
116424		- fts3EvalZeroPoslist(pPhrase);
	117155	+ fts3EvalInvalidatePoslist(pPhrase);
116425	117156	memset(&pPhrase->doclist, 0, sizeof(Fts3Doclist));
116426	117157	for(i=0; i<pPhrase->nToken; i++){
116427	117158	fts3SegReaderCursorFree(pPhrase->aToken[i].pSegcsr);
116428	117159	pPhrase->aToken[i].pSegcsr = 0;
116429	117160	}
116430	117161	}
116431	117162	}
	117163	+
	117164	+#if !SQLITE_CORE
	117165	+/*
	117166	+** Initialize API pointer table, if required.
	117167	+*/
	117168	+SQLITE_API int sqlite3_extension_init(
	117169	+ sqlite3 *db,
	117170	+ char **pzErrMsg,
	117171	+ const sqlite3_api_routines *pApi
	117172	+){
	117173	+ SQLITE_EXTENSION_INIT2(pApi)
	117174	+ return sqlite3Fts3Init(db);
	117175	+}
	117176	+#endif
116432	117177
116433	117178	#endif
116434	117179
116435	117180	/************ End of fts3.c **********************************************/
116436	117181	/************ Begin file fts3_aux.c **************************************/
		@@ -118917,14 +119662,10 @@
118917	119662	** (in which case SQLITE_CORE is not defined), or
118918	119663	**
118919	119664	** * The FTS3 module is being built into the core of
118920	119665	** SQLite (in which case SQLITE_ENABLE_FTS3 is defined).
118921	119666	*/
118922		-#ifndef SQLITE_CORE
118923		- SQLITE_EXTENSION_INIT1
118924		-#endif
118925		-
118926	119667	#if !defined(SQLITE_CORE) \|\| defined(SQLITE_ENABLE_FTS3)
118927	119668
118928	119669
118929	119670	/*
118930	119671	** Implementation of the SQL scalar function for accessing the underlying
118931	119672

	--- src/sqlite3.c
	+++ src/sqlite3.c
	@@ -1,8 +1,8 @@
1	/******************************************************************************
2	** This file is an amalgamation of many separate C source files from SQLite
3	** version 3.7.7. By combining all the individual C code files into this
4	** single large file, the entire code can be compiled as a single translation
5	** unit. This allows many compilers to do optimizations that would not be
6	** possible if the files were compiled separately. Performance improvements
7	** of 5% or more are commonly seen when SQLite is compiled as a single
8	** translation unit.
	@@ -648,13 +648,13 @@
648	**
649	** See also: [sqlite3_libversion()],
650	** [sqlite3_libversion_number()], [sqlite3_sourceid()],
651	** [sqlite_version()] and [sqlite_source_id()].
652	*/
653	#define SQLITE_VERSION "3.7.7"
654	#define SQLITE_VERSION_NUMBER 3007007
655	#define SQLITE_SOURCE_ID "2011-06-24 11:29:51 9b191bb4c7c1e1b12b188c0b3eee1f8f587887c8"
656
657	/*
658	** CAPI3REF: Run-Time Library Version Numbers
659	** KEYWORDS: sqlite3_version, sqlite3_sourceid
660	**
	@@ -1282,20 +1282,37 @@
1282	** when [PRAGMA synchronous \| PRAGMA synchronous=OFF] is set, but most
1283	** VFSes do not need this signal and should silently ignore this opcode.
1284	** Applications should not call [sqlite3_file_control()] with this
1285	** opcode as doing so may disrupt the operation of the specialized VFSes
1286	** that do require it.

















1287	*/
1288	#define SQLITE_FCNTL_LOCKSTATE 1
1289	#define SQLITE_GET_LOCKPROXYFILE 2
1290	#define SQLITE_SET_LOCKPROXYFILE 3
1291	#define SQLITE_LAST_ERRNO 4
1292	#define SQLITE_FCNTL_SIZE_HINT 5
1293	#define SQLITE_FCNTL_CHUNK_SIZE 6
1294	#define SQLITE_FCNTL_FILE_POINTER 7
1295	#define SQLITE_FCNTL_SYNC_OMITTED 8
1296
1297
1298	/*
1299	** CAPI3REF: Mutex Handle
1300	**
1301	** The mutex module within SQLite defines [sqlite3_mutex] to be an
	@@ -9558,10 +9575,11 @@
9558	#define SQLITE_IndexSearch 0x08 /* Disable indexes for searching */
9559	#define SQLITE_IndexCover 0x10 /* Disable index covering table */
9560	#define SQLITE_GroupByOrder 0x20 /* Disable GROUPBY cover of ORDERBY */
9561	#define SQLITE_FactorOutConst 0x40 /* Disable factoring out constants */
9562	#define SQLITE_IdxRealAsInt 0x80 /* Store REAL as INT in indices */

9563	#define SQLITE_OptMask 0xff /* Mask of all disablable opts */
9564
9565	/*
9566	** Possible values for the sqlite.magic field.
9567	** The numbers are obtained at random and have no special meaning, other
	@@ -10449,10 +10467,11 @@
10449	Table pTab; / An SQL table corresponding to zName */
10450	Select pSelect; / A SELECT statement used in place of a table name */
10451	u8 isPopulated; /* Temporary table associated with SELECT is populated */
10452	u8 jointype; /* Type of join between this able and the previous */
10453	u8 notIndexed; /* True if there is a NOT INDEXED clause */

10454	#ifndef SQLITE_OMIT_EXPLAIN
10455	u8 iSelectId; /* If pSelect!=0, the id of the sub-select in EQP */
10456	#endif
10457	int iCursor; /* The VDBE cursor number used to access this table */
10458	Expr pOn; / The ON clause of a join */
	@@ -10568,10 +10587,11 @@
10568	struct WhereInfo {
10569	Parse pParse; / Parsing and code generating context */
10570	u16 wctrlFlags; /* Flags originally passed to sqlite3WhereBegin() */
10571	u8 okOnePass; /* Ok to use one-pass algorithm for UPDATE or DELETE */
10572	u8 untestedTerms; /* Not all WHERE terms resolved by outer loop */

10573	SrcList pTabList; / List of tables in the join */
10574	int iTop; /* The very beginning of the WHERE loop */
10575	int iContinue; /* Jump here to continue with next record */
10576	int iBreak; /* Jump here to break out of the loop */
10577	int nLevel; /* Number of nested loop */
	@@ -10579,10 +10599,13 @@
10579	double savedNQueryLoop; /* pParse->nQueryLoop outside the WHERE loop */
10580	double nRowOut; /* Estimated number of output rows */
10581	WhereLevel a[1]; /* Information about each nest loop in WHERE */
10582	};
10583



10584	/*
10585	** A NameContext defines a context in which to resolve table and column
10586	** names. The context consists of a list of tables (the pSrcList) field and
10587	** a list of named expression (pEList). The named expression list may
10588	** be NULL. The pSrc corresponds to the FROM clause of a SELECT or
	@@ -11340,11 +11363,11 @@
11340	#if defined(SQLITE_ENABLE_UPDATE_DELETE_LIMIT) && !defined(SQLITE_OMIT_SUBQUERY)
11341	SQLITE_PRIVATE Expr sqlite3LimitWhere(Parse , SrcList , Expr , ExprList , Expr , Expr , char );
11342	#endif
11343	SQLITE_PRIVATE void sqlite3DeleteFrom(Parse, SrcList, Expr*);
11344	SQLITE_PRIVATE void sqlite3Update(Parse, SrcList, ExprList, Expr, int);
11345	SQLITE_PRIVATE WhereInfo sqlite3WhereBegin(Parse, SrcList, Expr, ExprList**, u16);
11346	SQLITE_PRIVATE void sqlite3WhereEnd(WhereInfo*);
11347	SQLITE_PRIVATE int sqlite3ExprCodeGetColumn(Parse, Table, int, int, int);
11348	SQLITE_PRIVATE void sqlite3ExprCodeGetColumnOfTable(Vdbe, Table, int, int, int);
11349	SQLITE_PRIVATE void sqlite3ExprCodeMove(Parse*, int, int, int);
11350	SQLITE_PRIVATE void sqlite3ExprCodeCopy(Parse*, int, int, int);
	@@ -15841,11 +15864,11 @@
15841	** Free an outstanding memory allocation.
15842	**
15843	** This function assumes that the necessary mutexes, if any, are
15844	** already held by the caller. Hence "Unsafe".
15845	*/
15846	void memsys3FreeUnsafe(void *pOld){
15847	Mem3Block p = (Mem3Block)pOld;
15848	int i;
15849	u32 size, x;
15850	assert( sqlite3_mutex_held(mem3.mutex) );
15851	assert( p>mem3.aPool && p<&mem3.aPool[mem3.nPool] );
	@@ -15916,21 +15939,21 @@
15916	}
15917
15918	/*
15919	** Free memory.
15920	*/
15921	void memsys3Free(void *pPrior){
15922	assert( pPrior );
15923	memsys3Enter();
15924	memsys3FreeUnsafe(pPrior);
15925	memsys3Leave();
15926	}
15927
15928	/*
15929	** Change the size of an existing memory allocation
15930	*/
15931	void memsys3Realloc(void pPrior, int nBytes){
15932	int nOld;
15933	void *p;
15934	if( pPrior==0 ){
15935	return sqlite3_malloc(nBytes);
15936	}
	@@ -25143,11 +25166,13 @@
25143	case EINVAL:
25144	case ENOTCONN:
25145	case ENODEV:
25146	case ENXIO:
25147	case ENOENT:

25148	case ESTALE:

25149	case ENOSYS:
25150	/* these should force the client to close the file and reconnect */
25151
25152	default:
25153	return sqliteIOErr;
	@@ -31820,10 +31845,58 @@
31820	iLine, iErrno, zFunc, zPath, zMsg
31821	);
31822
31823	return errcode;
31824	}
















































31825
31826	#if SQLITE_OS_WINCE
31827	/*************************************************************************
31828	** This section contains code for WinCE only.
31829	*/
	@@ -32238,22 +32311,25 @@
32238	int amt, /* Number of bytes to read */
32239	sqlite3_int64 offset /* Begin reading at this offset */
32240	){
32241	winFile pFile = (winFile)id; /* file handle */
32242	DWORD nRead; /* Number of bytes actually read from file */

32243
32244	assert( id!=0 );
32245	SimulateIOError(return SQLITE_IOERR_READ);
32246	OSTRACE(("READ %d lock=%d\n", pFile->h, pFile->locktype));
32247
32248	if( seekWinFile(pFile, offset) ){
32249	return SQLITE_FULL;
32250	}
32251	if( !ReadFile(pFile->h, pBuf, amt, &nRead, 0) ){

32252	pFile->lastErrno = GetLastError();
32253	return winLogError(SQLITE_IOERR_READ, "winRead", pFile->zPath);
32254	}

32255	if( nRead<(DWORD)amt ){
32256	/* Unread parts of the buffer must be zero-filled */
32257	memset(&((char*)pBuf)[nRead], 0, amt-nRead);
32258	return SQLITE_IOERR_SHORT_READ;
32259	}
	@@ -32271,10 +32347,11 @@
32271	int amt, /* Number of bytes to write */
32272	sqlite3_int64 offset /* Offset into the file to begin writing at */
32273	){
32274	int rc; /* True if error has occured, else false */
32275	winFile pFile = (winFile)id; /* File handle */

32276
32277	assert( amt>0 );
32278	assert( pFile );
32279	SimulateIOError(return SQLITE_IOERR_WRITE);
32280	SimulateDiskfullError(return SQLITE_FULL);
	@@ -32285,11 +32362,16 @@
32285	if( rc==0 ){
32286	u8 aRem = (u8 )pBuf; /* Data yet to be written */
32287	int nRem = amt; /* Number of bytes yet to be written */
32288	DWORD nWrite; /* Bytes written by each WriteFile() call */
32289
32290	while( nRem>0 && WriteFile(pFile->h, aRem, nRem, &nWrite, 0) && nWrite>0 ){





32291	aRem += nWrite;
32292	nRem -= nWrite;
32293	}
32294	if( nRem>0 ){
32295	pFile->lastErrno = GetLastError();
	@@ -32301,10 +32383,12 @@
32301	if( ( pFile->lastErrno==ERROR_HANDLE_DISK_FULL )
32302	\|\| ( pFile->lastErrno==ERROR_DISK_FULL )){
32303	return SQLITE_FULL;
32304	}
32305	return winLogError(SQLITE_IOERR_WRITE, "winWrite", pFile->zPath);


32306	}
32307	return SQLITE_OK;
32308	}
32309
32310	/*
	@@ -32716,10 +32800,24 @@
32716	SimulateIOErrorBenign(0);
32717	return SQLITE_OK;
32718	}
32719	case SQLITE_FCNTL_SYNC_OMITTED: {
32720	return SQLITE_OK;














32721	}
32722	}
32723	return SQLITE_NOTFOUND;
32724	}
32725
	@@ -33734,19 +33832,17 @@
33734	** file open, we will be unable to delete it. To work around this
33735	** problem, we delay 100 milliseconds and try to delete again. Up
33736	** to MX_DELETION_ATTEMPTs deletion attempts are run before giving
33737	** up and returning an error.
33738	*/
33739	#define MX_DELETION_ATTEMPTS 5
33740	static int winDelete(
33741	sqlite3_vfs pVfs, / Not used on win32 */
33742	const char zFilename, / Name of file to delete */
33743	int syncDir /* Not used on win32 */
33744	){
33745	int cnt = 0;
33746	DWORD rc;
33747	DWORD error = 0;
33748	void *zConverted;
33749	UNUSED_PARAMETER(pVfs);
33750	UNUSED_PARAMETER(syncDir);
33751
33752	SimulateIOError(return SQLITE_IOERR_DELETE);
	@@ -33753,38 +33849,34 @@
33753	zConverted = convertUtf8Filename(zFilename);
33754	if( zConverted==0 ){
33755	return SQLITE_NOMEM;
33756	}
33757	if( isNT() ){
33758	do{
33759	DeleteFileW(zConverted);
33760	}while( ( ((rc = GetFileAttributesW(zConverted)) != INVALID_FILE_ATTRIBUTES)
33761	\|\| ((error = GetLastError()) == ERROR_ACCESS_DENIED))
33762	&& (++cnt < MX_DELETION_ATTEMPTS)
33763	&& (Sleep(100), 1) );
33764	/* isNT() is 1 if SQLITE_OS_WINCE==1, so this else is never executed.
33765	** Since the ASCII version of these Windows API do not exist for WINCE,
33766	** it's important to not reference them for WINCE builds.
33767	*/
33768	#if SQLITE_OS_WINCE==0
33769	}else{
33770	do{
33771	DeleteFileA(zConverted);
33772	}while( ( ((rc = GetFileAttributesA(zConverted)) != INVALID_FILE_ATTRIBUTES)
33773	\|\| ((error = GetLastError()) == ERROR_ACCESS_DENIED))
33774	&& (++cnt < MX_DELETION_ATTEMPTS)
33775	&& (Sleep(100), 1) );
33776	#endif
33777	}





33778	free(zConverted);
33779	OSTRACE(("DELETE \"%s\" %s\n", zFilename,
33780	( (rc==INVALID_FILE_ATTRIBUTES) && (error==ERROR_FILE_NOT_FOUND)) ?
33781	"ok" : "failed" ));
33782
33783	return ( (rc == INVALID_FILE_ATTRIBUTES)
33784	&& (error == ERROR_FILE_NOT_FOUND)) ? SQLITE_OK :
33785	winLogError(SQLITE_IOERR_DELETE, "winDelete", zFilename);
33786	}
33787
33788	/*
33789	** Check the existance and status of a file.
33790	*/
	@@ -59049,10 +59141,11 @@
59049	nMem = 10;
59050	}
59051	memset(zCsr, 0, zEnd-zCsr);
59052	zCsr += (zCsr - (u8*)0)&7;
59053	assert( EIGHT_BYTE_ALIGNMENT(zCsr) );

59054
59055	/* Memory for registers, parameters, cursor, etc, is allocated in two
59056	** passes. On the first pass, we try to reuse unused space at the
59057	** end of the opcode array. If we are unable to satisfy all memory
59058	** requirements by reusing the opcode array tail, then the second
	@@ -61277,11 +61370,11 @@
61277	sqlite3_mutex_enter(db->mutex);
61278	while( (rc = sqlite3Step(v))==SQLITE_SCHEMA
61279	&& cnt++ < SQLITE_MAX_SCHEMA_RETRY
61280	&& (rc2 = rc = sqlite3Reprepare(v))==SQLITE_OK ){
61281	sqlite3_reset(pStmt);
61282	v->expired = 0;
61283	}
61284	if( rc2!=SQLITE_OK && ALWAYS(v->isPrepareV2) && ALWAYS(db->pErr) ){
61285	/* This case occurs after failing to recompile an sql statement.
61286	** The error message from the SQL compiler has already been loaded
61287	** into the database handle. This block copies the error message
	@@ -65968,11 +66061,11 @@
65968	** automatically created table with root-page 1 (an BLOB_INTKEY table).
65969	*/
65970	if( pOp->p4.pKeyInfo ){
65971	int pgno;
65972	assert( pOp->p4type==P4_KEYINFO );
65973	rc = sqlite3BtreeCreateTable(u.ax.pCx->pBt, &pgno, BTREE_BLOBKEY);
65974	if( rc==SQLITE_OK ){
65975	assert( pgno==MASTER_ROOT+1 );
65976	rc = sqlite3BtreeCursor(u.ax.pCx->pBt, pgno, 1,
65977	(KeyInfo*)pOp->p4.z, u.ax.pCx->pCursor);
65978	u.ax.pCx->pKeyInfo = pOp->p4.pKeyInfo;
	@@ -71007,15 +71100,29 @@
71007	/* Recursively resolve names in all subqueries
71008	*/
71009	for(i=0; i<p->pSrc->nSrc; i++){
71010	struct SrcList_item *pItem = &p->pSrc->a[i];
71011	if( pItem->pSelect ){


71012	const char *zSavedContext = pParse->zAuthContext;








71013	if( pItem->zName ) pParse->zAuthContext = pItem->zName;
71014	sqlite3ResolveSelectNames(pParse, pItem->pSelect, pOuterNC);
71015	pParse->zAuthContext = zSavedContext;
71016	if( pParse->nErr \|\| db->mallocFailed ) return WRC_Abort;




71017	}
71018	}
71019
71020	/* If there are no aggregate functions in the result-set, and no GROUP BY
71021	** expression, do not allow aggregates in any of the other expressions.
	@@ -72119,10 +72226,11 @@
72119	pNewItem->zName = sqlite3DbStrDup(db, pOldItem->zName);
72120	pNewItem->zAlias = sqlite3DbStrDup(db, pOldItem->zAlias);
72121	pNewItem->jointype = pOldItem->jointype;
72122	pNewItem->iCursor = pOldItem->iCursor;
72123	pNewItem->isPopulated = pOldItem->isPopulated;

72124	pNewItem->zIndex = sqlite3DbStrDup(db, pOldItem->zIndex);
72125	pNewItem->notIndexed = pOldItem->notIndexed;
72126	pNewItem->pIndex = pOldItem->pIndex;
72127	pTab = pNewItem->pTab = pOldItem->pTab;
72128	if( pTab ){
	@@ -80126,11 +80234,11 @@
80126	if( pStart ){
80127	assert( pEnd!=0 );
80128	/* A named index with an explicit CREATE INDEX statement */
80129	zStmt = sqlite3MPrintf(db, "CREATE%s INDEX %.*s",
80130	onError==OE_None ? "" : " UNIQUE",
80131	pEnd->z - pName->z + 1,
80132	pName->z);
80133	}else{
80134	/* An automatic index created by a PRIMARY KEY or UNIQUE constraint */
80135	/* zStmt = sqlite3MPrintf(""); */
80136	zStmt = 0;
	@@ -81921,11 +82029,13 @@
81921	int regRowid; /* Actual register containing rowids */
81922
81923	/* Collect rowids of every row to be deleted.
81924	*/
81925	sqlite3VdbeAddOp2(v, OP_Null, 0, iRowSet);
81926	pWInfo = sqlite3WhereBegin(pParse, pTabList, pWhere,0,WHERE_DUPLICATES_OK);


81927	if( pWInfo==0 ) goto delete_from_cleanup;
81928	regRowid = sqlite3ExprCodeGetColumn(pParse, pTab, -1, iCur, iRowid);
81929	sqlite3VdbeAddOp2(v, OP_RowSetAdd, iRowSet, regRowid);
81930	if( db->flags & SQLITE_CountRows ){
81931	sqlite3VdbeAddOp2(v, OP_AddImm, memCnt, 1);
	@@ -84368,11 +84478,11 @@
84368
84369	/* Create VDBE to loop through the entries in pSrc that match the WHERE
84370	** clause. If the constraint is not deferred, throw an exception for
84371	** each row found. Otherwise, for deferred constraints, increment the
84372	** deferred constraint counter by nIncr for each row selected. */
84373	pWInfo = sqlite3WhereBegin(pParse, pSrc, pWhere, 0, 0);
84374	if( nIncr>0 && pFKey->isDeferred==0 ){
84375	sqlite3ParseToplevel(pParse)->mayAbort = 1;
84376	}
84377	sqlite3VdbeAddOp2(v, OP_FkCounter, pFKey->isDeferred, nIncr);
84378	if( pWInfo ){
	@@ -87235,10 +87345,13 @@
87235	int (strnicmp)(const char,const char*,int);
87236	int (unlock_notify)(sqlite3,void()(void,int),void);
87237	int (wal_autocheckpoint)(sqlite3,int);
87238	int (wal_checkpoint)(sqlite3,const char*);
87239	void (wal_hook)(sqlite3,int()(void,sqlite3,const char,int),void);



87240	};
87241
87242	/*
87243	** The following macros redefine the API routines so that they are
87244	** redirected throught the global sqlite3_api structure.
	@@ -87435,10 +87548,13 @@
87435	#define sqlite3_strnicmp sqlite3_api->strnicmp
87436	#define sqlite3_unlock_notify sqlite3_api->unlock_notify
87437	#define sqlite3_wal_autocheckpoint sqlite3_api->wal_autocheckpoint
87438	#define sqlite3_wal_checkpoint sqlite3_api->wal_checkpoint
87439	#define sqlite3_wal_hook sqlite3_api->wal_hook



87440	#endif /* SQLITE_CORE */
87441
87442	#define SQLITE_EXTENSION_INIT1 const sqlite3_api_routines *sqlite3_api = 0;
87443	#define SQLITE_EXTENSION_INIT2(v) sqlite3_api = v;
87444
	@@ -87509,10 +87625,12 @@
87509
87510	#ifdef SQLITE_OMIT_VIRTUALTABLE
87511	# define sqlite3_create_module 0
87512	# define sqlite3_create_module_v2 0
87513	# define sqlite3_declare_vtab 0


87514	#endif
87515
87516	#ifdef SQLITE_OMIT_SHARED_CACHE
87517	# define sqlite3_enable_shared_cache 0
87518	#endif
	@@ -87532,10 +87650,11 @@
87532	#define sqlite3_blob_bytes 0
87533	#define sqlite3_blob_close 0
87534	#define sqlite3_blob_open 0
87535	#define sqlite3_blob_read 0
87536	#define sqlite3_blob_write 0

87537	#endif
87538
87539	/*
87540	** The following structure contains pointers to all SQLite API routines.
87541	** A pointer to this structure is passed into extensions when they are
	@@ -87797,10 +87916,13 @@
87797	#else
87798	0,
87799	0,
87800	0,
87801	#endif



87802	};
87803
87804	/*
87805	** Attempt to load an SQLite extension library contained in the file
87806	** zFile. The entry point is zProc. zProc may be 0 in which case a
	@@ -94186,10 +94308,11 @@
94186	Expr pHaving; / The HAVING clause. May be NULL */
94187	int isDistinct; /* True if the DISTINCT keyword is present */
94188	int distinct; /* Table to use for the distinct set */
94189	int rc = 1; /* Value to return from this function */
94190	int addrSortIndex; /* Address of an OP_OpenEphemeral instruction */

94191	AggInfo sAggInfo; /* Information used by aggregate queries */
94192	int iEnd; /* Address of the end of the query */
94193	sqlite3 db; / The database connection */
94194
94195	#ifndef SQLITE_OMIT_EXPLAIN
	@@ -94312,20 +94435,10 @@
94312	explainSetInteger(pParse->iSelectId, iRestoreSelectId);
94313	return rc;
94314	}
94315	#endif
94316
94317	/* If possible, rewrite the query to use GROUP BY instead of DISTINCT.
94318	** GROUP BY might use an index, DISTINCT never does.
94319	*/
94320	assert( p->pGroupBy==0 \|\| (p->selFlags & SF_Aggregate)!=0 );
94321	if( (p->selFlags & (SF_Distinct\|SF_Aggregate))==SF_Distinct ){
94322	p->pGroupBy = sqlite3ExprListDup(db, p->pEList, 0);
94323	pGroupBy = p->pGroupBy;
94324	p->selFlags &= ~SF_Distinct;
94325	}
94326
94327	/* If there is both a GROUP BY and an ORDER BY clause and they are
94328	** identical, then disable the ORDER BY clause since the GROUP BY
94329	** will cause elements to come out in the correct order. This is
94330	** an optimization - the correct answer should result regardless.
94331	** Use the SQLITE_GroupByOrder flag with SQLITE_TESTCTRL_OPTIMIZER
	@@ -94333,10 +94446,34 @@
94333	*/
94334	if( sqlite3ExprListCompare(p->pGroupBy, pOrderBy)==0
94335	&& (db->flags & SQLITE_GroupByOrder)==0 ){
94336	pOrderBy = 0;
94337	}
























94338
94339	/* If there is an ORDER BY clause, then this sorting
94340	** index might end up being unused if the data can be
94341	** extracted in pre-sorted order. If that is the case, then the
94342	** OP_OpenEphemeral instruction will be changed to an OP_Noop once
	@@ -94369,26 +94506,25 @@
94369
94370	/* Open a virtual index to use for the distinct set.
94371	*/
94372	if( p->selFlags & SF_Distinct ){
94373	KeyInfo *pKeyInfo;
94374	assert( isAgg \|\| pGroupBy );
94375	distinct = pParse->nTab++;
94376	pKeyInfo = keyInfoFromExprList(pParse, p->pEList);
94377	sqlite3VdbeAddOp4(v, OP_OpenEphemeral, distinct, 0, 0,
94378	(char*)pKeyInfo, P4_KEYINFO_HANDOFF);
94379	sqlite3VdbeChangeP5(v, BTREE_UNORDERED);
94380	}else{
94381	distinct = -1;
94382	}
94383
94384	/* Aggregate and non-aggregate queries are handled differently */
94385	if( !isAgg && pGroupBy==0 ){
94386	/* This case is for non-aggregate queries
94387	** Begin the database scan
94388	*/
94389	pWInfo = sqlite3WhereBegin(pParse, pTabList, pWhere, &pOrderBy, 0);
94390	if( pWInfo==0 ) goto select_end;
94391	if( pWInfo->nRowOut < p->nSelectRow ) p->nSelectRow = pWInfo->nRowOut;
94392
94393	/* If sorting index that was created by a prior OP_OpenEphemeral
94394	** instruction ended up not being needed, then change the OP_OpenEphemeral
	@@ -94397,14 +94533,56 @@
94397	if( addrSortIndex>=0 && pOrderBy==0 ){
94398	sqlite3VdbeChangeToNoop(v, addrSortIndex, 1);
94399	p->addrOpenEphm[2] = -1;
94400	}
94401
94402	/* Use the standard inner loop
94403	*/
94404	assert(!isDistinct);
94405	selectInnerLoop(pParse, p, pEList, 0, 0, pOrderBy, -1, pDest,










































94406	pWInfo->iContinue, pWInfo->iBreak);
94407
94408	/* End the database scan loop.
94409	*/
94410	sqlite3WhereEnd(pWInfo);
	@@ -94510,11 +94688,11 @@
94510	** This might involve two separate loops with an OP_Sort in between, or
94511	** it might be a single loop that uses an index to extract information
94512	** in the right order to begin with.
94513	*/
94514	sqlite3VdbeAddOp2(v, OP_Gosub, regReset, addrReset);
94515	pWInfo = sqlite3WhereBegin(pParse, pTabList, pWhere, &pGroupBy, 0);
94516	if( pWInfo==0 ) goto select_end;
94517	if( pGroupBy==0 ){
94518	/* The optimizer is able to deliver rows in group by order so
94519	** we do not have to sort. The OP_OpenEphemeral table will be
94520	** cancelled later because we still need to use the pKeyInfo
	@@ -94772,11 +94950,11 @@
94772	/* This case runs if the aggregate has no GROUP BY clause. The
94773	** processing is much simpler since there is only a single row
94774	** of output.
94775	*/
94776	resetAccumulator(pParse, &sAggInfo);
94777	pWInfo = sqlite3WhereBegin(pParse, pTabList, pWhere, &pMinMax, flag);
94778	if( pWInfo==0 ){
94779	sqlite3ExprListDelete(db, pDel);
94780	goto select_end;
94781	}
94782	updateAccumulator(pParse, &sAggInfo);
	@@ -95248,19 +95426,32 @@
95248	iDb = sqlite3TwoPartName(pParse, pName1, pName2, &pName);
95249	if( iDb<0 ){
95250	goto trigger_cleanup;
95251	}
95252	}
















95253
95254	/* If the trigger name was unqualified, and the table is a temp table,
95255	** then set iDb to 1 to create the trigger in the temporary database.
95256	** If sqlite3SrcListLookup() returns 0, indicating the table does not
95257	** exist, the error is caught by the block below.
95258	*/
95259	if( !pTableName \|\| db->mallocFailed ){
95260	goto trigger_cleanup;
95261	}
95262	pTab = sqlite3SrcListLookup(pParse, pTableName);
95263	if( db->init.busy==0 && pName2->n==0 && pTab
95264	&& pTab->pSchema==db->aDb[1].pSchema ){
95265	iDb = 1;
95266	}
	@@ -96554,11 +96745,13 @@
96554	}
96555
96556	/* Begin the database scan
96557	*/
96558	sqlite3VdbeAddOp2(v, OP_Null, 0, regOldRowid);
96559	pWInfo = sqlite3WhereBegin(pParse, pTabList, pWhere,0, WHERE_ONEPASS_DESIRED);


96560	if( pWInfo==0 ) goto update_cleanup;
96561	okOnePass = pWInfo->okOnePass;
96562
96563	/* Remember the rowid of every item to be updated.
96564	*/
	@@ -98580,10 +98773,11 @@
98580	#define WHERE_REVERSE 0x02000000 /* Scan in reverse order */
98581	#define WHERE_UNIQUE 0x04000000 /* Selects no more than one row */
98582	#define WHERE_VIRTUALTABLE 0x08000000 /* Use virtual-table processing */
98583	#define WHERE_MULTI_OR 0x10000000 /* OR using multiple indices */
98584	#define WHERE_TEMP_INDEX 0x20000000 /* Uses an ephemeral index */

98585
98586	/*
98587	** Initialize a preallocated WhereClause structure.
98588	*/
98589	static void whereClauseInit(
	@@ -99724,10 +99918,166 @@
99724	}
99725	}
99726	return 0;
99727	}
99728




























































































































































99729
99730	/*
99731	** This routine decides if pIdx can be used to satisfy the ORDER BY
99732	** clause. If it can, it returns 1. If pIdx cannot satisfy the
99733	** ORDER BY clause, this routine returns 0.
	@@ -99760,11 +100110,14 @@
99760	int sortOrder = 0; /* XOR of index and ORDER BY sort direction */
99761	int nTerm; /* Number of ORDER BY terms */
99762	struct ExprList_item pTerm; / A term of the ORDER BY clause */
99763	sqlite3 *db = pParse->db;
99764
99765	assert( pOrderBy!=0 );



99766	nTerm = pOrderBy->nExpr;
99767	assert( nTerm>0 );
99768
99769	/* Argument pIdx must either point to a 'real' named index structure,
99770	** or an index structure allocated on the stack by bestBtreeIndex() to
	@@ -100073,10 +100426,14 @@
100073	double costTempIdx; /* per-query cost of the transient index */
100074	WhereTerm pTerm; / A single term of the WHERE clause */
100075	WhereTerm pWCEnd; / End of pWC->a[] */
100076	Table pTable; / Table tht might be indexed */
100077




100078	if( (pParse->db->flags & SQLITE_AutoIndex)==0 ){
100079	/* Automatic indices are disabled at run-time */
100080	return;
100081	}
100082	if( (pCost->plan.wsFlags & WHERE_NOT_FULLSCAN)!=0 ){
	@@ -100084,10 +100441,14 @@
100084	return;
100085	}
100086	if( pSrc->notIndexed ){
100087	/* The NOT INDEXED clause appears in the SQL. */
100088	return;




100089	}
100090
100091	assert( pParse->nQueryLoop >= (double)1 );
100092	pTable = pSrc->pTab;
100093	nTableRow = pTable->nRowEst;
	@@ -101016,10 +101377,11 @@
101016	WhereClause pWC, / The WHERE clause */
101017	struct SrcList_item pSrc, / The FROM clause term to search */
101018	Bitmask notReady, /* Mask of cursors not available for indexing */
101019	Bitmask notValid, /* Cursors not available for any purpose */
101020	ExprList pOrderBy, / The ORDER BY clause */

101021	WhereCost pCost / Lowest cost query plan */
101022	){
101023	int iCur = pSrc->iCursor; /* The cursor of the table to be accessed */
101024	Index pProbe; / An index we are evaluating */
101025	Index pIdx; / Copy of pProbe, or zero for IPK index */
	@@ -101156,11 +101518,12 @@
101156	int nEq; /* Number of == or IN terms matching index */
101157	int bInEst = 0; /* True if "x IN (SELECT...)" seen */
101158	int nInMul = 1; /* Number of distinct equalities to lookup */
101159	int estBound = 100; /* Estimated reduction in search space */
101160	int nBound = 0; /* Number of range constraints seen */
101161	int bSort = 0; /* True if external sort required */

101162	int bLookup = 0; /* True if not a covering index */
101163	WhereTerm pTerm; / A single term of the WHERE clause */
101164	#ifdef SQLITE_ENABLE_STAT2
101165	WhereTerm pFirstTerm = 0; / First term matching the index */
101166	#endif
	@@ -101220,21 +101583,24 @@
101220
101221	/* If there is an ORDER BY clause and the index being considered will
101222	** naturally scan rows in the required order, set the appropriate flags
101223	** in wsFlags. Otherwise, if there is an ORDER BY clause but the index
101224	** will scan rows in a different order, set the bSort variable. */
101225	if( pOrderBy ){
101226	if( (wsFlags & WHERE_COLUMN_IN)==0
101227	&& pProbe->bUnordered==0
101228	&& isSortingIndex(pParse, pWC->pMaskSet, pProbe, iCur, pOrderBy,
101229	nEq, wsFlags, &rev)
101230	){
101231	wsFlags \|= WHERE_ROWID_RANGE\|WHERE_COLUMN_RANGE\|WHERE_ORDERBY;
101232	wsFlags \|= (rev ? WHERE_REVERSE : 0);
101233	}else{
101234	bSort = 1;
101235	}



101236	}
101237
101238	/* If currently calculating the cost of using an index (not the IPK
101239	** index), determine if all required column data may be obtained without
101240	** using the main table (i.e. if the index is a covering
	@@ -101265,16 +101631,17 @@
101265	nRow = aiRowEst[0]/2;
101266	nInMul = (int)(nRow / aiRowEst[nEq]);
101267	}
101268
101269	#ifdef SQLITE_ENABLE_STAT2
101270	/* If the constraint is of the form x=VALUE and histogram

101271	** data is available for column x, then it might be possible
101272	** to get a better estimate on the number of rows based on
101273	** VALUE and how common that value is according to the histogram.
101274	*/
101275	if( nRow>(double)1 && nEq==1 && pFirstTerm!=0 ){
101276	if( pFirstTerm->eOperator & (WO_EQ\|WO_ISNULL) ){
101277	testcase( pFirstTerm->eOperator==WO_EQ );
101278	testcase( pFirstTerm->eOperator==WO_ISNULL );
101279	whereEqualScanEst(pParse, pProbe, pFirstTerm->pExpr->pRight, &nRow);
101280	}else if( pFirstTerm->eOperator==WO_IN && bInEst==0 ){
	@@ -101347,10 +101714,13 @@
101347	** difference and select C of 3.0.
101348	*/
101349	if( bSort ){
101350	cost += nRowestLog(nRow)3;
101351	}



101352
101353	/** Cost of using this index has now been computed **/
101354
101355	/* If there are additional constraints on this table that cannot
101356	** be used with the current index, but which might lower the number
	@@ -101492,11 +101862,11 @@
101492	}
101493	sqlite3DbFree(pParse->db, p);
101494	}else
101495	#endif
101496	{
101497	bestBtreeIndex(pParse, pWC, pSrc, notReady, notValid, pOrderBy, pCost);
101498	}
101499	}
101500
101501	/*
101502	** Disable a term in the WHERE clause. Except, do not disable the term
	@@ -102454,11 +102824,11 @@
102454	for(ii=0; ii<pOrWc->nTerm; ii++){
102455	WhereTerm *pOrTerm = &pOrWc->a[ii];
102456	if( pOrTerm->leftCursor==iCur \|\| pOrTerm->eOperator==WO_AND ){
102457	WhereInfo pSubWInfo; / Info for single OR-term scan */
102458	/* Loop through table entries that match term pOrTerm. */
102459	pSubWInfo = sqlite3WhereBegin(pParse, pOrTab, pOrTerm->pExpr, 0,
102460	WHERE_OMIT_OPEN \| WHERE_OMIT_CLOSE \|
102461	WHERE_FORCE_TABLE \| WHERE_ONETABLE_ONLY);
102462	if( pSubWInfo ){
102463	explainOneScan(
102464	pParse, pOrTab, &pSubWInfo->a[0], iLevel, pLevel->iFrom, 0
	@@ -102695,10 +103065,11 @@
102695	SQLITE_PRIVATE WhereInfo *sqlite3WhereBegin(
102696	Parse pParse, / The parser context */
102697	SrcList pTabList, / A list of all tables to be scanned */
102698	Expr pWhere, / The WHERE clause */
102699	ExprList *ppOrderBy, / An ORDER BY clause, or NULL */

102700	u16 wctrlFlags /* One of the WHERE_* flags defined in sqliteInt.h */
102701	){
102702	int i; /* Loop counter */
102703	int nByteWInfo; /* Num. bytes allocated for WhereInfo struct */
102704	int nTabList; /* Number of elements in pTabList */
	@@ -102754,10 +103125,14 @@
102754	pWInfo->iBreak = sqlite3VdbeMakeLabel(v);
102755	pWInfo->pWC = pWC = (WhereClause )&((u8 )pWInfo)[nByteWInfo];
102756	pWInfo->wctrlFlags = wctrlFlags;
102757	pWInfo->savedNQueryLoop = pParse->nQueryLoop;
102758	pMaskSet = (WhereMaskSet*)&pWC[1];




102759
102760	/* Split the WHERE clause into separate subexpressions where each
102761	** subexpression is separated by an AND operator.
102762	*/
102763	initMaskSet(pMaskSet);
	@@ -102821,10 +103196,19 @@
102821	*/
102822	exprAnalyzeAll(pTabList, pWC);
102823	if( db->mallocFailed ){
102824	goto whereBeginError;
102825	}









102826
102827	/* Chose the best index to use for each table in the FROM clause.
102828	**
102829	** This loop fills in the following fields:
102830	**
	@@ -102905,10 +103289,11 @@
102905	Bitmask mask; /* Mask of tables not yet ready */
102906	for(j=iFrom, pTabItem=&pTabList->a[j]; j<nTabList; j++, pTabItem++){
102907	int doNotReorder; /* True if this table should not be reordered */
102908	WhereCost sCost; /* Cost information from best[Virtual]Index() */
102909	ExprList pOrderBy; / ORDER BY clause for index to optimize */

102910
102911	doNotReorder = (pTabItem->jointype & (JT_LEFT\|JT_CROSS))!=0;
102912	if( j!=iFrom && doNotReorder ) break;
102913	m = getMask(pMaskSet, pTabItem->iCursor);
102914	if( (m & notReady)==0 ){
	@@ -102915,10 +103300,11 @@
102915	if( j==iFrom ) iFrom++;
102916	continue;
102917	}
102918	mask = (isOptimal ? m : notReady);
102919	pOrderBy = ((i==0 && ppOrderBy )?*ppOrderBy:0);

102920	if( pTabItem->pIndex==0 ) nUnconstrained++;
102921
102922	WHERETRACE(("=== trying table %d with isOptimal=%d ===\n",
102923	j, isOptimal));
102924	assert( pTabItem->pTab );
	@@ -102929,11 +103315,11 @@
102929	&sCost, pp);
102930	}else
102931	#endif
102932	{
102933	bestBtreeIndex(pParse, pWC, pTabItem, mask, notReady, pOrderBy,
102934	&sCost);
102935	}
102936	assert( isOptimal \|\| (sCost.used&notReady)==0 );
102937
102938	/* If an INDEXED BY clause is present, then the plan must use that
102939	** index if it uses any index at all */
	@@ -102989,10 +103375,14 @@
102989	WHERETRACE(("*** Optimizer selects table %d for loop %d"
102990	" with cost=%g and nRow=%g\n",
102991	bestJ, pLevel-pWInfo->a, bestPlan.rCost, bestPlan.plan.nRow));
102992	if( (bestPlan.plan.wsFlags & WHERE_ORDERBY)!=0 ){
102993	*ppOrderBy = 0;




102994	}
102995	andFlags &= bestPlan.plan.wsFlags;
102996	pLevel->plan = bestPlan.plan;
102997	testcase( bestPlan.plan.wsFlags & WHERE_INDEXED );
102998	testcase( bestPlan.plan.wsFlags & WHERE_TEMP_INDEX );
	@@ -111519,11 +111909,17 @@
111519	*/
111520	#if defined(SQLITE_ENABLE_FTS4) && !defined(SQLITE_ENABLE_FTS3)
111521	# define SQLITE_ENABLE_FTS3
111522	#endif
111523
111524	#ifdef SQLITE_ENABLE_FTS3






111525	/************ Include fts3_tokenizer.h in the middle of fts3Int.h ********/
111526	/************ Begin file fts3_tokenizer.h ********************************/
111527	/*
111528	** 2006 July 10
111529	**
	@@ -112052,11 +112448,11 @@
112052
112053	sqlite3_int64 iDocid; /* Current docid (if pList!=0) */
112054	int bFreeList; /* True if pList should be sqlite3_free()d */
112055	char pList; / Pointer to position list following iDocid */
112056	int nList; /* Length of position list */
112057	} doclist;
112058
112059	/*
112060	** A "phrase" is a sequence of one or more tokens that must match in
112061	** sequence. A single token is the base case and the most common case.
112062	** For a sequence of tokens contained in double-quotes (i.e. "one two three")
	@@ -112252,23 +112648,12 @@
112252	#endif
112253
112254	/* fts3_aux.c */
112255	SQLITE_PRIVATE int sqlite3Fts3InitAux(sqlite3 *db);
112256
112257	SQLITE_PRIVATE int sqlite3Fts3TermSegReaderCursor(
112258	Fts3Cursor pCsr, / Virtual table cursor handle */
112259	const char zTerm, / Term to query for */
112260	int nTerm, /* Size of zTerm in bytes */
112261	int isPrefix, /* True for a prefix search */
112262	Fts3MultiSegReader *ppSegcsr / OUT: Allocated seg-reader cursor */
112263	);
112264
112265	SQLITE_PRIVATE void sqlite3Fts3EvalPhraseCleanup(Fts3Phrase *);
112266
112267	SQLITE_PRIVATE int sqlite3Fts3EvalStart(Fts3Cursor , Fts3Expr , int);
112268	SQLITE_PRIVATE int sqlite3Fts3EvalNext(Fts3Cursor *pCsr);
112269
112270	SQLITE_PRIVATE int sqlite3Fts3MsrIncrStart(
112271	Fts3Table, Fts3MultiSegReader, int, const char*, int);
112272	SQLITE_PRIVATE int sqlite3Fts3MsrIncrNext(
112273	Fts3Table , Fts3MultiSegReader , sqlite3_int64 , char , int );
112274	SQLITE_PRIVATE char sqlite3Fts3EvalPhrasePoslist(Fts3Cursor , Fts3Expr *, int iCol);
	@@ -112275,11 +112660,11 @@
112275	SQLITE_PRIVATE int sqlite3Fts3MsrOvfl(Fts3Cursor , Fts3MultiSegReader , int *);
112276	SQLITE_PRIVATE int sqlite3Fts3MsrIncrRestart(Fts3MultiSegReader *pCsr);
112277
112278	SQLITE_PRIVATE int sqlite3Fts3DeferredTokenList(Fts3DeferredToken , char , int );
112279
112280	#endif /* SQLITE_ENABLE_FTS3 */
112281	#endif /* _FTSINT_H */
112282
112283	/************ End of fts3Int.h *******************************************/
112284	/************ Continuing where we left off in fts3.c *********************/
112285	#if !defined(SQLITE_CORE) \|\| defined(SQLITE_ENABLE_FTS3)
	@@ -112291,10 +112676,15 @@
112291
112292	#ifndef SQLITE_CORE
112293	SQLITE_EXTENSION_INIT1
112294	#endif
112295





112296	/*
112297	** Write a 64-bit variable-length integer to memory starting at p[0].
112298	** The length of data written will be between 1 and FTS3_VARINT_MAX bytes.
112299	** The number of bytes written is returned.
112300	*/
	@@ -112799,31 +113189,60 @@
112799	}
112800	sqlite3_free(zFree);
112801	return zRet;
112802	}
112803













112804	static int fts3GobbleInt(const char *pp, int pnOut){
112805	const char p = pp;
112806	int nInt = 0;

112807	for(p=*pp; p[0]>='0' && p[0]<='9'; p++){
112808	nInt = nInt * 10 + (p[0] - '0');
112809	}
112810	if( p==*pp ) return SQLITE_ERROR;
112811	*pnOut = nInt;
112812	*pp = p;
112813	return SQLITE_OK;
112814	}
112815
112816
















112817	static int fts3PrefixParameter(
112818	const char zParam, / ABC in prefix=ABC parameter to parse */
112819	int pnIndex, / OUT: size of apIndex[] array /
112820	struct Fts3Index *apIndex, / OUT: Array of indexes for this table */
112821	struct Fts3Index *apFree / OUT: Free this with sqlite3_free() */
112822	){
112823	struct Fts3Index *aIndex;
112824	int nIndex = 1;
112825
112826	if( zParam && zParam[0] ){
112827	const char *p;
112828	nIndex++;
112829	for(p=zParam; *p; p++){
	@@ -112830,11 +113249,11 @@
112830	if( *p==',' ) nIndex++;
112831	}
112832	}
112833
112834	aIndex = sqlite3_malloc(sizeof(struct Fts3Index) * nIndex);
112835	apIndex = apFree = aIndex;
112836	*pnIndex = nIndex;
112837	if( !aIndex ){
112838	return SQLITE_NOMEM;
112839	}
112840
	@@ -112887,12 +113306,11 @@
112887	int isFts4 = (argv[0][3]=='4'); /* True for FTS4, false for FTS3 */
112888	const char *aCol; / Array of column names */
112889	sqlite3_tokenizer pTokenizer = 0; / Tokenizer for this table */
112890
112891	int nIndex; /* Size of aIndex[] array */
112892	struct Fts3Index aIndex; / Array of indexes for this table */
112893	struct Fts3Index aFree = 0; / Free this before returning */
112894
112895	/* The results of parsing supported FTS4 key=value options: */
112896	int bNoDocsize = 0; /* True to omit %_docsize table */
112897	int bDescIdx = 0; /* True to store descending indexes */
112898	char zPrefix = 0; / Prefix parameter value (or NULL) */
	@@ -113025,11 +113443,11 @@
113025	rc = sqlite3Fts3InitTokenizer(pHash, "simple", &pTokenizer, pzErr);
113026	if( rc!=SQLITE_OK ) goto fts3_init_out;
113027	}
113028	assert( pTokenizer );
113029
113030	rc = fts3PrefixParameter(zPrefix, &nIndex, &aIndex, &aFree);
113031	if( rc==SQLITE_ERROR ){
113032	assert( zPrefix );
113033	*pzErr = sqlite3_mprintf("error parsing prefix parameter: %s", zPrefix);
113034	}
113035	if( rc!=SQLITE_OK ) goto fts3_init_out;
	@@ -113112,11 +113530,11 @@
113112	/* Declare the table schema to SQLite. */
113113	fts3DeclareVtab(&rc, p);
113114
113115	fts3_init_out:
113116	sqlite3_free(zPrefix);
113117	sqlite3_free(aFree);
113118	sqlite3_free(zCompress);
113119	sqlite3_free(zUncompress);
113120	sqlite3_free((void *)aCol);
113121	if( rc!=SQLITE_OK ){
113122	if( p ){
	@@ -113703,12 +114121,10 @@
113703	*pp1 = p1 + 1;
113704	*pp2 = p2 + 1;
113705	}
113706
113707	/*
113708	** nToken==1 searches for adjacent positions.
113709	**
113710	** This function is used to merge two position lists into one. When it is
113711	** called, pp1 and pp2 must both point to position lists. A position-list is
113712	** the part of a doclist that follows each document id. For example, if a row
113713	** contains:
113714	**
	@@ -113724,10 +114140,12 @@
113724	** If isSaveLeft is 0, an entry is added to the output position list for
113725	** each position in *pp2 for which there exists one or more positions in
113726	** pp1 so that (pos(pp2)>pos(pp1) && pos(pp2)-pos(*pp1)<=nToken). i.e.
113727	** when the pp1 token appears before the pp2 token, but not more than nToken
113728	** slots before it.


113729	*/
113730	static int fts3PoslistPhraseMerge(
113731	char *pp, / IN/OUT: Preallocated output buffer */
113732	int nToken, /* Maximum difference in token positions */
113733	int isSaveLeft, /* Save the left position */
	@@ -113890,26 +114308,38 @@
113890
113891	return res;
113892	}
113893
113894	/*
113895	** A pointer to an instance of this structure is used as the context
113896	** argument to sqlite3Fts3SegReaderIterate()

113897	*/
113898	typedef struct TermSelect TermSelect;
113899	struct TermSelect {
113900	int isReqPos;
113901	char aaOutput[16]; / Malloc'd output buffer */
113902	int anOutput[16]; /* Size of output in bytes */
113903	};
113904
113905












113906	static void fts3GetDeltaVarint3(
113907	char **pp,
113908	char *pEnd,
113909	int bDescIdx,
113910	sqlite3_int64 *pVal
113911	){
113912	if( *pp>=pEnd ){
113913	*pp = 0;
113914	}else{
113915	sqlite3_int64 iVal;
	@@ -113920,10 +114350,25 @@
113920	*pVal += iVal;
113921	}
113922	}
113923	}
113924















113925	static void fts3PutDeltaVarint3(
113926	char *pp, / IN/OUT: Output pointer */
113927	int bDescIdx, /* True for descending docids */
113928	sqlite3_int64 piPrev, / IN/OUT: Previous value written to list */
113929	int pbFirst, / IN/OUT: True after first int written */
	@@ -113940,14 +114385,38 @@
113940	pp += sqlite3Fts3PutVarint(pp, iWrite);
113941	*piPrev = iVal;
113942	*pbFirst = 1;
113943	}
113944
113945	#define COMPARE_DOCID(i1, i2) ((bDescIdx?-1:1) * (i1-i2))
113946

























113947	static int fts3DoclistOrMerge(
113948	int bDescIdx, /* True if arguments are desc */
113949	char a1, int n1, / First doclist */
113950	char a2, int n2, / Second doclist */
113951	char *paOut, int pnOut /* OUT: Malloc'd doclist */
113952	){
113953	sqlite3_int64 i1 = 0;
	@@ -113968,35 +114437,47 @@
113968
113969	p = aOut;
113970	fts3GetDeltaVarint3(&p1, pEnd1, 0, &i1);
113971	fts3GetDeltaVarint3(&p2, pEnd2, 0, &i2);
113972	while( p1 \|\| p2 ){
113973	sqlite3_int64 iDiff = COMPARE_DOCID(i1, i2);
113974
113975	if( p2 && p1 && iDiff==0 ){
113976	fts3PutDeltaVarint3(&p, bDescIdx, &iPrev, &bFirstOut, i1);
113977	fts3PoslistMerge(&p, &p1, &p2);
113978	fts3GetDeltaVarint3(&p1, pEnd1, bDescIdx, &i1);
113979	fts3GetDeltaVarint3(&p2, pEnd2, bDescIdx, &i2);
113980	}else if( !p2 \|\| (p1 && iDiff<0) ){
113981	fts3PutDeltaVarint3(&p, bDescIdx, &iPrev, &bFirstOut, i1);
113982	fts3PoslistCopy(&p, &p1);
113983	fts3GetDeltaVarint3(&p1, pEnd1, bDescIdx, &i1);
113984	}else{
113985	fts3PutDeltaVarint3(&p, bDescIdx, &iPrev, &bFirstOut, i2);
113986	fts3PoslistCopy(&p, &p2);
113987	fts3GetDeltaVarint3(&p2, pEnd2, bDescIdx, &i2);
113988	}
113989	}
113990
113991	*paOut = aOut;
113992	*pnOut = (p-aOut);
113993	return SQLITE_OK;
113994	}
113995












113996	static void fts3DoclistPhraseMerge(
113997	int bDescIdx, /* True if arguments are desc */
113998	int nDist, /* Distance from left to right (1=adjacent) */
113999	char aLeft, int nLeft, / Left doclist */
114000	char aRight, int pnRight /* IN/OUT: Right/output doclist */
114001	){
114002	sqlite3_int64 i1 = 0;
	@@ -114015,30 +114496,30 @@
114015	p = aOut;
114016	fts3GetDeltaVarint3(&p1, pEnd1, 0, &i1);
114017	fts3GetDeltaVarint3(&p2, pEnd2, 0, &i2);
114018
114019	while( p1 && p2 ){
114020	sqlite3_int64 iDiff = COMPARE_DOCID(i1, i2);
114021	if( iDiff==0 ){
114022	char *pSave = p;
114023	sqlite3_int64 iPrevSave = iPrev;
114024	int bFirstOutSave = bFirstOut;
114025
114026	fts3PutDeltaVarint3(&p, bDescIdx, &iPrev, &bFirstOut, i1);
114027	if( 0==fts3PoslistPhraseMerge(&p, nDist, 0, 1, &p1, &p2) ){
114028	p = pSave;
114029	iPrev = iPrevSave;
114030	bFirstOut = bFirstOutSave;
114031	}
114032	fts3GetDeltaVarint3(&p1, pEnd1, bDescIdx, &i1);
114033	fts3GetDeltaVarint3(&p2, pEnd2, bDescIdx, &i2);
114034	}else if( iDiff<0 ){
114035	fts3PoslistCopy(0, &p1);
114036	fts3GetDeltaVarint3(&p1, pEnd1, bDescIdx, &i1);
114037	}else{
114038	fts3PoslistCopy(0, &p2);
114039	fts3GetDeltaVarint3(&p2, pEnd2, bDescIdx, &i2);
114040	}
114041	}
114042
114043	*pnRight = p - aOut;
114044	}
	@@ -114051,11 +114532,11 @@
114051	**
114052	** If an OOM error occurs, return SQLITE_NOMEM. In this case it is
114053	** the responsibility of the caller to free any doclists left in the
114054	** TermSelect.aaOutput[] array.
114055	*/
114056	static int fts3TermSelectMerge(Fts3Table p, TermSelect pTS){
114057	char *aOut = 0;
114058	int nOut = 0;
114059	int i;
114060
114061	/* Loop through the doclists in the aaOutput[] array. Merge them all
	@@ -114092,28 +114573,29 @@
114092	pTS->anOutput[0] = nOut;
114093	return SQLITE_OK;
114094	}
114095
114096	/*
114097	** This function is used as the sqlite3Fts3SegReaderIterate() callback when
114098	** querying the full-text index for a doclist associated with a term or
114099	** term-prefix.









114100	*/
114101	static int fts3TermSelectCb(
114102	Fts3Table p, / Virtual table object */
114103	void pContext, / Pointer to TermSelect structure */
114104	char *zTerm,
114105	int nTerm,
114106	char *aDoclist,
114107	int nDoclist
114108	){
114109	TermSelect pTS = (TermSelect )pContext;
114110
114111	UNUSED_PARAMETER(p);
114112	UNUSED_PARAMETER(zTerm);
114113	UNUSED_PARAMETER(nTerm);
114114
114115	if( pTS->aaOutput[0]==0 ){
114116	/* If this is the first term selected, copy the doclist to the output
114117	** buffer using memcpy(). */
114118	pTS->aaOutput[0] = sqlite3_malloc(nDoclist);
114119	pTS->anOutput[0] = nDoclist;
	@@ -114180,23 +114662,30 @@
114180	}
114181	pCsr->apSegment[pCsr->nSegment++] = pNew;
114182	return SQLITE_OK;
114183	}
114184







114185	static int fts3SegReaderCursor(
114186	Fts3Table p, / FTS3 table handle */
114187	int iIndex, /* Index to search (from 0 to p->nIndex-1) */
114188	int iLevel, /* Level of segments to scan */
114189	const char zTerm, / Term to query for */
114190	int nTerm, /* Size of zTerm in bytes */
114191	int isPrefix, /* True for a prefix search */
114192	int isScan, /* True to scan from zTerm to EOF */
114193	Fts3MultiSegReader pCsr / Cursor object to populate */
114194	){
114195	int rc = SQLITE_OK;
114196	int rc2;
114197	sqlite3_stmt *pStmt = 0;
114198
114199	/* If iLevel is less than 0 and this is not a scan, include a seg-reader
114200	** for the pending-terms. If this is a scan, then this call must be being
114201	** made by an fts4aux module, not an FTS table. In this case calling
114202	** Fts3SegReaderPending might segfault, as the data structures used by
	@@ -114281,28 +114770,46 @@
114281	return fts3SegReaderCursor(
114282	p, iIndex, iLevel, zTerm, nTerm, isPrefix, isScan, pCsr
114283	);
114284	}
114285






114286	static int fts3SegReaderCursorAddZero(
114287	Fts3Table *p,
114288	const char *zTerm,
114289	int nTerm,
114290	Fts3MultiSegReader *pCsr
114291	){
114292	return fts3SegReaderCursor(p, 0, FTS3_SEGCURSOR_ALL, zTerm, nTerm, 0, 0,pCsr);
114293	}
114294
114295
114296	SQLITE_PRIVATE int sqlite3Fts3TermSegReaderCursor(












114297	Fts3Cursor pCsr, / Virtual table cursor handle */
114298	const char zTerm, / Term to query for */
114299	int nTerm, /* Size of zTerm in bytes */
114300	int isPrefix, /* True for a prefix search */
114301	Fts3MultiSegReader *ppSegcsr / OUT: Allocated seg-reader cursor */
114302	){
114303	Fts3MultiSegReader pSegcsr; / Object to allocate and return */
114304	int rc = SQLITE_NOMEM; /* Return code */
114305
114306	pSegcsr = sqlite3_malloc(sizeof(Fts3MultiSegReader));
114307	if( pSegcsr ){
114308	int i;
	@@ -114342,62 +114849,53 @@
114342
114343	*ppSegcsr = pSegcsr;
114344	return rc;
114345	}
114346



114347	static void fts3SegReaderCursorFree(Fts3MultiSegReader *pSegcsr){
114348	sqlite3Fts3SegReaderFinish(pSegcsr);
114349	sqlite3_free(pSegcsr);
114350	}
114351
114352	/*
114353	** This function retreives the doclist for the specified term (or term
114354	** prefix) from the database.
114355	**
114356	** The returned doclist may be in one of two formats, depending on the
114357	** value of parameter isReqPos. If isReqPos is zero, then the doclist is
114358	** a sorted list of delta-compressed docids (a bare doclist). If isReqPos
114359	** is non-zero, then the returned list is in the same format as is stored
114360	** in the database without the found length specifier at the start of on-disk
114361	** doclists.
114362	*/
114363	static int fts3TermSelect(
114364	Fts3Table p, / Virtual table handle */
114365	Fts3PhraseToken pTok, / Token to query for */
114366	int iColumn, /* Column to query (or -ve for all columns) */
114367	int isReqPos, /* True to include position lists in output */
114368	int pnOut, / OUT: Size of buffer at ppOut /
114369	char *ppOut / OUT: Malloced result buffer */
114370	){
114371	int rc; /* Return code */
114372	Fts3MultiSegReader pSegcsr; / Seg-reader cursor for this term */
114373	TermSelect tsc; /* Context object for fts3TermSelectCb() */
114374	Fts3SegFilter filter; /* Segment term filter configuration */
114375
114376	pSegcsr = pTok->pSegcsr;
114377	memset(&tsc, 0, sizeof(TermSelect));
114378	tsc.isReqPos = isReqPos;
114379
114380	filter.flags = FTS3_SEGMENT_IGNORE_EMPTY
114381	\| (pTok->isPrefix ? FTS3_SEGMENT_PREFIX : 0)
114382	\| (isReqPos ? FTS3_SEGMENT_REQUIRE_POS : 0)
114383	\| (iColumn<p->nColumn ? FTS3_SEGMENT_COLUMN_FILTER : 0);
114384	filter.iCol = iColumn;
114385	filter.zTerm = pTok->z;
114386	filter.nTerm = pTok->n;
114387
114388	rc = sqlite3Fts3SegReaderStart(p, pSegcsr, &filter);
114389	while( SQLITE_OK==rc
114390	&& SQLITE_ROW==(rc = sqlite3Fts3SegReaderStep(p, pSegcsr))
114391	){
114392	rc = fts3TermSelectCb(p, (void *)&tsc,
114393	pSegcsr->zTerm, pSegcsr->nTerm, pSegcsr->aDoclist, pSegcsr->nDoclist
114394	);
114395	}
114396
114397	if( rc==SQLITE_OK ){
114398	rc = fts3TermSelectMerge(p, &tsc);
114399	}
114400	if( rc==SQLITE_OK ){
114401	*ppOut = tsc.aaOutput[0];
114402	*pnOut = tsc.anOutput[0];
114403	}else{
	@@ -114419,28 +114917,19 @@
114419	** If the isPoslist argument is true, then it is assumed that the doclist
114420	** contains a position-list following each docid. Otherwise, it is assumed
114421	** that the doclist is simply a list of docids stored as delta encoded
114422	** varints.
114423	*/
114424	static int fts3DoclistCountDocids(int isPoslist, char *aList, int nList){
114425	int nDoc = 0; /* Return value */
114426	if( aList ){
114427	char aEnd = &aList[nList]; / Pointer to one byte after EOF */
114428	char p = aList; / Cursor */
114429	if( !isPoslist ){
114430	/* The number of docids in the list is the same as the number of
114431	** varints. In FTS3 a varint consists of a single byte with the 0x80
114432	** bit cleared and zero or more bytes with the 0x80 bit set. So to
114433	** count the varints in the buffer, just count the number of bytes
114434	** with the 0x80 bit clear. */
114435	while( p<aEnd ) nDoc += (((*p++)&0x80)==0);
114436	}else{
114437	while( p<aEnd ){
114438	nDoc++;
114439	while( (p++)&0x80 ); / Skip docid varint */
114440	fts3PoslistCopy(0, &p); /* Skip over position list */
114441	}
114442	}
114443	}
114444
114445	return nDoc;
114446	}
	@@ -114466,11 +114955,11 @@
114466	}else{
114467	pCsr->iPrevId = sqlite3_column_int64(pCsr->pStmt, 0);
114468	rc = SQLITE_OK;
114469	}
114470	}else{
114471	rc = sqlite3Fts3EvalNext((Fts3Cursor *)pCursor);
114472	}
114473	assert( ((Fts3Table *)pCsr->base.pVtab)->pSegments==0 );
114474	return rc;
114475	}
114476
	@@ -114543,11 +115032,11 @@
114543	}
114544
114545	rc = sqlite3Fts3ReadLock(p);
114546	if( rc!=SQLITE_OK ) return rc;
114547
114548	rc = sqlite3Fts3EvalStart(pCsr, pCsr->pExpr, 1);
114549
114550	sqlite3Fts3SegmentsClose(p);
114551	if( rc!=SQLITE_OK ) return rc;
114552	pCsr->pNextId = pCsr->aDoclist;
114553	pCsr->iPrevId = 0;
	@@ -114950,26 +115439,43 @@
114950	p->zDb, p->zName, zName
114951	);
114952	return rc;
114953	}
114954





114955	static int fts3SavepointMethod(sqlite3_vtab *pVtab, int iSavepoint){
114956	UNUSED_PARAMETER(iSavepoint);
114957	assert( ((Fts3Table *)pVtab)->inTransaction );
114958	assert( ((Fts3Table *)pVtab)->mxSavepoint < iSavepoint );
114959	TESTONLY( ((Fts3Table *)pVtab)->mxSavepoint = iSavepoint );
114960	return fts3SyncMethod(pVtab);
114961	}






114962	static int fts3ReleaseMethod(sqlite3_vtab *pVtab, int iSavepoint){
114963	TESTONLY( Fts3Table p = (Fts3Table)pVtab );
114964	UNUSED_PARAMETER(iSavepoint);
114965	UNUSED_PARAMETER(pVtab);
114966	assert( p->inTransaction );
114967	assert( p->mxSavepoint >= iSavepoint );
114968	TESTONLY( p->mxSavepoint = iSavepoint-1 );
114969	return SQLITE_OK;
114970	}






114971	static int fts3RollbackToMethod(sqlite3_vtab *pVtab, int iSavepoint){
114972	Fts3Table p = (Fts3Table)pVtab;
114973	UNUSED_PARAMETER(iSavepoint);
114974	assert( p->inTransaction );
114975	assert( p->mxSavepoint >= iSavepoint );
	@@ -115114,22 +115620,10 @@
115114	sqlite3Fts3HashClear(pHash);
115115	sqlite3_free(pHash);
115116	}
115117	return rc;
115118	}
115119
115120	#if !SQLITE_CORE
115121	SQLITE_API int sqlite3_extension_init(
115122	sqlite3 *db,
115123	char **pzErrMsg,
115124	const sqlite3_api_routines *pApi
115125	){
115126	SQLITE_EXTENSION_INIT2(pApi)
115127	return sqlite3Fts3Init(db);
115128	}
115129	#endif
115130
115131
115132	/*
115133	** Allocate an Fts3MultiSegReader for each token in the expression headed
115134	** by pExpr.
115135	**
	@@ -115143,24 +115637,24 @@
115143	** there exists prefix b-tree of the right length) then it may be traversed
115144	** and merged incrementally. Otherwise, it has to be merged into an in-memory
115145	** doclist and then traversed.
115146	*/
115147	static void fts3EvalAllocateReaders(
115148	Fts3Cursor *pCsr,
115149	Fts3Expr *pExpr,
115150	int pnToken, / OUT: Total number of tokens in phrase. */
115151	int pnOr, / OUT: Total number of OR nodes in expr. */
115152	int *pRc
115153	){
115154	if( pExpr && SQLITE_OK==*pRc ){
115155	if( pExpr->eType==FTSQUERY_PHRASE ){
115156	int i;
115157	int nToken = pExpr->pPhrase->nToken;
115158	*pnToken += nToken;
115159	for(i=0; i<nToken; i++){
115160	Fts3PhraseToken *pToken = &pExpr->pPhrase->aToken[i];
115161	int rc = sqlite3Fts3TermSegReaderCursor(pCsr,
115162	pToken->z, pToken->n, pToken->isPrefix, &pToken->pSegcsr
115163	);
115164	if( rc!=SQLITE_OK ){
115165	*pRc = rc;
115166	return;
	@@ -115174,16 +115668,24 @@
115174	fts3EvalAllocateReaders(pCsr, pExpr->pRight, pnToken, pnOr, pRc);
115175	}
115176	}
115177	}
115178








115179	static void fts3EvalPhraseMergeToken(
115180	Fts3Table *pTab,
115181	Fts3Phrase *p,
115182	int iToken,
115183	char *pList,
115184	int nList
115185	){
115186	assert( iToken!=p->iDoclistToken );
115187
115188	if( pList==0 ){
115189	sqlite3_free(p->doclist.aAll);
	@@ -115228,13 +115730,19 @@
115228	}
115229
115230	if( iToken>p->iDoclistToken ) p->iDoclistToken = iToken;
115231	}
115232






115233	static int fts3EvalPhraseLoad(
115234	Fts3Cursor *pCsr,
115235	Fts3Phrase *p
115236	){
115237	Fts3Table pTab = (Fts3Table )pCsr->base.pVtab;
115238	int iToken;
115239	int rc = SQLITE_OK;
115240
	@@ -115243,11 +115751,11 @@
115243	assert( pToken->pDeferred==0 \|\| pToken->pSegcsr==0 );
115244
115245	if( pToken->pSegcsr ){
115246	int nThis = 0;
115247	char *pThis = 0;
115248	rc = fts3TermSelect(pTab, pToken, p->iColumn, 1, &nThis, &pThis);
115249	if( rc==SQLITE_OK ){
115250	fts3EvalPhraseMergeToken(pTab, p, iToken, pThis, nThis);
115251	}
115252	}
115253	assert( pToken->pSegcsr==0 );
	@@ -115254,18 +115762,26 @@
115254	}
115255
115256	return rc;
115257	}
115258










115259	static int fts3EvalDeferredPhrase(Fts3Cursor pCsr, Fts3Phrase pPhrase){
115260	int iToken;
115261	int rc = SQLITE_OK;
115262
115263	int nMaxUndeferred = pPhrase->iDoclistToken;
115264	char *aPoslist = 0;
115265	int nPoslist = 0;
115266	int iPrev = -1;
115267
115268	assert( pPhrase->doclist.bFreeList==0 );
115269
115270	for(iToken=0; rc==SQLITE_OK && iToken<pPhrase->nToken; iToken++){
115271	Fts3PhraseToken *pToken = &pPhrase->aToken[iToken];
	@@ -115307,10 +115823,11 @@
115307	iPrev = iToken;
115308	}
115309	}
115310
115311	if( iPrev>=0 ){

115312	if( nMaxUndeferred<0 ){
115313	pPhrase->doclist.pList = aPoslist;
115314	pPhrase->doclist.nList = nPoslist;
115315	pPhrase->doclist.iDocid = pCsr->iPrevId;
115316	pPhrase->doclist.bFreeList = 1;
	@@ -115355,13 +115872,19 @@
115355	/*
115356	** This function is called for each Fts3Phrase in a full-text query
115357	** expression to initialize the mechanism for returning rows. Once this
115358	** function has been called successfully on an Fts3Phrase, it may be
115359	** used with fts3EvalPhraseNext() to iterate through the matching docids.






115360	*/
115361	static int fts3EvalPhraseStart(Fts3Cursor pCsr, int bOptOk, Fts3Phrase p){
115362	int rc;
115363	Fts3PhraseToken *pFirst = &p->aToken[0];
115364	Fts3Table pTab = (Fts3Table )pCsr->base.pVtab;
115365
115366	if( pCsr->bDesc==pTab->bDescIdx
115367	&& bOptOk==1
	@@ -115385,11 +115908,17 @@
115385	return rc;
115386	}
115387
115388	/*
115389	** This function is used to iterate backwards (from the end to start)
115390	** through doclists.






115391	*/
115392	SQLITE_PRIVATE void sqlite3Fts3DoclistPrev(
115393	int bDescIdx, /* True if the doclist is desc */
115394	char aDoclist, / Pointer to entire doclist */
115395	int nDoclist, /* Length of aDoclist in bytes */
	@@ -115450,13 +115979,13 @@
115450	** If there is no "next" entry and no error occurs, then *pbEof is set to
115451	** 1 before returning. Otherwise, if no error occurs and the iterator is
115452	** successfully advanced, *pbEof is set to 0.
115453	*/
115454	static int fts3EvalPhraseNext(
115455	Fts3Cursor *pCsr,
115456	Fts3Phrase *p,
115457	u8 *pbEof
115458	){
115459	int rc = SQLITE_OK;
115460	Fts3Doclist *pDL = &p->doclist;
115461	Fts3Table pTab = (Fts3Table )pCsr->base.pVtab;
115462
	@@ -115498,14 +116027,14 @@
115498	fts3PoslistCopy(0, &pIter);
115499	pDL->nList = (pIter - pDL->pList);
115500
115501	/* pIter now points just past the 0x00 that terminates the position-
115502	** list for document pDL->iDocid. However, if this position-list was
115503	** edited in place by fts3EvalNearTrim2(), then pIter may not actually
115504	** point to the start of the next docid value. The following line deals
115505	** with this case by advancing pIter past the zero-padding added by
115506	** fts3EvalNearTrim2(). */
115507	while( pIter<pEnd && *pIter==0 ) pIter++;
115508
115509	pDL->pNextDocid = pIter;
115510	assert( pIter>=&pDL->aAll[pDL->nAll] \|\| *pIter );
115511	*pbEof = 0;
	@@ -115513,15 +116042,31 @@
115513	}
115514
115515	return rc;
115516	}
115517
















115518	static void fts3EvalStartReaders(
115519	Fts3Cursor *pCsr,
115520	Fts3Expr *pExpr,
115521	int bOptOk,
115522	int *pRc
115523	){
115524	if( pExpr && SQLITE_OK==*pRc ){
115525	if( pExpr->eType==FTSQUERY_PHRASE ){
115526	int i;
115527	int nToken = pExpr->pPhrase->nToken;
	@@ -115536,27 +116081,46 @@
115536	pExpr->bDeferred = (pExpr->pLeft->bDeferred && pExpr->pRight->bDeferred);
115537	}
115538	}
115539	}
115540












115541	typedef struct Fts3TokenAndCost Fts3TokenAndCost;
115542	struct Fts3TokenAndCost {
115543	Fts3Phrase pPhrase; / The phrase the token belongs to */
115544	int iToken; /* Position of token in phrase */
115545	Fts3PhraseToken pToken; / The token itself */
115546	Fts3Expr *pRoot;
115547	int nOvfl;
115548	int iCol; /* The column the token must match */
115549	};
115550







115551	static void fts3EvalTokenCosts(
115552	Fts3Cursor *pCsr,
115553	Fts3Expr *pRoot,
115554	Fts3Expr *pExpr,
115555	Fts3TokenAndCost **ppTC,
115556	Fts3Expr ***ppOr,
115557	int *pRc
115558	){
115559	if( *pRc==SQLITE_OK && pExpr ){
115560	if( pExpr->eType==FTSQUERY_PHRASE ){
115561	Fts3Phrase *pPhrase = pExpr->pPhrase;
115562	int i;
	@@ -115584,23 +116148,34 @@
115584	fts3EvalTokenCosts(pCsr, pRoot, pExpr->pRight, ppTC, ppOr, pRc);
115585	}
115586	}
115587	}
115588











115589	static int fts3EvalAverageDocsize(Fts3Cursor pCsr, int pnPage){
115590	if( pCsr->nRowAvg==0 ){
115591	/* The average document size, which is required to calculate the cost
115592	** of each doclist, has not yet been determined. Read the required
115593	** data from the %_stat table to calculate it.
115594	**
115595	** Entry 0 of the %_stat table is a blob containing (nCol+1) FTS3
115596	** varints, where nCol is the number of columns in the FTS3 table.
115597	** The first varint is the number of documents currently stored in
115598	** the table. The following nCol varints contain the total amount of
115599	** data stored in all rows of each column of the table, from left
115600	** to right.
115601	*/
115602	int rc;
115603	Fts3Table p = (Fts3Table)pCsr->base.pVtab;
115604	sqlite3_stmt *pStmt;
115605	sqlite3_int64 nDoc = 0;
115606	sqlite3_int64 nByte = 0;
	@@ -115631,109 +116206,151 @@
115631
115632	*pnPage = pCsr->nRowAvg;
115633	return SQLITE_OK;
115634	}
115635














115636	static int fts3EvalSelectDeferred(
115637	Fts3Cursor *pCsr,
115638	Fts3Expr *pRoot,
115639	Fts3TokenAndCost *aTC,
115640	int nTC
115641	){
115642	int nDocSize = 0;
115643	int nDocEst = 0;
115644	int rc = SQLITE_OK;
115645	Fts3Table pTab = (Fts3Table )pCsr->base.pVtab;
115646	int ii;




115647
115648	int nOvfl = 0;
115649	int nTerm = 0;
115650



115651	for(ii=0; ii<nTC; ii++){
115652	if( aTC[ii].pRoot==pRoot ){
115653	nOvfl += aTC[ii].nOvfl;
115654	nTerm++;
115655	}
115656	}
115657	if( nOvfl==0 \|\| nTerm<2 ) return SQLITE_OK;
115658

115659	rc = fts3EvalAverageDocsize(pCsr, &nDocSize);
115660
115661	for(ii=0; ii<nTerm && rc==SQLITE_OK; ii++){
115662	int jj;
115663	Fts3TokenAndCost *pTC = 0;
115664
115665	for(jj=0; jj<nTC; jj++){
115666	if( aTC[jj].pToken && aTC[jj].pRoot==pRoot
115667	&& (!pTC \|\| aTC[jj].nOvfl<pTC->nOvfl)

























115668	){
115669	pTC = &aTC[jj];
115670	}
115671	}
115672	assert( pTC );
115673
115674	/* At this point pTC points to the cheapest remaining token. */
115675	if( ii==0 ){
115676	if( pTC->nOvfl ){
115677	nDocEst = (pTC->nOvfl * pTab->nPgsz + pTab->nPgsz) / 10;
115678	}else{










115679	Fts3PhraseToken *pToken = pTC->pToken;
115680	int nList = 0;
115681	char *pList = 0;
115682	rc = fts3TermSelect(pTab, pToken, pTC->iCol, 1, &nList, &pList);
115683	assert( rc==SQLITE_OK \|\| pList==0 );
115684
115685	if( rc==SQLITE_OK ){
115686	nDocEst = fts3DoclistCountDocids(1, pList, nList);
115687	fts3EvalPhraseMergeToken(pTab, pTC->pPhrase, pTC->iToken,pList,nList);




115688	}
115689	}
115690	}else{
115691	if( pTC->nOvfl>=(nDocEst*nDocSize) ){
115692	Fts3PhraseToken *pToken = pTC->pToken;
115693	rc = sqlite3Fts3DeferToken(pCsr, pToken, pTC->iCol);
115694	fts3SegReaderCursorFree(pToken->pSegcsr);
115695	pToken->pSegcsr = 0;
115696	}
115697	nDocEst = 1 + (nDocEst/4);
115698	}
115699	pTC->pToken = 0;
115700	}
115701
115702	return rc;
115703	}
115704
115705	SQLITE_PRIVATE int sqlite3Fts3EvalStart(Fts3Cursor pCsr, Fts3Expr pExpr, int bOptOk){












115706	Fts3Table pTab = (Fts3Table )pCsr->base.pVtab;
115707	int rc = SQLITE_OK;
115708	int nToken = 0;
115709	int nOr = 0;
115710
115711	/* Allocate a MultiSegReader for each token in the expression. */
115712	fts3EvalAllocateReaders(pCsr, pExpr, &nToken, &nOr, &rc);
115713
115714	/* Call fts3EvalPhraseStart() on all phrases in the expression. TODO:
115715	** This call will eventually also be responsible for determining which
115716	** tokens are 'deferred' until the document text is loaded into memory.
115717	**
115718	** Each token in each phrase is dealt with using one of the following
115719	** three strategies:
115720	**
115721	** 1. Entire doclist loaded into memory as part of the
115722	** fts3EvalStartReaders() call.
115723	**
115724	** 2. Doclist loaded into memory incrementally, as part of each
115725	** sqlite3Fts3EvalNext() call.
115726	**
115727	** 3. Token doclist is never loaded. Instead, documents are loaded into
115728	** memory and scanned for the token as part of the sqlite3Fts3EvalNext()
115729	** call. This is known as a "deferred" token.
115730	*/
115731
115732	/* If bOptOk is true, check if there are any tokens that should be deferred.
115733	*/
115734	if( rc==SQLITE_OK && bOptOk && nToken>1 && pTab->bHasStat ){
115735	Fts3TokenAndCost *aTC;
115736	Fts3Expr **apOr;
115737	aTC = (Fts3TokenAndCost *)sqlite3_malloc(
115738	sizeof(Fts3TokenAndCost) * nToken
115739	+ sizeof(Fts3Expr ) nOr * 2
	@@ -115745,11 +116362,11 @@
115745	}else{
115746	int ii;
115747	Fts3TokenAndCost *pTC = aTC;
115748	Fts3Expr **ppOr = apOr;
115749
115750	fts3EvalTokenCosts(pCsr, 0, pExpr, &pTC, &ppOr, &rc);
115751	nToken = pTC-aTC;
115752	nOr = ppOr-apOr;
115753
115754	if( rc==SQLITE_OK ){
115755	rc = fts3EvalSelectDeferred(pCsr, 0, aTC, nToken);
	@@ -115760,25 +116377,50 @@
115760
115761	sqlite3_free(aTC);
115762	}
115763	}
115764
115765	fts3EvalStartReaders(pCsr, pExpr, bOptOk, &rc);
115766	return rc;
115767	}
115768
115769	static void fts3EvalZeroPoslist(Fts3Phrase *pPhrase){



115770	if( pPhrase->doclist.bFreeList ){
115771	sqlite3_free(pPhrase->doclist.pList);
115772	}
115773	pPhrase->doclist.pList = 0;
115774	pPhrase->doclist.nList = 0;
115775	pPhrase->doclist.bFreeList = 0;
115776	}
115777
115778	static int fts3EvalNearTrim2(
115779	int nNear,






















115780	char aTmp, / Temporary space to use */
115781	char *paPoslist, / IN/OUT: Position list */
115782	int pnToken, / IN/OUT: Tokens in phrase of paPoslist /
115783	Fts3Phrase pPhrase / The phrase object to trim the doclist of */
115784	){
	@@ -115806,10 +116448,176 @@
115806	}
115807
115808	return res;
115809	}
115810






































































































































































115811	static int fts3EvalNearTest(Fts3Expr pExpr, int pRc){
115812	int res = 1;
115813
115814	/* The following block runs if pExpr is the root of a NEAR query.
115815	** For example, the query:
	@@ -115827,11 +116635,11 @@
115827	** \| \|
115828	** "w" "x"
115829	**
115830	** The right-hand child of a NEAR node is always a phrase. The
115831	** left-hand child may be either a phrase or a NEAR node. There are
115832	** no exceptions to this.
115833	*/
115834	if( *pRc==SQLITE_OK
115835	&& pExpr->eType==FTSQUERY_NEAR
115836	&& pExpr->bEof==0
115837	&& (pExpr->pParent==0 \|\| pExpr->pParent->eType!=FTSQUERY_NEAR)
	@@ -115854,21 +116662,21 @@
115854	int nToken = p->pPhrase->nToken;
115855
115856	for(p=p->pParent;res && p && p->eType==FTSQUERY_NEAR; p=p->pParent){
115857	Fts3Phrase *pPhrase = p->pRight->pPhrase;
115858	int nNear = p->nNear;
115859	res = fts3EvalNearTrim2(nNear, aTmp, &aPoslist, &nToken, pPhrase);
115860	}
115861
115862	aPoslist = pExpr->pRight->pPhrase->doclist.pList;
115863	nToken = pExpr->pRight->pPhrase->nToken;
115864	for(p=pExpr->pLeft; p && res; p=p->pLeft){
115865	int nNear = p->pParent->nNear;
115866	Fts3Phrase *pPhrase = (
115867	p->eType==FTSQUERY_NEAR ? p->pRight->pPhrase : p->pPhrase
115868	);
115869	res = fts3EvalNearTrim2(nNear, aTmp, &aPoslist, &nToken, pPhrase);
115870	}
115871	}
115872
115873	sqlite3_free(aTmp);
115874	}
	@@ -115875,132 +116683,33 @@
115875
115876	return res;
115877	}
115878
115879	/*
115880	** This macro is used by the fts3EvalNext() function. The two arguments are
115881	** 64-bit docid values. If the current query is "ORDER BY docid ASC", then
115882	** the macro returns (i1 - i2). Or if it is "ORDER BY docid DESC", then
115883	** it returns (i2 - i1). This allows the same code to be used for merging
115884	** doclists in ascending or descending order.




115885	*/
115886	#define DOCID_CMP(i1, i2) ((pCsr->bDesc?-1:1) * (i1-i2))
115887
115888	static void fts3EvalNext(
115889	Fts3Cursor *pCsr,
115890	Fts3Expr *pExpr,
115891	int *pRc
115892	){
115893	if( *pRc==SQLITE_OK ){
115894	assert( pExpr->bEof==0 );
115895	pExpr->bStart = 1;
115896
115897	switch( pExpr->eType ){
115898	case FTSQUERY_NEAR:
115899	case FTSQUERY_AND: {
115900	Fts3Expr *pLeft = pExpr->pLeft;
115901	Fts3Expr *pRight = pExpr->pRight;
115902	assert( !pLeft->bDeferred \|\| !pRight->bDeferred );
115903	if( pLeft->bDeferred ){
115904	fts3EvalNext(pCsr, pRight, pRc);
115905	pExpr->iDocid = pRight->iDocid;
115906	pExpr->bEof = pRight->bEof;
115907	}else if( pRight->bDeferred ){
115908	fts3EvalNext(pCsr, pLeft, pRc);
115909	pExpr->iDocid = pLeft->iDocid;
115910	pExpr->bEof = pLeft->bEof;
115911	}else{
115912	fts3EvalNext(pCsr, pLeft, pRc);
115913	fts3EvalNext(pCsr, pRight, pRc);
115914
115915	while( !pLeft->bEof && !pRight->bEof && *pRc==SQLITE_OK ){
115916	sqlite3_int64 iDiff = DOCID_CMP(pLeft->iDocid, pRight->iDocid);
115917	if( iDiff==0 ) break;
115918	if( iDiff<0 ){
115919	fts3EvalNext(pCsr, pLeft, pRc);
115920	}else{
115921	fts3EvalNext(pCsr, pRight, pRc);
115922	}
115923	}
115924
115925	pExpr->iDocid = pLeft->iDocid;
115926	pExpr->bEof = (pLeft->bEof \|\| pRight->bEof);
115927	}
115928	break;
115929	}
115930
115931	case FTSQUERY_OR: {
115932	Fts3Expr *pLeft = pExpr->pLeft;
115933	Fts3Expr *pRight = pExpr->pRight;
115934	sqlite3_int64 iCmp = DOCID_CMP(pLeft->iDocid, pRight->iDocid);
115935
115936	assert( pLeft->bStart \|\| pLeft->iDocid==pRight->iDocid );
115937	assert( pRight->bStart \|\| pLeft->iDocid==pRight->iDocid );
115938
115939	if( pRight->bEof \|\| (pLeft->bEof==0 && iCmp<0) ){
115940	fts3EvalNext(pCsr, pLeft, pRc);
115941	}else if( pLeft->bEof \|\| (pRight->bEof==0 && iCmp>0) ){
115942	fts3EvalNext(pCsr, pRight, pRc);
115943	}else{
115944	fts3EvalNext(pCsr, pLeft, pRc);
115945	fts3EvalNext(pCsr, pRight, pRc);
115946	}
115947
115948	pExpr->bEof = (pLeft->bEof && pRight->bEof);
115949	iCmp = DOCID_CMP(pLeft->iDocid, pRight->iDocid);
115950	if( pRight->bEof \|\| (pLeft->bEof==0 && iCmp<0) ){
115951	pExpr->iDocid = pLeft->iDocid;
115952	}else{
115953	pExpr->iDocid = pRight->iDocid;
115954	}
115955
115956	break;
115957	}
115958
115959	case FTSQUERY_NOT: {
115960	Fts3Expr *pLeft = pExpr->pLeft;
115961	Fts3Expr *pRight = pExpr->pRight;
115962
115963	if( pRight->bStart==0 ){
115964	fts3EvalNext(pCsr, pRight, pRc);
115965	assert( *pRc!=SQLITE_OK \|\| pRight->bStart );
115966	}
115967
115968	fts3EvalNext(pCsr, pLeft, pRc);
115969	if( pLeft->bEof==0 ){
115970	while( !*pRc
115971	&& !pRight->bEof
115972	&& DOCID_CMP(pLeft->iDocid, pRight->iDocid)>0
115973	){
115974	fts3EvalNext(pCsr, pRight, pRc);
115975	}
115976	}
115977	pExpr->iDocid = pLeft->iDocid;
115978	pExpr->bEof = pLeft->bEof;
115979	break;
115980	}
115981
115982	default: {
115983	Fts3Phrase *pPhrase = pExpr->pPhrase;
115984	fts3EvalZeroPoslist(pPhrase);
115985	*pRc = fts3EvalPhraseNext(pCsr, pPhrase, &pExpr->bEof);
115986	pExpr->iDocid = pPhrase->doclist.iDocid;
115987	break;
115988	}
115989	}
115990	}
115991	}
115992
115993	static int fts3EvalDeferredTest(Fts3Cursor pCsr, Fts3Expr pExpr, int *pRc){
115994	int bHit = 1;
115995	if( *pRc==SQLITE_OK ){
115996	switch( pExpr->eType ){
115997	case FTSQUERY_NEAR:
115998	case FTSQUERY_AND:
115999	bHit = (
116000	fts3EvalDeferredTest(pCsr, pExpr->pLeft, pRc)
116001	&& fts3EvalDeferredTest(pCsr, pExpr->pRight, pRc)
116002	&& fts3EvalNearTest(pExpr, pRc)
116003	);
116004
116005	/* If the NEAR expression does not match any rows, zero the doclist for
116006	** all phrases involved in the NEAR. This is because the snippet(),
	@@ -116022,31 +116731,31 @@
116022	&& (pExpr->pParent==0 \|\| pExpr->pParent->eType!=FTSQUERY_NEAR)
116023	){
116024	Fts3Expr *p;
116025	for(p=pExpr; p->pPhrase==0; p=p->pLeft){
116026	if( p->pRight->iDocid==pCsr->iPrevId ){
116027	fts3EvalZeroPoslist(p->pRight->pPhrase);
116028	}
116029	}
116030	if( p->iDocid==pCsr->iPrevId ){
116031	fts3EvalZeroPoslist(p->pPhrase);
116032	}
116033	}
116034
116035	break;
116036
116037	case FTSQUERY_OR: {
116038	int bHit1 = fts3EvalDeferredTest(pCsr, pExpr->pLeft, pRc);
116039	int bHit2 = fts3EvalDeferredTest(pCsr, pExpr->pRight, pRc);
116040	bHit = bHit1 \|\| bHit2;
116041	break;
116042	}
116043
116044	case FTSQUERY_NOT:
116045	bHit = (
116046	fts3EvalDeferredTest(pCsr, pExpr->pLeft, pRc)
116047	&& !fts3EvalDeferredTest(pCsr, pExpr->pRight, pRc)
116048	);
116049	break;
116050
116051	default: {
116052	if( pCsr->pDeferred
	@@ -116053,11 +116762,11 @@
116053	&& (pExpr->iDocid==pCsr->iPrevId \|\| pExpr->bDeferred)
116054	){
116055	Fts3Phrase *pPhrase = pExpr->pPhrase;
116056	assert( pExpr->bDeferred \|\| pPhrase->doclist.bFreeList==0 );
116057	if( pExpr->bDeferred ){
116058	fts3EvalZeroPoslist(pPhrase);
116059	}
116060	*pRc = fts3EvalDeferredPhrase(pCsr, pPhrase);
116061	bHit = (pPhrase->doclist.pList!=0);
116062	pExpr->iDocid = pCsr->iPrevId;
116063	}else{
	@@ -116069,31 +116778,53 @@
116069	}
116070	return bHit;
116071	}
116072
116073	/*
116074	** Return 1 if both of the following are true:













116075	**
116076	** 1. *pRc is SQLITE_OK when this function returns, and
116077	**
116078	** 2. After scanning the current FTS table row for the deferred tokens,
116079	** it is determined that the row does not match the query.
116080	**
116081	** Or, if no error occurs and it seems the current row does match the FTS
116082	** query, return 0.
116083	*/
116084	static int fts3EvalLoadDeferred(Fts3Cursor pCsr, int pRc){
116085	int rc = *pRc;
116086	int bMiss = 0;
116087	if( rc==SQLITE_OK ){







116088	if( pCsr->pDeferred ){
116089	rc = fts3CursorSeek(0, pCsr);
116090	if( rc==SQLITE_OK ){
116091	rc = sqlite3Fts3CacheDeferredDoclists(pCsr);
116092	}
116093	}
116094	bMiss = (0==fts3EvalDeferredTest(pCsr, pCsr->pExpr, &rc));


116095	sqlite3Fts3FreeDeferredDoclists(pCsr);
116096	*pRc = rc;
116097	}
116098	return (rc==SQLITE_OK && bMiss);
116099	}
	@@ -116100,11 +116831,11 @@
116100
116101	/*
116102	** Advance to the next document that matches the FTS expression in
116103	** Fts3Cursor.pExpr.
116104	*/
116105	SQLITE_PRIVATE int sqlite3Fts3EvalNext(Fts3Cursor *pCsr){
116106	int rc = SQLITE_OK; /* Return Code */
116107	Fts3Expr *pExpr = pCsr->pExpr;
116108	assert( pCsr->isEof==0 );
116109	if( pExpr==0 ){
116110	pCsr->isEof = 1;
	@@ -116112,23 +116843,23 @@
116112	do {
116113	if( pCsr->isRequireSeek==0 ){
116114	sqlite3_reset(pCsr->pStmt);
116115	}
116116	assert( sqlite3_data_count(pCsr->pStmt)==0 );
116117	fts3EvalNext(pCsr, pExpr, &rc);
116118	pCsr->isEof = pExpr->bEof;
116119	pCsr->isRequireSeek = 1;
116120	pCsr->isMatchinfoNeeded = 1;
116121	pCsr->iPrevId = pExpr->iDocid;
116122	}while( pCsr->isEof==0 && fts3EvalLoadDeferred(pCsr, &rc) );
116123	}
116124	return rc;
116125	}
116126
116127	/*
116128	** Restart interation for expression pExpr so that the next call to
116129	** sqlite3Fts3EvalNext() visits the first row. Do not allow incremental
116130	** loading or merging of phrase doclists for this iteration.
116131	**
116132	** If *pRc is other than SQLITE_OK when this function is called, it is
116133	** a no-op. If an error occurs within this function, *pRc is set to an
116134	** SQLite error code before returning.
	@@ -116140,11 +116871,11 @@
116140	){
116141	if( pExpr && *pRc==SQLITE_OK ){
116142	Fts3Phrase *pPhrase = pExpr->pPhrase;
116143
116144	if( pPhrase ){
116145	fts3EvalZeroPoslist(pPhrase);
116146	if( pPhrase->bIncr ){
116147	assert( pPhrase->nToken==1 );
116148	assert( pPhrase->aToken[0].pSegcsr );
116149	sqlite3Fts3MsrIncrRestart(pPhrase->aToken[0].pSegcsr);
116150	*pRc = fts3EvalPhraseStart(pCsr, 0, pPhrase);
	@@ -116256,18 +116987,18 @@
116256	/* Ensure the %_content statement is reset. */
116257	if( pCsr->isRequireSeek==0 ) sqlite3_reset(pCsr->pStmt);
116258	assert( sqlite3_data_count(pCsr->pStmt)==0 );
116259
116260	/* Advance to the next document */
116261	fts3EvalNext(pCsr, pRoot, &rc);
116262	pCsr->isEof = pRoot->bEof;
116263	pCsr->isRequireSeek = 1;
116264	pCsr->isMatchinfoNeeded = 1;
116265	pCsr->iPrevId = pRoot->iDocid;
116266	}while( pCsr->isEof==0
116267	&& pRoot->eType==FTSQUERY_NEAR
116268	&& fts3EvalLoadDeferred(pCsr, &rc)
116269	);
116270
116271	if( rc==SQLITE_OK && pCsr->isEof==0 ){
116272	fts3EvalUpdateCounts(pRoot);
116273	}
	@@ -116285,14 +117016,14 @@
116285	**
116286	** do {...} while( pRoot->iDocid<iDocid && rc==SQLITE_OK );
116287	*/
116288	fts3EvalRestart(pCsr, pRoot, &rc);
116289	do {
116290	fts3EvalNext(pCsr, pRoot, &rc);
116291	assert( pRoot->bEof==0 );
116292	}while( pRoot->iDocid!=iDocid && rc==SQLITE_OK );
116293	fts3EvalLoadDeferred(pCsr, &rc);
116294	}
116295	}
116296	return rc;
116297	}
116298
	@@ -116419,18 +117150,32 @@
116419	*/
116420	SQLITE_PRIVATE void sqlite3Fts3EvalPhraseCleanup(Fts3Phrase *pPhrase){
116421	if( pPhrase ){
116422	int i;
116423	sqlite3_free(pPhrase->doclist.aAll);
116424	fts3EvalZeroPoslist(pPhrase);
116425	memset(&pPhrase->doclist, 0, sizeof(Fts3Doclist));
116426	for(i=0; i<pPhrase->nToken; i++){
116427	fts3SegReaderCursorFree(pPhrase->aToken[i].pSegcsr);
116428	pPhrase->aToken[i].pSegcsr = 0;
116429	}
116430	}
116431	}














116432
116433	#endif
116434
116435	/************ End of fts3.c **********************************************/
116436	/************ Begin file fts3_aux.c **************************************/
	@@ -118917,14 +119662,10 @@
118917	** (in which case SQLITE_CORE is not defined), or
118918	**
118919	** * The FTS3 module is being built into the core of
118920	** SQLite (in which case SQLITE_ENABLE_FTS3 is defined).
118921	*/
118922	#ifndef SQLITE_CORE
118923	SQLITE_EXTENSION_INIT1
118924	#endif
118925
118926	#if !defined(SQLITE_CORE) \|\| defined(SQLITE_ENABLE_FTS3)
118927
118928
118929	/*
118930	** Implementation of the SQL scalar function for accessing the underlying
118931

	--- src/sqlite3.c
	+++ src/sqlite3.c
	@@ -1,8 +1,8 @@
1	/******************************************************************************
2	** This file is an amalgamation of many separate C source files from SQLite
3	** version 3.7.8. By combining all the individual C code files into this
4	** single large file, the entire code can be compiled as a single translation
5	** unit. This allows many compilers to do optimizations that would not be
6	** possible if the files were compiled separately. Performance improvements
7	** of 5% or more are commonly seen when SQLite is compiled as a single
8	** translation unit.
	@@ -648,13 +648,13 @@
648	**
649	** See also: [sqlite3_libversion()],
650	** [sqlite3_libversion_number()], [sqlite3_sourceid()],
651	** [sqlite_version()] and [sqlite_source_id()].
652	*/
653	#define SQLITE_VERSION "3.7.8"
654	#define SQLITE_VERSION_NUMBER 3007008
655	#define SQLITE_SOURCE_ID "2011-07-19 18:29:00 ed5f0aad6b21066bacd01521e82c22e96991f400"
656
657	/*
658	** CAPI3REF: Run-Time Library Version Numbers
659	** KEYWORDS: sqlite3_version, sqlite3_sourceid
660	**
	@@ -1282,20 +1282,37 @@
1282	** when [PRAGMA synchronous \| PRAGMA synchronous=OFF] is set, but most
1283	** VFSes do not need this signal and should silently ignore this opcode.
1284	** Applications should not call [sqlite3_file_control()] with this
1285	** opcode as doing so may disrupt the operation of the specialized VFSes
1286	** that do require it.
1287	**
1288	** ^The [SQLITE_FCNTL_WIN32_AV_RETRY] opcode is used to configure automatic
1289	** retry counts and intervals for certain disk I/O operations for the
1290	** windows [VFS] in order to work to provide robustness against
1291	** anti-virus programs. By default, the windows VFS will retry file read,
1292	** file write, and file delete opertions up to 10 times, with a delay
1293	** of 25 milliseconds before the first retry and with the delay increasing
1294	** by an additional 25 milliseconds with each subsequent retry. This
1295	** opcode allows those to values (10 retries and 25 milliseconds of delay)
1296	** to be adjusted. The values are changed for all database connections
1297	** within the same process. The argument is a pointer to an array of two
1298	** integers where the first integer i the new retry count and the second
1299	** integer is the delay. If either integer is negative, then the setting
1300	** is not changed but instead the prior value of that setting is written
1301	** into the array entry, allowing the current retry settings to be
1302	** interrogated. The zDbName parameter is ignored.
1303	**
1304	*/
1305	#define SQLITE_FCNTL_LOCKSTATE 1
1306	#define SQLITE_GET_LOCKPROXYFILE 2
1307	#define SQLITE_SET_LOCKPROXYFILE 3
1308	#define SQLITE_LAST_ERRNO 4
1309	#define SQLITE_FCNTL_SIZE_HINT 5
1310	#define SQLITE_FCNTL_CHUNK_SIZE 6
1311	#define SQLITE_FCNTL_FILE_POINTER 7
1312	#define SQLITE_FCNTL_SYNC_OMITTED 8
1313	#define SQLITE_FCNTL_WIN32_AV_RETRY 9
1314
1315	/*
1316	** CAPI3REF: Mutex Handle
1317	**
1318	** The mutex module within SQLite defines [sqlite3_mutex] to be an
	@@ -9558,10 +9575,11 @@
9575	#define SQLITE_IndexSearch 0x08 /* Disable indexes for searching */
9576	#define SQLITE_IndexCover 0x10 /* Disable index covering table */
9577	#define SQLITE_GroupByOrder 0x20 /* Disable GROUPBY cover of ORDERBY */
9578	#define SQLITE_FactorOutConst 0x40 /* Disable factoring out constants */
9579	#define SQLITE_IdxRealAsInt 0x80 /* Store REAL as INT in indices */
9580	#define SQLITE_DistinctOpt 0x80 /* DISTINCT using indexes */
9581	#define SQLITE_OptMask 0xff /* Mask of all disablable opts */
9582
9583	/*
9584	** Possible values for the sqlite.magic field.
9585	** The numbers are obtained at random and have no special meaning, other
	@@ -10449,10 +10467,11 @@
10467	Table pTab; / An SQL table corresponding to zName */
10468	Select pSelect; / A SELECT statement used in place of a table name */
10469	u8 isPopulated; /* Temporary table associated with SELECT is populated */
10470	u8 jointype; /* Type of join between this able and the previous */
10471	u8 notIndexed; /* True if there is a NOT INDEXED clause */
10472	u8 isCorrelated; /* True if sub-query is correlated */
10473	#ifndef SQLITE_OMIT_EXPLAIN
10474	u8 iSelectId; /* If pSelect!=0, the id of the sub-select in EQP */
10475	#endif
10476	int iCursor; /* The VDBE cursor number used to access this table */
10477	Expr pOn; / The ON clause of a join */
	@@ -10568,10 +10587,11 @@
10587	struct WhereInfo {
10588	Parse pParse; / Parsing and code generating context */
10589	u16 wctrlFlags; /* Flags originally passed to sqlite3WhereBegin() */
10590	u8 okOnePass; /* Ok to use one-pass algorithm for UPDATE or DELETE */
10591	u8 untestedTerms; /* Not all WHERE terms resolved by outer loop */
10592	u8 eDistinct;
10593	SrcList pTabList; / List of tables in the join */
10594	int iTop; /* The very beginning of the WHERE loop */
10595	int iContinue; /* Jump here to continue with next record */
10596	int iBreak; /* Jump here to break out of the loop */
10597	int nLevel; /* Number of nested loop */
	@@ -10579,10 +10599,13 @@
10599	double savedNQueryLoop; /* pParse->nQueryLoop outside the WHERE loop */
10600	double nRowOut; /* Estimated number of output rows */
10601	WhereLevel a[1]; /* Information about each nest loop in WHERE */
10602	};
10603
10604	#define WHERE_DISTINCT_UNIQUE 1
10605	#define WHERE_DISTINCT_ORDERED 2
10606
10607	/*
10608	** A NameContext defines a context in which to resolve table and column
10609	** names. The context consists of a list of tables (the pSrcList) field and
10610	** a list of named expression (pEList). The named expression list may
10611	** be NULL. The pSrc corresponds to the FROM clause of a SELECT or
	@@ -11340,11 +11363,11 @@
11363	#if defined(SQLITE_ENABLE_UPDATE_DELETE_LIMIT) && !defined(SQLITE_OMIT_SUBQUERY)
11364	SQLITE_PRIVATE Expr sqlite3LimitWhere(Parse , SrcList , Expr , ExprList , Expr , Expr , char );
11365	#endif
11366	SQLITE_PRIVATE void sqlite3DeleteFrom(Parse, SrcList, Expr*);
11367	SQLITE_PRIVATE void sqlite3Update(Parse, SrcList, ExprList, Expr, int);
11368	SQLITE_PRIVATE WhereInfo sqlite3WhereBegin(Parse, SrcList, Expr, ExprList*,ExprList,u16);
11369	SQLITE_PRIVATE void sqlite3WhereEnd(WhereInfo*);
11370	SQLITE_PRIVATE int sqlite3ExprCodeGetColumn(Parse, Table, int, int, int);
11371	SQLITE_PRIVATE void sqlite3ExprCodeGetColumnOfTable(Vdbe, Table, int, int, int);
11372	SQLITE_PRIVATE void sqlite3ExprCodeMove(Parse*, int, int, int);
11373	SQLITE_PRIVATE void sqlite3ExprCodeCopy(Parse*, int, int, int);
	@@ -15841,11 +15864,11 @@
15864	** Free an outstanding memory allocation.
15865	**
15866	** This function assumes that the necessary mutexes, if any, are
15867	** already held by the caller. Hence "Unsafe".
15868	*/
15869	static void memsys3FreeUnsafe(void *pOld){
15870	Mem3Block p = (Mem3Block)pOld;
15871	int i;
15872	u32 size, x;
15873	assert( sqlite3_mutex_held(mem3.mutex) );
15874	assert( p>mem3.aPool && p<&mem3.aPool[mem3.nPool] );
	@@ -15916,21 +15939,21 @@
15939	}
15940
15941	/*
15942	** Free memory.
15943	*/
15944	static void memsys3Free(void *pPrior){
15945	assert( pPrior );
15946	memsys3Enter();
15947	memsys3FreeUnsafe(pPrior);
15948	memsys3Leave();
15949	}
15950
15951	/*
15952	** Change the size of an existing memory allocation
15953	*/
15954	static void memsys3Realloc(void pPrior, int nBytes){
15955	int nOld;
15956	void *p;
15957	if( pPrior==0 ){
15958	return sqlite3_malloc(nBytes);
15959	}
	@@ -25143,11 +25166,13 @@
25166	case EINVAL:
25167	case ENOTCONN:
25168	case ENODEV:
25169	case ENXIO:
25170	case ENOENT:
25171	#ifdef ESTALE /* ESTALE is not defined on Interix systems */
25172	case ESTALE:
25173	#endif
25174	case ENOSYS:
25175	/* these should force the client to close the file and reconnect */
25176
25177	default:
25178	return sqliteIOErr;
	@@ -31820,10 +31845,58 @@
31845	iLine, iErrno, zFunc, zPath, zMsg
31846	);
31847
31848	return errcode;
31849	}
31850
31851	/*
31852	** The number of times that a ReadFile(), WriteFile(), and DeleteFile()
31853	** will be retried following a locking error - probably caused by
31854	** antivirus software. Also the initial delay before the first retry.
31855	** The delay increases linearly with each retry.
31856	*/
31857	#ifndef SQLITE_WIN32_IOERR_RETRY
31858	# define SQLITE_WIN32_IOERR_RETRY 10
31859	#endif
31860	#ifndef SQLITE_WIN32_IOERR_RETRY_DELAY
31861	# define SQLITE_WIN32_IOERR_RETRY_DELAY 25
31862	#endif
31863	static int win32IoerrRetry = SQLITE_WIN32_IOERR_RETRY;
31864	static int win32IoerrRetryDelay = SQLITE_WIN32_IOERR_RETRY_DELAY;
31865
31866	/*
31867	** If a ReadFile() or WriteFile() error occurs, invoke this routine
31868	** to see if it should be retried. Return TRUE to retry. Return FALSE
31869	** to give up with an error.
31870	*/
31871	static int retryIoerr(int *pnRetry){
31872	DWORD e;
31873	if( *pnRetry>=win32IoerrRetry ){
31874	return 0;
31875	}
31876	e = GetLastError();
31877	if( e==ERROR_ACCESS_DENIED \|\|
31878	e==ERROR_LOCK_VIOLATION \|\|
31879	e==ERROR_SHARING_VIOLATION ){
31880	Sleep(win32IoerrRetryDelay(1+pnRetry));
31881	++*pnRetry;
31882	return 1;
31883	}
31884	return 0;
31885	}
31886
31887	/*
31888	** Log a I/O error retry episode.
31889	*/
31890	static void logIoerr(int nRetry){
31891	if( nRetry ){
31892	sqlite3_log(SQLITE_IOERR,
31893	"delayed %dms for lock/sharing conflict",
31894	win32IoerrRetryDelaynRetry(nRetry+1)/2
31895	);
31896	}
31897	}
31898
31899	#if SQLITE_OS_WINCE
31900	/*************************************************************************
31901	** This section contains code for WinCE only.
31902	*/
	@@ -32238,22 +32311,25 @@
32311	int amt, /* Number of bytes to read */
32312	sqlite3_int64 offset /* Begin reading at this offset */
32313	){
32314	winFile pFile = (winFile)id; /* file handle */
32315	DWORD nRead; /* Number of bytes actually read from file */
32316	int nRetry = 0; /* Number of retrys */
32317
32318	assert( id!=0 );
32319	SimulateIOError(return SQLITE_IOERR_READ);
32320	OSTRACE(("READ %d lock=%d\n", pFile->h, pFile->locktype));
32321
32322	if( seekWinFile(pFile, offset) ){
32323	return SQLITE_FULL;
32324	}
32325	while( !ReadFile(pFile->h, pBuf, amt, &nRead, 0) ){
32326	if( retryIoerr(&nRetry) ) continue;
32327	pFile->lastErrno = GetLastError();
32328	return winLogError(SQLITE_IOERR_READ, "winRead", pFile->zPath);
32329	}
32330	logIoerr(nRetry);
32331	if( nRead<(DWORD)amt ){
32332	/* Unread parts of the buffer must be zero-filled */
32333	memset(&((char*)pBuf)[nRead], 0, amt-nRead);
32334	return SQLITE_IOERR_SHORT_READ;
32335	}
	@@ -32271,10 +32347,11 @@
32347	int amt, /* Number of bytes to write */
32348	sqlite3_int64 offset /* Offset into the file to begin writing at */
32349	){
32350	int rc; /* True if error has occured, else false */
32351	winFile pFile = (winFile)id; /* File handle */
32352	int nRetry = 0; /* Number of retries */
32353
32354	assert( amt>0 );
32355	assert( pFile );
32356	SimulateIOError(return SQLITE_IOERR_WRITE);
32357	SimulateDiskfullError(return SQLITE_FULL);
	@@ -32285,11 +32362,16 @@
32362	if( rc==0 ){
32363	u8 aRem = (u8 )pBuf; /* Data yet to be written */
32364	int nRem = amt; /* Number of bytes yet to be written */
32365	DWORD nWrite; /* Bytes written by each WriteFile() call */
32366
32367	while( nRem>0 ){
32368	if( !WriteFile(pFile->h, aRem, nRem, &nWrite, 0) ){
32369	if( retryIoerr(&nRetry) ) continue;
32370	break;
32371	}
32372	if( nWrite<=0 ) break;
32373	aRem += nWrite;
32374	nRem -= nWrite;
32375	}
32376	if( nRem>0 ){
32377	pFile->lastErrno = GetLastError();
	@@ -32301,10 +32383,12 @@
32383	if( ( pFile->lastErrno==ERROR_HANDLE_DISK_FULL )
32384	\|\| ( pFile->lastErrno==ERROR_DISK_FULL )){
32385	return SQLITE_FULL;
32386	}
32387	return winLogError(SQLITE_IOERR_WRITE, "winWrite", pFile->zPath);
32388	}else{
32389	logIoerr(nRetry);
32390	}
32391	return SQLITE_OK;
32392	}
32393
32394	/*
	@@ -32716,10 +32800,24 @@
32800	SimulateIOErrorBenign(0);
32801	return SQLITE_OK;
32802	}
32803	case SQLITE_FCNTL_SYNC_OMITTED: {
32804	return SQLITE_OK;
32805	}
32806	case SQLITE_FCNTL_WIN32_AV_RETRY: {
32807	int a = (int)pArg;
32808	if( a[0]>0 ){
32809	win32IoerrRetry = a[0];
32810	}else{
32811	a[0] = win32IoerrRetry;
32812	}
32813	if( a[1]>0 ){
32814	win32IoerrRetryDelay = a[1];
32815	}else{
32816	a[1] = win32IoerrRetryDelay;
32817	}
32818	return SQLITE_OK;
32819	}
32820	}
32821	return SQLITE_NOTFOUND;
32822	}
32823
	@@ -33734,19 +33832,17 @@
33832	** file open, we will be unable to delete it. To work around this
33833	** problem, we delay 100 milliseconds and try to delete again. Up
33834	** to MX_DELETION_ATTEMPTs deletion attempts are run before giving
33835	** up and returning an error.
33836	*/

33837	static int winDelete(
33838	sqlite3_vfs pVfs, / Not used on win32 */
33839	const char zFilename, / Name of file to delete */
33840	int syncDir /* Not used on win32 */
33841	){
33842	int cnt = 0;
33843	int rc;

33844	void *zConverted;
33845	UNUSED_PARAMETER(pVfs);
33846	UNUSED_PARAMETER(syncDir);
33847
33848	SimulateIOError(return SQLITE_IOERR_DELETE);
	@@ -33753,38 +33849,34 @@
33849	zConverted = convertUtf8Filename(zFilename);
33850	if( zConverted==0 ){
33851	return SQLITE_NOMEM;
33852	}
33853	if( isNT() ){
33854	rc = 1;
33855	while( GetFileAttributesW(zConverted)!=INVALID_FILE_ATTRIBUTES &&
33856	(rc = DeleteFileW(zConverted))==0 && retryIoerr(&cnt) ){}
33857	rc = rc ? SQLITE_OK : SQLITE_ERROR;


33858	/* isNT() is 1 if SQLITE_OS_WINCE==1, so this else is never executed.
33859	** Since the ASCII version of these Windows API do not exist for WINCE,
33860	** it's important to not reference them for WINCE builds.
33861	*/
33862	#if SQLITE_OS_WINCE==0
33863	}else{
33864	rc = 1;
33865	while( GetFileAttributesA(zConverted)!=INVALID_FILE_ATTRIBUTES &&
33866	(rc = DeleteFileA(zConverted))==0 && retryIoerr(&cnt) ){}
33867	rc = rc ? SQLITE_OK : SQLITE_ERROR;


33868	#endif
33869	}
33870	if( rc ){
33871	rc = winLogError(SQLITE_IOERR_DELETE, "winDelete", zFilename);
33872	}else{
33873	logIoerr(cnt);
33874	}
33875	free(zConverted);
33876	OSTRACE(("DELETE \"%s\" %s\n", zFilename, (rc ? "failed" : "ok" )));
33877	return rc;





33878	}
33879
33880	/*
33881	** Check the existance and status of a file.
33882	*/
	@@ -59049,10 +59141,11 @@
59141	nMem = 10;
59142	}
59143	memset(zCsr, 0, zEnd-zCsr);
59144	zCsr += (zCsr - (u8*)0)&7;
59145	assert( EIGHT_BYTE_ALIGNMENT(zCsr) );
59146	p->expired = 0;
59147
59148	/* Memory for registers, parameters, cursor, etc, is allocated in two
59149	** passes. On the first pass, we try to reuse unused space at the
59150	** end of the opcode array. If we are unable to satisfy all memory
59151	** requirements by reusing the opcode array tail, then the second
	@@ -61277,11 +61370,11 @@
61370	sqlite3_mutex_enter(db->mutex);
61371	while( (rc = sqlite3Step(v))==SQLITE_SCHEMA
61372	&& cnt++ < SQLITE_MAX_SCHEMA_RETRY
61373	&& (rc2 = rc = sqlite3Reprepare(v))==SQLITE_OK ){
61374	sqlite3_reset(pStmt);
61375	assert( v->expired==0 );
61376	}
61377	if( rc2!=SQLITE_OK && ALWAYS(v->isPrepareV2) && ALWAYS(db->pErr) ){
61378	/* This case occurs after failing to recompile an sql statement.
61379	** The error message from the SQL compiler has already been loaded
61380	** into the database handle. This block copies the error message
	@@ -65968,11 +66061,11 @@
66061	** automatically created table with root-page 1 (an BLOB_INTKEY table).
66062	*/
66063	if( pOp->p4.pKeyInfo ){
66064	int pgno;
66065	assert( pOp->p4type==P4_KEYINFO );
66066	rc = sqlite3BtreeCreateTable(u.ax.pCx->pBt, &pgno, BTREE_BLOBKEY \| pOp->p5);
66067	if( rc==SQLITE_OK ){
66068	assert( pgno==MASTER_ROOT+1 );
66069	rc = sqlite3BtreeCursor(u.ax.pCx->pBt, pgno, 1,
66070	(KeyInfo*)pOp->p4.z, u.ax.pCx->pCursor);
66071	u.ax.pCx->pKeyInfo = pOp->p4.pKeyInfo;
	@@ -71007,15 +71100,29 @@
71100	/* Recursively resolve names in all subqueries
71101	*/
71102	for(i=0; i<p->pSrc->nSrc; i++){
71103	struct SrcList_item *pItem = &p->pSrc->a[i];
71104	if( pItem->pSelect ){
71105	NameContext pNC; / Used to iterate name contexts */
71106	int nRef = 0; /* Refcount for pOuterNC and outer contexts */
71107	const char *zSavedContext = pParse->zAuthContext;
71108
71109	/* Count the total number of references to pOuterNC and all of its
71110	** parent contexts. After resolving references to expressions in
71111	** pItem->pSelect, check if this value has changed. If so, then
71112	** SELECT statement pItem->pSelect must be correlated. Set the
71113	** pItem->isCorrelated flag if this is the case. */
71114	for(pNC=pOuterNC; pNC; pNC=pNC->pNext) nRef += pNC->nRef;
71115
71116	if( pItem->zName ) pParse->zAuthContext = pItem->zName;
71117	sqlite3ResolveSelectNames(pParse, pItem->pSelect, pOuterNC);
71118	pParse->zAuthContext = zSavedContext;
71119	if( pParse->nErr \|\| db->mallocFailed ) return WRC_Abort;
71120
71121	for(pNC=pOuterNC; pNC; pNC=pNC->pNext) nRef -= pNC->nRef;
71122	assert( pItem->isCorrelated==0 && nRef<=0 );
71123	pItem->isCorrelated = (nRef!=0);
71124	}
71125	}
71126
71127	/* If there are no aggregate functions in the result-set, and no GROUP BY
71128	** expression, do not allow aggregates in any of the other expressions.
	@@ -72119,10 +72226,11 @@
72226	pNewItem->zName = sqlite3DbStrDup(db, pOldItem->zName);
72227	pNewItem->zAlias = sqlite3DbStrDup(db, pOldItem->zAlias);
72228	pNewItem->jointype = pOldItem->jointype;
72229	pNewItem->iCursor = pOldItem->iCursor;
72230	pNewItem->isPopulated = pOldItem->isPopulated;
72231	pNewItem->isCorrelated = pOldItem->isCorrelated;
72232	pNewItem->zIndex = sqlite3DbStrDup(db, pOldItem->zIndex);
72233	pNewItem->notIndexed = pOldItem->notIndexed;
72234	pNewItem->pIndex = pOldItem->pIndex;
72235	pTab = pNewItem->pTab = pOldItem->pTab;
72236	if( pTab ){
	@@ -80126,11 +80234,11 @@
80234	if( pStart ){
80235	assert( pEnd!=0 );
80236	/* A named index with an explicit CREATE INDEX statement */
80237	zStmt = sqlite3MPrintf(db, "CREATE%s INDEX %.*s",
80238	onError==OE_None ? "" : " UNIQUE",
80239	(int)(pEnd->z - pName->z) + 1,
80240	pName->z);
80241	}else{
80242	/* An automatic index created by a PRIMARY KEY or UNIQUE constraint */
80243	/* zStmt = sqlite3MPrintf(""); */
80244	zStmt = 0;
	@@ -81921,11 +82029,13 @@
82029	int regRowid; /* Actual register containing rowids */
82030
82031	/* Collect rowids of every row to be deleted.
82032	*/
82033	sqlite3VdbeAddOp2(v, OP_Null, 0, iRowSet);
82034	pWInfo = sqlite3WhereBegin(
82035	pParse, pTabList, pWhere, 0, 0, WHERE_DUPLICATES_OK
82036	);
82037	if( pWInfo==0 ) goto delete_from_cleanup;
82038	regRowid = sqlite3ExprCodeGetColumn(pParse, pTab, -1, iCur, iRowid);
82039	sqlite3VdbeAddOp2(v, OP_RowSetAdd, iRowSet, regRowid);
82040	if( db->flags & SQLITE_CountRows ){
82041	sqlite3VdbeAddOp2(v, OP_AddImm, memCnt, 1);
	@@ -84368,11 +84478,11 @@
84478
84479	/* Create VDBE to loop through the entries in pSrc that match the WHERE
84480	** clause. If the constraint is not deferred, throw an exception for
84481	** each row found. Otherwise, for deferred constraints, increment the
84482	** deferred constraint counter by nIncr for each row selected. */
84483	pWInfo = sqlite3WhereBegin(pParse, pSrc, pWhere, 0, 0, 0);
84484	if( nIncr>0 && pFKey->isDeferred==0 ){
84485	sqlite3ParseToplevel(pParse)->mayAbort = 1;
84486	}
84487	sqlite3VdbeAddOp2(v, OP_FkCounter, pFKey->isDeferred, nIncr);
84488	if( pWInfo ){
	@@ -87235,10 +87345,13 @@
87345	int (strnicmp)(const char,const char*,int);
87346	int (unlock_notify)(sqlite3,void()(void,int),void);
87347	int (wal_autocheckpoint)(sqlite3,int);
87348	int (wal_checkpoint)(sqlite3,const char*);
87349	void (wal_hook)(sqlite3,int()(void,sqlite3,const char,int),void);
87350	int (blob_reopen)(sqlite3_blob,sqlite3_int64);
87351	int (vtab_config)(sqlite3,int op,...);
87352	int (vtab_on_conflict)(sqlite3);
87353	};
87354
87355	/*
87356	** The following macros redefine the API routines so that they are
87357	** redirected throught the global sqlite3_api structure.
	@@ -87435,10 +87548,13 @@
87548	#define sqlite3_strnicmp sqlite3_api->strnicmp
87549	#define sqlite3_unlock_notify sqlite3_api->unlock_notify
87550	#define sqlite3_wal_autocheckpoint sqlite3_api->wal_autocheckpoint
87551	#define sqlite3_wal_checkpoint sqlite3_api->wal_checkpoint
87552	#define sqlite3_wal_hook sqlite3_api->wal_hook
87553	#define sqlite3_blob_reopen sqlite3_api->blob_reopen
87554	#define sqlite3_vtab_config sqlite3_api->vtab_config
87555	#define sqlite3_vtab_on_conflict sqlite3_api->vtab_on_conflict
87556	#endif /* SQLITE_CORE */
87557
87558	#define SQLITE_EXTENSION_INIT1 const sqlite3_api_routines *sqlite3_api = 0;
87559	#define SQLITE_EXTENSION_INIT2(v) sqlite3_api = v;
87560
	@@ -87509,10 +87625,12 @@
87625
87626	#ifdef SQLITE_OMIT_VIRTUALTABLE
87627	# define sqlite3_create_module 0
87628	# define sqlite3_create_module_v2 0
87629	# define sqlite3_declare_vtab 0
87630	# define sqlite3_vtab_config 0
87631	# define sqlite3_vtab_on_conflict 0
87632	#endif
87633
87634	#ifdef SQLITE_OMIT_SHARED_CACHE
87635	# define sqlite3_enable_shared_cache 0
87636	#endif
	@@ -87532,10 +87650,11 @@
87650	#define sqlite3_blob_bytes 0
87651	#define sqlite3_blob_close 0
87652	#define sqlite3_blob_open 0
87653	#define sqlite3_blob_read 0
87654	#define sqlite3_blob_write 0
87655	#define sqlite3_blob_reopen 0
87656	#endif
87657
87658	/*
87659	** The following structure contains pointers to all SQLite API routines.
87660	** A pointer to this structure is passed into extensions when they are
	@@ -87797,10 +87916,13 @@
87916	#else
87917	0,
87918	0,
87919	0,
87920	#endif
87921	sqlite3_blob_reopen,
87922	sqlite3_vtab_config,
87923	sqlite3_vtab_on_conflict,
87924	};
87925
87926	/*
87927	** Attempt to load an SQLite extension library contained in the file
87928	** zFile. The entry point is zProc. zProc may be 0 in which case a
	@@ -94186,10 +94308,11 @@
94308	Expr pHaving; / The HAVING clause. May be NULL */
94309	int isDistinct; /* True if the DISTINCT keyword is present */
94310	int distinct; /* Table to use for the distinct set */
94311	int rc = 1; /* Value to return from this function */
94312	int addrSortIndex; /* Address of an OP_OpenEphemeral instruction */
94313	int addrDistinctIndex; /* Address of an OP_OpenEphemeral instruction */
94314	AggInfo sAggInfo; /* Information used by aggregate queries */
94315	int iEnd; /* Address of the end of the query */
94316	sqlite3 db; / The database connection */
94317
94318	#ifndef SQLITE_OMIT_EXPLAIN
	@@ -94312,20 +94435,10 @@
94435	explainSetInteger(pParse->iSelectId, iRestoreSelectId);
94436	return rc;
94437	}
94438	#endif
94439










94440	/* If there is both a GROUP BY and an ORDER BY clause and they are
94441	** identical, then disable the ORDER BY clause since the GROUP BY
94442	** will cause elements to come out in the correct order. This is
94443	** an optimization - the correct answer should result regardless.
94444	** Use the SQLITE_GroupByOrder flag with SQLITE_TESTCTRL_OPTIMIZER
	@@ -94333,10 +94446,34 @@
94446	*/
94447	if( sqlite3ExprListCompare(p->pGroupBy, pOrderBy)==0
94448	&& (db->flags & SQLITE_GroupByOrder)==0 ){
94449	pOrderBy = 0;
94450	}
94451
94452	/* If the query is DISTINCT with an ORDER BY but is not an aggregate, and
94453	** if the select-list is the same as the ORDER BY list, then this query
94454	** can be rewritten as a GROUP BY. In other words, this:
94455	**
94456	** SELECT DISTINCT xyz FROM ... ORDER BY xyz
94457	**
94458	** is transformed to:
94459	**
94460	** SELECT xyz FROM ... GROUP BY xyz
94461	**
94462	** The second form is preferred as a single index (or temp-table) may be
94463	** used for both the ORDER BY and DISTINCT processing. As originally
94464	** written the query must use a temp-table for at least one of the ORDER
94465	** BY and DISTINCT, and an index or separate temp-table for the other.
94466	*/
94467	if( (p->selFlags & (SF_Distinct\|SF_Aggregate))==SF_Distinct
94468	&& sqlite3ExprListCompare(pOrderBy, p->pEList)==0
94469	){
94470	p->selFlags &= ~SF_Distinct;
94471	p->pGroupBy = sqlite3ExprListDup(db, p->pEList, 0);
94472	pGroupBy = p->pGroupBy;
94473	pOrderBy = 0;
94474	}
94475
94476	/* If there is an ORDER BY clause, then this sorting
94477	** index might end up being unused if the data can be
94478	** extracted in pre-sorted order. If that is the case, then the
94479	** OP_OpenEphemeral instruction will be changed to an OP_Noop once
	@@ -94369,26 +94506,25 @@
94506
94507	/* Open a virtual index to use for the distinct set.
94508	*/
94509	if( p->selFlags & SF_Distinct ){
94510	KeyInfo *pKeyInfo;

94511	distinct = pParse->nTab++;
94512	pKeyInfo = keyInfoFromExprList(pParse, p->pEList);
94513	addrDistinctIndex = sqlite3VdbeAddOp4(v, OP_OpenEphemeral, distinct, 0, 0,
94514	(char*)pKeyInfo, P4_KEYINFO_HANDOFF);
94515	sqlite3VdbeChangeP5(v, BTREE_UNORDERED);
94516	}else{
94517	distinct = addrDistinctIndex = -1;
94518	}
94519
94520	/* Aggregate and non-aggregate queries are handled differently */
94521	if( !isAgg && pGroupBy==0 ){
94522	ExprList *pDist = (isDistinct ? p->pEList : 0);
94523
94524	/* Begin the database scan. */
94525	pWInfo = sqlite3WhereBegin(pParse, pTabList, pWhere, &pOrderBy, pDist, 0);
94526	if( pWInfo==0 ) goto select_end;
94527	if( pWInfo->nRowOut < p->nSelectRow ) p->nSelectRow = pWInfo->nRowOut;
94528
94529	/* If sorting index that was created by a prior OP_OpenEphemeral
94530	** instruction ended up not being needed, then change the OP_OpenEphemeral
	@@ -94397,14 +94533,56 @@
94533	if( addrSortIndex>=0 && pOrderBy==0 ){
94534	sqlite3VdbeChangeToNoop(v, addrSortIndex, 1);
94535	p->addrOpenEphm[2] = -1;
94536	}
94537
94538	if( pWInfo->eDistinct ){
94539	VdbeOp pOp; / No longer required OpenEphemeral instr. */
94540
94541	assert( addrDistinctIndex>0 );
94542	pOp = sqlite3VdbeGetOp(v, addrDistinctIndex);
94543
94544	assert( isDistinct );
94545	assert( pWInfo->eDistinct==WHERE_DISTINCT_ORDERED
94546	\|\| pWInfo->eDistinct==WHERE_DISTINCT_UNIQUE
94547	);
94548	distinct = -1;
94549	if( pWInfo->eDistinct==WHERE_DISTINCT_ORDERED ){
94550	int iJump;
94551	int iExpr;
94552	int iFlag = ++pParse->nMem;
94553	int iBase = pParse->nMem+1;
94554	int iBase2 = iBase + pEList->nExpr;
94555	pParse->nMem += (pEList->nExpr*2);
94556
94557	/* Change the OP_OpenEphemeral coded earlier to an OP_Integer. The
94558	** OP_Integer initializes the "first row" flag. */
94559	pOp->opcode = OP_Integer;
94560	pOp->p1 = 1;
94561	pOp->p2 = iFlag;
94562
94563	sqlite3ExprCodeExprList(pParse, pEList, iBase, 1);
94564	iJump = sqlite3VdbeCurrentAddr(v) + 1 + pEList->nExpr + 1 + 1;
94565	sqlite3VdbeAddOp2(v, OP_If, iFlag, iJump-1);
94566	for(iExpr=0; iExpr<pEList->nExpr; iExpr++){
94567	CollSeq *pColl = sqlite3ExprCollSeq(pParse, pEList->a[iExpr].pExpr);
94568	sqlite3VdbeAddOp3(v, OP_Ne, iBase+iExpr, iJump, iBase2+iExpr);
94569	sqlite3VdbeChangeP4(v, -1, (const char *)pColl, P4_COLLSEQ);
94570	sqlite3VdbeChangeP5(v, SQLITE_NULLEQ);
94571	}
94572	sqlite3VdbeAddOp2(v, OP_Goto, 0, pWInfo->iContinue);
94573
94574	sqlite3VdbeAddOp2(v, OP_Integer, 0, iFlag);
94575	assert( sqlite3VdbeCurrentAddr(v)==iJump );
94576	sqlite3VdbeAddOp3(v, OP_Move, iBase, iBase2, pEList->nExpr);
94577	}else{
94578	pOp->opcode = OP_Noop;
94579	}
94580	}
94581
94582	/* Use the standard inner loop. */
94583	selectInnerLoop(pParse, p, pEList, 0, 0, pOrderBy, distinct, pDest,
94584	pWInfo->iContinue, pWInfo->iBreak);
94585
94586	/* End the database scan loop.
94587	*/
94588	sqlite3WhereEnd(pWInfo);
	@@ -94510,11 +94688,11 @@
94688	** This might involve two separate loops with an OP_Sort in between, or
94689	** it might be a single loop that uses an index to extract information
94690	** in the right order to begin with.
94691	*/
94692	sqlite3VdbeAddOp2(v, OP_Gosub, regReset, addrReset);
94693	pWInfo = sqlite3WhereBegin(pParse, pTabList, pWhere, &pGroupBy, 0, 0);
94694	if( pWInfo==0 ) goto select_end;
94695	if( pGroupBy==0 ){
94696	/* The optimizer is able to deliver rows in group by order so
94697	** we do not have to sort. The OP_OpenEphemeral table will be
94698	** cancelled later because we still need to use the pKeyInfo
	@@ -94772,11 +94950,11 @@
94950	/* This case runs if the aggregate has no GROUP BY clause. The
94951	** processing is much simpler since there is only a single row
94952	** of output.
94953	*/
94954	resetAccumulator(pParse, &sAggInfo);
94955	pWInfo = sqlite3WhereBegin(pParse, pTabList, pWhere, &pMinMax, 0, flag);
94956	if( pWInfo==0 ){
94957	sqlite3ExprListDelete(db, pDel);
94958	goto select_end;
94959	}
94960	updateAccumulator(pParse, &sAggInfo);
	@@ -95248,19 +95426,32 @@
95426	iDb = sqlite3TwoPartName(pParse, pName1, pName2, &pName);
95427	if( iDb<0 ){
95428	goto trigger_cleanup;
95429	}
95430	}
95431	if( !pTableName \|\| db->mallocFailed ){
95432	goto trigger_cleanup;
95433	}
95434
95435	/* A long-standing parser bug is that this syntax was allowed:
95436	**
95437	** CREATE TRIGGER attached.demo AFTER INSERT ON attached.tab ....
95438	** ^^^^^^^^
95439	**
95440	** To maintain backwards compatibility, ignore the database
95441	** name on pTableName if we are reparsing our of SQLITE_MASTER.
95442	*/
95443	if( db->init.busy && iDb!=1 ){
95444	sqlite3DbFree(db, pTableName->a[0].zDatabase);
95445	pTableName->a[0].zDatabase = 0;
95446	}
95447
95448	/* If the trigger name was unqualified, and the table is a temp table,
95449	** then set iDb to 1 to create the trigger in the temporary database.
95450	** If sqlite3SrcListLookup() returns 0, indicating the table does not
95451	** exist, the error is caught by the block below.
95452	*/



95453	pTab = sqlite3SrcListLookup(pParse, pTableName);
95454	if( db->init.busy==0 && pName2->n==0 && pTab
95455	&& pTab->pSchema==db->aDb[1].pSchema ){
95456	iDb = 1;
95457	}
	@@ -96554,11 +96745,13 @@
96745	}
96746
96747	/* Begin the database scan
96748	*/
96749	sqlite3VdbeAddOp2(v, OP_Null, 0, regOldRowid);
96750	pWInfo = sqlite3WhereBegin(
96751	pParse, pTabList, pWhere, 0, 0, WHERE_ONEPASS_DESIRED
96752	);
96753	if( pWInfo==0 ) goto update_cleanup;
96754	okOnePass = pWInfo->okOnePass;
96755
96756	/* Remember the rowid of every item to be updated.
96757	*/
	@@ -98580,10 +98773,11 @@
98773	#define WHERE_REVERSE 0x02000000 /* Scan in reverse order */
98774	#define WHERE_UNIQUE 0x04000000 /* Selects no more than one row */
98775	#define WHERE_VIRTUALTABLE 0x08000000 /* Use virtual-table processing */
98776	#define WHERE_MULTI_OR 0x10000000 /* OR using multiple indices */
98777	#define WHERE_TEMP_INDEX 0x20000000 /* Uses an ephemeral index */
98778	#define WHERE_DISTINCT 0x40000000 /* Correct order for DISTINCT */
98779
98780	/*
98781	** Initialize a preallocated WhereClause structure.
98782	*/
98783	static void whereClauseInit(
	@@ -99724,10 +99918,166 @@
99918	}
99919	}
99920	return 0;
99921	}
99922
99923	/*
99924	** This function searches the expression list passed as the second argument
99925	** for an expression of type TK_COLUMN that refers to the same column and
99926	** uses the same collation sequence as the iCol'th column of index pIdx.
99927	** Argument iBase is the cursor number used for the table that pIdx refers
99928	** to.
99929	**
99930	** If such an expression is found, its index in pList->a[] is returned. If
99931	** no expression is found, -1 is returned.
99932	*/
99933	static int findIndexCol(
99934	Parse pParse, / Parse context */
99935	ExprList pList, / Expression list to search */
99936	int iBase, /* Cursor for table associated with pIdx */
99937	Index pIdx, / Index to match column of */
99938	int iCol /* Column of index to match */
99939	){
99940	int i;
99941	const char *zColl = pIdx->azColl[iCol];
99942
99943	for(i=0; i<pList->nExpr; i++){
99944	Expr *p = pList->a[i].pExpr;
99945	if( p->op==TK_COLUMN
99946	&& p->iColumn==pIdx->aiColumn[iCol]
99947	&& p->iTable==iBase
99948	){
99949	CollSeq *pColl = sqlite3ExprCollSeq(pParse, p);
99950	if( ALWAYS(pColl) && 0==sqlite3StrICmp(pColl->zName, zColl) ){
99951	return i;
99952	}
99953	}
99954	}
99955
99956	return -1;
99957	}
99958
99959	/*
99960	** This routine determines if pIdx can be used to assist in processing a
99961	** DISTINCT qualifier. In other words, it tests whether or not using this
99962	** index for the outer loop guarantees that rows with equal values for
99963	** all expressions in the pDistinct list are delivered grouped together.
99964	**
99965	** For example, the query
99966	**
99967	** SELECT DISTINCT a, b, c FROM tbl WHERE a = ?
99968	**
99969	** can benefit from any index on columns "b" and "c".
99970	*/
99971	static int isDistinctIndex(
99972	Parse pParse, / Parsing context */
99973	WhereClause pWC, / The WHERE clause */
99974	Index pIdx, / The index being considered */
99975	int base, /* Cursor number for the table pIdx is on */
99976	ExprList pDistinct, / The DISTINCT expressions */
99977	int nEqCol /* Number of index columns with == */
99978	){
99979	Bitmask mask = 0; /* Mask of unaccounted for pDistinct exprs */
99980	int i; /* Iterator variable */
99981
99982	if( pIdx->zName==0 \|\| pDistinct==0 \|\| pDistinct->nExpr>=BMS ) return 0;
99983	testcase( pDistinct->nExpr==BMS-1 );
99984
99985	/* Loop through all the expressions in the distinct list. If any of them
99986	** are not simple column references, return early. Otherwise, test if the
99987	** WHERE clause contains a "col=X" clause. If it does, the expression
99988	** can be ignored. If it does not, and the column does not belong to the
99989	** same table as index pIdx, return early. Finally, if there is no
99990	** matching "col=X" expression and the column is on the same table as pIdx,
99991	** set the corresponding bit in variable mask.
99992	*/
99993	for(i=0; i<pDistinct->nExpr; i++){
99994	WhereTerm *pTerm;
99995	Expr *p = pDistinct->a[i].pExpr;
99996	if( p->op!=TK_COLUMN ) return 0;
99997	pTerm = findTerm(pWC, p->iTable, p->iColumn, ~(Bitmask)0, WO_EQ, 0);
99998	if( pTerm ){
99999	Expr *pX = pTerm->pExpr;
100000	CollSeq *p1 = sqlite3BinaryCompareCollSeq(pParse, pX->pLeft, pX->pRight);
100001	CollSeq *p2 = sqlite3ExprCollSeq(pParse, p);
100002	if( p1==p2 ) continue;
100003	}
100004	if( p->iTable!=base ) return 0;
100005	mask \|= (((Bitmask)1) << i);
100006	}
100007
100008	for(i=nEqCol; mask && i<pIdx->nColumn; i++){
100009	int iExpr = findIndexCol(pParse, pDistinct, base, pIdx, i);
100010	if( iExpr<0 ) break;
100011	mask &= ~(((Bitmask)1) << iExpr);
100012	}
100013
100014	return (mask==0);
100015	}
100016
100017
100018	/*
100019	** Return true if the DISTINCT expression-list passed as the third argument
100020	** is redundant. A DISTINCT list is redundant if the database contains a
100021	** UNIQUE index that guarantees that the result of the query will be distinct
100022	** anyway.
100023	*/
100024	static int isDistinctRedundant(
100025	Parse *pParse,
100026	SrcList *pTabList,
100027	WhereClause *pWC,
100028	ExprList *pDistinct
100029	){
100030	Table *pTab;
100031	Index *pIdx;
100032	int i;
100033	int iBase;
100034
100035	/* If there is more than one table or sub-select in the FROM clause of
100036	** this query, then it will not be possible to show that the DISTINCT
100037	** clause is redundant. */
100038	if( pTabList->nSrc!=1 ) return 0;
100039	iBase = pTabList->a[0].iCursor;
100040	pTab = pTabList->a[0].pTab;
100041
100042	/* If any of the expressions is an IPK column on table iBase, then return
100043	** true. Note: The (p->iTable==iBase) part of this test may be false if the
100044	** current SELECT is a correlated sub-query.
100045	*/
100046	for(i=0; i<pDistinct->nExpr; i++){
100047	Expr *p = pDistinct->a[i].pExpr;
100048	if( p->op==TK_COLUMN && p->iTable==iBase && p->iColumn<0 ) return 1;
100049	}
100050
100051	/* Loop through all indices on the table, checking each to see if it makes
100052	** the DISTINCT qualifier redundant. It does so if:
100053	**
100054	** 1. The index is itself UNIQUE, and
100055	**
100056	** 2. All of the columns in the index are either part of the pDistinct
100057	** list, or else the WHERE clause contains a term of the form "col=X",
100058	** where X is a constant value. The collation sequences of the
100059	** comparison and select-list expressions must match those of the index.
100060	*/
100061	for(pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext){
100062	if( pIdx->onError==OE_None ) continue;
100063	for(i=0; i<pIdx->nColumn; i++){
100064	int iCol = pIdx->aiColumn[i];
100065	if( 0==findTerm(pWC, iBase, iCol, ~(Bitmask)0, WO_EQ, pIdx)
100066	&& 0>findIndexCol(pParse, pDistinct, iBase, pIdx, i)
100067	){
100068	break;
100069	}
100070	}
100071	if( i==pIdx->nColumn ){
100072	/* This index implies that the DISTINCT qualifier is redundant. */
100073	return 1;
100074	}
100075	}
100076
100077	return 0;
100078	}
100079
100080	/*
100081	** This routine decides if pIdx can be used to satisfy the ORDER BY
100082	** clause. If it can, it returns 1. If pIdx cannot satisfy the
100083	** ORDER BY clause, this routine returns 0.
	@@ -99760,11 +100110,14 @@
100110	int sortOrder = 0; /* XOR of index and ORDER BY sort direction */
100111	int nTerm; /* Number of ORDER BY terms */
100112	struct ExprList_item pTerm; / A term of the ORDER BY clause */
100113	sqlite3 *db = pParse->db;
100114
100115	if( !pOrderBy ) return 0;
100116	if( wsFlags & WHERE_COLUMN_IN ) return 0;
100117	if( pIdx->bUnordered ) return 0;
100118
100119	nTerm = pOrderBy->nExpr;
100120	assert( nTerm>0 );
100121
100122	/* Argument pIdx must either point to a 'real' named index structure,
100123	** or an index structure allocated on the stack by bestBtreeIndex() to
	@@ -100073,10 +100426,14 @@
100426	double costTempIdx; /* per-query cost of the transient index */
100427	WhereTerm pTerm; / A single term of the WHERE clause */
100428	WhereTerm pWCEnd; / End of pWC->a[] */
100429	Table pTable; / Table tht might be indexed */
100430
100431	if( pParse->nQueryLoop<=(double)1 ){
100432	/* There is no point in building an automatic index for a single scan */
100433	return;
100434	}
100435	if( (pParse->db->flags & SQLITE_AutoIndex)==0 ){
100436	/* Automatic indices are disabled at run-time */
100437	return;
100438	}
100439	if( (pCost->plan.wsFlags & WHERE_NOT_FULLSCAN)!=0 ){
	@@ -100084,10 +100441,14 @@
100441	return;
100442	}
100443	if( pSrc->notIndexed ){
100444	/* The NOT INDEXED clause appears in the SQL. */
100445	return;
100446	}
100447	if( pSrc->isCorrelated ){
100448	/* The source is a correlated sub-query. No point in indexing it. */
100449	return;
100450	}
100451
100452	assert( pParse->nQueryLoop >= (double)1 );
100453	pTable = pSrc->pTab;
100454	nTableRow = pTable->nRowEst;
	@@ -101016,10 +101377,11 @@
101377	WhereClause pWC, / The WHERE clause */
101378	struct SrcList_item pSrc, / The FROM clause term to search */
101379	Bitmask notReady, /* Mask of cursors not available for indexing */
101380	Bitmask notValid, /* Cursors not available for any purpose */
101381	ExprList pOrderBy, / The ORDER BY clause */
101382	ExprList pDistinct, / The select-list if query is DISTINCT */
101383	WhereCost pCost / Lowest cost query plan */
101384	){
101385	int iCur = pSrc->iCursor; /* The cursor of the table to be accessed */
101386	Index pProbe; / An index we are evaluating */
101387	Index pIdx; / Copy of pProbe, or zero for IPK index */
	@@ -101156,11 +101518,12 @@
101518	int nEq; /* Number of == or IN terms matching index */
101519	int bInEst = 0; /* True if "x IN (SELECT...)" seen */
101520	int nInMul = 1; /* Number of distinct equalities to lookup */
101521	int estBound = 100; /* Estimated reduction in search space */
101522	int nBound = 0; /* Number of range constraints seen */
101523	int bSort = !!pOrderBy; /* True if external sort required */
101524	int bDist = !!pDistinct; /* True if index cannot help with DISTINCT */
101525	int bLookup = 0; /* True if not a covering index */
101526	WhereTerm pTerm; / A single term of the WHERE clause */
101527	#ifdef SQLITE_ENABLE_STAT2
101528	WhereTerm pFirstTerm = 0; / First term matching the index */
101529	#endif
	@@ -101220,21 +101583,24 @@
101583
101584	/* If there is an ORDER BY clause and the index being considered will
101585	** naturally scan rows in the required order, set the appropriate flags
101586	** in wsFlags. Otherwise, if there is an ORDER BY clause but the index
101587	** will scan rows in a different order, set the bSort variable. */
101588	if( isSortingIndex(
101589	pParse, pWC->pMaskSet, pProbe, iCur, pOrderBy, nEq, wsFlags, &rev)
101590	){
101591	bSort = 0;
101592	wsFlags \|= WHERE_ROWID_RANGE\|WHERE_COLUMN_RANGE\|WHERE_ORDERBY;
101593	wsFlags \|= (rev ? WHERE_REVERSE : 0);
101594	}
101595
101596	/* If there is a DISTINCT qualifier and this index will scan rows in
101597	** order of the DISTINCT expressions, clear bDist and set the appropriate
101598	** flags in wsFlags. */
101599	if( isDistinctIndex(pParse, pWC, pProbe, iCur, pDistinct, nEq) ){
101600	bDist = 0;
101601	wsFlags \|= WHERE_ROWID_RANGE\|WHERE_COLUMN_RANGE\|WHERE_DISTINCT;
101602	}
101603
101604	/* If currently calculating the cost of using an index (not the IPK
101605	** index), determine if all required column data may be obtained without
101606	** using the main table (i.e. if the index is a covering
	@@ -101265,16 +101631,17 @@
101631	nRow = aiRowEst[0]/2;
101632	nInMul = (int)(nRow / aiRowEst[nEq]);
101633	}
101634
101635	#ifdef SQLITE_ENABLE_STAT2
101636	/* If the constraint is of the form x=VALUE or x IN (E1,E2,...)
101637	** and we do not think that values of x are unique and if histogram
101638	** data is available for column x, then it might be possible
101639	** to get a better estimate on the number of rows based on
101640	** VALUE and how common that value is according to the histogram.
101641	*/
101642	if( nRow>(double)1 && nEq==1 && pFirstTerm!=0 && aiRowEst[1]>1 ){
101643	if( pFirstTerm->eOperator & (WO_EQ\|WO_ISNULL) ){
101644	testcase( pFirstTerm->eOperator==WO_EQ );
101645	testcase( pFirstTerm->eOperator==WO_ISNULL );
101646	whereEqualScanEst(pParse, pProbe, pFirstTerm->pExpr->pRight, &nRow);
101647	}else if( pFirstTerm->eOperator==WO_IN && bInEst==0 ){
	@@ -101347,10 +101714,13 @@
101714	** difference and select C of 3.0.
101715	*/
101716	if( bSort ){
101717	cost += nRowestLog(nRow)3;
101718	}
101719	if( bDist ){
101720	cost += nRowestLog(nRow)3;
101721	}
101722
101723	/** Cost of using this index has now been computed **/
101724
101725	/* If there are additional constraints on this table that cannot
101726	** be used with the current index, but which might lower the number
	@@ -101492,11 +101862,11 @@
101862	}
101863	sqlite3DbFree(pParse->db, p);
101864	}else
101865	#endif
101866	{
101867	bestBtreeIndex(pParse, pWC, pSrc, notReady, notValid, pOrderBy, 0, pCost);
101868	}
101869	}
101870
101871	/*
101872	** Disable a term in the WHERE clause. Except, do not disable the term
	@@ -102454,11 +102824,11 @@
102824	for(ii=0; ii<pOrWc->nTerm; ii++){
102825	WhereTerm *pOrTerm = &pOrWc->a[ii];
102826	if( pOrTerm->leftCursor==iCur \|\| pOrTerm->eOperator==WO_AND ){
102827	WhereInfo pSubWInfo; / Info for single OR-term scan */
102828	/* Loop through table entries that match term pOrTerm. */
102829	pSubWInfo = sqlite3WhereBegin(pParse, pOrTab, pOrTerm->pExpr, 0, 0,
102830	WHERE_OMIT_OPEN \| WHERE_OMIT_CLOSE \|
102831	WHERE_FORCE_TABLE \| WHERE_ONETABLE_ONLY);
102832	if( pSubWInfo ){
102833	explainOneScan(
102834	pParse, pOrTab, &pSubWInfo->a[0], iLevel, pLevel->iFrom, 0
	@@ -102695,10 +103065,11 @@
103065	SQLITE_PRIVATE WhereInfo *sqlite3WhereBegin(
103066	Parse pParse, / The parser context */
103067	SrcList pTabList, / A list of all tables to be scanned */
103068	Expr pWhere, / The WHERE clause */
103069	ExprList *ppOrderBy, / An ORDER BY clause, or NULL */
103070	ExprList pDistinct, / The select-list for DISTINCT queries - or NULL */
103071	u16 wctrlFlags /* One of the WHERE_* flags defined in sqliteInt.h */
103072	){
103073	int i; /* Loop counter */
103074	int nByteWInfo; /* Num. bytes allocated for WhereInfo struct */
103075	int nTabList; /* Number of elements in pTabList */
	@@ -102754,10 +103125,14 @@
103125	pWInfo->iBreak = sqlite3VdbeMakeLabel(v);
103126	pWInfo->pWC = pWC = (WhereClause )&((u8 )pWInfo)[nByteWInfo];
103127	pWInfo->wctrlFlags = wctrlFlags;
103128	pWInfo->savedNQueryLoop = pParse->nQueryLoop;
103129	pMaskSet = (WhereMaskSet*)&pWC[1];
103130
103131	/* Disable the DISTINCT optimization if SQLITE_DistinctOpt is set via
103132	** sqlite3_test_ctrl(SQLITE_TESTCTRL_OPTIMIZATIONS,...) */
103133	if( db->flags & SQLITE_DistinctOpt ) pDistinct = 0;
103134
103135	/* Split the WHERE clause into separate subexpressions where each
103136	** subexpression is separated by an AND operator.
103137	*/
103138	initMaskSet(pMaskSet);
	@@ -102821,10 +103196,19 @@
103196	*/
103197	exprAnalyzeAll(pTabList, pWC);
103198	if( db->mallocFailed ){
103199	goto whereBeginError;
103200	}
103201
103202	/* Check if the DISTINCT qualifier, if there is one, is redundant.
103203	** If it is, then set pDistinct to NULL and WhereInfo.eDistinct to
103204	** WHERE_DISTINCT_UNIQUE to tell the caller to ignore the DISTINCT.
103205	*/
103206	if( pDistinct && isDistinctRedundant(pParse, pTabList, pWC, pDistinct) ){
103207	pDistinct = 0;
103208	pWInfo->eDistinct = WHERE_DISTINCT_UNIQUE;
103209	}
103210
103211	/* Chose the best index to use for each table in the FROM clause.
103212	**
103213	** This loop fills in the following fields:
103214	**
	@@ -102905,10 +103289,11 @@
103289	Bitmask mask; /* Mask of tables not yet ready */
103290	for(j=iFrom, pTabItem=&pTabList->a[j]; j<nTabList; j++, pTabItem++){
103291	int doNotReorder; /* True if this table should not be reordered */
103292	WhereCost sCost; /* Cost information from best[Virtual]Index() */
103293	ExprList pOrderBy; / ORDER BY clause for index to optimize */
103294	ExprList pDist; / DISTINCT clause for index to optimize */
103295
103296	doNotReorder = (pTabItem->jointype & (JT_LEFT\|JT_CROSS))!=0;
103297	if( j!=iFrom && doNotReorder ) break;
103298	m = getMask(pMaskSet, pTabItem->iCursor);
103299	if( (m & notReady)==0 ){
	@@ -102915,10 +103300,11 @@
103300	if( j==iFrom ) iFrom++;
103301	continue;
103302	}
103303	mask = (isOptimal ? m : notReady);
103304	pOrderBy = ((i==0 && ppOrderBy )?*ppOrderBy:0);
103305	pDist = (i==0 ? pDistinct : 0);
103306	if( pTabItem->pIndex==0 ) nUnconstrained++;
103307
103308	WHERETRACE(("=== trying table %d with isOptimal=%d ===\n",
103309	j, isOptimal));
103310	assert( pTabItem->pTab );
	@@ -102929,11 +103315,11 @@
103315	&sCost, pp);
103316	}else
103317	#endif
103318	{
103319	bestBtreeIndex(pParse, pWC, pTabItem, mask, notReady, pOrderBy,
103320	pDist, &sCost);
103321	}
103322	assert( isOptimal \|\| (sCost.used&notReady)==0 );
103323
103324	/* If an INDEXED BY clause is present, then the plan must use that
103325	** index if it uses any index at all */
	@@ -102989,10 +103375,14 @@
103375	WHERETRACE(("*** Optimizer selects table %d for loop %d"
103376	" with cost=%g and nRow=%g\n",
103377	bestJ, pLevel-pWInfo->a, bestPlan.rCost, bestPlan.plan.nRow));
103378	if( (bestPlan.plan.wsFlags & WHERE_ORDERBY)!=0 ){
103379	*ppOrderBy = 0;
103380	}
103381	if( (bestPlan.plan.wsFlags & WHERE_DISTINCT)!=0 ){
103382	assert( pWInfo->eDistinct==0 );
103383	pWInfo->eDistinct = WHERE_DISTINCT_ORDERED;
103384	}
103385	andFlags &= bestPlan.plan.wsFlags;
103386	pLevel->plan = bestPlan.plan;
103387	testcase( bestPlan.plan.wsFlags & WHERE_INDEXED );
103388	testcase( bestPlan.plan.wsFlags & WHERE_TEMP_INDEX );
	@@ -111519,11 +111909,17 @@
111909	*/
111910	#if defined(SQLITE_ENABLE_FTS4) && !defined(SQLITE_ENABLE_FTS3)
111911	# define SQLITE_ENABLE_FTS3
111912	#endif
111913
111914	#if !defined(SQLITE_CORE) \|\| defined(SQLITE_ENABLE_FTS3)
111915
111916	/* If not building as part of the core, include sqlite3ext.h. */
111917	#ifndef SQLITE_CORE
111918	SQLITE_API extern const sqlite3_api_routines *sqlite3_api;
111919	#endif
111920
111921	/************ Include fts3_tokenizer.h in the middle of fts3Int.h ********/
111922	/************ Begin file fts3_tokenizer.h ********************************/
111923	/*
111924	** 2006 July 10
111925	**
	@@ -112052,11 +112448,11 @@
112448
112449	sqlite3_int64 iDocid; /* Current docid (if pList!=0) */
112450	int bFreeList; /* True if pList should be sqlite3_free()d */
112451	char pList; / Pointer to position list following iDocid */
112452	int nList; /* Length of position list */
112453	};
112454
112455	/*
112456	** A "phrase" is a sequence of one or more tokens that must match in
112457	** sequence. A single token is the base case and the most common case.
112458	** For a sequence of tokens contained in double-quotes (i.e. "one two three")
	@@ -112252,23 +112648,12 @@
112648	#endif
112649
112650	/* fts3_aux.c */
112651	SQLITE_PRIVATE int sqlite3Fts3InitAux(sqlite3 *db);
112652








112653	SQLITE_PRIVATE void sqlite3Fts3EvalPhraseCleanup(Fts3Phrase *);
112654



112655	SQLITE_PRIVATE int sqlite3Fts3MsrIncrStart(
112656	Fts3Table, Fts3MultiSegReader, int, const char*, int);
112657	SQLITE_PRIVATE int sqlite3Fts3MsrIncrNext(
112658	Fts3Table , Fts3MultiSegReader , sqlite3_int64 , char , int );
112659	SQLITE_PRIVATE char sqlite3Fts3EvalPhrasePoslist(Fts3Cursor , Fts3Expr *, int iCol);
	@@ -112275,11 +112660,11 @@
112660	SQLITE_PRIVATE int sqlite3Fts3MsrOvfl(Fts3Cursor , Fts3MultiSegReader , int *);
112661	SQLITE_PRIVATE int sqlite3Fts3MsrIncrRestart(Fts3MultiSegReader *pCsr);
112662
112663	SQLITE_PRIVATE int sqlite3Fts3DeferredTokenList(Fts3DeferredToken , char , int );
112664
112665	#endif /* !SQLITE_CORE \|\| SQLITE_ENABLE_FTS3 */
112666	#endif /* _FTSINT_H */
112667
112668	/************ End of fts3Int.h *******************************************/
112669	/************ Continuing where we left off in fts3.c *********************/
112670	#if !defined(SQLITE_CORE) \|\| defined(SQLITE_ENABLE_FTS3)
	@@ -112291,10 +112676,15 @@
112676
112677	#ifndef SQLITE_CORE
112678	SQLITE_EXTENSION_INIT1
112679	#endif
112680
112681	static int fts3EvalNext(Fts3Cursor *pCsr);
112682	static int fts3EvalStart(Fts3Cursor *pCsr);
112683	static int fts3TermSegReaderCursor(
112684	Fts3Cursor , const char , int, int, Fts3MultiSegReader **);
112685
112686	/*
112687	** Write a 64-bit variable-length integer to memory starting at p[0].
112688	** The length of data written will be between 1 and FTS3_VARINT_MAX bytes.
112689	** The number of bytes written is returned.
112690	*/
	@@ -112799,31 +113189,60 @@
113189	}
113190	sqlite3_free(zFree);
113191	return zRet;
113192	}
113193
113194	/*
113195	** This function interprets the string at (*pp) as a non-negative integer
113196	** value. It reads the integer and sets *pnOut to the value read, then
113197	** sets *pp to point to the byte immediately following the last byte of
113198	** the integer value.
113199	**
113200	** Only decimal digits ('0'..'9') may be part of an integer value.
113201	**
113202	** If *pp does not being with a decimal digit SQLITE_ERROR is returned and
113203	** the output value undefined. Otherwise SQLITE_OK is returned.
113204	**
113205	** This function is used when parsing the "prefix=" FTS4 parameter.
113206	*/
113207	static int fts3GobbleInt(const char *pp, int pnOut){
113208	const char p = pp; /* Iterator pointer */
113209	int nInt = 0; /* Output value */
113210
113211	for(p=*pp; p[0]>='0' && p[0]<='9'; p++){
113212	nInt = nInt * 10 + (p[0] - '0');
113213	}
113214	if( p==*pp ) return SQLITE_ERROR;
113215	*pnOut = nInt;
113216	*pp = p;
113217	return SQLITE_OK;
113218	}
113219
113220	/*
113221	** This function is called to allocate an array of Fts3Index structures
113222	** representing the indexes maintained by the current FTS table. FTS tables
113223	** always maintain the main "terms" index, but may also maintain one or
113224	** more "prefix" indexes, depending on the value of the "prefix=" parameter
113225	** (if any) specified as part of the CREATE VIRTUAL TABLE statement.
113226	**
113227	** Argument zParam is passed the value of the "prefix=" option if one was
113228	** specified, or NULL otherwise.
113229	**
113230	** If no error occurs, SQLITE_OK is returned and *apIndex set to point to
113231	** the allocated array. *pnIndex is set to the number of elements in the
113232	** array. If an error does occur, an SQLite error code is returned.
113233	**
113234	** Regardless of whether or not an error is returned, it is the responsibility
113235	** of the caller to call sqlite3_free() on the output array to free it.
113236	*/
113237	static int fts3PrefixParameter(
113238	const char zParam, / ABC in prefix=ABC parameter to parse */
113239	int pnIndex, / OUT: size of apIndex[] array /
113240	struct Fts3Index *apIndex / OUT: Array of indexes for this table */

113241	){
113242	struct Fts3Index aIndex; / Allocated array */
113243	int nIndex = 1; /* Number of entries in array */
113244
113245	if( zParam && zParam[0] ){
113246	const char *p;
113247	nIndex++;
113248	for(p=zParam; *p; p++){
	@@ -112830,11 +113249,11 @@
113249	if( *p==',' ) nIndex++;
113250	}
113251	}
113252
113253	aIndex = sqlite3_malloc(sizeof(struct Fts3Index) * nIndex);
113254	*apIndex = aIndex;
113255	*pnIndex = nIndex;
113256	if( !aIndex ){
113257	return SQLITE_NOMEM;
113258	}
113259
	@@ -112887,12 +113306,11 @@
113306	int isFts4 = (argv[0][3]=='4'); /* True for FTS4, false for FTS3 */
113307	const char *aCol; / Array of column names */
113308	sqlite3_tokenizer pTokenizer = 0; / Tokenizer for this table */
113309
113310	int nIndex; /* Size of aIndex[] array */
113311	struct Fts3Index aIndex = 0; / Array of indexes for this table */

113312
113313	/* The results of parsing supported FTS4 key=value options: */
113314	int bNoDocsize = 0; /* True to omit %_docsize table */
113315	int bDescIdx = 0; /* True to store descending indexes */
113316	char zPrefix = 0; / Prefix parameter value (or NULL) */
	@@ -113025,11 +113443,11 @@
113443	rc = sqlite3Fts3InitTokenizer(pHash, "simple", &pTokenizer, pzErr);
113444	if( rc!=SQLITE_OK ) goto fts3_init_out;
113445	}
113446	assert( pTokenizer );
113447
113448	rc = fts3PrefixParameter(zPrefix, &nIndex, &aIndex);
113449	if( rc==SQLITE_ERROR ){
113450	assert( zPrefix );
113451	*pzErr = sqlite3_mprintf("error parsing prefix parameter: %s", zPrefix);
113452	}
113453	if( rc!=SQLITE_OK ) goto fts3_init_out;
	@@ -113112,11 +113530,11 @@
113530	/* Declare the table schema to SQLite. */
113531	fts3DeclareVtab(&rc, p);
113532
113533	fts3_init_out:
113534	sqlite3_free(zPrefix);
113535	sqlite3_free(aIndex);
113536	sqlite3_free(zCompress);
113537	sqlite3_free(zUncompress);
113538	sqlite3_free((void *)aCol);
113539	if( rc!=SQLITE_OK ){
113540	if( p ){
	@@ -113703,12 +114121,10 @@
114121	*pp1 = p1 + 1;
114122	*pp2 = p2 + 1;
114123	}
114124
114125	/*


114126	** This function is used to merge two position lists into one. When it is
114127	** called, pp1 and pp2 must both point to position lists. A position-list is
114128	** the part of a doclist that follows each document id. For example, if a row
114129	** contains:
114130	**
	@@ -113724,10 +114140,12 @@
114140	** If isSaveLeft is 0, an entry is added to the output position list for
114141	** each position in *pp2 for which there exists one or more positions in
114142	** pp1 so that (pos(pp2)>pos(pp1) && pos(pp2)-pos(*pp1)<=nToken). i.e.
114143	** when the pp1 token appears before the pp2 token, but not more than nToken
114144	** slots before it.
114145	**
114146	** e.g. nToken==1 searches for adjacent positions.
114147	*/
114148	static int fts3PoslistPhraseMerge(
114149	char *pp, / IN/OUT: Preallocated output buffer */
114150	int nToken, /* Maximum difference in token positions */
114151	int isSaveLeft, /* Save the left position */
	@@ -113890,26 +114308,38 @@
114308
114309	return res;
114310	}
114311
114312	/*
114313	** An instance of this function is used to merge together the (potentially
114314	** large number of) doclists for each term that matches a prefix query.
114315	** See function fts3TermSelectMerge() for details.
114316	*/
114317	typedef struct TermSelect TermSelect;
114318	struct TermSelect {
114319	char aaOutput[16]; / Malloc'd output buffers */
114320	int anOutput[16]; /* Size each output buffer in bytes */

114321	};
114322
114323	/*
114324	** This function is used to read a single varint from a buffer. Parameter
114325	** pEnd points 1 byte past the end of the buffer. When this function is
114326	** called, if *pp points to pEnd or greater, then the end of the buffer
114327	** has been reached. In this case *pp is set to 0 and the function returns.
114328	**
114329	** If *pp does not point to or past pEnd, then a single varint is read
114330	** from pp. pp is then set to point 1 byte past the end of the read varint.
114331	**
114332	** If bDescIdx is false, the value read is added to *pVal before returning.
114333	** If it is true, the value read is subtracted from *pVal before this
114334	** function returns.
114335	*/
114336	static void fts3GetDeltaVarint3(
114337	char *pp, / IN/OUT: Point to read varint from */
114338	char pEnd, / End of buffer */
114339	int bDescIdx, /* True if docids are descending */
114340	sqlite3_int64 pVal / IN/OUT: Integer value */
114341	){
114342	if( *pp>=pEnd ){
114343	*pp = 0;
114344	}else{
114345	sqlite3_int64 iVal;
	@@ -113920,10 +114350,25 @@
114350	*pVal += iVal;
114351	}
114352	}
114353	}
114354
114355	/*
114356	** This function is used to write a single varint to a buffer. The varint
114357	** is written to pp. Before returning, pp is set to point 1 byte past the
114358	** end of the value written.
114359	**
114360	** If *pbFirst is zero when this function is called, the value written to
114361	** the buffer is that of parameter iVal.
114362	**
114363	** If *pbFirst is non-zero when this function is called, then the value
114364	** written is either (iVal-piPrev) (if bDescIdx is zero) or (piPrev-iVal)
114365	** (if bDescIdx is non-zero).
114366	**
114367	** Before returning, this function always sets pbFirst to 1 and piPrev
114368	** to the value of parameter iVal.
114369	*/
114370	static void fts3PutDeltaVarint3(
114371	char *pp, / IN/OUT: Output pointer */
114372	int bDescIdx, /* True for descending docids */
114373	sqlite3_int64 piPrev, / IN/OUT: Previous value written to list */
114374	int pbFirst, / IN/OUT: True after first int written */
	@@ -113940,14 +114385,38 @@
114385	pp += sqlite3Fts3PutVarint(pp, iWrite);
114386	*piPrev = iVal;
114387	*pbFirst = 1;
114388	}
114389

114390
114391	/*
114392	** This macro is used by various functions that merge doclists. The two
114393	** arguments are 64-bit docid values. If the value of the stack variable
114394	** bDescDoclist is 0 when this macro is invoked, then it returns (i1-i2).
114395	** Otherwise, (i2-i1).
114396	**
114397	** Using this makes it easier to write code that can merge doclists that are
114398	** sorted in either ascending or descending order.
114399	*/
114400	#define DOCID_CMP(i1, i2) ((bDescDoclist?-1:1) * (i1-i2))
114401
114402	/*
114403	** This function does an "OR" merge of two doclists (output contains all
114404	** positions contained in either argument doclist). If the docids in the
114405	** input doclists are sorted in ascending order, parameter bDescDoclist
114406	** should be false. If they are sorted in ascending order, it should be
114407	** passed a non-zero value.
114408	**
114409	** If no error occurs, *paOut is set to point at an sqlite3_malloc'd buffer
114410	** containing the output doclist and SQLITE_OK is returned. In this case
114411	** *pnOut is set to the number of bytes in the output doclist.
114412	**
114413	** If an error occurs, an SQLite error code is returned. The output values
114414	** are undefined in this case.
114415	*/
114416	static int fts3DoclistOrMerge(
114417	int bDescDoclist, /* True if arguments are desc */
114418	char a1, int n1, / First doclist */
114419	char a2, int n2, / Second doclist */
114420	char *paOut, int pnOut /* OUT: Malloc'd doclist */
114421	){
114422	sqlite3_int64 i1 = 0;
	@@ -113968,35 +114437,47 @@
114437
114438	p = aOut;
114439	fts3GetDeltaVarint3(&p1, pEnd1, 0, &i1);
114440	fts3GetDeltaVarint3(&p2, pEnd2, 0, &i2);
114441	while( p1 \|\| p2 ){
114442	sqlite3_int64 iDiff = DOCID_CMP(i1, i2);
114443
114444	if( p2 && p1 && iDiff==0 ){
114445	fts3PutDeltaVarint3(&p, bDescDoclist, &iPrev, &bFirstOut, i1);
114446	fts3PoslistMerge(&p, &p1, &p2);
114447	fts3GetDeltaVarint3(&p1, pEnd1, bDescDoclist, &i1);
114448	fts3GetDeltaVarint3(&p2, pEnd2, bDescDoclist, &i2);
114449	}else if( !p2 \|\| (p1 && iDiff<0) ){
114450	fts3PutDeltaVarint3(&p, bDescDoclist, &iPrev, &bFirstOut, i1);
114451	fts3PoslistCopy(&p, &p1);
114452	fts3GetDeltaVarint3(&p1, pEnd1, bDescDoclist, &i1);
114453	}else{
114454	fts3PutDeltaVarint3(&p, bDescDoclist, &iPrev, &bFirstOut, i2);
114455	fts3PoslistCopy(&p, &p2);
114456	fts3GetDeltaVarint3(&p2, pEnd2, bDescDoclist, &i2);
114457	}
114458	}
114459
114460	*paOut = aOut;
114461	*pnOut = (p-aOut);
114462	return SQLITE_OK;
114463	}
114464
114465	/*
114466	** This function does a "phrase" merge of two doclists. In a phrase merge,
114467	** the output contains a copy of each position from the right-hand input
114468	** doclist for which there is a position in the left-hand input doclist
114469	** exactly nDist tokens before it.
114470	**
114471	** If the docids in the input doclists are sorted in ascending order,
114472	** parameter bDescDoclist should be false. If they are sorted in ascending
114473	** order, it should be passed a non-zero value.
114474	**
114475	** The right-hand input doclist is overwritten by this function.
114476	*/
114477	static void fts3DoclistPhraseMerge(
114478	int bDescDoclist, /* True if arguments are desc */
114479	int nDist, /* Distance from left to right (1=adjacent) */
114480	char aLeft, int nLeft, / Left doclist */
114481	char aRight, int pnRight /* IN/OUT: Right/output doclist */
114482	){
114483	sqlite3_int64 i1 = 0;
	@@ -114015,30 +114496,30 @@
114496	p = aOut;
114497	fts3GetDeltaVarint3(&p1, pEnd1, 0, &i1);
114498	fts3GetDeltaVarint3(&p2, pEnd2, 0, &i2);
114499
114500	while( p1 && p2 ){
114501	sqlite3_int64 iDiff = DOCID_CMP(i1, i2);
114502	if( iDiff==0 ){
114503	char *pSave = p;
114504	sqlite3_int64 iPrevSave = iPrev;
114505	int bFirstOutSave = bFirstOut;
114506
114507	fts3PutDeltaVarint3(&p, bDescDoclist, &iPrev, &bFirstOut, i1);
114508	if( 0==fts3PoslistPhraseMerge(&p, nDist, 0, 1, &p1, &p2) ){
114509	p = pSave;
114510	iPrev = iPrevSave;
114511	bFirstOut = bFirstOutSave;
114512	}
114513	fts3GetDeltaVarint3(&p1, pEnd1, bDescDoclist, &i1);
114514	fts3GetDeltaVarint3(&p2, pEnd2, bDescDoclist, &i2);
114515	}else if( iDiff<0 ){
114516	fts3PoslistCopy(0, &p1);
114517	fts3GetDeltaVarint3(&p1, pEnd1, bDescDoclist, &i1);
114518	}else{
114519	fts3PoslistCopy(0, &p2);
114520	fts3GetDeltaVarint3(&p2, pEnd2, bDescDoclist, &i2);
114521	}
114522	}
114523
114524	*pnRight = p - aOut;
114525	}
	@@ -114051,11 +114532,11 @@
114532	**
114533	** If an OOM error occurs, return SQLITE_NOMEM. In this case it is
114534	** the responsibility of the caller to free any doclists left in the
114535	** TermSelect.aaOutput[] array.
114536	*/
114537	static int fts3TermSelectFinishMerge(Fts3Table p, TermSelect pTS){
114538	char *aOut = 0;
114539	int nOut = 0;
114540	int i;
114541
114542	/* Loop through the doclists in the aaOutput[] array. Merge them all
	@@ -114092,28 +114573,29 @@
114573	pTS->anOutput[0] = nOut;
114574	return SQLITE_OK;
114575	}
114576
114577	/*
114578	** Merge the doclist aDoclist/nDoclist into the TermSelect object passed
114579	** as the first argument. The merge is an "OR" merge (see function
114580	** fts3DoclistOrMerge() for details).
114581	**
114582	** This function is called with the doclist for each term that matches
114583	** a queried prefix. It merges all these doclists into one, the doclist
114584	** for the specified prefix. Since there can be a very large number of
114585	** doclists to merge, the merging is done pair-wise using the TermSelect
114586	** object.
114587	**
114588	** This function returns SQLITE_OK if the merge is successful, or an
114589	** SQLite error code (SQLITE_NOMEM) if an error occurs.
114590	*/
114591	static int fts3TermSelectMerge(
114592	Fts3Table p, / FTS table handle */
114593	TermSelect pTS, / TermSelect object to merge into */
114594	char aDoclist, / Pointer to doclist */
114595	int nDoclist /* Size of aDoclist in bytes */


114596	){






114597	if( pTS->aaOutput[0]==0 ){
114598	/* If this is the first term selected, copy the doclist to the output
114599	** buffer using memcpy(). */
114600	pTS->aaOutput[0] = sqlite3_malloc(nDoclist);
114601	pTS->anOutput[0] = nDoclist;
	@@ -114180,23 +114662,30 @@
114662	}
114663	pCsr->apSegment[pCsr->nSegment++] = pNew;
114664	return SQLITE_OK;
114665	}
114666
114667	/*
114668	** Add seg-reader objects to the Fts3MultiSegReader object passed as the
114669	** 8th argument.
114670	**
114671	** This function returns SQLITE_OK if successful, or an SQLite error code
114672	** otherwise.
114673	*/
114674	static int fts3SegReaderCursor(
114675	Fts3Table p, / FTS3 table handle */
114676	int iIndex, /* Index to search (from 0 to p->nIndex-1) */
114677	int iLevel, /* Level of segments to scan */
114678	const char zTerm, / Term to query for */
114679	int nTerm, /* Size of zTerm in bytes */
114680	int isPrefix, /* True for a prefix search */
114681	int isScan, /* True to scan from zTerm to EOF */
114682	Fts3MultiSegReader pCsr / Cursor object to populate */
114683	){
114684	int rc = SQLITE_OK; /* Error code */
114685	sqlite3_stmt pStmt = 0; / Statement to iterate through segments */
114686	int rc2; /* Result of sqlite3_reset() */
114687
114688	/* If iLevel is less than 0 and this is not a scan, include a seg-reader
114689	** for the pending-terms. If this is a scan, then this call must be being
114690	** made by an fts4aux module, not an FTS table. In this case calling
114691	** Fts3SegReaderPending might segfault, as the data structures used by
	@@ -114281,28 +114770,46 @@
114770	return fts3SegReaderCursor(
114771	p, iIndex, iLevel, zTerm, nTerm, isPrefix, isScan, pCsr
114772	);
114773	}
114774
114775	/*
114776	** In addition to its current configuration, have the Fts3MultiSegReader
114777	** passed as the 4th argument also scan the doclist for term zTerm/nTerm.
114778	**
114779	** SQLITE_OK is returned if no error occurs, otherwise an SQLite error code.
114780	*/
114781	static int fts3SegReaderCursorAddZero(
114782	Fts3Table p, / FTS virtual table handle */
114783	const char zTerm, / Term to scan doclist of */
114784	int nTerm, /* Number of bytes in zTerm */
114785	Fts3MultiSegReader pCsr / Fts3MultiSegReader to modify */
114786	){
114787	return fts3SegReaderCursor(p, 0, FTS3_SEGCURSOR_ALL, zTerm, nTerm, 0, 0,pCsr);
114788	}
114789
114790	/*
114791	** Open an Fts3MultiSegReader to scan the doclist for term zTerm/nTerm. Or,
114792	** if isPrefix is true, to scan the doclist for all terms for which
114793	** zTerm/nTerm is a prefix. If successful, return SQLITE_OK and write
114794	** a pointer to the new Fts3MultiSegReader to *ppSegcsr. Otherwise, return
114795	** an SQLite error code.
114796	**
114797	** It is the responsibility of the caller to free this object by eventually
114798	** passing it to fts3SegReaderCursorFree()
114799	**
114800	** SQLITE_OK is returned if no error occurs, otherwise an SQLite error code.
114801	** Output parameter *ppSegcsr is set to 0 if an error occurs.
114802	*/
114803	static int fts3TermSegReaderCursor(
114804	Fts3Cursor pCsr, / Virtual table cursor handle */
114805	const char zTerm, / Term to query for */
114806	int nTerm, /* Size of zTerm in bytes */
114807	int isPrefix, /* True for a prefix search */
114808	Fts3MultiSegReader *ppSegcsr / OUT: Allocated seg-reader cursor */
114809	){
114810	Fts3MultiSegReader pSegcsr; / Object to allocate and return */
114811	int rc = SQLITE_NOMEM; /* Return code */
114812
114813	pSegcsr = sqlite3_malloc(sizeof(Fts3MultiSegReader));
114814	if( pSegcsr ){
114815	int i;
	@@ -114342,62 +114849,53 @@
114849
114850	*ppSegcsr = pSegcsr;
114851	return rc;
114852	}
114853
114854	/*
114855	** Free an Fts3MultiSegReader allocated by fts3TermSegReaderCursor().
114856	*/
114857	static void fts3SegReaderCursorFree(Fts3MultiSegReader *pSegcsr){
114858	sqlite3Fts3SegReaderFinish(pSegcsr);
114859	sqlite3_free(pSegcsr);
114860	}
114861
114862	/*
114863	** This function retreives the doclist for the specified term (or term
114864	** prefix) from the database.







114865	*/
114866	static int fts3TermSelect(
114867	Fts3Table p, / Virtual table handle */
114868	Fts3PhraseToken pTok, / Token to query for */
114869	int iColumn, /* Column to query (or -ve for all columns) */

114870	int pnOut, / OUT: Size of buffer at ppOut /
114871	char *ppOut / OUT: Malloced result buffer */
114872	){
114873	int rc; /* Return code */
114874	Fts3MultiSegReader pSegcsr; / Seg-reader cursor for this term */
114875	TermSelect tsc; /* Object for pair-wise doclist merging */
114876	Fts3SegFilter filter; /* Segment term filter configuration */
114877
114878	pSegcsr = pTok->pSegcsr;
114879	memset(&tsc, 0, sizeof(TermSelect));

114880
114881	filter.flags = FTS3_SEGMENT_IGNORE_EMPTY \| FTS3_SEGMENT_REQUIRE_POS
114882	\| (pTok->isPrefix ? FTS3_SEGMENT_PREFIX : 0)

114883	\| (iColumn<p->nColumn ? FTS3_SEGMENT_COLUMN_FILTER : 0);
114884	filter.iCol = iColumn;
114885	filter.zTerm = pTok->z;
114886	filter.nTerm = pTok->n;
114887
114888	rc = sqlite3Fts3SegReaderStart(p, pSegcsr, &filter);
114889	while( SQLITE_OK==rc
114890	&& SQLITE_ROW==(rc = sqlite3Fts3SegReaderStep(p, pSegcsr))
114891	){
114892	rc = fts3TermSelectMerge(p, &tsc, pSegcsr->aDoclist, pSegcsr->nDoclist);


114893	}
114894
114895	if( rc==SQLITE_OK ){
114896	rc = fts3TermSelectFinishMerge(p, &tsc);
114897	}
114898	if( rc==SQLITE_OK ){
114899	*ppOut = tsc.aaOutput[0];
114900	*pnOut = tsc.anOutput[0];
114901	}else{
	@@ -114419,28 +114917,19 @@
114917	** If the isPoslist argument is true, then it is assumed that the doclist
114918	** contains a position-list following each docid. Otherwise, it is assumed
114919	** that the doclist is simply a list of docids stored as delta encoded
114920	** varints.
114921	*/
114922	static int fts3DoclistCountDocids(char *aList, int nList){
114923	int nDoc = 0; /* Return value */
114924	if( aList ){
114925	char aEnd = &aList[nList]; / Pointer to one byte after EOF */
114926	char p = aList; / Cursor */
114927	while( p<aEnd ){
114928	nDoc++;
114929	while( (p++)&0x80 ); / Skip docid varint */
114930	fts3PoslistCopy(0, &p); /* Skip over position list */









114931	}
114932	}
114933
114934	return nDoc;
114935	}
	@@ -114466,11 +114955,11 @@
114955	}else{
114956	pCsr->iPrevId = sqlite3_column_int64(pCsr->pStmt, 0);
114957	rc = SQLITE_OK;
114958	}
114959	}else{
114960	rc = fts3EvalNext((Fts3Cursor *)pCursor);
114961	}
114962	assert( ((Fts3Table *)pCsr->base.pVtab)->pSegments==0 );
114963	return rc;
114964	}
114965
	@@ -114543,11 +115032,11 @@
115032	}
115033
115034	rc = sqlite3Fts3ReadLock(p);
115035	if( rc!=SQLITE_OK ) return rc;
115036
115037	rc = fts3EvalStart(pCsr);
115038
115039	sqlite3Fts3SegmentsClose(p);
115040	if( rc!=SQLITE_OK ) return rc;
115041	pCsr->pNextId = pCsr->aDoclist;
115042	pCsr->iPrevId = 0;
	@@ -114950,26 +115439,43 @@
115439	p->zDb, p->zName, zName
115440	);
115441	return rc;
115442	}
115443
115444	/*
115445	** The xSavepoint() method.
115446	**
115447	** Flush the contents of the pending-terms table to disk.
115448	*/
115449	static int fts3SavepointMethod(sqlite3_vtab *pVtab, int iSavepoint){
115450	UNUSED_PARAMETER(iSavepoint);
115451	assert( ((Fts3Table *)pVtab)->inTransaction );
115452	assert( ((Fts3Table *)pVtab)->mxSavepoint < iSavepoint );
115453	TESTONLY( ((Fts3Table *)pVtab)->mxSavepoint = iSavepoint );
115454	return fts3SyncMethod(pVtab);
115455	}
115456
115457	/*
115458	** The xRelease() method.
115459	**
115460	** This is a no-op.
115461	*/
115462	static int fts3ReleaseMethod(sqlite3_vtab *pVtab, int iSavepoint){
115463	TESTONLY( Fts3Table p = (Fts3Table)pVtab );
115464	UNUSED_PARAMETER(iSavepoint);
115465	UNUSED_PARAMETER(pVtab);
115466	assert( p->inTransaction );
115467	assert( p->mxSavepoint >= iSavepoint );
115468	TESTONLY( p->mxSavepoint = iSavepoint-1 );
115469	return SQLITE_OK;
115470	}
115471
115472	/*
115473	** The xRollbackTo() method.
115474	**
115475	** Discard the contents of the pending terms table.
115476	*/
115477	static int fts3RollbackToMethod(sqlite3_vtab *pVtab, int iSavepoint){
115478	Fts3Table p = (Fts3Table)pVtab;
115479	UNUSED_PARAMETER(iSavepoint);
115480	assert( p->inTransaction );
115481	assert( p->mxSavepoint >= iSavepoint );
	@@ -115114,22 +115620,10 @@
115620	sqlite3Fts3HashClear(pHash);
115621	sqlite3_free(pHash);
115622	}
115623	return rc;
115624	}












115625
115626	/*
115627	** Allocate an Fts3MultiSegReader for each token in the expression headed
115628	** by pExpr.
115629	**
	@@ -115143,24 +115637,24 @@
115637	** there exists prefix b-tree of the right length) then it may be traversed
115638	** and merged incrementally. Otherwise, it has to be merged into an in-memory
115639	** doclist and then traversed.
115640	*/
115641	static void fts3EvalAllocateReaders(
115642	Fts3Cursor pCsr, / FTS cursor handle */
115643	Fts3Expr pExpr, / Allocate readers for this expression */
115644	int pnToken, / OUT: Total number of tokens in phrase. */
115645	int pnOr, / OUT: Total number of OR nodes in expr. */
115646	int pRc / IN/OUT: Error code */
115647	){
115648	if( pExpr && SQLITE_OK==*pRc ){
115649	if( pExpr->eType==FTSQUERY_PHRASE ){
115650	int i;
115651	int nToken = pExpr->pPhrase->nToken;
115652	*pnToken += nToken;
115653	for(i=0; i<nToken; i++){
115654	Fts3PhraseToken *pToken = &pExpr->pPhrase->aToken[i];
115655	int rc = fts3TermSegReaderCursor(pCsr,
115656	pToken->z, pToken->n, pToken->isPrefix, &pToken->pSegcsr
115657	);
115658	if( rc!=SQLITE_OK ){
115659	*pRc = rc;
115660	return;
	@@ -115174,16 +115668,24 @@
115668	fts3EvalAllocateReaders(pCsr, pExpr->pRight, pnToken, pnOr, pRc);
115669	}
115670	}
115671	}
115672
115673	/*
115674	** Arguments pList/nList contain the doclist for token iToken of phrase p.
115675	** It is merged into the main doclist stored in p->doclist.aAll/nAll.
115676	**
115677	** This function assumes that pList points to a buffer allocated using
115678	** sqlite3_malloc(). This function takes responsibility for eventually
115679	** freeing the buffer.
115680	*/
115681	static void fts3EvalPhraseMergeToken(
115682	Fts3Table pTab, / FTS Table pointer */
115683	Fts3Phrase p, / Phrase to merge pList/nList into */
115684	int iToken, /* Token pList/nList corresponds to */
115685	char pList, / Pointer to doclist */
115686	int nList /* Number of bytes in pList */
115687	){
115688	assert( iToken!=p->iDoclistToken );
115689
115690	if( pList==0 ){
115691	sqlite3_free(p->doclist.aAll);
	@@ -115228,13 +115730,19 @@
115730	}
115731
115732	if( iToken>p->iDoclistToken ) p->iDoclistToken = iToken;
115733	}
115734
115735	/*
115736	** Load the doclist for phrase p into p->doclist.aAll/nAll. The loaded doclist
115737	** does not take deferred tokens into account.
115738	**
115739	** SQLITE_OK is returned if no error occurs, otherwise an SQLite error code.
115740	*/
115741	static int fts3EvalPhraseLoad(
115742	Fts3Cursor pCsr, / FTS Cursor handle */
115743	Fts3Phrase p / Phrase object */
115744	){
115745	Fts3Table pTab = (Fts3Table )pCsr->base.pVtab;
115746	int iToken;
115747	int rc = SQLITE_OK;
115748
	@@ -115243,11 +115751,11 @@
115751	assert( pToken->pDeferred==0 \|\| pToken->pSegcsr==0 );
115752
115753	if( pToken->pSegcsr ){
115754	int nThis = 0;
115755	char *pThis = 0;
115756	rc = fts3TermSelect(pTab, pToken, p->iColumn, &nThis, &pThis);
115757	if( rc==SQLITE_OK ){
115758	fts3EvalPhraseMergeToken(pTab, p, iToken, pThis, nThis);
115759	}
115760	}
115761	assert( pToken->pSegcsr==0 );
	@@ -115254,18 +115762,26 @@
115762	}
115763
115764	return rc;
115765	}
115766
115767	/*
115768	** This function is called on each phrase after the position lists for
115769	** any deferred tokens have been loaded into memory. It updates the phrases
115770	** current position list to include only those positions that are really
115771	** instances of the phrase (after considering deferred tokens). If this
115772	** means that the phrase does not appear in the current row, doclist.pList
115773	** and doclist.nList are both zeroed.
115774	**
115775	** SQLITE_OK is returned if no error occurs, otherwise an SQLite error code.
115776	*/
115777	static int fts3EvalDeferredPhrase(Fts3Cursor pCsr, Fts3Phrase pPhrase){
115778	int iToken; /* Used to iterate through phrase tokens */
115779	int rc = SQLITE_OK; /* Return code */
115780	char aPoslist = 0; / Position list for deferred tokens */
115781	int nPoslist = 0; /* Number of bytes in aPoslist */
115782	int iPrev = -1; /* Token number of previous deferred token */


115783
115784	assert( pPhrase->doclist.bFreeList==0 );
115785
115786	for(iToken=0; rc==SQLITE_OK && iToken<pPhrase->nToken; iToken++){
115787	Fts3PhraseToken *pToken = &pPhrase->aToken[iToken];
	@@ -115307,10 +115823,11 @@
115823	iPrev = iToken;
115824	}
115825	}
115826
115827	if( iPrev>=0 ){
115828	int nMaxUndeferred = pPhrase->iDoclistToken;
115829	if( nMaxUndeferred<0 ){
115830	pPhrase->doclist.pList = aPoslist;
115831	pPhrase->doclist.nList = nPoslist;
115832	pPhrase->doclist.iDocid = pCsr->iPrevId;
115833	pPhrase->doclist.bFreeList = 1;
	@@ -115355,13 +115872,19 @@
115872	/*
115873	** This function is called for each Fts3Phrase in a full-text query
115874	** expression to initialize the mechanism for returning rows. Once this
115875	** function has been called successfully on an Fts3Phrase, it may be
115876	** used with fts3EvalPhraseNext() to iterate through the matching docids.
115877	**
115878	** If parameter bOptOk is true, then the phrase may (or may not) use the
115879	** incremental loading strategy. Otherwise, the entire doclist is loaded into
115880	** memory within this call.
115881	**
115882	** SQLITE_OK is returned if no error occurs, otherwise an SQLite error code.
115883	*/
115884	static int fts3EvalPhraseStart(Fts3Cursor pCsr, int bOptOk, Fts3Phrase p){
115885	int rc; /* Error code */
115886	Fts3PhraseToken *pFirst = &p->aToken[0];
115887	Fts3Table pTab = (Fts3Table )pCsr->base.pVtab;
115888
115889	if( pCsr->bDesc==pTab->bDescIdx
115890	&& bOptOk==1
	@@ -115385,11 +115908,17 @@
115908	return rc;
115909	}
115910
115911	/*
115912	** This function is used to iterate backwards (from the end to start)
115913	** through doclists. It is used by this module to iterate through phrase
115914	** doclists in reverse and by the fts3_write.c module to iterate through
115915	** pending-terms lists when writing to databases with "order=desc".
115916	**
115917	** The doclist may be sorted in ascending (parameter bDescIdx==0) or
115918	** descending (parameter bDescIdx==1) order of docid. Regardless, this
115919	** function iterates from the end of the doclist to the beginning.
115920	*/
115921	SQLITE_PRIVATE void sqlite3Fts3DoclistPrev(
115922	int bDescIdx, /* True if the doclist is desc */
115923	char aDoclist, / Pointer to entire doclist */
115924	int nDoclist, /* Length of aDoclist in bytes */
	@@ -115450,13 +115979,13 @@
115979	** If there is no "next" entry and no error occurs, then *pbEof is set to
115980	** 1 before returning. Otherwise, if no error occurs and the iterator is
115981	** successfully advanced, *pbEof is set to 0.
115982	*/
115983	static int fts3EvalPhraseNext(
115984	Fts3Cursor pCsr, / FTS Cursor handle */
115985	Fts3Phrase p, / Phrase object to advance to next docid */
115986	u8 pbEof / OUT: Set to 1 if EOF */
115987	){
115988	int rc = SQLITE_OK;
115989	Fts3Doclist *pDL = &p->doclist;
115990	Fts3Table pTab = (Fts3Table )pCsr->base.pVtab;
115991
	@@ -115498,14 +116027,14 @@
116027	fts3PoslistCopy(0, &pIter);
116028	pDL->nList = (pIter - pDL->pList);
116029
116030	/* pIter now points just past the 0x00 that terminates the position-
116031	** list for document pDL->iDocid. However, if this position-list was
116032	** edited in place by fts3EvalNearTrim(), then pIter may not actually
116033	** point to the start of the next docid value. The following line deals
116034	** with this case by advancing pIter past the zero-padding added by
116035	** fts3EvalNearTrim(). */
116036	while( pIter<pEnd && *pIter==0 ) pIter++;
116037
116038	pDL->pNextDocid = pIter;
116039	assert( pIter>=&pDL->aAll[pDL->nAll] \|\| *pIter );
116040	*pbEof = 0;
	@@ -115513,15 +116042,31 @@
116042	}
116043
116044	return rc;
116045	}
116046
116047	/*
116048	**
116049	** If *pRc is not SQLITE_OK when this function is called, it is a no-op.
116050	** Otherwise, fts3EvalPhraseStart() is called on all phrases within the
116051	** expression. Also the Fts3Expr.bDeferred variable is set to true for any
116052	** expressions for which all descendent tokens are deferred.
116053	**
116054	** If parameter bOptOk is zero, then it is guaranteed that the
116055	** Fts3Phrase.doclist.aAll/nAll variables contain the entire doclist for
116056	** each phrase in the expression (subject to deferred token processing).
116057	** Or, if bOptOk is non-zero, then one or more tokens within the expression
116058	** may be loaded incrementally, meaning doclist.aAll/nAll is not available.
116059	**
116060	** If an error occurs within this function, *pRc is set to an SQLite error
116061	** code before returning.
116062	*/
116063	static void fts3EvalStartReaders(
116064	Fts3Cursor pCsr, / FTS Cursor handle */
116065	Fts3Expr pExpr, / Expression to initialize phrases in */
116066	int bOptOk, /* True to enable incremental loading */
116067	int pRc / IN/OUT: Error code */
116068	){
116069	if( pExpr && SQLITE_OK==*pRc ){
116070	if( pExpr->eType==FTSQUERY_PHRASE ){
116071	int i;
116072	int nToken = pExpr->pPhrase->nToken;
	@@ -115536,27 +116081,46 @@
116081	pExpr->bDeferred = (pExpr->pLeft->bDeferred && pExpr->pRight->bDeferred);
116082	}
116083	}
116084	}
116085
116086	/*
116087	** An array of the following structures is assembled as part of the process
116088	** of selecting tokens to defer before the query starts executing (as part
116089	** of the xFilter() method). There is one element in the array for each
116090	** token in the FTS expression.
116091	**
116092	** Tokens are divided into AND/NEAR clusters. All tokens in a cluster belong
116093	** to phrases that are connected only by AND and NEAR operators (not OR or
116094	** NOT). When determining tokens to defer, each AND/NEAR cluster is considered
116095	** separately. The root of a tokens AND/NEAR cluster is stored in
116096	** Fts3TokenAndCost.pRoot.
116097	*/
116098	typedef struct Fts3TokenAndCost Fts3TokenAndCost;
116099	struct Fts3TokenAndCost {
116100	Fts3Phrase pPhrase; / The phrase the token belongs to */
116101	int iToken; /* Position of token in phrase */
116102	Fts3PhraseToken pToken; / The token itself */
116103	Fts3Expr pRoot; / Root of NEAR/AND cluster */
116104	int nOvfl; /* Number of overflow pages to load doclist */
116105	int iCol; /* The column the token must match */
116106	};
116107
116108	/*
116109	** This function is used to populate an allocated Fts3TokenAndCost array.
116110	**
116111	** If *pRc is not SQLITE_OK when this function is called, it is a no-op.
116112	** Otherwise, if an error occurs during execution, *pRc is set to an
116113	** SQLite error code.
116114	*/
116115	static void fts3EvalTokenCosts(
116116	Fts3Cursor pCsr, / FTS Cursor handle */
116117	Fts3Expr pRoot, / Root of current AND/NEAR cluster */
116118	Fts3Expr pExpr, / Expression to consider */
116119	Fts3TokenAndCost *ppTC, / Write new entries to (ppTC)++ */
116120	Fts3Expr **ppOr, / Write new OR root to (ppOr)++ */
116121	int pRc / IN/OUT: Error code */
116122	){
116123	if( *pRc==SQLITE_OK && pExpr ){
116124	if( pExpr->eType==FTSQUERY_PHRASE ){
116125	Fts3Phrase *pPhrase = pExpr->pPhrase;
116126	int i;
	@@ -115584,23 +116148,34 @@
116148	fts3EvalTokenCosts(pCsr, pRoot, pExpr->pRight, ppTC, ppOr, pRc);
116149	}
116150	}
116151	}
116152
116153	/*
116154	** Determine the average document (row) size in pages. If successful,
116155	** write this value to *pnPage and return SQLITE_OK. Otherwise, return
116156	** an SQLite error code.
116157	**
116158	** The average document size in pages is calculated by first calculating
116159	** determining the average size in bytes, B. If B is less than the amount
116160	** of data that will fit on a single leaf page of an intkey table in
116161	** this database, then the average docsize is 1. Otherwise, it is 1 plus
116162	** the number of overflow pages consumed by a record B bytes in size.
116163	*/
116164	static int fts3EvalAverageDocsize(Fts3Cursor pCsr, int pnPage){
116165	if( pCsr->nRowAvg==0 ){
116166	/* The average document size, which is required to calculate the cost
116167	** of each doclist, has not yet been determined. Read the required
116168	** data from the %_stat table to calculate it.
116169	**
116170	** Entry 0 of the %_stat table is a blob containing (nCol+1) FTS3
116171	** varints, where nCol is the number of columns in the FTS3 table.
116172	** The first varint is the number of documents currently stored in
116173	** the table. The following nCol varints contain the total amount of
116174	** data stored in all rows of each column of the table, from left
116175	** to right.
116176	*/
116177	int rc;
116178	Fts3Table p = (Fts3Table)pCsr->base.pVtab;
116179	sqlite3_stmt *pStmt;
116180	sqlite3_int64 nDoc = 0;
116181	sqlite3_int64 nByte = 0;
	@@ -115631,109 +116206,151 @@
116206
116207	*pnPage = pCsr->nRowAvg;
116208	return SQLITE_OK;
116209	}
116210
116211	/*
116212	** This function is called to select the tokens (if any) that will be
116213	** deferred. The array aTC[] has already been populated when this is
116214	** called.
116215	**
116216	** This function is called once for each AND/NEAR cluster in the
116217	** expression. Each invocation determines which tokens to defer within
116218	** the cluster with root node pRoot. See comments above the definition
116219	** of struct Fts3TokenAndCost for more details.
116220	**
116221	** If no error occurs, SQLITE_OK is returned and sqlite3Fts3DeferToken()
116222	** called on each token to defer. Otherwise, an SQLite error code is
116223	** returned.
116224	*/
116225	static int fts3EvalSelectDeferred(
116226	Fts3Cursor pCsr, / FTS Cursor handle */
116227	Fts3Expr pRoot, / Consider tokens with this root node */
116228	Fts3TokenAndCost aTC, / Array of expression tokens and costs */
116229	int nTC /* Number of entries in aTC[] */
116230	){



116231	Fts3Table pTab = (Fts3Table )pCsr->base.pVtab;
116232	int nDocSize = 0; /* Number of pages per doc loaded */
116233	int rc = SQLITE_OK; /* Return code */
116234	int ii; /* Iterator variable for various purposes */
116235	int nOvfl = 0; /* Total overflow pages used by doclists */
116236	int nToken = 0; /* Total number of tokens in cluster */
116237
116238	int nMinEst = 0; /* The minimum count for any phrase so far. */
116239	int nLoad4 = 1; /* (Phrases that will be loaded)^4. */
116240
116241	/* Count the tokens in this AND/NEAR cluster. If none of the doclists
116242	** associated with the tokens spill onto overflow pages, or if there is
116243	** only 1 token, exit early. No tokens to defer in this case. */
116244	for(ii=0; ii<nTC; ii++){
116245	if( aTC[ii].pRoot==pRoot ){
116246	nOvfl += aTC[ii].nOvfl;
116247	nToken++;
116248	}
116249	}
116250	if( nOvfl==0 \|\| nToken<2 ) return SQLITE_OK;
116251
116252	/* Obtain the average docsize (in pages). */
116253	rc = fts3EvalAverageDocsize(pCsr, &nDocSize);
116254	assert( rc!=SQLITE_OK \|\| nDocSize>0 );
116255
116256
116257	/* Iterate through all tokens in this AND/NEAR cluster, in ascending order
116258	** of the number of overflow pages that will be loaded by the pager layer
116259	** to retrieve the entire doclist for the token from the full-text index.
116260	** Load the doclists for tokens that are either:
116261	**
116262	** a. The cheapest token in the entire query (i.e. the one visited by the
116263	** first iteration of this loop), or
116264	**
116265	** b. Part of a multi-token phrase.
116266	**
116267	** After each token doclist is loaded, merge it with the others from the
116268	** same phrase and count the number of documents that the merged doclist
116269	** contains. Set variable "nMinEst" to the smallest number of documents in
116270	** any phrase doclist for which 1 or more token doclists have been loaded.
116271	** Let nOther be the number of other phrases for which it is certain that
116272	** one or more tokens will not be deferred.
116273	**
116274	** Then, for each token, defer it if loading the doclist would result in
116275	** loading N or more overflow pages into memory, where N is computed as:
116276	**
116277	** (nMinEst + 4^nOther - 1) / (4^nOther)
116278	*/
116279	for(ii=0; ii<nToken && rc==SQLITE_OK; ii++){
116280	int iTC; /* Used to iterate through aTC[] array. */
116281	Fts3TokenAndCost pTC = 0; / Set to cheapest remaining token. */
116282
116283	/* Set pTC to point to the cheapest remaining token. */
116284	for(iTC=0; iTC<nTC; iTC++){
116285	if( aTC[iTC].pToken && aTC[iTC].pRoot==pRoot
116286	&& (!pTC \|\| aTC[iTC].nOvfl<pTC->nOvfl)
116287	){
116288	pTC = &aTC[iTC];
116289	}
116290	}
116291	assert( pTC );
116292
116293	if( ii && pTC->nOvfl>=((nMinEst+(nLoad4/4)-1)/(nLoad4/4))*nDocSize ){
116294	/* The number of overflow pages to load for this (and therefore all
116295	** subsequent) tokens is greater than the estimated number of pages
116296	** that will be loaded if all subsequent tokens are deferred.
116297	*/
116298	Fts3PhraseToken *pToken = pTC->pToken;
116299	rc = sqlite3Fts3DeferToken(pCsr, pToken, pTC->iCol);
116300	fts3SegReaderCursorFree(pToken->pSegcsr);
116301	pToken->pSegcsr = 0;
116302	}else{
116303	nLoad4 = nLoad4*4;
116304	if( ii==0 \|\| pTC->pPhrase->nToken>1 ){
116305	/* Either this is the cheapest token in the entire query, or it is
116306	** part of a multi-token phrase. Either way, the entire doclist will
116307	** (eventually) be loaded into memory. It may as well be now. */
116308	Fts3PhraseToken *pToken = pTC->pToken;
116309	int nList = 0;
116310	char *pList = 0;
116311	rc = fts3TermSelect(pTab, pToken, pTC->iCol, &nList, &pList);
116312	assert( rc==SQLITE_OK \|\| pList==0 );

116313	if( rc==SQLITE_OK ){
116314	int nCount;
116315	fts3EvalPhraseMergeToken(pTab, pTC->pPhrase, pTC->iToken,pList,nList);
116316	nCount = fts3DoclistCountDocids(
116317	pTC->pPhrase->doclist.aAll, pTC->pPhrase->doclist.nAll
116318	);
116319	if( ii==0 \|\| nCount<nMinEst ) nMinEst = nCount;
116320	}
116321	}








116322	}
116323	pTC->pToken = 0;
116324	}
116325
116326	return rc;
116327	}
116328
116329	/*
116330	** This function is called from within the xFilter method. It initializes
116331	** the full-text query currently stored in pCsr->pExpr. To iterate through
116332	** the results of a query, the caller does:
116333	**
116334	** fts3EvalStart(pCsr);
116335	** while( 1 ){
116336	** fts3EvalNext(pCsr);
116337	** if( pCsr->bEof ) break;
116338	** ... return row pCsr->iPrevId to the caller ...
116339	** }
116340	*/
116341	static int fts3EvalStart(Fts3Cursor *pCsr){
116342	Fts3Table pTab = (Fts3Table )pCsr->base.pVtab;
116343	int rc = SQLITE_OK;
116344	int nToken = 0;
116345	int nOr = 0;
116346
116347	/* Allocate a MultiSegReader for each token in the expression. */
116348	fts3EvalAllocateReaders(pCsr, pCsr->pExpr, &nToken, &nOr, &rc);
116349
116350	/* Determine which, if any, tokens in the expression should be deferred. */
116351	if( rc==SQLITE_OK && nToken>1 && pTab->bHasStat ){



















116352	Fts3TokenAndCost *aTC;
116353	Fts3Expr **apOr;
116354	aTC = (Fts3TokenAndCost *)sqlite3_malloc(
116355	sizeof(Fts3TokenAndCost) * nToken
116356	+ sizeof(Fts3Expr ) nOr * 2
	@@ -115745,11 +116362,11 @@
116362	}else{
116363	int ii;
116364	Fts3TokenAndCost *pTC = aTC;
116365	Fts3Expr **ppOr = apOr;
116366
116367	fts3EvalTokenCosts(pCsr, 0, pCsr->pExpr, &pTC, &ppOr, &rc);
116368	nToken = pTC-aTC;
116369	nOr = ppOr-apOr;
116370
116371	if( rc==SQLITE_OK ){
116372	rc = fts3EvalSelectDeferred(pCsr, 0, aTC, nToken);
	@@ -115760,25 +116377,50 @@
116377
116378	sqlite3_free(aTC);
116379	}
116380	}
116381
116382	fts3EvalStartReaders(pCsr, pCsr->pExpr, 1, &rc);
116383	return rc;
116384	}
116385
116386	/*
116387	** Invalidate the current position list for phrase pPhrase.
116388	*/
116389	static void fts3EvalInvalidatePoslist(Fts3Phrase *pPhrase){
116390	if( pPhrase->doclist.bFreeList ){
116391	sqlite3_free(pPhrase->doclist.pList);
116392	}
116393	pPhrase->doclist.pList = 0;
116394	pPhrase->doclist.nList = 0;
116395	pPhrase->doclist.bFreeList = 0;
116396	}
116397
116398	/*
116399	** This function is called to edit the position list associated with
116400	** the phrase object passed as the fifth argument according to a NEAR
116401	** condition. For example:
116402	**
116403	** abc NEAR/5 "def ghi"
116404	**
116405	** Parameter nNear is passed the NEAR distance of the expression (5 in
116406	** the example above). When this function is called, *paPoslist points to
116407	** the position list, and *pnToken is the number of phrase tokens in, the
116408	** phrase on the other side of the NEAR operator to pPhrase. For example,
116409	** if pPhrase refers to the "def ghi" phrase, then *paPoslist points to
116410	** the position list associated with phrase "abc".
116411	**
116412	** All positions in the pPhrase position list that are not sufficiently
116413	** close to a position in the *paPoslist position list are removed. If this
116414	** leaves 0 positions, zero is returned. Otherwise, non-zero.
116415	**
116416	** Before returning, *paPoslist is set to point to the position lsit
116417	** associated with pPhrase. And *pnToken is set to the number of tokens in
116418	** pPhrase.
116419	*/
116420	static int fts3EvalNearTrim(
116421	int nNear, /* NEAR distance. As in "NEAR/nNear". */
116422	char aTmp, / Temporary space to use */
116423	char *paPoslist, / IN/OUT: Position list */
116424	int pnToken, / IN/OUT: Tokens in phrase of paPoslist /
116425	Fts3Phrase pPhrase / The phrase object to trim the doclist of */
116426	){
	@@ -115806,10 +116448,176 @@
116448	}
116449
116450	return res;
116451	}
116452
116453	/*
116454	** This function is a no-op if *pRc is other than SQLITE_OK when it is called.
116455	** Otherwise, it advances the expression passed as the second argument to
116456	** point to the next matching row in the database. Expressions iterate through
116457	** matching rows in docid order. Ascending order if Fts3Cursor.bDesc is zero,
116458	** or descending if it is non-zero.
116459	**
116460	** If an error occurs, *pRc is set to an SQLite error code. Otherwise, if
116461	** successful, the following variables in pExpr are set:
116462	**
116463	** Fts3Expr.bEof (non-zero if EOF - there is no next row)
116464	** Fts3Expr.iDocid (valid if bEof==0. The docid of the next row)
116465	**
116466	** If the expression is of type FTSQUERY_PHRASE, and the expression is not
116467	** at EOF, then the following variables are populated with the position list
116468	** for the phrase for the visited row:
116469	**
116470	** FTs3Expr.pPhrase->doclist.nList (length of pList in bytes)
116471	** FTs3Expr.pPhrase->doclist.pList (pointer to position list)
116472	**
116473	** It says above that this function advances the expression to the next
116474	** matching row. This is usually true, but there are the following exceptions:
116475	**
116476	** 1. Deferred tokens are not taken into account. If a phrase consists
116477	** entirely of deferred tokens, it is assumed to match every row in
116478	** the db. In this case the position-list is not populated at all.
116479	**
116480	** Or, if a phrase contains one or more deferred tokens and one or
116481	** more non-deferred tokens, then the expression is advanced to the
116482	** next possible match, considering only non-deferred tokens. In other
116483	** words, if the phrase is "A B C", and "B" is deferred, the expression
116484	** is advanced to the next row that contains an instance of "A * C",
116485	** where "*" may match any single token. The position list in this case
116486	** is populated as for "A * C" before returning.
116487	**
116488	** 2. NEAR is treated as AND. If the expression is "x NEAR y", it is
116489	** advanced to point to the next row that matches "x AND y".
116490	**
116491	** See fts3EvalTestDeferredAndNear() for details on testing if a row is
116492	** really a match, taking into account deferred tokens and NEAR operators.
116493	*/
116494	static void fts3EvalNextRow(
116495	Fts3Cursor pCsr, / FTS Cursor handle */
116496	Fts3Expr pExpr, / Expr. to advance to next matching row */
116497	int pRc / IN/OUT: Error code */
116498	){
116499	if( *pRc==SQLITE_OK ){
116500	int bDescDoclist = pCsr->bDesc; /* Used by DOCID_CMP() macro */
116501	assert( pExpr->bEof==0 );
116502	pExpr->bStart = 1;
116503
116504	switch( pExpr->eType ){
116505	case FTSQUERY_NEAR:
116506	case FTSQUERY_AND: {
116507	Fts3Expr *pLeft = pExpr->pLeft;
116508	Fts3Expr *pRight = pExpr->pRight;
116509	assert( !pLeft->bDeferred \|\| !pRight->bDeferred );
116510
116511	if( pLeft->bDeferred ){
116512	/* LHS is entirely deferred. So we assume it matches every row.
116513	** Advance the RHS iterator to find the next row visited. */
116514	fts3EvalNextRow(pCsr, pRight, pRc);
116515	pExpr->iDocid = pRight->iDocid;
116516	pExpr->bEof = pRight->bEof;
116517	}else if( pRight->bDeferred ){
116518	/* RHS is entirely deferred. So we assume it matches every row.
116519	** Advance the LHS iterator to find the next row visited. */
116520	fts3EvalNextRow(pCsr, pLeft, pRc);
116521	pExpr->iDocid = pLeft->iDocid;
116522	pExpr->bEof = pLeft->bEof;
116523	}else{
116524	/* Neither the RHS or LHS are deferred. */
116525	fts3EvalNextRow(pCsr, pLeft, pRc);
116526	fts3EvalNextRow(pCsr, pRight, pRc);
116527	while( !pLeft->bEof && !pRight->bEof && *pRc==SQLITE_OK ){
116528	sqlite3_int64 iDiff = DOCID_CMP(pLeft->iDocid, pRight->iDocid);
116529	if( iDiff==0 ) break;
116530	if( iDiff<0 ){
116531	fts3EvalNextRow(pCsr, pLeft, pRc);
116532	}else{
116533	fts3EvalNextRow(pCsr, pRight, pRc);
116534	}
116535	}
116536	pExpr->iDocid = pLeft->iDocid;
116537	pExpr->bEof = (pLeft->bEof \|\| pRight->bEof);
116538	}
116539	break;
116540	}
116541
116542	case FTSQUERY_OR: {
116543	Fts3Expr *pLeft = pExpr->pLeft;
116544	Fts3Expr *pRight = pExpr->pRight;
116545	sqlite3_int64 iCmp = DOCID_CMP(pLeft->iDocid, pRight->iDocid);
116546
116547	assert( pLeft->bStart \|\| pLeft->iDocid==pRight->iDocid );
116548	assert( pRight->bStart \|\| pLeft->iDocid==pRight->iDocid );
116549
116550	if( pRight->bEof \|\| (pLeft->bEof==0 && iCmp<0) ){
116551	fts3EvalNextRow(pCsr, pLeft, pRc);
116552	}else if( pLeft->bEof \|\| (pRight->bEof==0 && iCmp>0) ){
116553	fts3EvalNextRow(pCsr, pRight, pRc);
116554	}else{
116555	fts3EvalNextRow(pCsr, pLeft, pRc);
116556	fts3EvalNextRow(pCsr, pRight, pRc);
116557	}
116558
116559	pExpr->bEof = (pLeft->bEof && pRight->bEof);
116560	iCmp = DOCID_CMP(pLeft->iDocid, pRight->iDocid);
116561	if( pRight->bEof \|\| (pLeft->bEof==0 && iCmp<0) ){
116562	pExpr->iDocid = pLeft->iDocid;
116563	}else{
116564	pExpr->iDocid = pRight->iDocid;
116565	}
116566
116567	break;
116568	}
116569
116570	case FTSQUERY_NOT: {
116571	Fts3Expr *pLeft = pExpr->pLeft;
116572	Fts3Expr *pRight = pExpr->pRight;
116573
116574	if( pRight->bStart==0 ){
116575	fts3EvalNextRow(pCsr, pRight, pRc);
116576	assert( *pRc!=SQLITE_OK \|\| pRight->bStart );
116577	}
116578
116579	fts3EvalNextRow(pCsr, pLeft, pRc);
116580	if( pLeft->bEof==0 ){
116581	while( !*pRc
116582	&& !pRight->bEof
116583	&& DOCID_CMP(pLeft->iDocid, pRight->iDocid)>0
116584	){
116585	fts3EvalNextRow(pCsr, pRight, pRc);
116586	}
116587	}
116588	pExpr->iDocid = pLeft->iDocid;
116589	pExpr->bEof = pLeft->bEof;
116590	break;
116591	}
116592
116593	default: {
116594	Fts3Phrase *pPhrase = pExpr->pPhrase;
116595	fts3EvalInvalidatePoslist(pPhrase);
116596	*pRc = fts3EvalPhraseNext(pCsr, pPhrase, &pExpr->bEof);
116597	pExpr->iDocid = pPhrase->doclist.iDocid;
116598	break;
116599	}
116600	}
116601	}
116602	}
116603
116604	/*
116605	** If *pRc is not SQLITE_OK, or if pExpr is not the root node of a NEAR
116606	** cluster, then this function returns 1 immediately.
116607	**
116608	** Otherwise, it checks if the current row really does match the NEAR
116609	** expression, using the data currently stored in the position lists
116610	** (Fts3Expr->pPhrase.doclist.pList/nList) for each phrase in the expression.
116611	**
116612	** If the current row is a match, the position list associated with each
116613	** phrase in the NEAR expression is edited in place to contain only those
116614	** phrase instances sufficiently close to their peers to satisfy all NEAR
116615	** constraints. In this case it returns 1. If the NEAR expression does not
116616	** match the current row, 0 is returned. The position lists may or may not
116617	** be edited if 0 is returned.
116618	*/
116619	static int fts3EvalNearTest(Fts3Expr pExpr, int pRc){
116620	int res = 1;
116621
116622	/* The following block runs if pExpr is the root of a NEAR query.
116623	** For example, the query:
	@@ -115827,11 +116635,11 @@
116635	** \| \|
116636	** "w" "x"
116637	**
116638	** The right-hand child of a NEAR node is always a phrase. The
116639	** left-hand child may be either a phrase or a NEAR node. There are
116640	** no exceptions to this - it's the way the parser in fts3_expr.c works.
116641	*/
116642	if( *pRc==SQLITE_OK
116643	&& pExpr->eType==FTSQUERY_NEAR
116644	&& pExpr->bEof==0
116645	&& (pExpr->pParent==0 \|\| pExpr->pParent->eType!=FTSQUERY_NEAR)
	@@ -115854,21 +116662,21 @@
116662	int nToken = p->pPhrase->nToken;
116663
116664	for(p=p->pParent;res && p && p->eType==FTSQUERY_NEAR; p=p->pParent){
116665	Fts3Phrase *pPhrase = p->pRight->pPhrase;
116666	int nNear = p->nNear;
116667	res = fts3EvalNearTrim(nNear, aTmp, &aPoslist, &nToken, pPhrase);
116668	}
116669
116670	aPoslist = pExpr->pRight->pPhrase->doclist.pList;
116671	nToken = pExpr->pRight->pPhrase->nToken;
116672	for(p=pExpr->pLeft; p && res; p=p->pLeft){
116673	int nNear = p->pParent->nNear;
116674	Fts3Phrase *pPhrase = (
116675	p->eType==FTSQUERY_NEAR ? p->pRight->pPhrase : p->pPhrase
116676	);
116677	res = fts3EvalNearTrim(nNear, aTmp, &aPoslist, &nToken, pPhrase);
116678	}
116679	}
116680
116681	sqlite3_free(aTmp);
116682	}
	@@ -115875,132 +116683,33 @@
116683
116684	return res;
116685	}
116686
116687	/*
116688	** This function is a helper function for fts3EvalTestDeferredAndNear().
116689	** Assuming no error occurs or has occurred, It returns non-zero if the
116690	** expression passed as the second argument matches the row that pCsr
116691	** currently points to, or zero if it does not.
116692	**
116693	** If *pRc is not SQLITE_OK when this function is called, it is a no-op.
116694	** If an error occurs during execution of this function, *pRc is set to
116695	** the appropriate SQLite error code. In this case the returned value is
116696	** undefined.
116697	*/
116698	static int fts3EvalTestExpr(
116699	Fts3Cursor pCsr, / FTS cursor handle */
116700	Fts3Expr pExpr, / Expr to test. May or may not be root. */
116701	int pRc / IN/OUT: Error code */
116702	){
116703	int bHit = 1; /* Return value */







































































































116704	if( *pRc==SQLITE_OK ){
116705	switch( pExpr->eType ){
116706	case FTSQUERY_NEAR:
116707	case FTSQUERY_AND:
116708	bHit = (
116709	fts3EvalTestExpr(pCsr, pExpr->pLeft, pRc)
116710	&& fts3EvalTestExpr(pCsr, pExpr->pRight, pRc)
116711	&& fts3EvalNearTest(pExpr, pRc)
116712	);
116713
116714	/* If the NEAR expression does not match any rows, zero the doclist for
116715	** all phrases involved in the NEAR. This is because the snippet(),
	@@ -116022,31 +116731,31 @@
116731	&& (pExpr->pParent==0 \|\| pExpr->pParent->eType!=FTSQUERY_NEAR)
116732	){
116733	Fts3Expr *p;
116734	for(p=pExpr; p->pPhrase==0; p=p->pLeft){
116735	if( p->pRight->iDocid==pCsr->iPrevId ){
116736	fts3EvalInvalidatePoslist(p->pRight->pPhrase);
116737	}
116738	}
116739	if( p->iDocid==pCsr->iPrevId ){
116740	fts3EvalInvalidatePoslist(p->pPhrase);
116741	}
116742	}
116743
116744	break;
116745
116746	case FTSQUERY_OR: {
116747	int bHit1 = fts3EvalTestExpr(pCsr, pExpr->pLeft, pRc);
116748	int bHit2 = fts3EvalTestExpr(pCsr, pExpr->pRight, pRc);
116749	bHit = bHit1 \|\| bHit2;
116750	break;
116751	}
116752
116753	case FTSQUERY_NOT:
116754	bHit = (
116755	fts3EvalTestExpr(pCsr, pExpr->pLeft, pRc)
116756	&& !fts3EvalTestExpr(pCsr, pExpr->pRight, pRc)
116757	);
116758	break;
116759
116760	default: {
116761	if( pCsr->pDeferred
	@@ -116053,11 +116762,11 @@
116762	&& (pExpr->iDocid==pCsr->iPrevId \|\| pExpr->bDeferred)
116763	){
116764	Fts3Phrase *pPhrase = pExpr->pPhrase;
116765	assert( pExpr->bDeferred \|\| pPhrase->doclist.bFreeList==0 );
116766	if( pExpr->bDeferred ){
116767	fts3EvalInvalidatePoslist(pPhrase);
116768	}
116769	*pRc = fts3EvalDeferredPhrase(pCsr, pPhrase);
116770	bHit = (pPhrase->doclist.pList!=0);
116771	pExpr->iDocid = pCsr->iPrevId;
116772	}else{
	@@ -116069,31 +116778,53 @@
116778	}
116779	return bHit;
116780	}
116781
116782	/*
116783	** This function is called as the second part of each xNext operation when
116784	** iterating through the results of a full-text query. At this point the
116785	** cursor points to a row that matches the query expression, with the
116786	** following caveats:
116787	**
116788	** * Up until this point, "NEAR" operators in the expression have been
116789	** treated as "AND".
116790	**
116791	** * Deferred tokens have not yet been considered.
116792	**
116793	** If *pRc is not SQLITE_OK when this function is called, it immediately
116794	** returns 0. Otherwise, it tests whether or not after considering NEAR
116795	** operators and deferred tokens the current row is still a match for the
116796	** expression. It returns 1 if both of the following are true:
116797	**
116798	** 1. *pRc is SQLITE_OK when this function returns, and
116799	**
116800	** 2. After scanning the current FTS table row for the deferred tokens,
116801	** it is determined that the row does not match the query.
116802	**
116803	** Or, if no error occurs and it seems the current row does match the FTS
116804	** query, return 0.
116805	*/
116806	static int fts3EvalTestDeferredAndNear(Fts3Cursor pCsr, int pRc){
116807	int rc = *pRc;
116808	int bMiss = 0;
116809	if( rc==SQLITE_OK ){
116810
116811	/* If there are one or more deferred tokens, load the current row into
116812	** memory and scan it to determine the position list for each deferred
116813	** token. Then, see if this row is really a match, considering deferred
116814	** tokens and NEAR operators (neither of which were taken into account
116815	** earlier, by fts3EvalNextRow()).
116816	*/
116817	if( pCsr->pDeferred ){
116818	rc = fts3CursorSeek(0, pCsr);
116819	if( rc==SQLITE_OK ){
116820	rc = sqlite3Fts3CacheDeferredDoclists(pCsr);
116821	}
116822	}
116823	bMiss = (0==fts3EvalTestExpr(pCsr, pCsr->pExpr, &rc));
116824
116825	/* Free the position-lists accumulated for each deferred token above. */
116826	sqlite3Fts3FreeDeferredDoclists(pCsr);
116827	*pRc = rc;
116828	}
116829	return (rc==SQLITE_OK && bMiss);
116830	}
	@@ -116100,11 +116831,11 @@
116831
116832	/*
116833	** Advance to the next document that matches the FTS expression in
116834	** Fts3Cursor.pExpr.
116835	*/
116836	static int fts3EvalNext(Fts3Cursor *pCsr){
116837	int rc = SQLITE_OK; /* Return Code */
116838	Fts3Expr *pExpr = pCsr->pExpr;
116839	assert( pCsr->isEof==0 );
116840	if( pExpr==0 ){
116841	pCsr->isEof = 1;
	@@ -116112,23 +116843,23 @@
116843	do {
116844	if( pCsr->isRequireSeek==0 ){
116845	sqlite3_reset(pCsr->pStmt);
116846	}
116847	assert( sqlite3_data_count(pCsr->pStmt)==0 );
116848	fts3EvalNextRow(pCsr, pExpr, &rc);
116849	pCsr->isEof = pExpr->bEof;
116850	pCsr->isRequireSeek = 1;
116851	pCsr->isMatchinfoNeeded = 1;
116852	pCsr->iPrevId = pExpr->iDocid;
116853	}while( pCsr->isEof==0 && fts3EvalTestDeferredAndNear(pCsr, &rc) );
116854	}
116855	return rc;
116856	}
116857
116858	/*
116859	** Restart interation for expression pExpr so that the next call to
116860	** fts3EvalNext() visits the first row. Do not allow incremental
116861	** loading or merging of phrase doclists for this iteration.
116862	**
116863	** If *pRc is other than SQLITE_OK when this function is called, it is
116864	** a no-op. If an error occurs within this function, *pRc is set to an
116865	** SQLite error code before returning.
	@@ -116140,11 +116871,11 @@
116871	){
116872	if( pExpr && *pRc==SQLITE_OK ){
116873	Fts3Phrase *pPhrase = pExpr->pPhrase;
116874
116875	if( pPhrase ){
116876	fts3EvalInvalidatePoslist(pPhrase);
116877	if( pPhrase->bIncr ){
116878	assert( pPhrase->nToken==1 );
116879	assert( pPhrase->aToken[0].pSegcsr );
116880	sqlite3Fts3MsrIncrRestart(pPhrase->aToken[0].pSegcsr);
116881	*pRc = fts3EvalPhraseStart(pCsr, 0, pPhrase);
	@@ -116256,18 +116987,18 @@
116987	/* Ensure the %_content statement is reset. */
116988	if( pCsr->isRequireSeek==0 ) sqlite3_reset(pCsr->pStmt);
116989	assert( sqlite3_data_count(pCsr->pStmt)==0 );
116990
116991	/* Advance to the next document */
116992	fts3EvalNextRow(pCsr, pRoot, &rc);
116993	pCsr->isEof = pRoot->bEof;
116994	pCsr->isRequireSeek = 1;
116995	pCsr->isMatchinfoNeeded = 1;
116996	pCsr->iPrevId = pRoot->iDocid;
116997	}while( pCsr->isEof==0
116998	&& pRoot->eType==FTSQUERY_NEAR
116999	&& fts3EvalTestDeferredAndNear(pCsr, &rc)
117000	);
117001
117002	if( rc==SQLITE_OK && pCsr->isEof==0 ){
117003	fts3EvalUpdateCounts(pRoot);
117004	}
	@@ -116285,14 +117016,14 @@
117016	**
117017	** do {...} while( pRoot->iDocid<iDocid && rc==SQLITE_OK );
117018	*/
117019	fts3EvalRestart(pCsr, pRoot, &rc);
117020	do {
117021	fts3EvalNextRow(pCsr, pRoot, &rc);
117022	assert( pRoot->bEof==0 );
117023	}while( pRoot->iDocid!=iDocid && rc==SQLITE_OK );
117024	fts3EvalTestDeferredAndNear(pCsr, &rc);
117025	}
117026	}
117027	return rc;
117028	}
117029
	@@ -116419,18 +117150,32 @@
117150	*/
117151	SQLITE_PRIVATE void sqlite3Fts3EvalPhraseCleanup(Fts3Phrase *pPhrase){
117152	if( pPhrase ){
117153	int i;
117154	sqlite3_free(pPhrase->doclist.aAll);
117155	fts3EvalInvalidatePoslist(pPhrase);
117156	memset(&pPhrase->doclist, 0, sizeof(Fts3Doclist));
117157	for(i=0; i<pPhrase->nToken; i++){
117158	fts3SegReaderCursorFree(pPhrase->aToken[i].pSegcsr);
117159	pPhrase->aToken[i].pSegcsr = 0;
117160	}
117161	}
117162	}
117163
117164	#if !SQLITE_CORE
117165	/*
117166	** Initialize API pointer table, if required.
117167	*/
117168	SQLITE_API int sqlite3_extension_init(
117169	sqlite3 *db,
117170	char **pzErrMsg,
117171	const sqlite3_api_routines *pApi
117172	){
117173	SQLITE_EXTENSION_INIT2(pApi)
117174	return sqlite3Fts3Init(db);
117175	}
117176	#endif
117177
117178	#endif
117179
117180	/************ End of fts3.c **********************************************/
117181	/************ Begin file fts3_aux.c **************************************/
	@@ -118917,14 +119662,10 @@
119662	** (in which case SQLITE_CORE is not defined), or
119663	**
119664	** * The FTS3 module is being built into the core of
119665	** SQLite (in which case SQLITE_ENABLE_FTS3 is defined).
119666	*/




119667	#if !defined(SQLITE_CORE) \|\| defined(SQLITE_ENABLE_FTS3)
119668
119669
119670	/*
119671	** Implementation of the SQL scalar function for accessing the underlying
119672

M src/sqlite3.h

+21 -4

		--- src/sqlite3.h
		+++ src/sqlite3.h
		@@ -105,13 +105,13 @@
105	105	**
106	106	** See also: [sqlite3_libversion()],
107	107	** [sqlite3_libversion_number()], [sqlite3_sourceid()],
108	108	** [sqlite_version()] and [sqlite_source_id()].
109	109	*/
110		-#define SQLITE_VERSION "3.7.7"
111		-#define SQLITE_VERSION_NUMBER 3007007
112		-#define SQLITE_SOURCE_ID "2011-06-24 11:29:51 9b191bb4c7c1e1b12b188c0b3eee1f8f587887c8"
	110	+#define SQLITE_VERSION "3.7.8"
	111	+#define SQLITE_VERSION_NUMBER 3007008
	112	+#define SQLITE_SOURCE_ID "2011-07-19 18:29:00 ed5f0aad6b21066bacd01521e82c22e96991f400"
113	113
114	114	/*
115	115	** CAPI3REF: Run-Time Library Version Numbers
116	116	** KEYWORDS: sqlite3_version, sqlite3_sourceid
117	117	**
		@@ -739,20 +739,37 @@
739	739	** when [PRAGMA synchronous \| PRAGMA synchronous=OFF] is set, but most
740	740	** VFSes do not need this signal and should silently ignore this opcode.
741	741	** Applications should not call [sqlite3_file_control()] with this
742	742	** opcode as doing so may disrupt the operation of the specialized VFSes
743	743	** that do require it.
	744	+**
	745	+** ^The [SQLITE_FCNTL_WIN32_AV_RETRY] opcode is used to configure automatic
	746	+** retry counts and intervals for certain disk I/O operations for the
	747	+** windows [VFS] in order to work to provide robustness against
	748	+** anti-virus programs. By default, the windows VFS will retry file read,
	749	+** file write, and file delete opertions up to 10 times, with a delay
	750	+** of 25 milliseconds before the first retry and with the delay increasing
	751	+** by an additional 25 milliseconds with each subsequent retry. This
	752	+** opcode allows those to values (10 retries and 25 milliseconds of delay)
	753	+** to be adjusted. The values are changed for all database connections
	754	+** within the same process. The argument is a pointer to an array of two
	755	+** integers where the first integer i the new retry count and the second
	756	+** integer is the delay. If either integer is negative, then the setting
	757	+** is not changed but instead the prior value of that setting is written
	758	+** into the array entry, allowing the current retry settings to be
	759	+** interrogated. The zDbName parameter is ignored.
	760	+**
744	761	*/
745	762	#define SQLITE_FCNTL_LOCKSTATE 1
746	763	#define SQLITE_GET_LOCKPROXYFILE 2
747	764	#define SQLITE_SET_LOCKPROXYFILE 3
748	765	#define SQLITE_LAST_ERRNO 4
749	766	#define SQLITE_FCNTL_SIZE_HINT 5
750	767	#define SQLITE_FCNTL_CHUNK_SIZE 6
751	768	#define SQLITE_FCNTL_FILE_POINTER 7
752	769	#define SQLITE_FCNTL_SYNC_OMITTED 8
753		-
	770	+#define SQLITE_FCNTL_WIN32_AV_RETRY 9
754	771
755	772	/*
756	773	** CAPI3REF: Mutex Handle
757	774	**
758	775	** The mutex module within SQLite defines [sqlite3_mutex] to be an
759	776

	--- src/sqlite3.h
	+++ src/sqlite3.h
	@@ -105,13 +105,13 @@
105	**
106	** See also: [sqlite3_libversion()],
107	** [sqlite3_libversion_number()], [sqlite3_sourceid()],
108	** [sqlite_version()] and [sqlite_source_id()].
109	*/
110	#define SQLITE_VERSION "3.7.7"
111	#define SQLITE_VERSION_NUMBER 3007007
112	#define SQLITE_SOURCE_ID "2011-06-24 11:29:51 9b191bb4c7c1e1b12b188c0b3eee1f8f587887c8"
113
114	/*
115	** CAPI3REF: Run-Time Library Version Numbers
116	** KEYWORDS: sqlite3_version, sqlite3_sourceid
117	**
	@@ -739,20 +739,37 @@
739	** when [PRAGMA synchronous \| PRAGMA synchronous=OFF] is set, but most
740	** VFSes do not need this signal and should silently ignore this opcode.
741	** Applications should not call [sqlite3_file_control()] with this
742	** opcode as doing so may disrupt the operation of the specialized VFSes
743	** that do require it.

















744	*/
745	#define SQLITE_FCNTL_LOCKSTATE 1
746	#define SQLITE_GET_LOCKPROXYFILE 2
747	#define SQLITE_SET_LOCKPROXYFILE 3
748	#define SQLITE_LAST_ERRNO 4
749	#define SQLITE_FCNTL_SIZE_HINT 5
750	#define SQLITE_FCNTL_CHUNK_SIZE 6
751	#define SQLITE_FCNTL_FILE_POINTER 7
752	#define SQLITE_FCNTL_SYNC_OMITTED 8
753
754
755	/*
756	** CAPI3REF: Mutex Handle
757	**
758	** The mutex module within SQLite defines [sqlite3_mutex] to be an
759

	--- src/sqlite3.h
	+++ src/sqlite3.h
	@@ -105,13 +105,13 @@
105	**
106	** See also: [sqlite3_libversion()],
107	** [sqlite3_libversion_number()], [sqlite3_sourceid()],
108	** [sqlite_version()] and [sqlite_source_id()].
109	*/
110	#define SQLITE_VERSION "3.7.8"
111	#define SQLITE_VERSION_NUMBER 3007008
112	#define SQLITE_SOURCE_ID "2011-07-19 18:29:00 ed5f0aad6b21066bacd01521e82c22e96991f400"
113
114	/*
115	** CAPI3REF: Run-Time Library Version Numbers
116	** KEYWORDS: sqlite3_version, sqlite3_sourceid
117	**
	@@ -739,20 +739,37 @@
739	** when [PRAGMA synchronous \| PRAGMA synchronous=OFF] is set, but most
740	** VFSes do not need this signal and should silently ignore this opcode.
741	** Applications should not call [sqlite3_file_control()] with this
742	** opcode as doing so may disrupt the operation of the specialized VFSes
743	** that do require it.
744	**
745	** ^The [SQLITE_FCNTL_WIN32_AV_RETRY] opcode is used to configure automatic
746	** retry counts and intervals for certain disk I/O operations for the
747	** windows [VFS] in order to work to provide robustness against
748	** anti-virus programs. By default, the windows VFS will retry file read,
749	** file write, and file delete opertions up to 10 times, with a delay
750	** of 25 milliseconds before the first retry and with the delay increasing
751	** by an additional 25 milliseconds with each subsequent retry. This
752	** opcode allows those to values (10 retries and 25 milliseconds of delay)
753	** to be adjusted. The values are changed for all database connections
754	** within the same process. The argument is a pointer to an array of two
755	** integers where the first integer i the new retry count and the second
756	** integer is the delay. If either integer is negative, then the setting
757	** is not changed but instead the prior value of that setting is written
758	** into the array entry, allowing the current retry settings to be
759	** interrogated. The zDbName parameter is ignored.
760	**
761	*/
762	#define SQLITE_FCNTL_LOCKSTATE 1
763	#define SQLITE_GET_LOCKPROXYFILE 2
764	#define SQLITE_SET_LOCKPROXYFILE 3
765	#define SQLITE_LAST_ERRNO 4
766	#define SQLITE_FCNTL_SIZE_HINT 5
767	#define SQLITE_FCNTL_CHUNK_SIZE 6
768	#define SQLITE_FCNTL_FILE_POINTER 7
769	#define SQLITE_FCNTL_SYNC_OMITTED 8
770	#define SQLITE_FCNTL_WIN32_AV_RETRY 9
771
772	/*
773	** CAPI3REF: Mutex Handle
774	**
775	** The mutex module within SQLite defines [sqlite3_mutex] to be an
776

Fossil SCM

Keyboard Shortcuts