Fossil SCM

Update the built-in SQLite to the latest 3.7.7 beta.

drh 2011-06-15 15:22 trunk
Commit dbe1e68e43487bff2250b9a48c07cdda9339f7b4
Parent 1c4d1272a621bbd…
2 files changed +2822 -1836 +1 -1
~ src/sqlite3.c ~ src/sqlite3.h
M src/sqlite3.c
+2822 -1836
--- src/sqlite3.c
+++ src/sqlite3.c
@@ -650,11 +650,11 @@
650650
 ** [sqlite3_libversion_number()], [sqlite3_sourceid()],
651651
 ** [sqlite_version()] and [sqlite_source_id()].
652652
 */
653653
 #define SQLITE_VERSION "3.7.7"
654654
 #define SQLITE_VERSION_NUMBER 3007007
655
-#define SQLITE_SOURCE_ID "2011-06-03 14:19:10 0206bc6f87bb9393218a380fc5b18039d334a8d8"
655
+#define SQLITE_SOURCE_ID "2011-06-15 13:11:06 f9750870ee04935f338e4d808900fee5a8b2b389"
656656
657657
 /*
658658
 ** CAPI3REF: Run-Time Library Version Numbers
659659
 ** KEYWORDS: sqlite3_version, sqlite3_sourceid
660660
 **
@@ -8483,10 +8483,11 @@
84838483
 SQLITE_PRIVATE int sqlite3VdbeAddOp2(Vdbe*,int,int,int);
84848484
 SQLITE_PRIVATE int sqlite3VdbeAddOp3(Vdbe*,int,int,int,int);
84858485
 SQLITE_PRIVATE int sqlite3VdbeAddOp4(Vdbe*,int,int,int,int,const char *zP4,int);
84868486
 SQLITE_PRIVATE int sqlite3VdbeAddOp4Int(Vdbe*,int,int,int,int,int);
84878487
 SQLITE_PRIVATE int sqlite3VdbeAddOpList(Vdbe*, int nOp, VdbeOpList const *aOp);
8488
+SQLITE_PRIVATE void sqlite3VdbeAddParseSchemaOp(Vdbe*,int,char*);
84888489
 SQLITE_PRIVATE void sqlite3VdbeChangeP1(Vdbe*, int addr, int P1);
84898490
 SQLITE_PRIVATE void sqlite3VdbeChangeP2(Vdbe*, int addr, int P2);
84908491
 SQLITE_PRIVATE void sqlite3VdbeChangeP3(Vdbe*, int addr, int P3);
84918492
 SQLITE_PRIVATE void sqlite3VdbeChangeP5(Vdbe*, u8 P5);
84928493
 SQLITE_PRIVATE void sqlite3VdbeJumpHere(Vdbe*, int addr);
@@ -9282,11 +9283,11 @@
92829283
 ** sqlite3_close().
92839284
 *
92849285
 ** A thread must be holding a mutex on the corresponding Btree in order
92859286
 ** to access Schema content. This implies that the thread must also be
92869287
 ** holding a mutex on the sqlite3 connection pointer that owns the Btree.
9287
-** For a TEMP Schema, on the connection mutex is required.
9288
+** For a TEMP Schema, only the connection mutex is required.
92889289
 */
92899290
 struct Schema {
92909291
 int schema_cookie; /* Database schema version number for this file */
92919292
 int iGeneration; /* Generation counter. Incremented with each change */
92929293
 Hash tblHash; /* All tables indexed by name */
@@ -11479,11 +11480,11 @@
1147911480
 SQLITE_PRIVATE int sqlite3AtoF(const char *z, double*, int, u8);
1148011481
 SQLITE_PRIVATE int sqlite3GetInt32(const char *, int*);
1148111482
 SQLITE_PRIVATE int sqlite3Atoi(const char*);
1148211483
 SQLITE_PRIVATE int sqlite3Utf16ByteLen(const void *pData, int nChar);
1148311484
 SQLITE_PRIVATE int sqlite3Utf8CharLen(const char *pData, int nByte);
11484
-SQLITE_PRIVATE int sqlite3Utf8Read(const u8*, const u8**);
11485
+SQLITE_PRIVATE u32 sqlite3Utf8Read(const u8*, const u8**);
1148511486
1148611487
 /*
1148711488
 ** Routines to read and write variable-length integers. These used to
1148811489
 ** be defined locally, but now we use the varint routines in the util.c
1148911490
 ** file. Code should use the MACRO forms below, as the Varint32 versions
@@ -20086,11 +20087,11 @@
2008620087
 } \
2008720088
 if( c<0x80 \
2008820089
 || (c&0xFFFFF800)==0xD800 \
2008920090
 || (c&0xFFFFFFFE)==0xFFFE ){ c = 0xFFFD; } \
2009020091
 }
20091
-SQLITE_PRIVATE int sqlite3Utf8Read(
20092
+SQLITE_PRIVATE u32 sqlite3Utf8Read(
2009220093
 const unsigned char *zIn, /* First byte of UTF-8 character */
2009320094
 const unsigned char **pzNext /* Write first byte past UTF-8 char here */
2009420095
 ){
2009520096
 unsigned int c;
2009620097
@@ -35805,11 +35806,11 @@
3580535806
 memset(pCache, 0, sz);
3580635807
 if( separateCache ){
3580735808
 pGroup = (PGroup*)&pCache[1];
3580835809
 pGroup->mxPinned = 10;
3580935810
 }else{
35810
- pGroup = &pcache1_g.grp;
35811
+ pGroup = &pcache1.grp;
3581135812
 }
3581235813
 pCache->pGroup = pGroup;
3581335814
 pCache->szPage = szPage;
3581435815
 pCache->bPurgeable = (bPurgeable ? 1 : 0);
3581535816
 if( bPurgeable ){
@@ -48324,10 +48325,12 @@
4832448325
 **
4832548326
 ** This routine works only for pages that do not contain overflow cells.
4832648327
 */
4832748328
 #define findCell(P,I) \
4832848329
 ((P)->aData + ((P)->maskPage & get2byte(&(P)->aData[(P)->cellOffset+2*(I)])))
48330
+#define findCellv2(D,M,O,I) (D+(M&get2byte(D+(O+2*(I)))))
48331
+
4832948332
4833048333
 /*
4833148334
 ** This a more complex version of findCell() that works for
4833248335
 ** pages that do contain overflow cells.
4833348336
 */
@@ -51918,11 +51921,11 @@
5191851921
 assert( pCur->apPage[pCur->iPage]->nCell==0 );
5191951922
 return SQLITE_OK;
5192051923
 }
5192151924
 assert( pCur->apPage[0]->intKey || pIdxKey );
5192251925
 for(;;){
51923
- int lwr, upr;
51926
+ int lwr, upr, idx;
5192451927
 Pgno chldPg;
5192551928
 MemPage *pPage = pCur->apPage[pCur->iPage];
5192651929
 int c;
5192751930
5192851931
 /* pPage->nCell must be greater than zero. If this is the root-page
@@ -51934,18 +51937,18 @@
5193451937
 assert( pPage->nCell>0 );
5193551938
 assert( pPage->intKey==(pIdxKey==0) );
5193651939
 lwr = 0;
5193751940
 upr = pPage->nCell-1;
5193851941
 if( biasRight ){
51939
- pCur->aiIdx[pCur->iPage] = (u16)upr;
51942
+ pCur->aiIdx[pCur->iPage] = (u16)(idx = upr);
5194051943
 }else{
51941
- pCur->aiIdx[pCur->iPage] = (u16)((upr+lwr)/2);
51944
+ pCur->aiIdx[pCur->iPage] = (u16)(idx = (upr+lwr)/2);
5194251945
 }
5194351946
 for(;;){
51944
- int idx = pCur->aiIdx[pCur->iPage]; /* Index of current cell in pPage */
5194551947
 u8 *pCell; /* Pointer to current cell in pPage */
5194651948
51949
+ assert( idx==pCur->aiIdx[pCur->iPage] );
5194751950
 pCur->info.nSize = 0;
5194851951
 pCell = findCell(pPage, idx) + pPage->childPtrSize;
5194951952
 if( pPage->intKey ){
5195051953
 i64 nCellKey;
5195151954
 if( pPage->hasData ){
@@ -52024,11 +52027,11 @@
5202452027
 upr = idx-1;
5202552028
 }
5202652029
 if( lwr>upr ){
5202752030
 break;
5202852031
 }
52029
- pCur->aiIdx[pCur->iPage] = (u16)((lwr+upr)/2);
52032
+ pCur->aiIdx[pCur->iPage] = (u16)(idx = (lwr+upr)/2);
5203052033
 }
5203152034
 assert( lwr==upr+1 );
5203252035
 assert( pPage->isInit );
5203352036
 if( pPage->leaf ){
5203452037
 chldPg = 0;
@@ -52886,13 +52889,13 @@
5288652889
 if( rc ){
5288752890
 *pRC = rc;
5288852891
 return;
5288952892
 }
5289052893
 endPtr = &data[pPage->cellOffset + 2*pPage->nCell - 2];
52894
+ assert( (SQLITE_PTR_TO_INT(ptr)&1)==0 ); /* ptr is always 2-byte aligned */
5289152895
 while( ptr<endPtr ){
52892
- ptr[0] = ptr[2];
52893
- ptr[1] = ptr[3];
52896
+ *(u16*)ptr = *(u16*)&ptr[2];
5289452897
 ptr += 2;
5289552898
 }
5289652899
 pPage->nCell--;
5289752900
 put2byte(&data[hdr+3], pPage->nCell);
5289852901
 pPage->nFree += 2;
@@ -52929,10 +52932,11 @@
5292952932
 int end; /* First byte past the last cell pointer in data[] */
5293052933
 int ins; /* Index in data[] where new cell pointer is inserted */
5293152934
 int cellOffset; /* Address of first cell pointer in data[] */
5293252935
 u8 *data; /* The content of the whole page */
5293352936
 u8 *ptr; /* Used for moving information around in data[] */
52937
+ u8 *endPtr; /* End of the loop */
5293452938
5293552939
 int nSkip = (iChild ? 4 : 0);
5293652940
5293752941
 if( *pRC ) return;
5293852942
@@ -52979,13 +52983,16 @@
5297952983
 pPage->nFree -= (u16)(2 + sz);
5298052984
 memcpy(&data[idx+nSkip], pCell+nSkip, sz-nSkip);
5298152985
 if( iChild ){
5298252986
 put4byte(&data[idx], iChild);
5298352987
 }
52984
- for(j=end, ptr=&data[j]; j>ins; j-=2, ptr-=2){
52985
- ptr[0] = ptr[-2];
52986
- ptr[1] = ptr[-1];
52988
+ ptr = &data[end];
52989
+ endPtr = &data[ins];
52990
+ assert( (SQLITE_PTR_TO_INT(ptr)&1)==0 ); /* ptr is always 2-byte aligned */
52991
+ while( ptr>endPtr ){
52992
+ *(u16*)ptr = *(u16*)&ptr[-2];
52993
+ ptr -= 2;
5298752994
 }
5298852995
 put2byte(&data[ins], idx);
5298952996
 put2byte(&data[pPage->hdrOffset+3], pPage->nCell);
5299052997
 #ifndef SQLITE_OMIT_AUTOVACUUM
5299152998
 if( pPage->pBt->autoVacuum ){
@@ -53026,14 +53033,15 @@
5302653033
 assert( get2byteNotZero(&data[hdr+5])==nUsable );
5302753034
5302853035
 pCellptr = &data[pPage->cellOffset + nCell*2];
5302953036
 cellbody = nUsable;
5303053037
 for(i=nCell-1; i>=0; i--){
53038
+ u16 sz = aSize[i];
5303153039
 pCellptr -= 2;
53032
- cellbody -= aSize[i];
53040
+ cellbody -= sz;
5303353041
 put2byte(pCellptr, cellbody);
53034
- memcpy(&data[cellbody], apCell[i], aSize[i]);
53042
+ memcpy(&data[cellbody], apCell[i], sz);
5303553043
 }
5303653044
 put2byte(&data[hdr+3], nCell);
5303753045
 put2byte(&data[hdr+5], cellbody);
5303853046
 pPage->nFree -= (nCell*2 + nUsable - cellbody);
5303953047
 pPage->nCell = (u16)nCell;
@@ -53483,16 +53491,28 @@
5348353491
 memcpy(pOld, apOld[i], sizeof(MemPage));
5348453492
 pOld->aData = (void*)&pOld[1];
5348553493
 memcpy(pOld->aData, apOld[i]->aData, pBt->pageSize);
5348653494
5348753495
 limit = pOld->nCell+pOld->nOverflow;
53488
- for(j=0; j<limit; j++){
53489
- assert( nCell<nMaxCells );
53490
- apCell[nCell] = findOverflowCell(pOld, j);
53491
- szCell[nCell] = cellSizePtr(pOld, apCell[nCell]);
53492
- nCell++;
53493
- }
53496
+ if( pOld->nOverflow>0 ){
53497
+ for(j=0; j<limit; j++){
53498
+ assert( nCell<nMaxCells );
53499
+ apCell[nCell] = findOverflowCell(pOld, j);
53500
+ szCell[nCell] = cellSizePtr(pOld, apCell[nCell]);
53501
+ nCell++;
53502
+ }
53503
+ }else{
53504
+ u8 *aData = pOld->aData;
53505
+ u16 maskPage = pOld->maskPage;
53506
+ u16 cellOffset = pOld->cellOffset;
53507
+ for(j=0; j<limit; j++){
53508
+ assert( nCell<nMaxCells );
53509
+ apCell[nCell] = findCellv2(aData, maskPage, cellOffset, j);
53510
+ szCell[nCell] = cellSizePtr(pOld, apCell[nCell]);
53511
+ nCell++;
53512
+ }
53513
+ }
5349453514
 if( i<nOld-1 && !leafData){
5349553515
 u16 sz = (u16)szNew[i];
5349653516
 u8 *pTemp;
5349753517
 assert( nCell<nMaxCells );
5349853518
 szCell[nCell] = sz;
@@ -57637,17 +57657,10 @@
5763757657
 pOp->p1 = p1;
5763857658
 pOp->p2 = p2;
5763957659
 pOp->p3 = p3;
5764057660
 pOp->p4.p = 0;
5764157661
 pOp->p4type = P4_NOTUSED;
57642
- p->expired = 0;
57643
- if( op==OP_ParseSchema ){
57644
- /* Any program that uses the OP_ParseSchema opcode needs to lock
57645
- ** all btrees. */
57646
- int j;
57647
- for(j=0; j<p->db->nDb; j++) sqlite3VdbeUsesBtree(p, j);
57648
- }
5764957662
 #ifdef SQLITE_DEBUG
5765057663
 pOp->zComment = 0;
5765157664
 if( sqlite3VdbeAddopTrace ) sqlite3VdbePrintOp(0, i, &p->aOp[i]);
5765257665
 #endif
5765357666
 #ifdef VDBE_PROFILE
@@ -57681,10 +57694,24 @@
5768157694
 ){
5768257695
 int addr = sqlite3VdbeAddOp3(p, op, p1, p2, p3);
5768357696
 sqlite3VdbeChangeP4(p, addr, zP4, p4type);
5768457697
 return addr;
5768557698
 }
57699
+
57700
+/*
57701
+** Add an OP_ParseSchema opcode. This routine is broken out from
57702
+** sqlite3VdbeAddOp4() since it needs to also local all btrees.
57703
+**
57704
+** The zWhere string must have been obtained from sqlite3_malloc().
57705
+** This routine will take ownership of the allocated memory.
57706
+*/
57707
+SQLITE_PRIVATE void sqlite3VdbeAddParseSchemaOp(Vdbe *p, int iDb, char *zWhere){
57708
+ int j;
57709
+ int addr = sqlite3VdbeAddOp3(p, OP_ParseSchema, iDb, 0, 0);
57710
+ sqlite3VdbeChangeP4(p, addr, zWhere, P4_DYNAMIC);
57711
+ for(j=0; j<p->db->nDb; j++) sqlite3VdbeUsesBtree(p, j);
57712
+}
5768657713
5768757714
 /*
5768857715
 ** Add an opcode that includes the p4 value as an integer.
5768957716
 */
5769057717
 SQLITE_PRIVATE int sqlite3VdbeAddOp4Int(
@@ -64083,10 +64110,20 @@
6408364110
 u.ag.ctx.pColl = pOp[-1].p4.pColl;
6408464111
 }
6408564112
 db->lastRowid = lastRowid;
6408664113
 (*u.ag.ctx.pFunc->xFunc)(&u.ag.ctx, u.ag.n, u.ag.apVal); /* IMP: R-24505-23230 */
6408764114
 lastRowid = db->lastRowid;
64115
+
64116
+ /* If any auxiliary data functions have been called by this user function,
64117
+ ** immediately call the destructor for any non-static values.
64118
+ */
64119
+ if( u.ag.ctx.pVdbeFunc ){
64120
+ sqlite3VdbeDeleteAuxData(u.ag.ctx.pVdbeFunc, pOp->p1);
64121
+ pOp->p4.pVdbeFunc = u.ag.ctx.pVdbeFunc;
64122
+ pOp->p4type = P4_VDBEFUNC;
64123
+ }
64124
+
6408864125
 if( db->mallocFailed ){
6408964126
 /* Even though a malloc() has failed, the implementation of the
6409064127
 ** user function may have called an sqlite3_result_XXX() function
6409164128
 ** to return a value. The following call releases any resources
6409264129
 ** associated with such a value.
@@ -64093,19 +64130,10 @@
6409364130
 */
6409464131
 sqlite3VdbeMemRelease(&u.ag.ctx.s);
6409564132
 goto no_mem;
6409664133
 }
6409764134
64098
- /* If any auxiliary data functions have been called by this user function,
64099
- ** immediately call the destructor for any non-static values.
64100
- */
64101
- if( u.ag.ctx.pVdbeFunc ){
64102
- sqlite3VdbeDeleteAuxData(u.ag.ctx.pVdbeFunc, pOp->p1);
64103
- pOp->p4.pVdbeFunc = u.ag.ctx.pVdbeFunc;
64104
- pOp->p4type = P4_VDBEFUNC;
64105
- }
64106
-
6410764135
 /* If the function returned an error, throw an exception */
6410864136
 if( u.ag.ctx.isError ){
6410964137
 sqlite3SetString(&p->zErrMsg, db, "%s", sqlite3_value_text(&u.ag.ctx.s));
6411064138
 rc = u.ag.ctx.isError;
6411164139
 }
@@ -75255,18 +75283,18 @@
7525575283
 sqlite3VdbeAddOp4(v, OP_DropTable, iDb, 0, 0, pTab->zName, 0);
7525675284
7525775285
 /* Reload the table, index and permanent trigger schemas. */
7525875286
 zWhere = sqlite3MPrintf(pParse->db, "tbl_name=%Q", zName);
7525975287
 if( !zWhere ) return;
75260
- sqlite3VdbeAddOp4(v, OP_ParseSchema, iDb, 0, 0, zWhere, P4_DYNAMIC);
75288
+ sqlite3VdbeAddParseSchemaOp(v, iDb, zWhere);
7526175289
7526275290
 #ifndef SQLITE_OMIT_TRIGGER
7526375291
 /* Now, if the table is not stored in the temp database, reload any temp
7526475292
 ** triggers. Don't use IN(...) in case SQLITE_OMIT_SUBQUERY is defined.
7526575293
 */
7526675294
 if( (zWhere=whereTempTriggers(pParse, pTab))!=0 ){
75267
- sqlite3VdbeAddOp4(v, OP_ParseSchema, 1, 0, 0, zWhere, P4_DYNAMIC);
75295
+ sqlite3VdbeAddParseSchemaOp(v, 1, zWhere);
7526875296
 }
7526975297
 #endif
7527075298
 }
7527175299
7527275300
 /*
@@ -78875,12 +78903,12 @@
7887578903
 }
7887678904
 }
7887778905
 #endif
7887878906
7887978907
 /* Reparse everything to update our internal data structures */
78880
- sqlite3VdbeAddOp4(v, OP_ParseSchema, iDb, 0, 0,
78881
- sqlite3MPrintf(db, "tbl_name='%q'",p->zName), P4_DYNAMIC);
78908
+ sqlite3VdbeAddParseSchemaOp(v, iDb,
78909
+ sqlite3MPrintf(db, "tbl_name='%q'", p->zName));
7888278910
 }
7888378911
7888478912
7888578913
 /* Add the table to the in-memory representation of the database.
7888678914
 */
@@ -80073,13 +80101,12 @@
8007380101
 ** to invalidate all pre-compiled statements.
8007480102
 */
8007580103
 if( pTblName ){
8007680104
 sqlite3RefillIndex(pParse, pIndex, iMem);
8007780105
 sqlite3ChangeCookie(pParse, iDb);
80078
- sqlite3VdbeAddOp4(v, OP_ParseSchema, iDb, 0, 0,
80079
- sqlite3MPrintf(db, "name='%q' AND type='index'", pIndex->zName),
80080
- P4_DYNAMIC);
80106
+ sqlite3VdbeAddParseSchemaOp(v, iDb,
80107
+ sqlite3MPrintf(db, "name='%q' AND type='index'", pIndex->zName));
8008180108
 sqlite3VdbeAddOp1(v, OP_Expire, 0);
8008280109
 }
8008380110
 }
8008480111
8008580112
 /* When adding an index to the list of indices for a table, make
@@ -82621,14 +82648,14 @@
8262182648
 ** character is exactly one byte in size. Also, all characters are
8262282649
 ** able to participate in upper-case-to-lower-case mappings in EBCDIC
8262382650
 ** whereas only characters less than 0x80 do in ASCII.
8262482651
 */
8262582652
 #if defined(SQLITE_EBCDIC)
82626
-# define sqlite3Utf8Read(A,C) (*(A++))
82627
-# define GlogUpperToLower(A) A = sqlite3UpperToLower[A]
82653
+# define sqlite3Utf8Read(A,C) (*(A++))
82654
+# define GlogUpperToLower(A) A = sqlite3UpperToLower[A]
8262882655
 #else
82629
-# define GlogUpperToLower(A) if( A<0x80 ){ A = sqlite3UpperToLower[A]; }
82656
+# define GlogUpperToLower(A) if( !((A)&~0x7f) ){ A = sqlite3UpperToLower[A]; }
8263082657
 #endif
8263182658
8263282659
 static const struct compareInfo globInfo = { '*', '?', '[', 0 };
8263382660
 /* The correct SQL-92 behavior is for the LIKE operator to ignore
8263482661
 ** case. Thus 'a' LIKE 'A' would be true. */
@@ -82667,13 +82694,13 @@
8266782694
 */
8266882695
 static int patternCompare(
8266982696
 const u8 *zPattern, /* The glob pattern */
8267082697
 const u8 *zString, /* The string to compare against the glob */
8267182698
 const struct compareInfo *pInfo, /* Information about how to do the compare */
82672
- const int esc /* The escape character */
82699
+ u32 esc /* The escape character */
8267382700
 ){
82674
- int c, c2;
82701
+ u32 c, c2;
8267582702
 int invert;
8267682703
 int seen;
8267782704
 u8 matchOne = pInfo->matchOne;
8267882705
 u8 matchAll = pInfo->matchAll;
8267982706
 u8 matchSet = pInfo->matchSet;
@@ -82723,11 +82750,11 @@
8272382750
 }else if( !prevEscape && c==matchOne ){
8272482751
 if( sqlite3Utf8Read(zString, &zString)==0 ){
8272582752
 return 0;
8272682753
 }
8272782754
 }else if( c==matchSet ){
82728
- int prior_c = 0;
82755
+ u32 prior_c = 0;
8272982756
 assert( esc==0 ); /* This only occurs for GLOB, not LIKE */
8273082757
 seen = 0;
8273182758
 invert = 0;
8273282759
 c = sqlite3Utf8Read(zString, &zString);
8273382760
 if( c==0 ) return 0;
@@ -82799,11 +82826,11 @@
8279982826
 sqlite3_context *context,
8280082827
 int argc,
8280182828
 sqlite3_value **argv
8280282829
 ){
8280382830
 const unsigned char *zA, *zB;
82804
- int escape = 0;
82831
+ u32 escape = 0;
8280582832
 int nPat;
8280682833
 sqlite3 *db = sqlite3_context_db_handle(context);
8280782834
8280882835
 zB = sqlite3_value_text(argv[0]);
8280982836
 zA = sqlite3_value_text(argv[1]);
@@ -84110,17 +84137,29 @@
8411084137
 }
8411184138
8411284139
 /* If the parent table is the same as the child table, and we are about
8411384140
 ** to increment the constraint-counter (i.e. this is an INSERT operation),
8411484141
 ** then check if the row being inserted matches itself. If so, do not
84115
- ** increment the constraint-counter. */
84142
+ ** increment the constraint-counter.
84143
+ **
84144
+ ** If any of the parent-key values are NULL, then the row cannot match
84145
+ ** itself. So set JUMPIFNULL to make sure we do the OP_Found if any
84146
+ ** of the parent-key values are NULL (at this point it is known that
84147
+ ** none of the child key values are).
84148
+ */
8411684149
 if( pTab==pFKey->pFrom && nIncr==1 ){
8411784150
 int iJump = sqlite3VdbeCurrentAddr(v) + nCol + 1;
8411884151
 for(i=0; i<nCol; i++){
8411984152
 int iChild = aiCol[i]+1+regData;
8412084153
 int iParent = pIdx->aiColumn[i]+1+regData;
84154
+ assert( aiCol[i]!=pTab->iPKey );
84155
+ if( pIdx->aiColumn[i]==pTab->iPKey ){
84156
+ /* The parent key is a composite key that includes the IPK column */
84157
+ iParent = regData;
84158
+ }
8412184159
 sqlite3VdbeAddOp3(v, OP_Ne, iChild, iJump, iParent);
84160
+ sqlite3VdbeChangeP5(v, SQLITE_JUMPIFNULL);
8412284161
 }
8412384162
 sqlite3VdbeAddOp2(v, OP_Goto, 0, iOk);
8412484163
 }
8412584164
8412684165
 sqlite3VdbeAddOp3(v, OP_MakeRecord, regTemp, nCol, regRec);
@@ -95336,13 +95375,12 @@
9533695375
 "INSERT INTO %Q.%s VALUES('trigger',%Q,%Q,0,'CREATE TRIGGER %q')",
9533795376
 db->aDb[iDb].zName, SCHEMA_TABLE(iDb), zName,
9533895377
 pTrig->table, z);
9533995378
 sqlite3DbFree(db, z);
9534095379
 sqlite3ChangeCookie(pParse, iDb);
95341
- sqlite3VdbeAddOp4(v, OP_ParseSchema, iDb, 0, 0, sqlite3MPrintf(
95342
- db, "type='trigger' AND name='%q'", zName), P4_DYNAMIC
95343
- );
95380
+ sqlite3VdbeAddParseSchemaOp(v, iDb,
95381
+ sqlite3MPrintf(db, "type='trigger' AND name='%q'", zName));
9534495382
 }
9534595383
9534695384
 if( db->init.busy ){
9534795385
 Trigger *pLink = pTrig;
9534895386
 Hash *pHash = &db->aDb[iDb].pSchema->trigHash;
@@ -96392,11 +96430,11 @@
9639296430
 aRegIdx = sqlite3DbMallocRaw(db, sizeof(Index*) * nIdx );
9639396431
 if( aRegIdx==0 ) goto update_cleanup;
9639496432
 }
9639596433
 for(j=0, pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext, j++){
9639696434
 int reg;
96397
- if( chngRowid ){
96435
+ if( hasFK || chngRowid ){
9639896436
 reg = ++pParse->nMem;
9639996437
 }else{
9640096438
 reg = 0;
9640196439
 for(i=0; i<pIdx->nColumn; i++){
9640296440
 if( aXRef[pIdx->aiColumn[i]]>=0 ){
@@ -97547,11 +97585,11 @@
9754797585
 v = sqlite3GetVdbe(pParse);
9754897586
 sqlite3ChangeCookie(pParse, iDb);
9754997587
9755097588
 sqlite3VdbeAddOp2(v, OP_Expire, 0, 0);
9755197589
 zWhere = sqlite3MPrintf(db, "name='%q' AND type='table'", pTab->zName);
97552
- sqlite3VdbeAddOp4(v, OP_ParseSchema, iDb, 0, 0, zWhere, P4_DYNAMIC);
97590
+ sqlite3VdbeAddParseSchemaOp(v, iDb, zWhere);
9755397591
 sqlite3VdbeAddOp4(v, OP_VCreate, iDb, 0, 0,
9755497592
 pTab->zName, sqlite3Strlen30(pTab->zName) + 1);
9755597593
 }
9755697594
9755797595
 /* If we are rereading the sqlite_master table create the in-memory
@@ -107274,17 +107312,16 @@
107274107312
 #ifndef SQLITE_OMIT_BLOB_LITERAL
107275107313
 case 'x': case 'X': {
107276107314
 testcase( z[0]=='x' ); testcase( z[0]=='X' );
107277107315
 if( z[1]=='\'' ){
107278107316
 *tokenType = TK_BLOB;
107279
- for(i=2; (c=z[i])!=0 && c!='\''; i++){
107280
- if( !sqlite3Isxdigit(c) ){
107281
- *tokenType = TK_ILLEGAL;
107282
- }
107317
+ for(i=2; sqlite3Isxdigit(z[i]); i++){}
107318
+ if( z[i]!='\'' || i%2 ){
107319
+ *tokenType = TK_ILLEGAL;
107320
+ while( z[i] && z[i]!='\'' ){ i++; }
107283107321
 }
107284
- if( i%2 || !c ) *tokenType = TK_ILLEGAL;
107285
- if( c ) i++;
107322
+ if( z[i] ) i++;
107286107323
 return i;
107287107324
 }
107288107325
 /* Otherwise fall through to the next case */
107289107326
 }
107290107327
 #endif
@@ -111710,16 +111747,39 @@
111710111747
 ** similar macro called ArraySize(). Use a different name to avoid
111711111748
 ** a collision when building an amalgamation with built-in FTS3.
111712111749
 */
111713111750
 #define SizeofArray(X) ((int)(sizeof(X)/sizeof(X[0])))
111714111751
111752
+
111753
+#ifndef MIN
111754
+# define MIN(x,y) ((x)<(y)?(x):(y))
111755
+#endif
111756
+
111715111757
 /*
111716111758
 ** Maximum length of a varint encoded integer. The varint format is different
111717111759
 ** from that used by SQLite, so the maximum length is 10, not 9.
111718111760
 */
111719111761
 #define FTS3_VARINT_MAX 10
111720111762
111763
+/*
111764
+** FTS4 virtual tables may maintain multiple indexes - one index of all terms
111765
+** in the document set and zero or more prefix indexes. All indexes are stored
111766
+** as one or more b+-trees in the %_segments and %_segdir tables.
111767
+**
111768
+** It is possible to determine which index a b+-tree belongs to based on the
111769
+** value stored in the "%_segdir.level" column. Given this value L, the index
111770
+** that the b+-tree belongs to is (L<<10). In other words, all b+-trees with
111771
+** level values between 0 and 1023 (inclusive) belong to index 0, all levels
111772
+** between 1024 and 2047 to index 1, and so on.
111773
+**
111774
+** It is considered impossible for an index to use more than 1024 levels. In
111775
+** theory though this may happen, but only after at least
111776
+** (FTS3_MERGE_COUNT^1024) separate flushes of the pending-terms tables.
111777
+*/
111778
+#define FTS3_SEGDIR_MAXLEVEL 1024
111779
+#define FTS3_SEGDIR_MAXLEVEL_STR "1024"
111780
+
111721111781
 /*
111722111782
 ** The testcase() macro is only used by the amalgamation. If undefined,
111723111783
 ** make it a no-op.
111724111784
 */
111725111785
 #ifndef testcase
@@ -111787,14 +111847,15 @@
111787111847
 typedef struct Fts3Cursor Fts3Cursor;
111788111848
 typedef struct Fts3Expr Fts3Expr;
111789111849
 typedef struct Fts3Phrase Fts3Phrase;
111790111850
 typedef struct Fts3PhraseToken Fts3PhraseToken;
111791111851
111852
+typedef struct Fts3Doclist Fts3Doclist;
111792111853
 typedef struct Fts3SegFilter Fts3SegFilter;
111793111854
 typedef struct Fts3DeferredToken Fts3DeferredToken;
111794111855
 typedef struct Fts3SegReader Fts3SegReader;
111795
-typedef struct Fts3SegReaderCursor Fts3SegReaderCursor;
111856
+typedef struct Fts3MultiSegReader Fts3MultiSegReader;
111796111857
111797111858
 /*
111798111859
 ** A connection to a fulltext index is an instance of the following
111799111860
 ** structure. The xCreate and xConnect methods create an instance
111800111861
 ** of this structure and xDestroy and xDisconnect free that instance.
@@ -111811,33 +111872,45 @@
111811111872
 sqlite3_tokenizer *pTokenizer; /* tokenizer for inserts and queries */
111812111873
111813111874
 /* Precompiled statements used by the implementation. Each of these
111814111875
 ** statements is run and reset within a single virtual table API call.
111815111876
 */
111816
- sqlite3_stmt *aStmt[24];
111877
+ sqlite3_stmt *aStmt[27];
111817111878
111818111879
 char *zReadExprlist;
111819111880
 char *zWriteExprlist;
111820111881
111821111882
 int nNodeSize; /* Soft limit for node size */
111822111883
 u8 bHasStat; /* True if %_stat table exists */
111823111884
 u8 bHasDocsize; /* True if %_docsize table exists */
111885
+ u8 bDescIdx; /* True if doclists are in reverse order */
111824111886
 int nPgsz; /* Page size for host database */
111825111887
 char *zSegmentsTbl; /* Name of %_segments table */
111826111888
 sqlite3_blob *pSegments; /* Blob handle open on %_segments table */
111827111889
111828
- /* The following hash table is used to buffer pending index updates during
111890
+ /* TODO: Fix the first paragraph of this comment.
111891
+ **
111892
+ ** The following hash table is used to buffer pending index updates during
111829111893
 ** transactions. Variable nPendingData estimates the memory size of the
111830111894
 ** pending data, including hash table overhead, but not malloc overhead.
111831111895
 ** When nPendingData exceeds nMaxPendingData, the buffer is flushed
111832111896
 ** automatically. Variable iPrevDocid is the docid of the most recently
111833111897
 ** inserted record.
111898
+ **
111899
+ ** A single FTS4 table may have multiple full-text indexes. For each index
111900
+ ** there is an entry in the aIndex[] array. Index 0 is an index of all the
111901
+ ** terms that appear in the document set. Each subsequent index in aIndex[]
111902
+ ** is an index of prefixes of a specific length.
111834111903
 */
111835
- int nMaxPendingData;
111836
- int nPendingData;
111837
- sqlite_int64 iPrevDocid;
111838
- Fts3Hash pendingTerms;
111904
+ int nIndex; /* Size of aIndex[] */
111905
+ struct Fts3Index {
111906
+ int nPrefix; /* Prefix length (0 for main terms index) */
111907
+ Fts3Hash hPending; /* Pending terms table for this index */
111908
+ } *aIndex;
111909
+ int nMaxPendingData; /* Max pending data before flush to disk */
111910
+ int nPendingData; /* Current bytes of pending data */
111911
+ sqlite_int64 iPrevDocid; /* Docid of most recently inserted document */
111839111912
111840111913
 #if defined(SQLITE_DEBUG)
111841111914
 /* State variables used for validating that the transaction control
111842111915
 ** methods of the virtual table are called at appropriate times. These
111843111916
 ** values do not contribution to the FTS computation; they are used for
@@ -111864,13 +111937,14 @@
111864111937
 Fts3DeferredToken *pDeferred; /* Deferred search tokens, if any */
111865111938
 sqlite3_int64 iPrevId; /* Previous id read from aDoclist */
111866111939
 char *pNextId; /* Pointer into the body of aDoclist */
111867111940
 char *aDoclist; /* List of docids for full-text queries */
111868111941
 int nDoclist; /* Size of buffer at aDoclist */
111869
- int desc; /* True to sort in descending order */
111942
+ u8 bDesc; /* True to sort in descending order */
111870111943
 int eEvalmode; /* An FTS3_EVAL_XX constant */
111871111944
 int nRowAvg; /* Average size of database rows, in pages */
111945
+ int nDoc; /* Documents in table */
111872111946
111873111947
 int isMatchinfoNeeded; /* True when aMatchinfo[] needs filling in */
111874111948
 u32 *aMatchinfo; /* Information about most recent match */
111875111949
 int nMatchinfo; /* Number of elements in aMatchinfo[] */
111876111950
 char *zMatchinfo; /* Matchinfo specification */
@@ -111897,66 +111971,90 @@
111897111971
 */
111898111972
 #define FTS3_FULLSCAN_SEARCH 0 /* Linear scan of %_content table */
111899111973
 #define FTS3_DOCID_SEARCH 1 /* Lookup by rowid on %_content table */
111900111974
 #define FTS3_FULLTEXT_SEARCH 2 /* Full-text index search */
111901111975
111976
+
111977
+struct Fts3Doclist {
111978
+ char *aAll; /* Array containing doclist (or NULL) */
111979
+ int nAll; /* Size of a[] in bytes */
111980
+ char *pNextDocid; /* Pointer to next docid */
111981
+
111982
+ sqlite3_int64 iDocid; /* Current docid (if pList!=0) */
111983
+ int bFreeList; /* True if pList should be sqlite3_free()d */
111984
+ char *pList; /* Pointer to position list following iDocid */
111985
+ int nList; /* Length of position list */
111986
+} doclist;
111987
+
111902111988
 /*
111903111989
 ** A "phrase" is a sequence of one or more tokens that must match in
111904111990
 ** sequence. A single token is the base case and the most common case.
111905111991
 ** For a sequence of tokens contained in double-quotes (i.e. "one two three")
111906111992
 ** nToken will be the number of tokens in the string.
111907
-**
111908
-** The nDocMatch and nMatch variables contain data that may be used by the
111909
-** matchinfo() function. They are populated when the full-text index is
111910
-** queried for hits on the phrase. If one or more tokens in the phrase
111911
-** are deferred, the nDocMatch and nMatch variables are populated based
111912
-** on the assumption that the
111913111993
 */
111914111994
 struct Fts3PhraseToken {
111915111995
 char *z; /* Text of the token */
111916111996
 int n; /* Number of bytes in buffer z */
111917111997
 int isPrefix; /* True if token ends with a "*" character */
111998
+
111999
+ /* Variables above this point are populated when the expression is
112000
+ ** parsed (by code in fts3_expr.c). Below this point the variables are
112001
+ ** used when evaluating the expression. */
111918112002
 int bFulltext; /* True if full-text index was used */
111919
- Fts3SegReaderCursor *pSegcsr; /* Segment-reader for this token */
111920112003
 Fts3DeferredToken *pDeferred; /* Deferred token object for this token */
112004
+ Fts3MultiSegReader *pSegcsr; /* Segment-reader for this token */
111921112005
 };
111922112006
111923112007
 struct Fts3Phrase {
111924
- /* Variables populated by fts3_expr.c when parsing a MATCH expression */
112008
+ /* Cache of doclist for this phrase. */
112009
+ Fts3Doclist doclist;
112010
+ int bIncr; /* True if doclist is loaded incrementally */
112011
+
112012
+ /* Variables below this point are populated by fts3_expr.c when parsing
112013
+ ** a MATCH expression. Everything above is part of the evaluation phase.
112014
+ */
111925112015
 int nToken; /* Number of tokens in the phrase */
111926112016
 int iColumn; /* Index of column this phrase must match */
111927
- int isNot; /* Phrase prefixed by unary not (-) operator */
111928112017
 Fts3PhraseToken aToken[1]; /* One entry for each token in the phrase */
111929112018
 };
111930112019
111931112020
 /*
111932112021
 ** A tree of these objects forms the RHS of a MATCH operator.
111933112022
 **
111934
-** If Fts3Expr.eType is either FTSQUERY_NEAR or FTSQUERY_PHRASE and isLoaded
111935
-** is true, then aDoclist points to a malloced buffer, size nDoclist bytes,
111936
-** containing the results of the NEAR or phrase query in FTS3 doclist
111937
-** format. As usual, the initial "Length" field found in doclists stored
111938
-** on disk is omitted from this buffer.
112023
+** If Fts3Expr.eType is FTSQUERY_PHRASE and isLoaded is true, then aDoclist
112024
+** points to a malloced buffer, size nDoclist bytes, containing the results
112025
+** of this phrase query in FTS3 doclist format. As usual, the initial
112026
+** "Length" field found in doclists stored on disk is omitted from this
112027
+** buffer.
111939112028
 **
111940
-** Variable pCurrent always points to the start of a docid field within
111941
-** aDoclist. Since the doclist is usually scanned in docid order, this can
111942
-** be used to accelerate seeking to the required docid within the doclist.
112029
+** Variable aMI is used only for FTSQUERY_NEAR nodes to store the global
112030
+** matchinfo data. If it is not NULL, it points to an array of size nCol*3,
112031
+** where nCol is the number of columns in the queried FTS table. The array
112032
+** is populated as follows:
112033
+**
112034
+** aMI[iCol*3 + 0] = Undefined
112035
+** aMI[iCol*3 + 1] = Number of occurrences
112036
+** aMI[iCol*3 + 2] = Number of rows containing at least one instance
112037
+**
112038
+** The aMI array is allocated using sqlite3_malloc(). It should be freed
112039
+** when the expression node is.
111943112040
 */
111944112041
 struct Fts3Expr {
111945112042
 int eType; /* One of the FTSQUERY_XXX values defined below */
111946112043
 int nNear; /* Valid if eType==FTSQUERY_NEAR */
111947112044
 Fts3Expr *pParent; /* pParent->pLeft==this or pParent->pRight==this */
111948112045
 Fts3Expr *pLeft; /* Left operand */
111949112046
 Fts3Expr *pRight; /* Right operand */
111950112047
 Fts3Phrase *pPhrase; /* Valid if eType==FTSQUERY_PHRASE */
111951112048
111952
- int isLoaded; /* True if aDoclist/nDoclist are initialized. */
111953
- char *aDoclist; /* Buffer containing doclist */
111954
- int nDoclist; /* Size of aDoclist in bytes */
112049
+ /* The following are used by the fts3_eval.c module. */
112050
+ sqlite3_int64 iDocid; /* Current docid */
112051
+ u8 bEof; /* True this expression is at EOF already */
112052
+ u8 bStart; /* True if iDocid is valid */
112053
+ u8 bDeferred; /* True if this expression is entirely deferred */
111955112054
111956
- sqlite3_int64 iCurrent;
111957
- char *pCurrent;
112055
+ u32 *aMI;
111958112056
 };
111959112057
111960112058
 /*
111961112059
 ** Candidate values for Fts3Query.eType. Note that the order of the first
111962112060
 ** four values is in order of precedence when parsing expressions. For
@@ -111980,35 +112078,36 @@
111980112078
 SQLITE_PRIVATE int sqlite3Fts3PendingTermsFlush(Fts3Table *);
111981112079
 SQLITE_PRIVATE void sqlite3Fts3PendingTermsClear(Fts3Table *);
111982112080
 SQLITE_PRIVATE int sqlite3Fts3Optimize(Fts3Table *);
111983112081
 SQLITE_PRIVATE int sqlite3Fts3SegReaderNew(int, sqlite3_int64,
111984112082
 sqlite3_int64, sqlite3_int64, const char *, int, Fts3SegReader**);
111985
-SQLITE_PRIVATE int sqlite3Fts3SegReaderPending(Fts3Table*,const char*,int,int,Fts3SegReader**);
112083
+SQLITE_PRIVATE int sqlite3Fts3SegReaderPending(
112084
+ Fts3Table*,int,const char*,int,int,Fts3SegReader**);
111986112085
 SQLITE_PRIVATE void sqlite3Fts3SegReaderFree(Fts3SegReader *);
111987
-SQLITE_PRIVATE int sqlite3Fts3SegReaderCost(Fts3Cursor *, Fts3SegReader *, int *);
111988
-SQLITE_PRIVATE int sqlite3Fts3AllSegdirs(Fts3Table*, int, sqlite3_stmt **);
112086
+SQLITE_PRIVATE int sqlite3Fts3AllSegdirs(Fts3Table*, int, int, sqlite3_stmt **);
111989112087
 SQLITE_PRIVATE int sqlite3Fts3ReadLock(Fts3Table *);
111990
-SQLITE_PRIVATE int sqlite3Fts3ReadBlock(Fts3Table*, sqlite3_int64, char **, int*);
112088
+SQLITE_PRIVATE int sqlite3Fts3ReadBlock(Fts3Table*, sqlite3_int64, char **, int*, int*);
111991112089
111992112090
 SQLITE_PRIVATE int sqlite3Fts3SelectDoctotal(Fts3Table *, sqlite3_stmt **);
111993112091
 SQLITE_PRIVATE int sqlite3Fts3SelectDocsize(Fts3Table *, sqlite3_int64, sqlite3_stmt **);
111994112092
111995112093
 SQLITE_PRIVATE void sqlite3Fts3FreeDeferredTokens(Fts3Cursor *);
111996112094
 SQLITE_PRIVATE int sqlite3Fts3DeferToken(Fts3Cursor *, Fts3PhraseToken *, int);
111997112095
 SQLITE_PRIVATE int sqlite3Fts3CacheDeferredDoclists(Fts3Cursor *);
111998112096
 SQLITE_PRIVATE void sqlite3Fts3FreeDeferredDoclists(Fts3Cursor *);
111999
-SQLITE_PRIVATE char *sqlite3Fts3DeferredDoclist(Fts3DeferredToken *, int *);
112000112097
 SQLITE_PRIVATE void sqlite3Fts3SegmentsClose(Fts3Table *);
112001112098
112002
-#define FTS3_SEGCURSOR_PENDING -1
112003
-#define FTS3_SEGCURSOR_ALL -2
112099
+/* Special values interpreted by sqlite3SegReaderCursor() */
112100
+#define FTS3_SEGCURSOR_PENDING -1
112101
+#define FTS3_SEGCURSOR_ALL -2
112004112102
112005
-SQLITE_PRIVATE int sqlite3Fts3SegReaderStart(Fts3Table*, Fts3SegReaderCursor*, Fts3SegFilter*);
112006
-SQLITE_PRIVATE int sqlite3Fts3SegReaderStep(Fts3Table *, Fts3SegReaderCursor *);
112007
-SQLITE_PRIVATE void sqlite3Fts3SegReaderFinish(Fts3SegReaderCursor *);
112103
+SQLITE_PRIVATE int sqlite3Fts3SegReaderStart(Fts3Table*, Fts3MultiSegReader*, Fts3SegFilter*);
112104
+SQLITE_PRIVATE int sqlite3Fts3SegReaderStep(Fts3Table *, Fts3MultiSegReader *);
112105
+SQLITE_PRIVATE void sqlite3Fts3SegReaderFinish(Fts3MultiSegReader *);
112106
+
112008112107
 SQLITE_PRIVATE int sqlite3Fts3SegReaderCursor(
112009
- Fts3Table *, int, const char *, int, int, int, Fts3SegReaderCursor *);
112108
+ Fts3Table *, int, int, const char *, int, int, int, Fts3MultiSegReader *);
112010112109
112011112110
 /* Flags allowed as part of the 4th argument to SegmentReaderIterate() */
112012112111
 #define FTS3_SEGMENT_REQUIRE_POS 0x00000001
112013112112
 #define FTS3_SEGMENT_IGNORE_EMPTY 0x00000002
112014112113
 #define FTS3_SEGMENT_COLUMN_FILTER 0x00000004
@@ -112021,21 +112120,24 @@
112021112120
 int nTerm;
112022112121
 int iCol;
112023112122
 int flags;
112024112123
 };
112025112124
112026
-struct Fts3SegReaderCursor {
112125
+struct Fts3MultiSegReader {
112027112126
 /* Used internally by sqlite3Fts3SegReaderXXX() calls */
112028112127
 Fts3SegReader **apSegment; /* Array of Fts3SegReader objects */
112029112128
 int nSegment; /* Size of apSegment array */
112030112129
 int nAdvance; /* How many seg-readers to advance */
112031112130
 Fts3SegFilter *pFilter; /* Pointer to filter object */
112032112131
 char *aBuffer; /* Buffer to merge doclists in */
112033112132
 int nBuffer; /* Allocated size of aBuffer[] in bytes */
112034112133
112035
- /* Cost of running this iterator. Used by fts3.c only. */
112036
- int nCost;
112134
+ int iColFilter; /* If >=0, filter for this column */
112135
+
112136
+ /* Used by fts3.c only. */
112137
+ int nCost; /* Cost of running iterator */
112138
+ int bLookup; /* True if a lookup of a single entry. */
112037112139
112038112140
 /* Output values. Valid only after Fts3SegReaderStep() returns SQLITE_ROW. */
112039112141
 char *zTerm; /* Pointer to term buffer */
112040112142
 int nTerm; /* Size of zTerm in bytes */
112041112143
 char *aDoclist; /* Pointer to doclist buffer */
@@ -112046,15 +112148,13 @@
112046112148
 SQLITE_PRIVATE int sqlite3Fts3PutVarint(char *, sqlite3_int64);
112047112149
 SQLITE_PRIVATE int sqlite3Fts3GetVarint(const char *, sqlite_int64 *);
112048112150
 SQLITE_PRIVATE int sqlite3Fts3GetVarint32(const char *, int *);
112049112151
 SQLITE_PRIVATE int sqlite3Fts3VarintLen(sqlite3_uint64);
112050112152
 SQLITE_PRIVATE void sqlite3Fts3Dequote(char *);
112153
+SQLITE_PRIVATE void sqlite3Fts3DoclistPrev(int,char*,int,char**,sqlite3_int64*,int*,u8*);
112051112154
112052
-SQLITE_PRIVATE char *sqlite3Fts3FindPositions(Fts3Cursor *, Fts3Expr *, sqlite3_int64, int);
112053
-SQLITE_PRIVATE int sqlite3Fts3ExprLoadDoclist(Fts3Cursor *, Fts3Expr *);
112054
-SQLITE_PRIVATE int sqlite3Fts3ExprLoadFtDoclist(Fts3Cursor *, Fts3Expr *, char **, int *);
112055
-SQLITE_PRIVATE int sqlite3Fts3ExprNearTrim(Fts3Expr *, Fts3Expr *, int);
112155
+SQLITE_PRIVATE int sqlite3Fts3EvalPhraseStats(Fts3Cursor *, Fts3Expr *, u32 *);
112056112156
112057112157
 /* fts3_tokenizer.c */
112058112158
 SQLITE_PRIVATE const char *sqlite3Fts3NextToken(const char *, int *);
112059112159
 SQLITE_PRIVATE int sqlite3Fts3InitHashTable(sqlite3 *, Fts3Hash *, const char *);
112060112160
 SQLITE_PRIVATE int sqlite3Fts3InitTokenizer(Fts3Hash *pHash, const char *,
@@ -112079,10 +112179,33 @@
112079112179
 SQLITE_PRIVATE int sqlite3Fts3InitTerm(sqlite3 *db);
112080112180
 #endif
112081112181
112082112182
 /* fts3_aux.c */
112083112183
 SQLITE_PRIVATE int sqlite3Fts3InitAux(sqlite3 *db);
112184
+
112185
+SQLITE_PRIVATE int sqlite3Fts3TermSegReaderCursor(
112186
+ Fts3Cursor *pCsr, /* Virtual table cursor handle */
112187
+ const char *zTerm, /* Term to query for */
112188
+ int nTerm, /* Size of zTerm in bytes */
112189
+ int isPrefix, /* True for a prefix search */
112190
+ Fts3MultiSegReader **ppSegcsr /* OUT: Allocated seg-reader cursor */
112191
+);
112192
+
112193
+SQLITE_PRIVATE void sqlite3Fts3EvalPhraseCleanup(Fts3Phrase *);
112194
+
112195
+SQLITE_PRIVATE int sqlite3Fts3EvalStart(Fts3Cursor *, Fts3Expr *, int);
112196
+SQLITE_PRIVATE int sqlite3Fts3EvalNext(Fts3Cursor *pCsr);
112197
+
112198
+SQLITE_PRIVATE int sqlite3Fts3MsrIncrStart(
112199
+ Fts3Table*, Fts3MultiSegReader*, int, const char*, int);
112200
+SQLITE_PRIVATE int sqlite3Fts3MsrIncrNext(
112201
+ Fts3Table *, Fts3MultiSegReader *, sqlite3_int64 *, char **, int *);
112202
+SQLITE_PRIVATE char *sqlite3Fts3EvalPhrasePoslist(Fts3Cursor *, Fts3Expr *, int iCol);
112203
+SQLITE_PRIVATE int sqlite3Fts3MsrOvfl(Fts3Cursor *, Fts3MultiSegReader *, int *);
112204
+
112205
+SQLITE_PRIVATE int sqlite3Fts3DeferredTokenList(Fts3DeferredToken *, char **, int *);
112206
+
112084112207
112085112208
 #endif /* _FTSINT_H */
112086112209
112087112210
 /************** End of fts3Int.h *********************************************/
112088112211
 /************** Continuing where we left off in fts3.c ***********************/
@@ -112200,16 +112323,16 @@
112200112323
112201112324
 /*
112202112325
 ** When this function is called, *pp points to the first byte following a
112203112326
 ** varint that is part of a doclist (or position-list, or any other list
112204112327
 ** of varints). This function moves *pp to point to the start of that varint,
112205
-** and decrements the value stored in *pVal by the varint value.
112328
+** and sets *pVal by the varint value.
112206112329
 **
112207112330
 ** Argument pStart points to the first byte of the doclist that the
112208112331
 ** varint is part of.
112209112332
 */
112210
-static void fts3GetReverseDeltaVarint(
112333
+static void fts3GetReverseVarint(
112211112334
 char **pp,
112212112335
 char *pStart,
112213112336
 sqlite3_int64 *pVal
112214112337
 ){
112215112338
 sqlite3_int64 iVal;
@@ -112221,25 +112344,11 @@
112221112344
 for(p = (*pp)-2; p>=pStart && *p&0x80; p--);
112222112345
 p++;
112223112346
 *pp = p;
112224112347
112225112348
 sqlite3Fts3GetVarint(p, &iVal);
112226
- *pVal -= iVal;
112227
-}
112228
-
112229
-/*
112230
-** As long as *pp has not reached its end (pEnd), then do the same
112231
-** as fts3GetDeltaVarint(): read a single varint and add it to *pVal.
112232
-** But if we have reached the end of the varint, just set *pp=0 and
112233
-** leave *pVal unchanged.
112234
-*/
112235
-static void fts3GetDeltaVarint2(char **pp, char *pEnd, sqlite3_int64 *pVal){
112236
- if( *pp>=pEnd ){
112237
- *pp = 0;
112238
- }else{
112239
- fts3GetDeltaVarint(pp, pVal);
112240
- }
112349
+ *pVal = iVal;
112241112350
 }
112242112351
112243112352
 /*
112244112353
 ** The xDisconnect() virtual table method.
112245112354
 */
@@ -112608,10 +112717,62 @@
112608112717
 fts3Appendf(pRc, &zRet, ",%s(?)", zFunction);
112609112718
 }
112610112719
 sqlite3_free(zFree);
112611112720
 return zRet;
112612112721
 }
112722
+
112723
+static int fts3GobbleInt(const char **pp, int *pnOut){
112724
+ const char *p = *pp;
112725
+ int nInt = 0;
112726
+ for(p=*pp; p[0]>='0' && p[0]<='9'; p++){
112727
+ nInt = nInt * 10 + (p[0] - '0');
112728
+ }
112729
+ if( p==*pp ) return SQLITE_ERROR;
112730
+ *pnOut = nInt;
112731
+ *pp = p;
112732
+ return SQLITE_OK;
112733
+}
112734
+
112735
+
112736
+static int fts3PrefixParameter(
112737
+ const char *zParam, /* ABC in prefix=ABC parameter to parse */
112738
+ int *pnIndex, /* OUT: size of *apIndex[] array */
112739
+ struct Fts3Index **apIndex, /* OUT: Array of indexes for this table */
112740
+ struct Fts3Index **apFree /* OUT: Free this with sqlite3_free() */
112741
+){
112742
+ struct Fts3Index *aIndex;
112743
+ int nIndex = 1;
112744
+
112745
+ if( zParam && zParam[0] ){
112746
+ const char *p;
112747
+ nIndex++;
112748
+ for(p=zParam; *p; p++){
112749
+ if( *p==',' ) nIndex++;
112750
+ }
112751
+ }
112752
+
112753
+ aIndex = sqlite3_malloc(sizeof(struct Fts3Index) * nIndex);
112754
+ *apIndex = *apFree = aIndex;
112755
+ *pnIndex = nIndex;
112756
+ if( !aIndex ){
112757
+ return SQLITE_NOMEM;
112758
+ }
112759
+
112760
+ memset(aIndex, 0, sizeof(struct Fts3Index) * nIndex);
112761
+ if( zParam ){
112762
+ const char *p = zParam;
112763
+ int i;
112764
+ for(i=1; i<nIndex; i++){
112765
+ int nPrefix;
112766
+ if( fts3GobbleInt(&p, &nPrefix) ) return SQLITE_ERROR;
112767
+ aIndex[i].nPrefix = nPrefix;
112768
+ p++;
112769
+ }
112770
+ }
112771
+
112772
+ return SQLITE_OK;
112773
+}
112613112774
112614112775
 /*
112615112776
 ** This function is the implementation of both the xConnect and xCreate
112616112777
 ** methods of the FTS3 virtual table.
112617112778
 **
@@ -112641,16 +112802,23 @@
112641112802
 int nCol = 0; /* Number of columns in the FTS table */
112642112803
 char *zCsr; /* Space for holding column names */
112643112804
 int nDb; /* Bytes required to hold database name */
112644112805
 int nName; /* Bytes required to hold table name */
112645112806
 int isFts4 = (argv[0][3]=='4'); /* True for FTS4, false for FTS3 */
112646
- int bNoDocsize = 0; /* True to omit %_docsize table */
112647112807
 const char **aCol; /* Array of column names */
112648112808
 sqlite3_tokenizer *pTokenizer = 0; /* Tokenizer for this table */
112649112809
112650
- char *zCompress = 0;
112651
- char *zUncompress = 0;
112810
+ int nIndex; /* Size of aIndex[] array */
112811
+ struct Fts3Index *aIndex; /* Array of indexes for this table */
112812
+ struct Fts3Index *aFree = 0; /* Free this before returning */
112813
+
112814
+ /* The results of parsing supported FTS4 key=value options: */
112815
+ int bNoDocsize = 0; /* True to omit %_docsize table */
112816
+ int bDescIdx = 0; /* True to store descending indexes */
112817
+ char *zPrefix = 0; /* Prefix parameter value (or NULL) */
112818
+ char *zCompress = 0; /* compress=? parameter (or NULL) */
112819
+ char *zUncompress = 0; /* uncompress=? parameter (or NULL) */
112652112820
112653112821
 assert( strlen(argv[0])==4 );
112654112822
 assert( (sqlite3_strnicmp(argv[0], "fts4", 4)==0 && isFts4)
112655112823
 || (sqlite3_strnicmp(argv[0], "fts3", 4)==0 && !isFts4)
112656112824
 );
@@ -112687,32 +112855,76 @@
112687112855
 rc = sqlite3Fts3InitTokenizer(pHash, &z[9], &pTokenizer, pzErr);
112688112856
 }
112689112857
112690112858
 /* Check if it is an FTS4 special argument. */
112691112859
 else if( isFts4 && fts3IsSpecialColumn(z, &nKey, &zVal) ){
112860
+ struct Fts4Option {
112861
+ const char *zOpt;
112862
+ int nOpt;
112863
+ char **pzVar;
112864
+ } aFts4Opt[] = {
112865
+ { "matchinfo", 9, 0 }, /* 0 -> MATCHINFO */
112866
+ { "prefix", 6, 0 }, /* 1 -> PREFIX */
112867
+ { "compress", 8, 0 }, /* 2 -> COMPRESS */
112868
+ { "uncompress", 10, 0 }, /* 3 -> UNCOMPRESS */
112869
+ { "order", 5, 0 } /* 4 -> ORDER */
112870
+ };
112871
+
112872
+ int iOpt;
112692112873
 if( !zVal ){
112693112874
 rc = SQLITE_NOMEM;
112694
- goto fts3_init_out;
112695
- }
112696
- if( nKey==9 && 0==sqlite3_strnicmp(z, "matchinfo", 9) ){
112697
- if( strlen(zVal)==4 && 0==sqlite3_strnicmp(zVal, "fts3", 4) ){
112698
- bNoDocsize = 1;
112699
- }else{
112700
- *pzErr = sqlite3_mprintf("unrecognized matchinfo: %s", zVal);
112701
- rc = SQLITE_ERROR;
112702
- }
112703
- }else if( nKey==8 && 0==sqlite3_strnicmp(z, "compress", 8) ){
112704
- zCompress = zVal;
112705
- zVal = 0;
112706
- }else if( nKey==10 && 0==sqlite3_strnicmp(z, "uncompress", 10) ){
112707
- zUncompress = zVal;
112708
- zVal = 0;
112709
- }else{
112710
- *pzErr = sqlite3_mprintf("unrecognized parameter: %s", z);
112711
- rc = SQLITE_ERROR;
112712
- }
112713
- sqlite3_free(zVal);
112875
+ }else{
112876
+ for(iOpt=0; iOpt<SizeofArray(aFts4Opt); iOpt++){
112877
+ struct Fts4Option *pOp = &aFts4Opt[iOpt];
112878
+ if( nKey==pOp->nOpt && !sqlite3_strnicmp(z, pOp->zOpt, pOp->nOpt) ){
112879
+ break;
112880
+ }
112881
+ }
112882
+ if( iOpt==SizeofArray(aFts4Opt) ){
112883
+ *pzErr = sqlite3_mprintf("unrecognized parameter: %s", z);
112884
+ rc = SQLITE_ERROR;
112885
+ }else{
112886
+ switch( iOpt ){
112887
+ case 0: /* MATCHINFO */
112888
+ if( strlen(zVal)!=4 || sqlite3_strnicmp(zVal, "fts3", 4) ){
112889
+ *pzErr = sqlite3_mprintf("unrecognized matchinfo: %s", zVal);
112890
+ rc = SQLITE_ERROR;
112891
+ }
112892
+ bNoDocsize = 1;
112893
+ break;
112894
+
112895
+ case 1: /* PREFIX */
112896
+ sqlite3_free(zPrefix);
112897
+ zPrefix = zVal;
112898
+ zVal = 0;
112899
+ break;
112900
+
112901
+ case 2: /* COMPRESS */
112902
+ sqlite3_free(zCompress);
112903
+ zCompress = zVal;
112904
+ zVal = 0;
112905
+ break;
112906
+
112907
+ case 3: /* UNCOMPRESS */
112908
+ sqlite3_free(zUncompress);
112909
+ zUncompress = zVal;
112910
+ zVal = 0;
112911
+ break;
112912
+
112913
+ case 4: /* ORDER */
112914
+ if( (strlen(zVal)!=3 || sqlite3_strnicmp(zVal, "asc", 3))
112915
+ && (strlen(zVal)!=4 || sqlite3_strnicmp(zVal, "desc", 3))
112916
+ ){
112917
+ *pzErr = sqlite3_mprintf("unrecognized order: %s", zVal);
112918
+ rc = SQLITE_ERROR;
112919
+ }
112920
+ bDescIdx = (zVal[0]=='d' || zVal[0]=='D');
112921
+ break;
112922
+ }
112923
+ }
112924
+ sqlite3_free(zVal);
112925
+ }
112714112926
 }
112715112927
112716112928
 /* Otherwise, the argument is a column name. */
112717112929
 else {
112718112930
 nString += (int)(strlen(z) + 1);
@@ -112732,14 +112944,21 @@
112732112944
 rc = sqlite3Fts3InitTokenizer(pHash, "simple", &pTokenizer, pzErr);
112733112945
 if( rc!=SQLITE_OK ) goto fts3_init_out;
112734112946
 }
112735112947
 assert( pTokenizer );
112736112948
112949
+ rc = fts3PrefixParameter(zPrefix, &nIndex, &aIndex, &aFree);
112950
+ if( rc==SQLITE_ERROR ){
112951
+ assert( zPrefix );
112952
+ *pzErr = sqlite3_mprintf("error parsing prefix parameter: %s", zPrefix);
112953
+ }
112954
+ if( rc!=SQLITE_OK ) goto fts3_init_out;
112737112955
112738112956
 /* Allocate and populate the Fts3Table structure. */
112739
- nByte = sizeof(Fts3Table) + /* Fts3Table */
112957
+ nByte = sizeof(Fts3Table) + /* Fts3Table */
112740112958
 nCol * sizeof(char *) + /* azColumn */
112959
+ nIndex * sizeof(struct Fts3Index) + /* aIndex */
112741112960
 nName + /* zName */
112742112961
 nDb + /* zDb */
112743112962
 nString; /* Space for azColumn strings */
112744112963
 p = (Fts3Table*)sqlite3_malloc(nByte);
112745112964
 if( p==0 ){
@@ -112750,20 +112969,26 @@
112750112969
 p->db = db;
112751112970
 p->nColumn = nCol;
112752112971
 p->nPendingData = 0;
112753112972
 p->azColumn = (char **)&p[1];
112754112973
 p->pTokenizer = pTokenizer;
112755
- p->nNodeSize = 1000;
112756112974
 p->nMaxPendingData = FTS3_MAX_PENDING_DATA;
112757112975
 p->bHasDocsize = (isFts4 && bNoDocsize==0);
112758112976
 p->bHasStat = isFts4;
112977
+ p->bDescIdx = bDescIdx;
112759112978
 TESTONLY( p->inTransaction = -1 );
112760112979
 TESTONLY( p->mxSavepoint = -1 );
112761
- fts3HashInit(&p->pendingTerms, FTS3_HASH_STRING, 1);
112980
+
112981
+ p->aIndex = (struct Fts3Index *)&p->azColumn[nCol];
112982
+ memcpy(p->aIndex, aIndex, sizeof(struct Fts3Index) * nIndex);
112983
+ p->nIndex = nIndex;
112984
+ for(i=0; i<nIndex; i++){
112985
+ fts3HashInit(&p->aIndex[i].hPending, FTS3_HASH_STRING, 1);
112986
+ }
112762112987
112763112988
 /* Fill in the zName and zDb fields of the vtab structure. */
112764
- zCsr = (char *)&p->azColumn[nCol];
112989
+ zCsr = (char *)&p->aIndex[nIndex];
112765112990
 p->zName = zCsr;
112766112991
 memcpy(zCsr, argv[2], nName);
112767112992
 zCsr += nName;
112768112993
 p->zDb = zCsr;
112769112994
 memcpy(zCsr, argv[1], nDb);
@@ -112797,19 +113022,20 @@
112797113022
 if( isCreate ){
112798113023
 rc = fts3CreateTables(p);
112799113024
 }
112800113025
112801113026
 /* Figure out the page-size for the database. This is required in order to
112802
- ** estimate the cost of loading large doclists from the database (see
112803
- ** function sqlite3Fts3SegReaderCost() for details).
112804
- */
113027
+ ** estimate the cost of loading large doclists from the database. */
112805113028
 fts3DatabasePageSize(&rc, p);
113029
+ p->nNodeSize = p->nPgsz-35;
112806113030
112807113031
 /* Declare the table schema to SQLite. */
112808113032
 fts3DeclareVtab(&rc, p);
112809113033
112810113034
 fts3_init_out:
113035
+ sqlite3_free(zPrefix);
113036
+ sqlite3_free(aFree);
112811113037
 sqlite3_free(zCompress);
112812113038
 sqlite3_free(zUncompress);
112813113039
 sqlite3_free((void *)aCol);
112814113040
 if( rc!=SQLITE_OK ){
112815113041
 if( p ){
@@ -112816,10 +113042,11 @@
112816113042
 fts3DisconnectMethod((sqlite3_vtab *)p);
112817113043
 }else if( pTokenizer ){
112818113044
 pTokenizer->pModule->xDestroy(pTokenizer);
112819113045
 }
112820113046
 }else{
113047
+ assert( p->pSegments==0 );
112821113048
 *ppVTab = &p->base;
112822113049
 }
112823113050
 return rc;
112824113051
 }
112825113052
@@ -112913,14 +113140,15 @@
112913113140
 if( pOrder->desc ){
112914113141
 pInfo->idxStr = "DESC";
112915113142
 }else{
112916113143
 pInfo->idxStr = "ASC";
112917113144
 }
113145
+ pInfo->orderByConsumed = 1;
112918113146
 }
112919
- pInfo->orderByConsumed = 1;
112920113147
 }
112921113148
113149
+ assert( p->pSegments==0 );
112922113150
 return SQLITE_OK;
112923113151
 }
112924113152
112925113153
 /*
112926113154
 ** Implementation of xOpen method.
@@ -112952,10 +113180,11 @@
112952113180
 sqlite3_finalize(pCsr->pStmt);
112953113181
 sqlite3Fts3ExprFree(pCsr->pExpr);
112954113182
 sqlite3Fts3FreeDeferredTokens(pCsr);
112955113183
 sqlite3_free(pCsr->aDoclist);
112956113184
 sqlite3_free(pCsr->aMatchinfo);
113185
+ assert( ((Fts3Table *)pCsr->base.pVtab)->pSegments==0 );
112957113186
 sqlite3_free(pCsr);
112958113187
 return SQLITE_OK;
112959113188
 }
112960113189
112961113190
 /*
@@ -112963,12 +113192,12 @@
112963113192
 ** of the %_content table that contains the last match. Return
112964113193
 ** SQLITE_OK on success.
112965113194
 */
112966113195
 static int fts3CursorSeek(sqlite3_context *pContext, Fts3Cursor *pCsr){
112967113196
 if( pCsr->isRequireSeek ){
112968
- pCsr->isRequireSeek = 0;
112969113197
 sqlite3_bind_int64(pCsr->pStmt, 1, pCsr->iPrevId);
113198
+ pCsr->isRequireSeek = 0;
112970113199
 if( SQLITE_ROW==sqlite3_step(pCsr->pStmt) ){
112971113200
 return SQLITE_OK;
112972113201
 }else{
112973113202
 int rc = sqlite3_reset(pCsr->pStmt);
112974113203
 if( rc==SQLITE_OK ){
@@ -113145,21 +113374,21 @@
113145113374
 if( rc==SQLITE_OK && iHeight>1 ){
113146113375
 char *zBlob = 0; /* Blob read from %_segments table */
113147113376
 int nBlob; /* Size of zBlob in bytes */
113148113377
113149113378
 if( piLeaf && piLeaf2 && (*piLeaf!=*piLeaf2) ){
113150
- rc = sqlite3Fts3ReadBlock(p, *piLeaf, &zBlob, &nBlob);
113379
+ rc = sqlite3Fts3ReadBlock(p, *piLeaf, &zBlob, &nBlob, 0);
113151113380
 if( rc==SQLITE_OK ){
113152113381
 rc = fts3SelectLeaf(p, zTerm, nTerm, zBlob, nBlob, piLeaf, 0);
113153113382
 }
113154113383
 sqlite3_free(zBlob);
113155113384
 piLeaf = 0;
113156113385
 zBlob = 0;
113157113386
 }
113158113387
113159113388
 if( rc==SQLITE_OK ){
113160
- rc = sqlite3Fts3ReadBlock(p, piLeaf ? *piLeaf : *piLeaf2, &zBlob, &nBlob);
113389
+ rc = sqlite3Fts3ReadBlock(p, piLeaf?*piLeaf:*piLeaf2, &zBlob, &nBlob, 0);
113161113390
 }
113162113391
 if( rc==SQLITE_OK ){
113163113392
 rc = fts3SelectLeaf(p, zTerm, nTerm, zBlob, nBlob, piLeaf, piLeaf2);
113164113393
 }
113165113394
 sqlite3_free(zBlob);
@@ -113531,11 +113760,23 @@
113531113760
 *pp = p;
113532113761
 return 1;
113533113762
 }
113534113763
113535113764
 /*
113536
-** Merge two position-lists as required by the NEAR operator.
113765
+** Merge two position-lists as required by the NEAR operator. The argument
113766
+** position lists correspond to the left and right phrases of an expression
113767
+** like:
113768
+**
113769
+** "phrase 1" NEAR "phrase number 2"
113770
+**
113771
+** Position list *pp1 corresponds to the left-hand side of the NEAR
113772
+** expression and *pp2 to the right. As usual, the indexes in the position
113773
+** lists are the offsets of the last token in each phrase (tokens "1" and "2"
113774
+** in the example above).
113775
+**
113776
+** The output position list - written to *pp - is a copy of *pp2 with those
113777
+** entries that are not sufficiently NEAR entries in *pp1 removed.
113537113778
 */
113538113779
 static int fts3PoslistNearMerge(
113539113780
 char **pp, /* Output buffer */
113540113781
 char *aTmp, /* Temporary buffer space */
113541113782
 int nRight, /* Maximum difference in token positions */
@@ -113544,218 +113785,31 @@
113544113785
 char **pp2 /* IN/OUT: Right input list */
113545113786
 ){
113546113787
 char *p1 = *pp1;
113547113788
 char *p2 = *pp2;
113548113789
113549
- if( !pp ){
113550
- if( fts3PoslistPhraseMerge(0, nRight, 0, 0, pp1, pp2) ) return 1;
113551
- *pp1 = p1;
113552
- *pp2 = p2;
113553
- return fts3PoslistPhraseMerge(0, nLeft, 0, 0, pp2, pp1);
113554
- }else{
113555
- char *pTmp1 = aTmp;
113556
- char *pTmp2;
113557
- char *aTmp2;
113558
- int res = 1;
113559
-
113560
- fts3PoslistPhraseMerge(&pTmp1, nRight, 0, 0, pp1, pp2);
113561
- aTmp2 = pTmp2 = pTmp1;
113562
- *pp1 = p1;
113563
- *pp2 = p2;
113564
- fts3PoslistPhraseMerge(&pTmp2, nLeft, 1, 0, pp2, pp1);
113565
- if( pTmp1!=aTmp && pTmp2!=aTmp2 ){
113566
- fts3PoslistMerge(pp, &aTmp, &aTmp2);
113567
- }else if( pTmp1!=aTmp ){
113568
- fts3PoslistCopy(pp, &aTmp);
113569
- }else if( pTmp2!=aTmp2 ){
113570
- fts3PoslistCopy(pp, &aTmp2);
113571
- }else{
113572
- res = 0;
113573
- }
113574
-
113575
- return res;
113576
- }
113577
-}
113578
-
113579
-/*
113580
-** Values that may be used as the first parameter to fts3DoclistMerge().
113581
-*/
113582
-#define MERGE_NOT 2 /* D + D -> D */
113583
-#define MERGE_AND 3 /* D + D -> D */
113584
-#define MERGE_OR 4 /* D + D -> D */
113585
-#define MERGE_POS_OR 5 /* P + P -> P */
113586
-#define MERGE_PHRASE 6 /* P + P -> D */
113587
-#define MERGE_POS_PHRASE 7 /* P + P -> P */
113588
-#define MERGE_NEAR 8 /* P + P -> D */
113589
-#define MERGE_POS_NEAR 9 /* P + P -> P */
113590
-
113591
-/*
113592
-** Merge the two doclists passed in buffer a1 (size n1 bytes) and a2
113593
-** (size n2 bytes). The output is written to pre-allocated buffer aBuffer,
113594
-** which is guaranteed to be large enough to hold the results. The number
113595
-** of bytes written to aBuffer is stored in *pnBuffer before returning.
113596
-**
113597
-** If successful, SQLITE_OK is returned. Otherwise, if a malloc error
113598
-** occurs while allocating a temporary buffer as part of the merge operation,
113599
-** SQLITE_NOMEM is returned.
113600
-*/
113601
-static int fts3DoclistMerge(
113602
- int mergetype, /* One of the MERGE_XXX constants */
113603
- int nParam1, /* Used by MERGE_NEAR and MERGE_POS_NEAR */
113604
- int nParam2, /* Used by MERGE_NEAR and MERGE_POS_NEAR */
113605
- char *aBuffer, /* Pre-allocated output buffer */
113606
- int *pnBuffer, /* OUT: Bytes written to aBuffer */
113607
- char *a1, /* Buffer containing first doclist */
113608
- int n1, /* Size of buffer a1 */
113609
- char *a2, /* Buffer containing second doclist */
113610
- int n2, /* Size of buffer a2 */
113611
- int *pnDoc /* OUT: Number of docids in output */
113612
-){
113613
- sqlite3_int64 i1 = 0;
113614
- sqlite3_int64 i2 = 0;
113615
- sqlite3_int64 iPrev = 0;
113616
-
113617
- char *p = aBuffer;
113618
- char *p1 = a1;
113619
- char *p2 = a2;
113620
- char *pEnd1 = &a1[n1];
113621
- char *pEnd2 = &a2[n2];
113622
- int nDoc = 0;
113623
-
113624
- assert( mergetype==MERGE_OR || mergetype==MERGE_POS_OR
113625
- || mergetype==MERGE_AND || mergetype==MERGE_NOT
113626
- || mergetype==MERGE_PHRASE || mergetype==MERGE_POS_PHRASE
113627
- || mergetype==MERGE_NEAR || mergetype==MERGE_POS_NEAR
113628
- );
113629
-
113630
- if( !aBuffer ){
113631
- *pnBuffer = 0;
113632
- return SQLITE_NOMEM;
113633
- }
113634
-
113635
- /* Read the first docid from each doclist */
113636
- fts3GetDeltaVarint2(&p1, pEnd1, &i1);
113637
- fts3GetDeltaVarint2(&p2, pEnd2, &i2);
113638
-
113639
- switch( mergetype ){
113640
- case MERGE_OR:
113641
- case MERGE_POS_OR:
113642
- while( p1 || p2 ){
113643
- if( p2 && p1 && i1==i2 ){
113644
- fts3PutDeltaVarint(&p, &iPrev, i1);
113645
- if( mergetype==MERGE_POS_OR ) fts3PoslistMerge(&p, &p1, &p2);
113646
- fts3GetDeltaVarint2(&p1, pEnd1, &i1);
113647
- fts3GetDeltaVarint2(&p2, pEnd2, &i2);
113648
- }else if( !p2 || (p1 && i1<i2) ){
113649
- fts3PutDeltaVarint(&p, &iPrev, i1);
113650
- if( mergetype==MERGE_POS_OR ) fts3PoslistCopy(&p, &p1);
113651
- fts3GetDeltaVarint2(&p1, pEnd1, &i1);
113652
- }else{
113653
- fts3PutDeltaVarint(&p, &iPrev, i2);
113654
- if( mergetype==MERGE_POS_OR ) fts3PoslistCopy(&p, &p2);
113655
- fts3GetDeltaVarint2(&p2, pEnd2, &i2);
113656
- }
113657
- }
113658
- break;
113659
-
113660
- case MERGE_AND:
113661
- while( p1 && p2 ){
113662
- if( i1==i2 ){
113663
- fts3PutDeltaVarint(&p, &iPrev, i1);
113664
- fts3GetDeltaVarint2(&p1, pEnd1, &i1);
113665
- fts3GetDeltaVarint2(&p2, pEnd2, &i2);
113666
- nDoc++;
113667
- }else if( i1<i2 ){
113668
- fts3GetDeltaVarint2(&p1, pEnd1, &i1);
113669
- }else{
113670
- fts3GetDeltaVarint2(&p2, pEnd2, &i2);
113671
- }
113672
- }
113673
- break;
113674
-
113675
- case MERGE_NOT:
113676
- while( p1 ){
113677
- if( p2 && i1==i2 ){
113678
- fts3GetDeltaVarint2(&p1, pEnd1, &i1);
113679
- fts3GetDeltaVarint2(&p2, pEnd2, &i2);
113680
- }else if( !p2 || i1<i2 ){
113681
- fts3PutDeltaVarint(&p, &iPrev, i1);
113682
- fts3GetDeltaVarint2(&p1, pEnd1, &i1);
113683
- }else{
113684
- fts3GetDeltaVarint2(&p2, pEnd2, &i2);
113685
- }
113686
- }
113687
- break;
113688
-
113689
- case MERGE_POS_PHRASE:
113690
- case MERGE_PHRASE: {
113691
- char **ppPos = (mergetype==MERGE_PHRASE ? 0 : &p);
113692
- while( p1 && p2 ){
113693
- if( i1==i2 ){
113694
- char *pSave = p;
113695
- sqlite3_int64 iPrevSave = iPrev;
113696
- fts3PutDeltaVarint(&p, &iPrev, i1);
113697
- if( 0==fts3PoslistPhraseMerge(ppPos, nParam1, 0, 1, &p1, &p2) ){
113698
- p = pSave;
113699
- iPrev = iPrevSave;
113700
- }else{
113701
- nDoc++;
113702
- }
113703
- fts3GetDeltaVarint2(&p1, pEnd1, &i1);
113704
- fts3GetDeltaVarint2(&p2, pEnd2, &i2);
113705
- }else if( i1<i2 ){
113706
- fts3PoslistCopy(0, &p1);
113707
- fts3GetDeltaVarint2(&p1, pEnd1, &i1);
113708
- }else{
113709
- fts3PoslistCopy(0, &p2);
113710
- fts3GetDeltaVarint2(&p2, pEnd2, &i2);
113711
- }
113712
- }
113713
- break;
113714
- }
113715
-
113716
- default: assert( mergetype==MERGE_POS_NEAR || mergetype==MERGE_NEAR ); {
113717
- char *aTmp = 0;
113718
- char **ppPos = 0;
113719
-
113720
- if( mergetype==MERGE_POS_NEAR ){
113721
- ppPos = &p;
113722
- aTmp = sqlite3_malloc(2*(n1+n2+1));
113723
- if( !aTmp ){
113724
- return SQLITE_NOMEM;
113725
- }
113726
- }
113727
-
113728
- while( p1 && p2 ){
113729
- if( i1==i2 ){
113730
- char *pSave = p;
113731
- sqlite3_int64 iPrevSave = iPrev;
113732
- fts3PutDeltaVarint(&p, &iPrev, i1);
113733
-
113734
- if( !fts3PoslistNearMerge(ppPos, aTmp, nParam1, nParam2, &p1, &p2) ){
113735
- iPrev = iPrevSave;
113736
- p = pSave;
113737
- }
113738
-
113739
- fts3GetDeltaVarint2(&p1, pEnd1, &i1);
113740
- fts3GetDeltaVarint2(&p2, pEnd2, &i2);
113741
- }else if( i1<i2 ){
113742
- fts3PoslistCopy(0, &p1);
113743
- fts3GetDeltaVarint2(&p1, pEnd1, &i1);
113744
- }else{
113745
- fts3PoslistCopy(0, &p2);
113746
- fts3GetDeltaVarint2(&p2, pEnd2, &i2);
113747
- }
113748
- }
113749
- sqlite3_free(aTmp);
113750
- break;
113751
- }
113752
- }
113753
-
113754
- if( pnDoc ) *pnDoc = nDoc;
113755
- *pnBuffer = (int)(p-aBuffer);
113756
- return SQLITE_OK;
113790
+ char *pTmp1 = aTmp;
113791
+ char *pTmp2;
113792
+ char *aTmp2;
113793
+ int res = 1;
113794
+
113795
+ fts3PoslistPhraseMerge(&pTmp1, nRight, 0, 0, pp1, pp2);
113796
+ aTmp2 = pTmp2 = pTmp1;
113797
+ *pp1 = p1;
113798
+ *pp2 = p2;
113799
+ fts3PoslistPhraseMerge(&pTmp2, nLeft, 1, 0, pp2, pp1);
113800
+ if( pTmp1!=aTmp && pTmp2!=aTmp2 ){
113801
+ fts3PoslistMerge(pp, &aTmp, &aTmp2);
113802
+ }else if( pTmp1!=aTmp ){
113803
+ fts3PoslistCopy(pp, &aTmp);
113804
+ }else if( pTmp2!=aTmp2 ){
113805
+ fts3PoslistCopy(pp, &aTmp2);
113806
+ }else{
113807
+ res = 0;
113808
+ }
113809
+
113810
+ return res;
113757113811
 }
113758113812
113759113813
 /*
113760113814
 ** A pointer to an instance of this structure is used as the context
113761113815
 ** argument to sqlite3Fts3SegReaderIterate()
@@ -113764,10 +113818,152 @@
113764113818
 struct TermSelect {
113765113819
 int isReqPos;
113766113820
 char *aaOutput[16]; /* Malloc'd output buffer */
113767113821
 int anOutput[16]; /* Size of output in bytes */
113768113822
 };
113823
+
113824
+
113825
+static void fts3GetDeltaVarint3(
113826
+ char **pp,
113827
+ char *pEnd,
113828
+ int bDescIdx,
113829
+ sqlite3_int64 *pVal
113830
+){
113831
+ if( *pp>=pEnd ){
113832
+ *pp = 0;
113833
+ }else{
113834
+ sqlite3_int64 iVal;
113835
+ *pp += sqlite3Fts3GetVarint(*pp, &iVal);
113836
+ if( bDescIdx ){
113837
+ *pVal -= iVal;
113838
+ }else{
113839
+ *pVal += iVal;
113840
+ }
113841
+ }
113842
+}
113843
+
113844
+static void fts3PutDeltaVarint3(
113845
+ char **pp, /* IN/OUT: Output pointer */
113846
+ int bDescIdx, /* True for descending docids */
113847
+ sqlite3_int64 *piPrev, /* IN/OUT: Previous value written to list */
113848
+ int *pbFirst, /* IN/OUT: True after first int written */
113849
+ sqlite3_int64 iVal /* Write this value to the list */
113850
+){
113851
+ sqlite3_int64 iWrite;
113852
+ if( bDescIdx==0 || *pbFirst==0 ){
113853
+ iWrite = iVal - *piPrev;
113854
+ }else{
113855
+ iWrite = *piPrev - iVal;
113856
+ }
113857
+ assert( *pbFirst || *piPrev==0 );
113858
+ assert( *pbFirst==0 || iWrite>0 );
113859
+ *pp += sqlite3Fts3PutVarint(*pp, iWrite);
113860
+ *piPrev = iVal;
113861
+ *pbFirst = 1;
113862
+}
113863
+
113864
+#define COMPARE_DOCID(i1, i2) ((bDescIdx?-1:1) * (i1-i2))
113865
+
113866
+static int fts3DoclistOrMerge(
113867
+ int bDescIdx, /* True if arguments are desc */
113868
+ char *a1, int n1, /* First doclist */
113869
+ char *a2, int n2, /* Second doclist */
113870
+ char **paOut, int *pnOut /* OUT: Malloc'd doclist */
113871
+){
113872
+ sqlite3_int64 i1 = 0;
113873
+ sqlite3_int64 i2 = 0;
113874
+ sqlite3_int64 iPrev = 0;
113875
+ char *pEnd1 = &a1[n1];
113876
+ char *pEnd2 = &a2[n2];
113877
+ char *p1 = a1;
113878
+ char *p2 = a2;
113879
+ char *p;
113880
+ char *aOut;
113881
+ int bFirstOut = 0;
113882
+
113883
+ *paOut = 0;
113884
+ *pnOut = 0;
113885
+ aOut = sqlite3_malloc(n1+n2);
113886
+ if( !aOut ) return SQLITE_NOMEM;
113887
+
113888
+ p = aOut;
113889
+ fts3GetDeltaVarint3(&p1, pEnd1, 0, &i1);
113890
+ fts3GetDeltaVarint3(&p2, pEnd2, 0, &i2);
113891
+ while( p1 || p2 ){
113892
+ sqlite3_int64 iDiff = COMPARE_DOCID(i1, i2);
113893
+
113894
+ if( p2 && p1 && iDiff==0 ){
113895
+ fts3PutDeltaVarint3(&p, bDescIdx, &iPrev, &bFirstOut, i1);
113896
+ fts3PoslistMerge(&p, &p1, &p2);
113897
+ fts3GetDeltaVarint3(&p1, pEnd1, bDescIdx, &i1);
113898
+ fts3GetDeltaVarint3(&p2, pEnd2, bDescIdx, &i2);
113899
+ }else if( !p2 || (p1 && iDiff<0) ){
113900
+ fts3PutDeltaVarint3(&p, bDescIdx, &iPrev, &bFirstOut, i1);
113901
+ fts3PoslistCopy(&p, &p1);
113902
+ fts3GetDeltaVarint3(&p1, pEnd1, bDescIdx, &i1);
113903
+ }else{
113904
+ fts3PutDeltaVarint3(&p, bDescIdx, &iPrev, &bFirstOut, i2);
113905
+ fts3PoslistCopy(&p, &p2);
113906
+ fts3GetDeltaVarint3(&p2, pEnd2, bDescIdx, &i2);
113907
+ }
113908
+ }
113909
+
113910
+ *paOut = aOut;
113911
+ *pnOut = (p-aOut);
113912
+ return SQLITE_OK;
113913
+}
113914
+
113915
+static void fts3DoclistPhraseMerge(
113916
+ int bDescIdx, /* True if arguments are desc */
113917
+ int nDist, /* Distance from left to right (1=adjacent) */
113918
+ char *aLeft, int nLeft, /* Left doclist */
113919
+ char *aRight, int *pnRight /* IN/OUT: Right/output doclist */
113920
+){
113921
+ sqlite3_int64 i1 = 0;
113922
+ sqlite3_int64 i2 = 0;
113923
+ sqlite3_int64 iPrev = 0;
113924
+ char *pEnd1 = &aLeft[nLeft];
113925
+ char *pEnd2 = &aRight[*pnRight];
113926
+ char *p1 = aLeft;
113927
+ char *p2 = aRight;
113928
+ char *p;
113929
+ int bFirstOut = 0;
113930
+ char *aOut = aRight;
113931
+
113932
+ assert( nDist>0 );
113933
+
113934
+ p = aOut;
113935
+ fts3GetDeltaVarint3(&p1, pEnd1, 0, &i1);
113936
+ fts3GetDeltaVarint3(&p2, pEnd2, 0, &i2);
113937
+
113938
+ while( p1 && p2 ){
113939
+ sqlite3_int64 iDiff = COMPARE_DOCID(i1, i2);
113940
+ if( iDiff==0 ){
113941
+ char *pSave = p;
113942
+ sqlite3_int64 iPrevSave = iPrev;
113943
+ int bFirstOutSave = bFirstOut;
113944
+
113945
+ fts3PutDeltaVarint3(&p, bDescIdx, &iPrev, &bFirstOut, i1);
113946
+ if( 0==fts3PoslistPhraseMerge(&p, nDist, 0, 1, &p1, &p2) ){
113947
+ p = pSave;
113948
+ iPrev = iPrevSave;
113949
+ bFirstOut = bFirstOutSave;
113950
+ }
113951
+ fts3GetDeltaVarint3(&p1, pEnd1, bDescIdx, &i1);
113952
+ fts3GetDeltaVarint3(&p2, pEnd2, bDescIdx, &i2);
113953
+ }else if( iDiff<0 ){
113954
+ fts3PoslistCopy(0, &p1);
113955
+ fts3GetDeltaVarint3(&p1, pEnd1, bDescIdx, &i1);
113956
+ }else{
113957
+ fts3PoslistCopy(0, &p2);
113958
+ fts3GetDeltaVarint3(&p2, pEnd2, bDescIdx, &i2);
113959
+ }
113960
+ }
113961
+
113962
+ *pnRight = p - aOut;
113963
+}
113964
+
113769113965
113770113966
 /*
113771113967
 ** Merge all doclists in the TermSelect.aaOutput[] array into a single
113772113968
 ** doclist stored in TermSelect.aaOutput[0]. If successful, delete all
113773113969
 ** other doclists (except the aaOutput[0] one) and return SQLITE_OK.
@@ -113774,12 +113970,11 @@
113774113970
 **
113775113971
 ** If an OOM error occurs, return SQLITE_NOMEM. In this case it is
113776113972
 ** the responsibility of the caller to free any doclists left in the
113777113973
 ** TermSelect.aaOutput[] array.
113778113974
 */
113779
-static int fts3TermSelectMerge(TermSelect *pTS){
113780
- int mergetype = (pTS->isReqPos ? MERGE_POS_OR : MERGE_OR);
113975
+static int fts3TermSelectMerge(Fts3Table *p, TermSelect *pTS){
113781113976
 char *aOut = 0;
113782113977
 int nOut = 0;
113783113978
 int i;
113784113979
113785113980
 /* Loop through the doclists in the aaOutput[] array. Merge them all
@@ -113790,19 +113985,21 @@
113790113985
 if( !aOut ){
113791113986
 aOut = pTS->aaOutput[i];
113792113987
 nOut = pTS->anOutput[i];
113793113988
 pTS->aaOutput[i] = 0;
113794113989
 }else{
113795
- int nNew = nOut + pTS->anOutput[i];
113796
- char *aNew = sqlite3_malloc(nNew);
113797
- if( !aNew ){
113990
+ int nNew;
113991
+ char *aNew;
113992
+
113993
+ int rc = fts3DoclistOrMerge(p->bDescIdx,
113994
+ pTS->aaOutput[i], pTS->anOutput[i], aOut, nOut, &aNew, &nNew
113995
+ );
113996
+ if( rc!=SQLITE_OK ){
113798113997
 sqlite3_free(aOut);
113799
- return SQLITE_NOMEM;
113998
+ return rc;
113800113999
 }
113801
- fts3DoclistMerge(mergetype, 0, 0,
113802
- aNew, &nNew, pTS->aaOutput[i], pTS->anOutput[i], aOut, nOut, 0
113803
- );
114000
+
113804114001
 sqlite3_free(pTS->aaOutput[i]);
113805114002
 sqlite3_free(aOut);
113806114003
 pTS->aaOutput[i] = 0;
113807114004
 aOut = aNew;
113808114005
 nOut = nNew;
@@ -113834,217 +114031,241 @@
113834114031
 UNUSED_PARAMETER(zTerm);
113835114032
 UNUSED_PARAMETER(nTerm);
113836114033
113837114034
 if( pTS->aaOutput[0]==0 ){
113838114035
 /* If this is the first term selected, copy the doclist to the output
113839
- ** buffer using memcpy(). TODO: Add a way to transfer control of the
113840
- ** aDoclist buffer from the caller so as to avoid the memcpy().
113841
- */
114036
+ ** buffer using memcpy(). */
113842114037
 pTS->aaOutput[0] = sqlite3_malloc(nDoclist);
113843114038
 pTS->anOutput[0] = nDoclist;
113844114039
 if( pTS->aaOutput[0] ){
113845114040
 memcpy(pTS->aaOutput[0], aDoclist, nDoclist);
113846114041
 }else{
113847114042
 return SQLITE_NOMEM;
113848114043
 }
113849114044
 }else{
113850
- int mergetype = (pTS->isReqPos ? MERGE_POS_OR : MERGE_OR);
113851114045
 char *aMerge = aDoclist;
113852114046
 int nMerge = nDoclist;
113853114047
 int iOut;
113854114048
113855114049
 for(iOut=0; iOut<SizeofArray(pTS->aaOutput); iOut++){
113856
- char *aNew;
113857
- int nNew;
113858114050
 if( pTS->aaOutput[iOut]==0 ){
113859114051
 assert( iOut>0 );
113860114052
 pTS->aaOutput[iOut] = aMerge;
113861114053
 pTS->anOutput[iOut] = nMerge;
113862114054
 break;
113863
- }
113864
-
113865
- nNew = nMerge + pTS->anOutput[iOut];
113866
- aNew = sqlite3_malloc(nNew);
113867
- if( !aNew ){
113868
- if( aMerge!=aDoclist ){
113869
- sqlite3_free(aMerge);
113870
- }
113871
- return SQLITE_NOMEM;
113872
- }
113873
- fts3DoclistMerge(mergetype, 0, 0, aNew, &nNew,
113874
- pTS->aaOutput[iOut], pTS->anOutput[iOut], aMerge, nMerge, 0
113875
- );
113876
-
113877
- if( iOut>0 ) sqlite3_free(aMerge);
113878
- sqlite3_free(pTS->aaOutput[iOut]);
113879
- pTS->aaOutput[iOut] = 0;
113880
-
113881
- aMerge = aNew;
113882
- nMerge = nNew;
113883
- if( (iOut+1)==SizeofArray(pTS->aaOutput) ){
113884
- pTS->aaOutput[iOut] = aMerge;
113885
- pTS->anOutput[iOut] = nMerge;
113886
- }
113887
- }
113888
- }
113889
- return SQLITE_OK;
113890
-}
113891
-
113892
-static int fts3DeferredTermSelect(
113893
- Fts3DeferredToken *pToken, /* Phrase token */
113894
- int isTermPos, /* True to include positions */
113895
- int *pnOut, /* OUT: Size of list */
113896
- char **ppOut /* OUT: Body of list */
113897
-){
113898
- char *aSource;
113899
- int nSource;
113900
-
113901
- aSource = sqlite3Fts3DeferredDoclist(pToken, &nSource);
113902
- if( !aSource ){
113903
- *pnOut = 0;
113904
- *ppOut = 0;
113905
- }else if( isTermPos ){
113906
- *ppOut = sqlite3_malloc(nSource);
113907
- if( !*ppOut ) return SQLITE_NOMEM;
113908
- memcpy(*ppOut, aSource, nSource);
113909
- *pnOut = nSource;
113910
- }else{
113911
- sqlite3_int64 docid;
113912
- *pnOut = sqlite3Fts3GetVarint(aSource, &docid);
113913
- *ppOut = sqlite3_malloc(*pnOut);
113914
- if( !*ppOut ) return SQLITE_NOMEM;
113915
- sqlite3Fts3PutVarint(*ppOut, docid);
113916
- }
113917
-
113918
- return SQLITE_OK;
113919
-}
113920
-
113921
-SQLITE_PRIVATE int sqlite3Fts3SegReaderCursor(
114055
+ }else{
114056
+ char *aNew;
114057
+ int nNew;
114058
+
114059
+ int rc = fts3DoclistOrMerge(p->bDescIdx, aMerge, nMerge,
114060
+ pTS->aaOutput[iOut], pTS->anOutput[iOut], &aNew, &nNew
114061
+ );
114062
+ if( rc!=SQLITE_OK ){
114063
+ if( aMerge!=aDoclist ) sqlite3_free(aMerge);
114064
+ return rc;
114065
+ }
114066
+
114067
+ if( aMerge!=aDoclist ) sqlite3_free(aMerge);
114068
+ sqlite3_free(pTS->aaOutput[iOut]);
114069
+ pTS->aaOutput[iOut] = 0;
114070
+
114071
+ aMerge = aNew;
114072
+ nMerge = nNew;
114073
+ if( (iOut+1)==SizeofArray(pTS->aaOutput) ){
114074
+ pTS->aaOutput[iOut] = aMerge;
114075
+ pTS->anOutput[iOut] = nMerge;
114076
+ }
114077
+ }
114078
+ }
114079
+ }
114080
+ return SQLITE_OK;
114081
+}
114082
+
114083
+/*
114084
+** Append SegReader object pNew to the end of the pCsr->apSegment[] array.
114085
+*/
114086
+static int fts3SegReaderCursorAppend(
114087
+ Fts3MultiSegReader *pCsr,
114088
+ Fts3SegReader *pNew
114089
+){
114090
+ if( (pCsr->nSegment%16)==0 ){
114091
+ Fts3SegReader **apNew;
114092
+ int nByte = (pCsr->nSegment + 16)*sizeof(Fts3SegReader*);
114093
+ apNew = (Fts3SegReader **)sqlite3_realloc(pCsr->apSegment, nByte);
114094
+ if( !apNew ){
114095
+ sqlite3Fts3SegReaderFree(pNew);
114096
+ return SQLITE_NOMEM;
114097
+ }
114098
+ pCsr->apSegment = apNew;
114099
+ }
114100
+ pCsr->apSegment[pCsr->nSegment++] = pNew;
114101
+ return SQLITE_OK;
114102
+}
114103
+
114104
+static int fts3SegReaderCursor(
113922114105
 Fts3Table *p, /* FTS3 table handle */
114106
+ int iIndex, /* Index to search (from 0 to p->nIndex-1) */
113923114107
 int iLevel, /* Level of segments to scan */
113924114108
 const char *zTerm, /* Term to query for */
113925114109
 int nTerm, /* Size of zTerm in bytes */
113926114110
 int isPrefix, /* True for a prefix search */
113927114111
 int isScan, /* True to scan from zTerm to EOF */
113928
- Fts3SegReaderCursor *pCsr /* Cursor object to populate */
114112
+ Fts3MultiSegReader *pCsr /* Cursor object to populate */
113929114113
 ){
113930114114
 int rc = SQLITE_OK;
113931114115
 int rc2;
113932
- int iAge = 0;
113933114116
 sqlite3_stmt *pStmt = 0;
113934
- Fts3SegReader *pPending = 0;
113935
-
113936
- assert( iLevel==FTS3_SEGCURSOR_ALL
113937
- || iLevel==FTS3_SEGCURSOR_PENDING
113938
- || iLevel>=0
113939
- );
113940
- assert( FTS3_SEGCURSOR_PENDING<0 );
113941
- assert( FTS3_SEGCURSOR_ALL<0 );
113942
- assert( iLevel==FTS3_SEGCURSOR_ALL || (zTerm==0 && isPrefix==1) );
113943
- assert( isPrefix==0 || isScan==0 );
113944
-
113945
-
113946
- memset(pCsr, 0, sizeof(Fts3SegReaderCursor));
113947
-
113948
- /* If iLevel is less than 0, include a seg-reader for the pending-terms. */
113949
- assert( isScan==0 || fts3HashCount(&p->pendingTerms)==0 );
113950
- if( iLevel<0 && isScan==0 ){
113951
- rc = sqlite3Fts3SegReaderPending(p, zTerm, nTerm, isPrefix, &pPending);
113952
- if( rc==SQLITE_OK && pPending ){
113953
- int nByte = (sizeof(Fts3SegReader *) * 16);
113954
- pCsr->apSegment = (Fts3SegReader **)sqlite3_malloc(nByte);
113955
- if( pCsr->apSegment==0 ){
113956
- rc = SQLITE_NOMEM;
113957
- }else{
113958
- pCsr->apSegment[0] = pPending;
113959
- pCsr->nSegment = 1;
113960
- pPending = 0;
113961
- }
114117
+
114118
+ /* If iLevel is less than 0 and this is not a scan, include a seg-reader
114119
+ ** for the pending-terms. If this is a scan, then this call must be being
114120
+ ** made by an fts4aux module, not an FTS table. In this case calling
114121
+ ** Fts3SegReaderPending might segfault, as the data structures used by
114122
+ ** fts4aux are not completely populated. So it's easiest to filter these
114123
+ ** calls out here. */
114124
+ if( iLevel<0 && p->aIndex ){
114125
+ Fts3SegReader *pSeg = 0;
114126
+ rc = sqlite3Fts3SegReaderPending(p, iIndex, zTerm, nTerm, isPrefix, &pSeg);
114127
+ if( rc==SQLITE_OK && pSeg ){
114128
+ rc = fts3SegReaderCursorAppend(pCsr, pSeg);
113962114129
 }
113963114130
 }
113964114131
113965114132
 if( iLevel!=FTS3_SEGCURSOR_PENDING ){
113966114133
 if( rc==SQLITE_OK ){
113967
- rc = sqlite3Fts3AllSegdirs(p, iLevel, &pStmt);
114134
+ rc = sqlite3Fts3AllSegdirs(p, iIndex, iLevel, &pStmt);
113968114135
 }
114136
+
113969114137
 while( rc==SQLITE_OK && SQLITE_ROW==(rc = sqlite3_step(pStmt)) ){
114138
+ Fts3SegReader *pSeg = 0;
113970114139
113971114140
 /* Read the values returned by the SELECT into local variables. */
113972114141
 sqlite3_int64 iStartBlock = sqlite3_column_int64(pStmt, 1);
113973114142
 sqlite3_int64 iLeavesEndBlock = sqlite3_column_int64(pStmt, 2);
113974114143
 sqlite3_int64 iEndBlock = sqlite3_column_int64(pStmt, 3);
113975114144
 int nRoot = sqlite3_column_bytes(pStmt, 4);
113976114145
 char const *zRoot = sqlite3_column_blob(pStmt, 4);
113977114146
113978
- /* If nSegment is a multiple of 16 the array needs to be extended. */
113979
- if( (pCsr->nSegment%16)==0 ){
113980
- Fts3SegReader **apNew;
113981
- int nByte = (pCsr->nSegment + 16)*sizeof(Fts3SegReader*);
113982
- apNew = (Fts3SegReader **)sqlite3_realloc(pCsr->apSegment, nByte);
113983
- if( !apNew ){
113984
- rc = SQLITE_NOMEM;
113985
- goto finished;
113986
- }
113987
- pCsr->apSegment = apNew;
113988
- }
113989
-
113990114147
 /* If zTerm is not NULL, and this segment is not stored entirely on its
113991114148
 ** root node, the range of leaves scanned can be reduced. Do this. */
113992114149
 if( iStartBlock && zTerm ){
113993114150
 sqlite3_int64 *pi = (isPrefix ? &iLeavesEndBlock : 0);
113994114151
 rc = fts3SelectLeaf(p, zTerm, nTerm, zRoot, nRoot, &iStartBlock, pi);
113995114152
 if( rc!=SQLITE_OK ) goto finished;
113996114153
 if( isPrefix==0 && isScan==0 ) iLeavesEndBlock = iStartBlock;
113997114154
 }
113998114155
113999
- rc = sqlite3Fts3SegReaderNew(iAge, iStartBlock, iLeavesEndBlock,
114000
- iEndBlock, zRoot, nRoot, &pCsr->apSegment[pCsr->nSegment]
114156
+ rc = sqlite3Fts3SegReaderNew(pCsr->nSegment+1,
114157
+ iStartBlock, iLeavesEndBlock, iEndBlock, zRoot, nRoot, &pSeg
114001114158
 );
114002114159
 if( rc!=SQLITE_OK ) goto finished;
114003
- pCsr->nSegment++;
114004
- iAge++;
114160
+ rc = fts3SegReaderCursorAppend(pCsr, pSeg);
114005114161
 }
114006114162
 }
114007114163
114008114164
 finished:
114009114165
 rc2 = sqlite3_reset(pStmt);
114010114166
 if( rc==SQLITE_DONE ) rc = rc2;
114011
- sqlite3Fts3SegReaderFree(pPending);
114012114167
114013114168
 return rc;
114014114169
 }
114015114170
114171
+/*
114172
+** Set up a cursor object for iterating through a full-text index or a
114173
+** single level therein.
114174
+*/
114175
+SQLITE_PRIVATE int sqlite3Fts3SegReaderCursor(
114176
+ Fts3Table *p, /* FTS3 table handle */
114177
+ int iIndex, /* Index to search (from 0 to p->nIndex-1) */
114178
+ int iLevel, /* Level of segments to scan */
114179
+ const char *zTerm, /* Term to query for */
114180
+ int nTerm, /* Size of zTerm in bytes */
114181
+ int isPrefix, /* True for a prefix search */
114182
+ int isScan, /* True to scan from zTerm to EOF */
114183
+ Fts3MultiSegReader *pCsr /* Cursor object to populate */
114184
+){
114185
+ assert( iIndex>=0 && iIndex<p->nIndex );
114186
+ assert( iLevel==FTS3_SEGCURSOR_ALL
114187
+ || iLevel==FTS3_SEGCURSOR_PENDING
114188
+ || iLevel>=0
114189
+ );
114190
+ assert( iLevel<FTS3_SEGDIR_MAXLEVEL );
114191
+ assert( FTS3_SEGCURSOR_ALL<0 && FTS3_SEGCURSOR_PENDING<0 );
114192
+ assert( isPrefix==0 || isScan==0 );
114016114193
114017
-static int fts3TermSegReaderCursor(
114194
+ /* "isScan" is only set to true by the ft4aux module, an ordinary
114195
+ ** full-text tables. */
114196
+ assert( isScan==0 || p->aIndex==0 );
114197
+
114198
+ memset(pCsr, 0, sizeof(Fts3MultiSegReader));
114199
+
114200
+ return fts3SegReaderCursor(
114201
+ p, iIndex, iLevel, zTerm, nTerm, isPrefix, isScan, pCsr
114202
+ );
114203
+}
114204
+
114205
+static int fts3SegReaderCursorAddZero(
114206
+ Fts3Table *p,
114207
+ const char *zTerm,
114208
+ int nTerm,
114209
+ Fts3MultiSegReader *pCsr
114210
+){
114211
+ return fts3SegReaderCursor(p, 0, FTS3_SEGCURSOR_ALL, zTerm, nTerm, 0, 0,pCsr);
114212
+}
114213
+
114214
+
114215
+SQLITE_PRIVATE int sqlite3Fts3TermSegReaderCursor(
114018114216
 Fts3Cursor *pCsr, /* Virtual table cursor handle */
114019114217
 const char *zTerm, /* Term to query for */
114020114218
 int nTerm, /* Size of zTerm in bytes */
114021114219
 int isPrefix, /* True for a prefix search */
114022
- Fts3SegReaderCursor **ppSegcsr /* OUT: Allocated seg-reader cursor */
114220
+ Fts3MultiSegReader **ppSegcsr /* OUT: Allocated seg-reader cursor */
114023114221
 ){
114024
- Fts3SegReaderCursor *pSegcsr; /* Object to allocate and return */
114222
+ Fts3MultiSegReader *pSegcsr; /* Object to allocate and return */
114025114223
 int rc = SQLITE_NOMEM; /* Return code */
114026114224
114027
- pSegcsr = sqlite3_malloc(sizeof(Fts3SegReaderCursor));
114225
+ pSegcsr = sqlite3_malloc(sizeof(Fts3MultiSegReader));
114028114226
 if( pSegcsr ){
114227
+ int i;
114228
+ int bFound = 0; /* True once an index has been found */
114029114229
 Fts3Table *p = (Fts3Table *)pCsr->base.pVtab;
114030
- int i;
114031
- int nCost = 0;
114032
- rc = sqlite3Fts3SegReaderCursor(
114033
- p, FTS3_SEGCURSOR_ALL, zTerm, nTerm, isPrefix, 0, pSegcsr);
114034
-
114035
- for(i=0; rc==SQLITE_OK && i<pSegcsr->nSegment; i++){
114036
- rc = sqlite3Fts3SegReaderCost(pCsr, pSegcsr->apSegment[i], &nCost);
114037
- }
114038
- pSegcsr->nCost = nCost;
114230
+
114231
+ if( isPrefix ){
114232
+ for(i=1; bFound==0 && i<p->nIndex; i++){
114233
+ if( p->aIndex[i].nPrefix==nTerm ){
114234
+ bFound = 1;
114235
+ rc = sqlite3Fts3SegReaderCursor(
114236
+ p, i, FTS3_SEGCURSOR_ALL, zTerm, nTerm, 0, 0, pSegcsr);
114237
+ pSegcsr->bLookup = 1;
114238
+ }
114239
+ }
114240
+
114241
+ for(i=1; bFound==0 && i<p->nIndex; i++){
114242
+ if( p->aIndex[i].nPrefix==nTerm+1 ){
114243
+ bFound = 1;
114244
+ rc = sqlite3Fts3SegReaderCursor(
114245
+ p, i, FTS3_SEGCURSOR_ALL, zTerm, nTerm, 1, 0, pSegcsr
114246
+ );
114247
+ if( rc==SQLITE_OK ){
114248
+ rc = fts3SegReaderCursorAddZero(p, zTerm, nTerm, pSegcsr);
114249
+ }
114250
+ }
114251
+ }
114252
+ }
114253
+
114254
+ if( bFound==0 ){
114255
+ rc = sqlite3Fts3SegReaderCursor(
114256
+ p, 0, FTS3_SEGCURSOR_ALL, zTerm, nTerm, isPrefix, 0, pSegcsr
114257
+ );
114258
+ pSegcsr->bLookup = !isPrefix;
114259
+ }
114039114260
 }
114040114261
114041114262
 *ppSegcsr = pSegcsr;
114042114263
 return rc;
114043114264
 }
114044114265
114045
-static void fts3SegReaderCursorFree(Fts3SegReaderCursor *pSegcsr){
114266
+static void fts3SegReaderCursorFree(Fts3MultiSegReader *pSegcsr){
114046114267
 sqlite3Fts3SegReaderFinish(pSegcsr);
114047114268
 sqlite3_free(pSegcsr);
114048114269
 }
114049114270
114050114271
 /*
@@ -114065,11 +114286,11 @@
114065114286
 int isReqPos, /* True to include position lists in output */
114066114287
 int *pnOut, /* OUT: Size of buffer at *ppOut */
114067114288
 char **ppOut /* OUT: Malloced result buffer */
114068114289
 ){
114069114290
 int rc; /* Return code */
114070
- Fts3SegReaderCursor *pSegcsr; /* Seg-reader cursor for this term */
114291
+ Fts3MultiSegReader *pSegcsr; /* Seg-reader cursor for this term */
114071114292
 TermSelect tsc; /* Context object for fts3TermSelectCb() */
114072114293
 Fts3SegFilter filter; /* Segment term filter configuration */
114073114294
114074114295
 pSegcsr = pTok->pSegcsr;
114075114296
 memset(&tsc, 0, sizeof(TermSelect));
@@ -114091,11 +114312,11 @@
114091114312
 pSegcsr->zTerm, pSegcsr->nTerm, pSegcsr->aDoclist, pSegcsr->nDoclist
114092114313
 );
114093114314
 }
114094114315
114095114316
 if( rc==SQLITE_OK ){
114096
- rc = fts3TermSelectMerge(&tsc);
114317
+ rc = fts3TermSelectMerge(p, &tsc);
114097114318
 }
114098114319
 if( rc==SQLITE_OK ){
114099114320
 *ppOut = tsc.aaOutput[0];
114100114321
 *pnOut = tsc.anOutput[0];
114101114322
 }else{
@@ -114141,664 +114362,10 @@
114141114362
 }
114142114363
114143114364
 return nDoc;
114144114365
 }
114145114366
114146
-/*
114147
-** Call sqlite3Fts3DeferToken() for each token in the expression pExpr.
114148
-*/
114149
-static int fts3DeferExpression(Fts3Cursor *pCsr, Fts3Expr *pExpr){
114150
- int rc = SQLITE_OK;
114151
- if( pExpr ){
114152
- rc = fts3DeferExpression(pCsr, pExpr->pLeft);
114153
- if( rc==SQLITE_OK ){
114154
- rc = fts3DeferExpression(pCsr, pExpr->pRight);
114155
- }
114156
- if( pExpr->eType==FTSQUERY_PHRASE ){
114157
- int iCol = pExpr->pPhrase->iColumn;
114158
- int i;
114159
- for(i=0; rc==SQLITE_OK && i<pExpr->pPhrase->nToken; i++){
114160
- Fts3PhraseToken *pToken = &pExpr->pPhrase->aToken[i];
114161
- if( pToken->pDeferred==0 ){
114162
- rc = sqlite3Fts3DeferToken(pCsr, pToken, iCol);
114163
- }
114164
- }
114165
- }
114166
- }
114167
- return rc;
114168
-}
114169
-
114170
-/*
114171
-** This function removes the position information from a doclist. When
114172
-** called, buffer aList (size *pnList bytes) contains a doclist that includes
114173
-** position information. This function removes the position information so
114174
-** that aList contains only docids, and adjusts *pnList to reflect the new
114175
-** (possibly reduced) size of the doclist.
114176
-*/
114177
-static void fts3DoclistStripPositions(
114178
- char *aList, /* IN/OUT: Buffer containing doclist */
114179
- int *pnList /* IN/OUT: Size of doclist in bytes */
114180
-){
114181
- if( aList ){
114182
- char *aEnd = &aList[*pnList]; /* Pointer to one byte after EOF */
114183
- char *p = aList; /* Input cursor */
114184
- char *pOut = aList; /* Output cursor */
114185
-
114186
- while( p<aEnd ){
114187
- sqlite3_int64 delta;
114188
- p += sqlite3Fts3GetVarint(p, &delta);
114189
- fts3PoslistCopy(0, &p);
114190
- pOut += sqlite3Fts3PutVarint(pOut, delta);
114191
- }
114192
-
114193
- *pnList = (int)(pOut - aList);
114194
- }
114195
-}
114196
-
114197
-/*
114198
-** Return a DocList corresponding to the phrase *pPhrase.
114199
-**
114200
-** If this function returns SQLITE_OK, but *pnOut is set to a negative value,
114201
-** then no tokens in the phrase were looked up in the full-text index. This
114202
-** is only possible when this function is called from within xFilter(). The
114203
-** caller should assume that all documents match the phrase. The actual
114204
-** filtering will take place in xNext().
114205
-*/
114206
-static int fts3PhraseSelect(
114207
- Fts3Cursor *pCsr, /* Virtual table cursor handle */
114208
- Fts3Phrase *pPhrase, /* Phrase to return a doclist for */
114209
- int isReqPos, /* True if output should contain positions */
114210
- char **paOut, /* OUT: Pointer to malloc'd result buffer */
114211
- int *pnOut /* OUT: Size of buffer at *paOut */
114212
-){
114213
- char *pOut = 0;
114214
- int nOut = 0;
114215
- int rc = SQLITE_OK;
114216
- int ii;
114217
- int iCol = pPhrase->iColumn;
114218
- int isTermPos = (pPhrase->nToken>1 || isReqPos);
114219
- Fts3Table *p = (Fts3Table *)pCsr->base.pVtab;
114220
- int isFirst = 1;
114221
-
114222
- int iPrevTok = 0;
114223
- int nDoc = 0;
114224
-
114225
- /* If this is an xFilter() evaluation, create a segment-reader for each
114226
- ** phrase token. Or, if this is an xNext() or snippet/offsets/matchinfo
114227
- ** evaluation, only create segment-readers if there are no Fts3DeferredToken
114228
- ** objects attached to the phrase-tokens.
114229
- */
114230
- for(ii=0; ii<pPhrase->nToken; ii++){
114231
- Fts3PhraseToken *pTok = &pPhrase->aToken[ii];
114232
- if( pTok->pSegcsr==0 ){
114233
- if( (pCsr->eEvalmode==FTS3_EVAL_FILTER)
114234
- || (pCsr->eEvalmode==FTS3_EVAL_NEXT && pCsr->pDeferred==0)
114235
- || (pCsr->eEvalmode==FTS3_EVAL_MATCHINFO && pTok->bFulltext)
114236
- ){
114237
- rc = fts3TermSegReaderCursor(
114238
- pCsr, pTok->z, pTok->n, pTok->isPrefix, &pTok->pSegcsr
114239
- );
114240
- if( rc!=SQLITE_OK ) return rc;
114241
- }
114242
- }
114243
- }
114244
-
114245
- for(ii=0; ii<pPhrase->nToken; ii++){
114246
- Fts3PhraseToken *pTok; /* Token to find doclist for */
114247
- int iTok = 0; /* The token being queried this iteration */
114248
- char *pList = 0; /* Pointer to token doclist */
114249
- int nList = 0; /* Size of buffer at pList */
114250
-
114251
- /* Select a token to process. If this is an xFilter() call, then tokens
114252
- ** are processed in order from least to most costly. Otherwise, tokens
114253
- ** are processed in the order in which they occur in the phrase.
114254
- */
114255
- if( pCsr->eEvalmode==FTS3_EVAL_MATCHINFO ){
114256
- assert( isReqPos );
114257
- iTok = ii;
114258
- pTok = &pPhrase->aToken[iTok];
114259
- if( pTok->bFulltext==0 ) continue;
114260
- }else if( pCsr->eEvalmode==FTS3_EVAL_NEXT || isReqPos ){
114261
- iTok = ii;
114262
- pTok = &pPhrase->aToken[iTok];
114263
- }else{
114264
- int nMinCost = 0x7FFFFFFF;
114265
- int jj;
114266
-
114267
- /* Find the remaining token with the lowest cost. */
114268
- for(jj=0; jj<pPhrase->nToken; jj++){
114269
- Fts3SegReaderCursor *pSegcsr = pPhrase->aToken[jj].pSegcsr;
114270
- if( pSegcsr && pSegcsr->nCost<nMinCost ){
114271
- iTok = jj;
114272
- nMinCost = pSegcsr->nCost;
114273
- }
114274
- }
114275
- pTok = &pPhrase->aToken[iTok];
114276
-
114277
- /* This branch is taken if it is determined that loading the doclist
114278
- ** for the next token would require more IO than loading all documents
114279
- ** currently identified by doclist pOut/nOut. No further doclists will
114280
- ** be loaded from the full-text index for this phrase.
114281
- */
114282
- if( nMinCost>nDoc && ii>0 ){
114283
- rc = fts3DeferExpression(pCsr, pCsr->pExpr);
114284
- break;
114285
- }
114286
- }
114287
-
114288
- if( pCsr->eEvalmode==FTS3_EVAL_NEXT && pTok->pDeferred ){
114289
- rc = fts3DeferredTermSelect(pTok->pDeferred, isTermPos, &nList, &pList);
114290
- }else{
114291
- if( pTok->pSegcsr ){
114292
- rc = fts3TermSelect(p, pTok, iCol, isTermPos, &nList, &pList);
114293
- }
114294
- pTok->bFulltext = 1;
114295
- }
114296
- assert( rc!=SQLITE_OK || pCsr->eEvalmode || pTok->pSegcsr==0 );
114297
- if( rc!=SQLITE_OK ) break;
114298
-
114299
- if( isFirst ){
114300
- pOut = pList;
114301
- nOut = nList;
114302
- if( pCsr->eEvalmode==FTS3_EVAL_FILTER && pPhrase->nToken>1 ){
114303
- nDoc = fts3DoclistCountDocids(1, pOut, nOut);
114304
- }
114305
- isFirst = 0;
114306
- iPrevTok = iTok;
114307
- }else{
114308
- /* Merge the new term list and the current output. */
114309
- char *aLeft, *aRight;
114310
- int nLeft, nRight;
114311
- int nDist;
114312
- int mt;
114313
-
114314
- /* If this is the final token of the phrase, and positions were not
114315
- ** requested by the caller, use MERGE_PHRASE instead of POS_PHRASE.
114316
- ** This drops the position information from the output list.
114317
- */
114318
- mt = MERGE_POS_PHRASE;
114319
- if( ii==pPhrase->nToken-1 && !isReqPos ) mt = MERGE_PHRASE;
114320
-
114321
- assert( iPrevTok!=iTok );
114322
- if( iPrevTok<iTok ){
114323
- aLeft = pOut;
114324
- nLeft = nOut;
114325
- aRight = pList;
114326
- nRight = nList;
114327
- nDist = iTok-iPrevTok;
114328
- iPrevTok = iTok;
114329
- }else{
114330
- aRight = pOut;
114331
- nRight = nOut;
114332
- aLeft = pList;
114333
- nLeft = nList;
114334
- nDist = iPrevTok-iTok;
114335
- }
114336
- pOut = aRight;
114337
- fts3DoclistMerge(
114338
- mt, nDist, 0, pOut, &nOut, aLeft, nLeft, aRight, nRight, &nDoc
114339
- );
114340
- sqlite3_free(aLeft);
114341
- }
114342
- assert( nOut==0 || pOut!=0 );
114343
- }
114344
-
114345
- if( rc==SQLITE_OK ){
114346
- if( ii!=pPhrase->nToken ){
114347
- assert( pCsr->eEvalmode==FTS3_EVAL_FILTER && isReqPos==0 );
114348
- fts3DoclistStripPositions(pOut, &nOut);
114349
- }
114350
- *paOut = pOut;
114351
- *pnOut = nOut;
114352
- }else{
114353
- sqlite3_free(pOut);
114354
- }
114355
- return rc;
114356
-}
114357
-
114358
-/*
114359
-** This function merges two doclists according to the requirements of a
114360
-** NEAR operator.
114361
-**
114362
-** Both input doclists must include position information. The output doclist
114363
-** includes position information if the first argument to this function
114364
-** is MERGE_POS_NEAR, or does not if it is MERGE_NEAR.
114365
-*/
114366
-static int fts3NearMerge(
114367
- int mergetype, /* MERGE_POS_NEAR or MERGE_NEAR */
114368
- int nNear, /* Parameter to NEAR operator */
114369
- int nTokenLeft, /* Number of tokens in LHS phrase arg */
114370
- char *aLeft, /* Doclist for LHS (incl. positions) */
114371
- int nLeft, /* Size of LHS doclist in bytes */
114372
- int nTokenRight, /* As nTokenLeft */
114373
- char *aRight, /* As aLeft */
114374
- int nRight, /* As nRight */
114375
- char **paOut, /* OUT: Results of merge (malloced) */
114376
- int *pnOut /* OUT: Sized of output buffer */
114377
-){
114378
- char *aOut; /* Buffer to write output doclist to */
114379
- int rc; /* Return code */
114380
-
114381
- assert( mergetype==MERGE_POS_NEAR || MERGE_NEAR );
114382
-
114383
- aOut = sqlite3_malloc(nLeft+nRight+1);
114384
- if( aOut==0 ){
114385
- rc = SQLITE_NOMEM;
114386
- }else{
114387
- rc = fts3DoclistMerge(mergetype, nNear+nTokenRight, nNear+nTokenLeft,
114388
- aOut, pnOut, aLeft, nLeft, aRight, nRight, 0
114389
- );
114390
- if( rc!=SQLITE_OK ){
114391
- sqlite3_free(aOut);
114392
- aOut = 0;
114393
- }
114394
- }
114395
-
114396
- *paOut = aOut;
114397
- return rc;
114398
-}
114399
-
114400
-/*
114401
-** This function is used as part of the processing for the snippet() and
114402
-** offsets() functions.
114403
-**
114404
-** Both pLeft and pRight are expression nodes of type FTSQUERY_PHRASE. Both
114405
-** have their respective doclists (including position information) loaded
114406
-** in Fts3Expr.aDoclist/nDoclist. This function removes all entries from
114407
-** each doclist that are not within nNear tokens of a corresponding entry
114408
-** in the other doclist.
114409
-*/
114410
-SQLITE_PRIVATE int sqlite3Fts3ExprNearTrim(Fts3Expr *pLeft, Fts3Expr *pRight, int nNear){
114411
- int rc; /* Return code */
114412
-
114413
- assert( pLeft->eType==FTSQUERY_PHRASE );
114414
- assert( pRight->eType==FTSQUERY_PHRASE );
114415
- assert( pLeft->isLoaded && pRight->isLoaded );
114416
-
114417
- if( pLeft->aDoclist==0 || pRight->aDoclist==0 ){
114418
- sqlite3_free(pLeft->aDoclist);
114419
- sqlite3_free(pRight->aDoclist);
114420
- pRight->aDoclist = 0;
114421
- pLeft->aDoclist = 0;
114422
- rc = SQLITE_OK;
114423
- }else{
114424
- char *aOut; /* Buffer in which to assemble new doclist */
114425
- int nOut; /* Size of buffer aOut in bytes */
114426
-
114427
- rc = fts3NearMerge(MERGE_POS_NEAR, nNear,
114428
- pLeft->pPhrase->nToken, pLeft->aDoclist, pLeft->nDoclist,
114429
- pRight->pPhrase->nToken, pRight->aDoclist, pRight->nDoclist,
114430
- &aOut, &nOut
114431
- );
114432
- if( rc!=SQLITE_OK ) return rc;
114433
- sqlite3_free(pRight->aDoclist);
114434
- pRight->aDoclist = aOut;
114435
- pRight->nDoclist = nOut;
114436
-
114437
- rc = fts3NearMerge(MERGE_POS_NEAR, nNear,
114438
- pRight->pPhrase->nToken, pRight->aDoclist, pRight->nDoclist,
114439
- pLeft->pPhrase->nToken, pLeft->aDoclist, pLeft->nDoclist,
114440
- &aOut, &nOut
114441
- );
114442
- sqlite3_free(pLeft->aDoclist);
114443
- pLeft->aDoclist = aOut;
114444
- pLeft->nDoclist = nOut;
114445
- }
114446
- return rc;
114447
-}
114448
-
114449
-
114450
-/*
114451
-** Allocate an Fts3SegReaderArray for each token in the expression pExpr.
114452
-** The allocated objects are stored in the Fts3PhraseToken.pArray member
114453
-** variables of each token structure.
114454
-*/
114455
-static int fts3ExprAllocateSegReaders(
114456
- Fts3Cursor *pCsr, /* FTS3 table */
114457
- Fts3Expr *pExpr, /* Expression to create seg-readers for */
114458
- int *pnExpr /* OUT: Number of AND'd expressions */
114459
-){
114460
- int rc = SQLITE_OK; /* Return code */
114461
-
114462
- assert( pCsr->eEvalmode==FTS3_EVAL_FILTER );
114463
- if( pnExpr && pExpr->eType!=FTSQUERY_AND ){
114464
- (*pnExpr)++;
114465
- pnExpr = 0;
114466
- }
114467
-
114468
- if( pExpr->eType==FTSQUERY_PHRASE ){
114469
- Fts3Phrase *pPhrase = pExpr->pPhrase;
114470
- int ii;
114471
-
114472
- for(ii=0; rc==SQLITE_OK && ii<pPhrase->nToken; ii++){
114473
- Fts3PhraseToken *pTok = &pPhrase->aToken[ii];
114474
- if( pTok->pSegcsr==0 ){
114475
- rc = fts3TermSegReaderCursor(
114476
- pCsr, pTok->z, pTok->n, pTok->isPrefix, &pTok->pSegcsr
114477
- );
114478
- }
114479
- }
114480
- }else{
114481
- rc = fts3ExprAllocateSegReaders(pCsr, pExpr->pLeft, pnExpr);
114482
- if( rc==SQLITE_OK ){
114483
- rc = fts3ExprAllocateSegReaders(pCsr, pExpr->pRight, pnExpr);
114484
- }
114485
- }
114486
- return rc;
114487
-}
114488
-
114489
-/*
114490
-** Free the Fts3SegReaderArray objects associated with each token in the
114491
-** expression pExpr. In other words, this function frees the resources
114492
-** allocated by fts3ExprAllocateSegReaders().
114493
-*/
114494
-static void fts3ExprFreeSegReaders(Fts3Expr *pExpr){
114495
- if( pExpr ){
114496
- Fts3Phrase *pPhrase = pExpr->pPhrase;
114497
- if( pPhrase ){
114498
- int kk;
114499
- for(kk=0; kk<pPhrase->nToken; kk++){
114500
- fts3SegReaderCursorFree(pPhrase->aToken[kk].pSegcsr);
114501
- pPhrase->aToken[kk].pSegcsr = 0;
114502
- }
114503
- }
114504
- fts3ExprFreeSegReaders(pExpr->pLeft);
114505
- fts3ExprFreeSegReaders(pExpr->pRight);
114506
- }
114507
-}
114508
-
114509
-/*
114510
-** Return the sum of the costs of all tokens in the expression pExpr. This
114511
-** function must be called after Fts3SegReaderArrays have been allocated
114512
-** for all tokens using fts3ExprAllocateSegReaders().
114513
-*/
114514
-static int fts3ExprCost(Fts3Expr *pExpr){
114515
- int nCost; /* Return value */
114516
- if( pExpr->eType==FTSQUERY_PHRASE ){
114517
- Fts3Phrase *pPhrase = pExpr->pPhrase;
114518
- int ii;
114519
- nCost = 0;
114520
- for(ii=0; ii<pPhrase->nToken; ii++){
114521
- Fts3SegReaderCursor *pSegcsr = pPhrase->aToken[ii].pSegcsr;
114522
- if( pSegcsr ) nCost += pSegcsr->nCost;
114523
- }
114524
- }else{
114525
- nCost = fts3ExprCost(pExpr->pLeft) + fts3ExprCost(pExpr->pRight);
114526
- }
114527
- return nCost;
114528
-}
114529
-
114530
-/*
114531
-** The following is a helper function (and type) for fts3EvalExpr(). It
114532
-** must be called after Fts3SegReaders have been allocated for every token
114533
-** in the expression. See the context it is called from in fts3EvalExpr()
114534
-** for further explanation.
114535
-*/
114536
-typedef struct ExprAndCost ExprAndCost;
114537
-struct ExprAndCost {
114538
- Fts3Expr *pExpr;
114539
- int nCost;
114540
-};
114541
-static void fts3ExprAssignCosts(
114542
- Fts3Expr *pExpr, /* Expression to create seg-readers for */
114543
- ExprAndCost **ppExprCost /* OUT: Write to *ppExprCost */
114544
-){
114545
- if( pExpr->eType==FTSQUERY_AND ){
114546
- fts3ExprAssignCosts(pExpr->pLeft, ppExprCost);
114547
- fts3ExprAssignCosts(pExpr->pRight, ppExprCost);
114548
- }else{
114549
- (*ppExprCost)->pExpr = pExpr;
114550
- (*ppExprCost)->nCost = fts3ExprCost(pExpr);
114551
- (*ppExprCost)++;
114552
- }
114553
-}
114554
-
114555
-/*
114556
-** Evaluate the full-text expression pExpr against FTS3 table pTab. Store
114557
-** the resulting doclist in *paOut and *pnOut. This routine mallocs for
114558
-** the space needed to store the output. The caller is responsible for
114559
-** freeing the space when it has finished.
114560
-**
114561
-** This function is called in two distinct contexts:
114562
-**
114563
-** * From within the virtual table xFilter() method. In this case, the
114564
-** output doclist contains entries for all rows in the table, based on
114565
-** data read from the full-text index.
114566
-**
114567
-** In this case, if the query expression contains one or more tokens that
114568
-** are very common, then the returned doclist may contain a superset of
114569
-** the documents that actually match the expression.
114570
-**
114571
-** * From within the virtual table xNext() method. This call is only made
114572
-** if the call from within xFilter() found that there were very common
114573
-** tokens in the query expression and did return a superset of the
114574
-** matching documents. In this case the returned doclist contains only
114575
-** entries that correspond to the current row of the table. Instead of
114576
-** reading the data for each token from the full-text index, the data is
114577
-** already available in-memory in the Fts3PhraseToken.pDeferred structures.
114578
-** See fts3EvalDeferred() for how it gets there.
114579
-**
114580
-** In the first case above, Fts3Cursor.doDeferred==0. In the second (if it is
114581
-** required) Fts3Cursor.doDeferred==1.
114582
-**
114583
-** If the SQLite invokes the snippet(), offsets() or matchinfo() function
114584
-** as part of a SELECT on an FTS3 table, this function is called on each
114585
-** individual phrase expression in the query. If there were very common tokens
114586
-** found in the xFilter() call, then this function is called once for phrase
114587
-** for each row visited, and the returned doclist contains entries for the
114588
-** current row only. Otherwise, if there were no very common tokens, then this
114589
-** function is called once only for each phrase in the query and the returned
114590
-** doclist contains entries for all rows of the table.
114591
-**
114592
-** Fts3Cursor.doDeferred==1 when this function is called on phrases as a
114593
-** result of a snippet(), offsets() or matchinfo() invocation.
114594
-*/
114595
-static int fts3EvalExpr(
114596
- Fts3Cursor *p, /* Virtual table cursor handle */
114597
- Fts3Expr *pExpr, /* Parsed fts3 expression */
114598
- char **paOut, /* OUT: Pointer to malloc'd result buffer */
114599
- int *pnOut, /* OUT: Size of buffer at *paOut */
114600
- int isReqPos /* Require positions in output buffer */
114601
-){
114602
- int rc = SQLITE_OK; /* Return code */
114603
-
114604
- /* Zero the output parameters. */
114605
- *paOut = 0;
114606
- *pnOut = 0;
114607
-
114608
- if( pExpr ){
114609
- assert( pExpr->eType==FTSQUERY_NEAR || pExpr->eType==FTSQUERY_OR
114610
- || pExpr->eType==FTSQUERY_AND || pExpr->eType==FTSQUERY_NOT
114611
- || pExpr->eType==FTSQUERY_PHRASE
114612
- );
114613
- assert( pExpr->eType==FTSQUERY_PHRASE || isReqPos==0 );
114614
-
114615
- if( pExpr->eType==FTSQUERY_PHRASE ){
114616
- rc = fts3PhraseSelect(p, pExpr->pPhrase,
114617
- isReqPos || (pExpr->pParent && pExpr->pParent->eType==FTSQUERY_NEAR),
114618
- paOut, pnOut
114619
- );
114620
- fts3ExprFreeSegReaders(pExpr);
114621
- }else if( p->eEvalmode==FTS3_EVAL_FILTER && pExpr->eType==FTSQUERY_AND ){
114622
- ExprAndCost *aExpr = 0; /* Array of AND'd expressions and costs */
114623
- int nExpr = 0; /* Size of aExpr[] */
114624
- char *aRet = 0; /* Doclist to return to caller */
114625
- int nRet = 0; /* Length of aRet[] in bytes */
114626
- int nDoc = 0x7FFFFFFF;
114627
-
114628
- assert( !isReqPos );
114629
-
114630
- rc = fts3ExprAllocateSegReaders(p, pExpr, &nExpr);
114631
- if( rc==SQLITE_OK ){
114632
- assert( nExpr>1 );
114633
- aExpr = sqlite3_malloc(sizeof(ExprAndCost) * nExpr);
114634
- if( !aExpr ) rc = SQLITE_NOMEM;
114635
- }
114636
- if( rc==SQLITE_OK ){
114637
- int ii; /* Used to iterate through expressions */
114638
-
114639
- fts3ExprAssignCosts(pExpr, &aExpr);
114640
- aExpr -= nExpr;
114641
- for(ii=0; ii<nExpr; ii++){
114642
- char *aNew;
114643
- int nNew;
114644
- int jj;
114645
- ExprAndCost *pBest = 0;
114646
-
114647
- for(jj=0; jj<nExpr; jj++){
114648
- ExprAndCost *pCand = &aExpr[jj];
114649
- if( pCand->pExpr && (pBest==0 || pCand->nCost<pBest->nCost) ){
114650
- pBest = pCand;
114651
- }
114652
- }
114653
-
114654
- if( pBest->nCost>nDoc ){
114655
- rc = fts3DeferExpression(p, p->pExpr);
114656
- break;
114657
- }else{
114658
- rc = fts3EvalExpr(p, pBest->pExpr, &aNew, &nNew, 0);
114659
- if( rc!=SQLITE_OK ) break;
114660
- pBest->pExpr = 0;
114661
- if( ii==0 ){
114662
- aRet = aNew;
114663
- nRet = nNew;
114664
- nDoc = fts3DoclistCountDocids(0, aRet, nRet);
114665
- }else{
114666
- fts3DoclistMerge(
114667
- MERGE_AND, 0, 0, aRet, &nRet, aRet, nRet, aNew, nNew, &nDoc
114668
- );
114669
- sqlite3_free(aNew);
114670
- }
114671
- }
114672
- }
114673
- }
114674
-
114675
- if( rc==SQLITE_OK ){
114676
- *paOut = aRet;
114677
- *pnOut = nRet;
114678
- }else{
114679
- assert( *paOut==0 );
114680
- sqlite3_free(aRet);
114681
- }
114682
- sqlite3_free(aExpr);
114683
- fts3ExprFreeSegReaders(pExpr);
114684
-
114685
- }else{
114686
- char *aLeft;
114687
- char *aRight;
114688
- int nLeft;
114689
- int nRight;
114690
-
114691
- assert( pExpr->eType==FTSQUERY_NEAR
114692
- || pExpr->eType==FTSQUERY_OR
114693
- || pExpr->eType==FTSQUERY_NOT
114694
- || (pExpr->eType==FTSQUERY_AND && p->eEvalmode==FTS3_EVAL_NEXT)
114695
- );
114696
-
114697
- if( 0==(rc = fts3EvalExpr(p, pExpr->pRight, &aRight, &nRight, isReqPos))
114698
- && 0==(rc = fts3EvalExpr(p, pExpr->pLeft, &aLeft, &nLeft, isReqPos))
114699
- ){
114700
- switch( pExpr->eType ){
114701
- case FTSQUERY_NEAR: {
114702
- Fts3Expr *pLeft;
114703
- Fts3Expr *pRight;
114704
- int mergetype = MERGE_NEAR;
114705
- if( pExpr->pParent && pExpr->pParent->eType==FTSQUERY_NEAR ){
114706
- mergetype = MERGE_POS_NEAR;
114707
- }
114708
- pLeft = pExpr->pLeft;
114709
- while( pLeft->eType==FTSQUERY_NEAR ){
114710
- pLeft=pLeft->pRight;
114711
- }
114712
- pRight = pExpr->pRight;
114713
- assert( pRight->eType==FTSQUERY_PHRASE );
114714
- assert( pLeft->eType==FTSQUERY_PHRASE );
114715
-
114716
- rc = fts3NearMerge(mergetype, pExpr->nNear,
114717
- pLeft->pPhrase->nToken, aLeft, nLeft,
114718
- pRight->pPhrase->nToken, aRight, nRight,
114719
- paOut, pnOut
114720
- );
114721
- sqlite3_free(aLeft);
114722
- break;
114723
- }
114724
-
114725
- case FTSQUERY_OR: {
114726
- /* Allocate a buffer for the output. The maximum size is the
114727
- ** sum of the sizes of the two input buffers. The +1 term is
114728
- ** so that a buffer of zero bytes is never allocated - this can
114729
- ** cause fts3DoclistMerge() to incorrectly return SQLITE_NOMEM.
114730
- */
114731
- char *aBuffer = sqlite3_malloc(nRight+nLeft+1);
114732
- rc = fts3DoclistMerge(MERGE_OR, 0, 0, aBuffer, pnOut,
114733
- aLeft, nLeft, aRight, nRight, 0
114734
- );
114735
- *paOut = aBuffer;
114736
- sqlite3_free(aLeft);
114737
- break;
114738
- }
114739
-
114740
- default: {
114741
- assert( FTSQUERY_NOT==MERGE_NOT && FTSQUERY_AND==MERGE_AND );
114742
- fts3DoclistMerge(pExpr->eType, 0, 0, aLeft, pnOut,
114743
- aLeft, nLeft, aRight, nRight, 0
114744
- );
114745
- *paOut = aLeft;
114746
- break;
114747
- }
114748
- }
114749
- }
114750
- sqlite3_free(aRight);
114751
- }
114752
- }
114753
-
114754
- assert( rc==SQLITE_OK || *paOut==0 );
114755
- return rc;
114756
-}
114757
-
114758
-/*
114759
-** This function is called from within xNext() for each row visited by
114760
-** an FTS3 query. If evaluating the FTS3 query expression within xFilter()
114761
-** was able to determine the exact set of matching rows, this function sets
114762
-** *pbRes to true and returns SQLITE_IO immediately.
114763
-**
114764
-** Otherwise, if evaluating the query expression within xFilter() returned a
114765
-** superset of the matching documents instead of an exact set (this happens
114766
-** when the query includes very common tokens and it is deemed too expensive to
114767
-** load their doclists from disk), this function tests if the current row
114768
-** really does match the FTS3 query.
114769
-**
114770
-** If an error occurs, an SQLite error code is returned. Otherwise, SQLITE_OK
114771
-** is returned and *pbRes is set to true if the current row matches the
114772
-** FTS3 query (and should be included in the results returned to SQLite), or
114773
-** false otherwise.
114774
-*/
114775
-static int fts3EvalDeferred(
114776
- Fts3Cursor *pCsr, /* FTS3 cursor pointing at row to test */
114777
- int *pbRes /* OUT: Set to true if row is a match */
114778
-){
114779
- int rc = SQLITE_OK;
114780
- if( pCsr->pDeferred==0 ){
114781
- *pbRes = 1;
114782
- }else{
114783
- rc = fts3CursorSeek(0, pCsr);
114784
- if( rc==SQLITE_OK ){
114785
- sqlite3Fts3FreeDeferredDoclists(pCsr);
114786
- rc = sqlite3Fts3CacheDeferredDoclists(pCsr);
114787
- }
114788
- if( rc==SQLITE_OK ){
114789
- char *a = 0;
114790
- int n = 0;
114791
- rc = fts3EvalExpr(pCsr, pCsr->pExpr, &a, &n, 0);
114792
- assert( n>=0 );
114793
- *pbRes = (n>0);
114794
- sqlite3_free(a);
114795
- }
114796
- }
114797
- return rc;
114798
-}
114799
-
114800114367
 /*
114801114368
 ** Advance the cursor to the next row in the %_content table that
114802114369
 ** matches the search criteria. For a MATCH search, this will be
114803114370
 ** the next row that matches. For a full-table scan, this will be
114804114371
 ** simply the next row in the %_content table. For a docid lookup,
@@ -114807,43 +114374,24 @@
114807114374
 ** Return SQLITE_OK if nothing goes wrong. SQLITE_OK is returned
114808114375
 ** even if we reach end-of-file. The fts3EofMethod() will be called
114809114376
 ** subsequently to determine whether or not an EOF was hit.
114810114377
 */
114811114378
 static int fts3NextMethod(sqlite3_vtab_cursor *pCursor){
114812
- int res;
114813
- int rc = SQLITE_OK; /* Return code */
114379
+ int rc;
114814114380
 Fts3Cursor *pCsr = (Fts3Cursor *)pCursor;
114815
-
114816
- pCsr->eEvalmode = FTS3_EVAL_NEXT;
114817
- do {
114818
- if( pCsr->aDoclist==0 ){
114819
- if( SQLITE_ROW!=sqlite3_step(pCsr->pStmt) ){
114820
- pCsr->isEof = 1;
114821
- rc = sqlite3_reset(pCsr->pStmt);
114822
- break;
114823
- }
114381
+ if( pCsr->eSearch==FTS3_DOCID_SEARCH || pCsr->eSearch==FTS3_FULLSCAN_SEARCH ){
114382
+ if( SQLITE_ROW!=sqlite3_step(pCsr->pStmt) ){
114383
+ pCsr->isEof = 1;
114384
+ rc = sqlite3_reset(pCsr->pStmt);
114385
+ }else{
114824114386
 pCsr->iPrevId = sqlite3_column_int64(pCsr->pStmt, 0);
114825
- }else{
114826
- if( pCsr->desc==0 ){
114827
- if( pCsr->pNextId>=&pCsr->aDoclist[pCsr->nDoclist] ){
114828
- pCsr->isEof = 1;
114829
- break;
114830
- }
114831
- fts3GetDeltaVarint(&pCsr->pNextId, &pCsr->iPrevId);
114832
- }else{
114833
- fts3GetReverseDeltaVarint(&pCsr->pNextId,pCsr->aDoclist,&pCsr->iPrevId);
114834
- if( pCsr->pNextId<=pCsr->aDoclist ){
114835
- pCsr->isEof = 1;
114836
- break;
114837
- }
114838
- }
114839
- sqlite3_reset(pCsr->pStmt);
114840
- pCsr->isRequireSeek = 1;
114841
- pCsr->isMatchinfoNeeded = 1;
114842
- }
114843
- }while( SQLITE_OK==(rc = fts3EvalDeferred(pCsr, &res)) && res==0 );
114844
-
114387
+ rc = SQLITE_OK;
114388
+ }
114389
+ }else{
114390
+ rc = sqlite3Fts3EvalNext((Fts3Cursor *)pCursor);
114391
+ }
114392
+ assert( ((Fts3Table *)pCsr->base.pVtab)->pSegments==0 );
114845114393
 return rc;
114846114394
 }
114847114395
114848114396
 /*
114849114397
 ** This is the xFilter interface for the virtual table. See
@@ -114866,15 +114414,11 @@
114866114414
 int idxNum, /* Strategy index */
114867114415
 const char *idxStr, /* Unused */
114868114416
 int nVal, /* Number of elements in apVal */
114869114417
 sqlite3_value **apVal /* Arguments for the indexing scheme */
114870114418
 ){
114871
- const char *azSql[] = {
114872
- "SELECT %s FROM %Q.'%q_content' AS x WHERE docid = ?", /* non-full-scan */
114873
- "SELECT %s FROM %Q.'%q_content' AS x ORDER BY docid %s", /* full-scan */
114874
- };
114875
- int rc; /* Return code */
114419
+ int rc;
114876114420
 char *zSql; /* SQL statement used to access %_content */
114877114421
 Fts3Table *p = (Fts3Table *)pCursor->pVtab;
114878114422
 Fts3Cursor *pCsr = (Fts3Cursor *)pCursor;
114879114423
114880114424
 UNUSED_PARAMETER(idxStr);
@@ -114889,10 +114433,17 @@
114889114433
 sqlite3_finalize(pCsr->pStmt);
114890114434
 sqlite3_free(pCsr->aDoclist);
114891114435
 sqlite3Fts3ExprFree(pCsr->pExpr);
114892114436
 memset(&pCursor[1], 0, sizeof(Fts3Cursor)-sizeof(sqlite3_vtab_cursor));
114893114437
114438
+ if( idxStr ){
114439
+ pCsr->bDesc = (idxStr[0]=='D');
114440
+ }else{
114441
+ pCsr->bDesc = p->bDescIdx;
114442
+ }
114443
+ pCsr->eSearch = (i16)idxNum;
114444
+
114894114445
 if( idxNum!=FTS3_DOCID_SEARCH && idxNum!=FTS3_FULLSCAN_SEARCH ){
114895114446
 int iCol = idxNum-FTS3_FULLTEXT_SEARCH;
114896114447
 const char *zQuery = (const char *)sqlite3_value_text(apVal[0]);
114897114448
114898114449
 if( zQuery==0 && sqlite3_value_type(apVal[0])!=SQLITE_NULL ){
@@ -114902,20 +114453,21 @@
114902114453
 rc = sqlite3Fts3ExprParse(p->pTokenizer, p->azColumn, p->nColumn,
114903114454
 iCol, zQuery, -1, &pCsr->pExpr
114904114455
 );
114905114456
 if( rc!=SQLITE_OK ){
114906114457
 if( rc==SQLITE_ERROR ){
114907
- p->base.zErrMsg = sqlite3_mprintf("malformed MATCH expression: [%s]",
114908
- zQuery);
114458
+ static const char *zErr = "malformed MATCH expression: [%s]";
114459
+ p->base.zErrMsg = sqlite3_mprintf(zErr, zQuery);
114909114460
 }
114910114461
 return rc;
114911114462
 }
114912114463
114913114464
 rc = sqlite3Fts3ReadLock(p);
114914114465
 if( rc!=SQLITE_OK ) return rc;
114915114466
114916
- rc = fts3EvalExpr(pCsr, pCsr->pExpr, &pCsr->aDoclist, &pCsr->nDoclist, 0);
114467
+ rc = sqlite3Fts3EvalStart(pCsr, pCsr->pExpr, 1);
114468
+
114917114469
 sqlite3Fts3SegmentsClose(p);
114918114470
 if( rc!=SQLITE_OK ) return rc;
114919114471
 pCsr->pNextId = pCsr->aDoclist;
114920114472
 pCsr->iPrevId = 0;
114921114473
 }
@@ -114923,41 +114475,28 @@
114923114475
 /* Compile a SELECT statement for this cursor. For a full-table-scan, the
114924114476
 ** statement loops through all rows of the %_content table. For a
114925114477
 ** full-text query or docid lookup, the statement retrieves a single
114926114478
 ** row by docid.
114927114479
 */
114928
- zSql = (char *)azSql[idxNum==FTS3_FULLSCAN_SEARCH];
114929
- zSql = sqlite3_mprintf(
114930
- zSql, p->zReadExprlist, p->zDb, p->zName, (idxStr ? idxStr : "ASC")
114931
- );
114932
- if( !zSql ){
114933
- rc = SQLITE_NOMEM;
114480
+ if( idxNum==FTS3_FULLSCAN_SEARCH ){
114481
+ const char *zSort = (pCsr->bDesc ? "DESC" : "ASC");
114482
+ const char *zTmpl = "SELECT %s FROM %Q.'%q_content' AS x ORDER BY docid %s";
114483
+ zSql = sqlite3_mprintf(zTmpl, p->zReadExprlist, p->zDb, p->zName, zSort);
114934114484
 }else{
114935
- rc = sqlite3_prepare_v2(p->db, zSql, -1, &pCsr->pStmt, 0);
114936
- sqlite3_free(zSql);
114485
+ const char *zTmpl = "SELECT %s FROM %Q.'%q_content' AS x WHERE docid = ?";
114486
+ zSql = sqlite3_mprintf(zTmpl, p->zReadExprlist, p->zDb, p->zName);
114937114487
 }
114938
- if( rc==SQLITE_OK && idxNum==FTS3_DOCID_SEARCH ){
114488
+ if( !zSql ) return SQLITE_NOMEM;
114489
+ rc = sqlite3_prepare_v2(p->db, zSql, -1, &pCsr->pStmt, 0);
114490
+ sqlite3_free(zSql);
114491
+ if( rc!=SQLITE_OK ) return rc;
114492
+
114493
+ if( idxNum==FTS3_DOCID_SEARCH ){
114939114494
 rc = sqlite3_bind_value(pCsr->pStmt, 1, apVal[0]);
114940
- }
114941
- pCsr->eSearch = (i16)idxNum;
114942
-
114943
- assert( pCsr->desc==0 );
114944
- if( rc!=SQLITE_OK ) return rc;
114945
- if( rc==SQLITE_OK && pCsr->nDoclist>0 && idxStr && idxStr[0]=='D' ){
114946
- sqlite3_int64 iDocid = 0;
114947
- char *csr = pCsr->aDoclist;
114948
- while( csr<&pCsr->aDoclist[pCsr->nDoclist] ){
114949
- fts3GetDeltaVarint(&csr, &iDocid);
114950
- }
114951
- pCsr->pNextId = csr;
114952
- pCsr->iPrevId = iDocid;
114953
- pCsr->desc = 1;
114954
- pCsr->isRequireSeek = 1;
114955
- pCsr->isMatchinfoNeeded = 1;
114956
- pCsr->eEvalmode = FTS3_EVAL_NEXT;
114957
- return SQLITE_OK;
114958
- }
114495
+ if( rc!=SQLITE_OK ) return rc;
114496
+ }
114497
+
114959114498
 return fts3NextMethod(pCursor);
114960114499
 }
114961114500
114962114501
 /*
114963114502
 ** This is the xEof method of the virtual table. SQLite calls this
@@ -114973,20 +114512,11 @@
114973114512
 ** exposes %_content.docid as the rowid for the virtual table. The
114974114513
 ** rowid should be written to *pRowid.
114975114514
 */
114976114515
 static int fts3RowidMethod(sqlite3_vtab_cursor *pCursor, sqlite_int64 *pRowid){
114977114516
 Fts3Cursor *pCsr = (Fts3Cursor *) pCursor;
114978
- if( pCsr->aDoclist ){
114979
- *pRowid = pCsr->iPrevId;
114980
- }else{
114981
- /* This branch runs if the query is implemented using a full-table scan
114982
- ** (not using the full-text index). In this case grab the rowid from the
114983
- ** SELECT statement.
114984
- */
114985
- assert( pCsr->isRequireSeek==0 );
114986
- *pRowid = sqlite3_column_int64(pCsr->pStmt, 0);
114987
- }
114517
+ *pRowid = pCsr->iPrevId;
114988114518
 return SQLITE_OK;
114989114519
 }
114990114520
114991114521
 /*
114992114522
 ** This is the xColumn method, called by SQLite to request a value from
@@ -114995,11 +114525,11 @@
114995114525
 static int fts3ColumnMethod(
114996114526
 sqlite3_vtab_cursor *pCursor, /* Cursor to retrieve value from */
114997114527
 sqlite3_context *pContext, /* Context for sqlite3_result_xxx() calls */
114998114528
 int iCol /* Index of column to read value from */
114999114529
 ){
115000
- int rc; /* Return Code */
114530
+ int rc = SQLITE_OK; /* Return Code */
115001114531
 Fts3Cursor *pCsr = (Fts3Cursor *) pCursor;
115002114532
 Fts3Table *p = (Fts3Table *)pCursor->pVtab;
115003114533
115004114534
 /* The column value supplied by SQLite must be in range. */
115005114535
 assert( iCol>=0 && iCol<=p->nColumn+1 );
@@ -115006,25 +114536,24 @@
115006114536
115007114537
 if( iCol==p->nColumn+1 ){
115008114538
 /* This call is a request for the "docid" column. Since "docid" is an
115009114539
 ** alias for "rowid", use the xRowid() method to obtain the value.
115010114540
 */
115011
- sqlite3_int64 iRowid;
115012
- rc = fts3RowidMethod(pCursor, &iRowid);
115013
- sqlite3_result_int64(pContext, iRowid);
114541
+ sqlite3_result_int64(pContext, pCsr->iPrevId);
115014114542
 }else if( iCol==p->nColumn ){
115015114543
 /* The extra column whose name is the same as the table.
115016114544
 ** Return a blob which is a pointer to the cursor.
115017114545
 */
115018114546
 sqlite3_result_blob(pContext, &pCsr, sizeof(pCsr), SQLITE_TRANSIENT);
115019
- rc = SQLITE_OK;
115020114547
 }else{
115021114548
 rc = fts3CursorSeek(0, pCsr);
115022114549
 if( rc==SQLITE_OK ){
115023114550
 sqlite3_result_value(pContext, sqlite3_column_value(pCsr->pStmt, iCol+1));
115024114551
 }
115025114552
 }
114553
+
114554
+ assert( ((Fts3Table *)pCsr->base.pVtab)->pSegments==0 );
115026114555
 return rc;
115027114556
 }
115028114557
115029114558
 /*
115030114559
 ** This function is the implementation of the xUpdate callback used by
@@ -115054,10 +114583,11 @@
115054114583
 ** Implementation of xBegin() method. This is a no-op.
115055114584
 */
115056114585
 static int fts3BeginMethod(sqlite3_vtab *pVtab){
115057114586
 UNUSED_PARAMETER(pVtab);
115058114587
 TESTONLY( Fts3Table *p = (Fts3Table*)pVtab );
114588
+ assert( p->pSegments==0 );
115059114589
 assert( p->nPendingData==0 );
115060114590
 assert( p->inTransaction!=1 );
115061114591
 TESTONLY( p->inTransaction = 1 );
115062114592
 TESTONLY( p->mxSavepoint = -1; );
115063114593
 return SQLITE_OK;
@@ -115071,10 +114601,11 @@
115071114601
 static int fts3CommitMethod(sqlite3_vtab *pVtab){
115072114602
 UNUSED_PARAMETER(pVtab);
115073114603
 TESTONLY( Fts3Table *p = (Fts3Table*)pVtab );
115074114604
 assert( p->nPendingData==0 );
115075114605
 assert( p->inTransaction!=0 );
114606
+ assert( p->pSegments==0 );
115076114607
 TESTONLY( p->inTransaction = 0 );
115077114608
 TESTONLY( p->mxSavepoint = -1; );
115078114609
 return SQLITE_OK;
115079114610
 }
115080114611
@@ -115088,132 +114619,30 @@
115088114619
 assert( p->inTransaction!=0 );
115089114620
 TESTONLY( p->inTransaction = 0 );
115090114621
 TESTONLY( p->mxSavepoint = -1; );
115091114622
 return SQLITE_OK;
115092114623
 }
115093
-
115094
-/*
115095
-** Load the doclist associated with expression pExpr to pExpr->aDoclist.
115096
-** The loaded doclist contains positions as well as the document ids.
115097
-** This is used by the matchinfo(), snippet() and offsets() auxillary
115098
-** functions.
115099
-*/
115100
-SQLITE_PRIVATE int sqlite3Fts3ExprLoadDoclist(Fts3Cursor *pCsr, Fts3Expr *pExpr){
115101
- int rc;
115102
- assert( pExpr->eType==FTSQUERY_PHRASE && pExpr->pPhrase );
115103
- assert( pCsr->eEvalmode==FTS3_EVAL_NEXT );
115104
- rc = fts3EvalExpr(pCsr, pExpr, &pExpr->aDoclist, &pExpr->nDoclist, 1);
115105
- return rc;
115106
-}
115107
-
115108
-SQLITE_PRIVATE int sqlite3Fts3ExprLoadFtDoclist(
115109
- Fts3Cursor *pCsr,
115110
- Fts3Expr *pExpr,
115111
- char **paDoclist,
115112
- int *pnDoclist
115113
-){
115114
- int rc;
115115
- assert( pCsr->eEvalmode==FTS3_EVAL_NEXT );
115116
- assert( pExpr->eType==FTSQUERY_PHRASE && pExpr->pPhrase );
115117
- pCsr->eEvalmode = FTS3_EVAL_MATCHINFO;
115118
- rc = fts3EvalExpr(pCsr, pExpr, paDoclist, pnDoclist, 1);
115119
- pCsr->eEvalmode = FTS3_EVAL_NEXT;
115120
- return rc;
115121
-}
115122
-
115123114624
115124114625
 /*
115125114626
 ** When called, *ppPoslist must point to the byte immediately following the
115126114627
 ** end of a position-list. i.e. ( (*ppPoslist)[-1]==POS_END ). This function
115127114628
 ** moves *ppPoslist so that it instead points to the first byte of the
115128114629
 ** same position list.
115129114630
 */
115130114631
 static void fts3ReversePoslist(char *pStart, char **ppPoslist){
115131
- char *p = &(*ppPoslist)[-3];
115132
- char c = p[1];
114632
+ char *p = &(*ppPoslist)[-2];
114633
+ char c;
114634
+
114635
+ while( p>pStart && (c=*p--)==0 );
115133114636
 while( p>pStart && (*p & 0x80) | c ){
115134114637
 c = *p--;
115135114638
 }
115136114639
 if( p>pStart ){ p = &p[2]; }
115137114640
 while( *p++&0x80 );
115138114641
 *ppPoslist = p;
115139114642
 }
115140114643
115141
-
115142
-/*
115143
-** After ExprLoadDoclist() (see above) has been called, this function is
115144
-** used to iterate/search through the position lists that make up the doclist
115145
-** stored in pExpr->aDoclist.
115146
-*/
115147
-SQLITE_PRIVATE char *sqlite3Fts3FindPositions(
115148
- Fts3Cursor *pCursor, /* Associate FTS3 cursor */
115149
- Fts3Expr *pExpr, /* Access this expressions doclist */
115150
- sqlite3_int64 iDocid, /* Docid associated with requested pos-list */
115151
- int iCol /* Column of requested pos-list */
115152
-){
115153
- assert( pExpr->isLoaded );
115154
- if( pExpr->aDoclist ){
115155
- char *pEnd = &pExpr->aDoclist[pExpr->nDoclist];
115156
- char *pCsr;
115157
-
115158
- if( pExpr->pCurrent==0 ){
115159
- if( pCursor->desc==0 ){
115160
- pExpr->pCurrent = pExpr->aDoclist;
115161
- pExpr->iCurrent = 0;
115162
- fts3GetDeltaVarint(&pExpr->pCurrent, &pExpr->iCurrent);
115163
- }else{
115164
- pCsr = pExpr->aDoclist;
115165
- while( pCsr<pEnd ){
115166
- fts3GetDeltaVarint(&pCsr, &pExpr->iCurrent);
115167
- fts3PoslistCopy(0, &pCsr);
115168
- }
115169
- fts3ReversePoslist(pExpr->aDoclist, &pCsr);
115170
- pExpr->pCurrent = pCsr;
115171
- }
115172
- }
115173
- pCsr = pExpr->pCurrent;
115174
- assert( pCsr );
115175
-
115176
- while( (pCursor->desc==0 && pCsr<pEnd)
115177
- || (pCursor->desc && pCsr>pExpr->aDoclist)
115178
- ){
115179
- if( pCursor->desc==0 && pExpr->iCurrent<iDocid ){
115180
- fts3PoslistCopy(0, &pCsr);
115181
- if( pCsr<pEnd ){
115182
- fts3GetDeltaVarint(&pCsr, &pExpr->iCurrent);
115183
- }
115184
- pExpr->pCurrent = pCsr;
115185
- }else if( pCursor->desc && pExpr->iCurrent>iDocid ){
115186
- fts3GetReverseDeltaVarint(&pCsr, pExpr->aDoclist, &pExpr->iCurrent);
115187
- fts3ReversePoslist(pExpr->aDoclist, &pCsr);
115188
- pExpr->pCurrent = pCsr;
115189
- }else{
115190
- if( pExpr->iCurrent==iDocid ){
115191
- int iThis = 0;
115192
- if( iCol<0 ){
115193
- /* If iCol is negative, return a pointer to the start of the
115194
- ** position-list (instead of a pointer to the start of a list
115195
- ** of offsets associated with a specific column).
115196
- */
115197
- return pCsr;
115198
- }
115199
- while( iThis<iCol ){
115200
- fts3ColumnlistCopy(0, &pCsr);
115201
- if( *pCsr==0x00 ) return 0;
115202
- pCsr++;
115203
- pCsr += sqlite3Fts3GetVarint32(pCsr, &iThis);
115204
- }
115205
- if( iCol==iThis && (*pCsr&0xFE) ) return pCsr;
115206
- }
115207
- return 0;
115208
- }
115209
- }
115210
- }
115211
-
115212
- return 0;
115213
-}
115214
-
115215114644
 /*
115216114645
 ** Helper function used by the implementation of the overloaded snippet(),
115217114646
 ** offsets() and optimize() SQL functions.
115218114647
 **
115219114648
 ** If the value passed as the third argument is a blob of size
@@ -115441,16 +114870,15 @@
115441114870
 );
115442114871
 return rc;
115443114872
 }
115444114873
115445114874
 static int fts3SavepointMethod(sqlite3_vtab *pVtab, int iSavepoint){
115446
- Fts3Table *p = (Fts3Table*)pVtab;
115447114875
 UNUSED_PARAMETER(iSavepoint);
115448
- assert( p->inTransaction );
115449
- assert( p->mxSavepoint < iSavepoint );
115450
- TESTONLY( p->mxSavepoint = iSavepoint );
115451
- return sqlite3Fts3PendingTermsFlush(p);
114876
+ assert( ((Fts3Table *)pVtab)->inTransaction );
114877
+ assert( ((Fts3Table *)pVtab)->mxSavepoint < iSavepoint );
114878
+ TESTONLY( ((Fts3Table *)pVtab)->mxSavepoint = iSavepoint );
114879
+ return fts3SyncMethod(pVtab);
115452114880
 }
115453114881
 static int fts3ReleaseMethod(sqlite3_vtab *pVtab, int iSavepoint){
115454114882
 TESTONLY( Fts3Table *p = (Fts3Table*)pVtab );
115455114883
 UNUSED_PARAMETER(iSavepoint);
115456114884
 UNUSED_PARAMETER(pVtab);
@@ -115617,10 +115045,1282 @@
115617115045
 SQLITE_EXTENSION_INIT2(pApi)
115618115046
 return sqlite3Fts3Init(db);
115619115047
 }
115620115048
 #endif
115621115049
115050
+
115051
+/*
115052
+** Allocate an Fts3MultiSegReader for each token in the expression headed
115053
+** by pExpr.
115054
+**
115055
+** An Fts3SegReader object is a cursor that can seek or scan a range of
115056
+** entries within a single segment b-tree. An Fts3MultiSegReader uses multiple
115057
+** Fts3SegReader objects internally to provide an interface to seek or scan
115058
+** within the union of all segments of a b-tree. Hence the name.
115059
+**
115060
+** If the allocated Fts3MultiSegReader just seeks to a single entry in a
115061
+** segment b-tree (if the term is not a prefix or it is a prefix for which
115062
+** there exists prefix b-tree of the right length) then it may be traversed
115063
+** and merged incrementally. Otherwise, it has to be merged into an in-memory
115064
+** doclist and then traversed.
115065
+*/
115066
+static void fts3EvalAllocateReaders(
115067
+ Fts3Cursor *pCsr,
115068
+ Fts3Expr *pExpr,
115069
+ int *pnToken, /* OUT: Total number of tokens in phrase. */
115070
+ int *pnOr, /* OUT: Total number of OR nodes in expr. */
115071
+ int *pRc
115072
+){
115073
+ if( pExpr && SQLITE_OK==*pRc ){
115074
+ if( pExpr->eType==FTSQUERY_PHRASE ){
115075
+ int i;
115076
+ int nToken = pExpr->pPhrase->nToken;
115077
+ *pnToken += nToken;
115078
+ for(i=0; i<nToken; i++){
115079
+ Fts3PhraseToken *pToken = &pExpr->pPhrase->aToken[i];
115080
+ int rc = sqlite3Fts3TermSegReaderCursor(pCsr,
115081
+ pToken->z, pToken->n, pToken->isPrefix, &pToken->pSegcsr
115082
+ );
115083
+ if( rc!=SQLITE_OK ){
115084
+ *pRc = rc;
115085
+ return;
115086
+ }
115087
+ }
115088
+ }else{
115089
+ *pnOr += (pExpr->eType==FTSQUERY_OR);
115090
+ fts3EvalAllocateReaders(pCsr, pExpr->pLeft, pnToken, pnOr, pRc);
115091
+ fts3EvalAllocateReaders(pCsr, pExpr->pRight, pnToken, pnOr, pRc);
115092
+ }
115093
+ }
115094
+}
115095
+
115096
+static int fts3EvalPhraseLoad(
115097
+ Fts3Cursor *pCsr,
115098
+ Fts3Phrase *p
115099
+){
115100
+ Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab;
115101
+ int iToken;
115102
+ int rc = SQLITE_OK;
115103
+
115104
+ char *aDoclist = 0;
115105
+ int nDoclist = 0;
115106
+ int iPrev = -1;
115107
+
115108
+ for(iToken=0; rc==SQLITE_OK && iToken<p->nToken; iToken++){
115109
+ Fts3PhraseToken *pToken = &p->aToken[iToken];
115110
+ assert( pToken->pSegcsr || pToken->pDeferred );
115111
+
115112
+ if( pToken->pDeferred==0 ){
115113
+ int nThis = 0;
115114
+ char *pThis = 0;
115115
+ rc = fts3TermSelect(pTab, pToken, p->iColumn, 1, &nThis, &pThis);
115116
+ if( rc==SQLITE_OK ){
115117
+ if( pThis==0 ){
115118
+ sqlite3_free(aDoclist);
115119
+ aDoclist = 0;
115120
+ nDoclist = 0;
115121
+ break;
115122
+ }else if( aDoclist==0 ){
115123
+ aDoclist = pThis;
115124
+ nDoclist = nThis;
115125
+ }else{
115126
+ assert( iPrev>=0 );
115127
+ fts3DoclistPhraseMerge(pTab->bDescIdx,
115128
+ iToken-iPrev, aDoclist, nDoclist, pThis, &nThis
115129
+ );
115130
+ sqlite3_free(aDoclist);
115131
+ aDoclist = pThis;
115132
+ nDoclist = nThis;
115133
+ }
115134
+ iPrev = iToken;
115135
+ }
115136
+ }
115137
+ }
115138
+
115139
+ if( rc==SQLITE_OK ){
115140
+ p->doclist.aAll = aDoclist;
115141
+ p->doclist.nAll = nDoclist;
115142
+ }else{
115143
+ sqlite3_free(aDoclist);
115144
+ }
115145
+ return rc;
115146
+}
115147
+
115148
+static int fts3EvalDeferredPhrase(Fts3Cursor *pCsr, Fts3Phrase *pPhrase){
115149
+ int iToken;
115150
+ int rc = SQLITE_OK;
115151
+
115152
+ int nMaxUndeferred = -1;
115153
+ char *aPoslist = 0;
115154
+ int nPoslist = 0;
115155
+ int iPrev = -1;
115156
+
115157
+ assert( pPhrase->doclist.bFreeList==0 );
115158
+
115159
+ for(iToken=0; rc==SQLITE_OK && iToken<pPhrase->nToken; iToken++){
115160
+ Fts3PhraseToken *pToken = &pPhrase->aToken[iToken];
115161
+ Fts3DeferredToken *pDeferred = pToken->pDeferred;
115162
+
115163
+ if( pDeferred ){
115164
+ char *pList;
115165
+ int nList;
115166
+ rc = sqlite3Fts3DeferredTokenList(pDeferred, &pList, &nList);
115167
+ if( rc!=SQLITE_OK ) return rc;
115168
+
115169
+ if( pList==0 ){
115170
+ sqlite3_free(aPoslist);
115171
+ pPhrase->doclist.pList = 0;
115172
+ pPhrase->doclist.nList = 0;
115173
+ return SQLITE_OK;
115174
+
115175
+ }else if( aPoslist==0 ){
115176
+ aPoslist = pList;
115177
+ nPoslist = nList;
115178
+
115179
+ }else{
115180
+ char *aOut = pList;
115181
+ char *p1 = aPoslist;
115182
+ char *p2 = aOut;
115183
+
115184
+ assert( iPrev>=0 );
115185
+ fts3PoslistPhraseMerge(&aOut, iToken-iPrev, 0, 1, &p1, &p2);
115186
+ sqlite3_free(aPoslist);
115187
+ aPoslist = pList;
115188
+ nPoslist = aOut - aPoslist;
115189
+ if( nPoslist==0 ){
115190
+ sqlite3_free(aPoslist);
115191
+ pPhrase->doclist.pList = 0;
115192
+ pPhrase->doclist.nList = 0;
115193
+ return SQLITE_OK;
115194
+ }
115195
+ }
115196
+ iPrev = iToken;
115197
+ }else{
115198
+ nMaxUndeferred = iToken;
115199
+ }
115200
+ }
115201
+
115202
+ if( iPrev>=0 ){
115203
+ if( nMaxUndeferred<0 ){
115204
+ pPhrase->doclist.pList = aPoslist;
115205
+ pPhrase->doclist.nList = nPoslist;
115206
+ pPhrase->doclist.iDocid = pCsr->iPrevId;
115207
+ pPhrase->doclist.bFreeList = 1;
115208
+ }else{
115209
+ int nDistance;
115210
+ char *p1;
115211
+ char *p2;
115212
+ char *aOut;
115213
+
115214
+ if( nMaxUndeferred>iPrev ){
115215
+ p1 = aPoslist;
115216
+ p2 = pPhrase->doclist.pList;
115217
+ nDistance = nMaxUndeferred - iPrev;
115218
+ }else{
115219
+ p1 = pPhrase->doclist.pList;
115220
+ p2 = aPoslist;
115221
+ nDistance = iPrev - nMaxUndeferred;
115222
+ }
115223
+
115224
+ aOut = (char *)sqlite3_malloc(nPoslist+8);
115225
+ if( !aOut ){
115226
+ sqlite3_free(aPoslist);
115227
+ return SQLITE_NOMEM;
115228
+ }
115229
+
115230
+ pPhrase->doclist.pList = aOut;
115231
+ if( fts3PoslistPhraseMerge(&aOut, nDistance, 0, 1, &p1, &p2) ){
115232
+ pPhrase->doclist.bFreeList = 1;
115233
+ pPhrase->doclist.nList = (aOut - pPhrase->doclist.pList);
115234
+ }else{
115235
+ sqlite3_free(aOut);
115236
+ pPhrase->doclist.pList = 0;
115237
+ pPhrase->doclist.nList = 0;
115238
+ }
115239
+ sqlite3_free(aPoslist);
115240
+ }
115241
+ }
115242
+
115243
+ return SQLITE_OK;
115244
+}
115245
+
115246
+/*
115247
+** This function is called for each Fts3Phrase in a full-text query
115248
+** expression to initialize the mechanism for returning rows. Once this
115249
+** function has been called successfully on an Fts3Phrase, it may be
115250
+** used with fts3EvalPhraseNext() to iterate through the matching docids.
115251
+*/
115252
+static int fts3EvalPhraseStart(Fts3Cursor *pCsr, int bOptOk, Fts3Phrase *p){
115253
+ int rc;
115254
+ Fts3PhraseToken *pFirst = &p->aToken[0];
115255
+ Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab;
115256
+
115257
+ assert( p->doclist.aAll==0 );
115258
+ if( pCsr->bDesc==pTab->bDescIdx && bOptOk==1 && p->nToken==1
115259
+ && pFirst->pSegcsr && pFirst->pSegcsr->bLookup
115260
+ ){
115261
+ /* Use the incremental approach. */
115262
+ int iCol = (p->iColumn >= pTab->nColumn ? -1 : p->iColumn);
115263
+ rc = sqlite3Fts3MsrIncrStart(
115264
+ pTab, pFirst->pSegcsr, iCol, pFirst->z, pFirst->n);
115265
+ p->bIncr = 1;
115266
+
115267
+ }else{
115268
+ /* Load the full doclist for the phrase into memory. */
115269
+ rc = fts3EvalPhraseLoad(pCsr, p);
115270
+ p->bIncr = 0;
115271
+ }
115272
+
115273
+ assert( rc!=SQLITE_OK || p->nToken<1 || p->aToken[0].pSegcsr==0 || p->bIncr );
115274
+ return rc;
115275
+}
115276
+
115277
+/*
115278
+** This function is used to iterate backwards (from the end to start)
115279
+** through doclists.
115280
+*/
115281
+SQLITE_PRIVATE void sqlite3Fts3DoclistPrev(
115282
+ int bDescIdx, /* True if the doclist is desc */
115283
+ char *aDoclist, /* Pointer to entire doclist */
115284
+ int nDoclist, /* Length of aDoclist in bytes */
115285
+ char **ppIter, /* IN/OUT: Iterator pointer */
115286
+ sqlite3_int64 *piDocid, /* IN/OUT: Docid pointer */
115287
+ int *pnList, /* IN/OUT: List length pointer */
115288
+ u8 *pbEof /* OUT: End-of-file flag */
115289
+){
115290
+ char *p = *ppIter;
115291
+
115292
+ assert( nDoclist>0 );
115293
+ assert( *pbEof==0 );
115294
+ assert( p || *piDocid==0 );
115295
+ assert( !p || (p>aDoclist && p<&aDoclist[nDoclist]) );
115296
+
115297
+ if( p==0 ){
115298
+ sqlite3_int64 iDocid = 0;
115299
+ char *pNext = 0;
115300
+ char *pDocid = aDoclist;
115301
+ char *pEnd = &aDoclist[nDoclist];
115302
+ int iMul = 1;
115303
+
115304
+ while( pDocid<pEnd ){
115305
+ sqlite3_int64 iDelta;
115306
+ pDocid += sqlite3Fts3GetVarint(pDocid, &iDelta);
115307
+ iDocid += (iMul * iDelta);
115308
+ pNext = pDocid;
115309
+ fts3PoslistCopy(0, &pDocid);
115310
+ while( pDocid<pEnd && *pDocid==0 ) pDocid++;
115311
+ iMul = (bDescIdx ? -1 : 1);
115312
+ }
115313
+
115314
+ *pnList = pEnd - pNext;
115315
+ *ppIter = pNext;
115316
+ *piDocid = iDocid;
115317
+ }else{
115318
+ int iMul = (bDescIdx ? -1 : 1);
115319
+ sqlite3_int64 iDelta;
115320
+ fts3GetReverseVarint(&p, aDoclist, &iDelta);
115321
+ *piDocid -= (iMul * iDelta);
115322
+
115323
+ if( p==aDoclist ){
115324
+ *pbEof = 1;
115325
+ }else{
115326
+ char *pSave = p;
115327
+ fts3ReversePoslist(aDoclist, &p);
115328
+ *pnList = (pSave - p);
115329
+ }
115330
+ *ppIter = p;
115331
+ }
115332
+}
115333
+
115334
+/*
115335
+** Attempt to move the phrase iterator to point to the next matching docid.
115336
+** If an error occurs, return an SQLite error code. Otherwise, return
115337
+** SQLITE_OK.
115338
+**
115339
+** If there is no "next" entry and no error occurs, then *pbEof is set to
115340
+** 1 before returning. Otherwise, if no error occurs and the iterator is
115341
+** successfully advanced, *pbEof is set to 0.
115342
+*/
115343
+static int fts3EvalPhraseNext(
115344
+ Fts3Cursor *pCsr,
115345
+ Fts3Phrase *p,
115346
+ u8 *pbEof
115347
+){
115348
+ int rc = SQLITE_OK;
115349
+ Fts3Doclist *pDL = &p->doclist;
115350
+ Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab;
115351
+
115352
+ if( p->bIncr ){
115353
+ assert( p->nToken==1 );
115354
+ assert( pDL->pNextDocid==0 );
115355
+ rc = sqlite3Fts3MsrIncrNext(pTab, p->aToken[0].pSegcsr,
115356
+ &pDL->iDocid, &pDL->pList, &pDL->nList
115357
+ );
115358
+ if( rc==SQLITE_OK && !pDL->pList ){
115359
+ *pbEof = 1;
115360
+ }
115361
+ }else if( pCsr->bDesc!=pTab->bDescIdx && pDL->nAll ){
115362
+ sqlite3Fts3DoclistPrev(pTab->bDescIdx, pDL->aAll, pDL->nAll,
115363
+ &pDL->pNextDocid, &pDL->iDocid, &pDL->nList, pbEof
115364
+ );
115365
+ pDL->pList = pDL->pNextDocid;
115366
+ }else{
115367
+ char *pIter; /* Used to iterate through aAll */
115368
+ char *pEnd = &pDL->aAll[pDL->nAll]; /* 1 byte past end of aAll */
115369
+ if( pDL->pNextDocid ){
115370
+ pIter = pDL->pNextDocid;
115371
+ }else{
115372
+ pIter = pDL->aAll;
115373
+ }
115374
+
115375
+ if( pIter>=pEnd ){
115376
+ /* We have already reached the end of this doclist. EOF. */
115377
+ *pbEof = 1;
115378
+ }else{
115379
+ sqlite3_int64 iDelta;
115380
+ pIter += sqlite3Fts3GetVarint(pIter, &iDelta);
115381
+ if( pTab->bDescIdx==0 || pDL->pNextDocid==0 ){
115382
+ pDL->iDocid += iDelta;
115383
+ }else{
115384
+ pDL->iDocid -= iDelta;
115385
+ }
115386
+ pDL->pList = pIter;
115387
+ fts3PoslistCopy(0, &pIter);
115388
+ pDL->nList = (pIter - pDL->pList);
115389
+
115390
+ /* pIter now points just past the 0x00 that terminates the position-
115391
+ ** list for document pDL->iDocid. However, if this position-list was
115392
+ ** edited in place by fts3EvalNearTrim2(), then pIter may not actually
115393
+ ** point to the start of the next docid value. The following line deals
115394
+ ** with this case by advancing pIter past the zero-padding added by
115395
+ ** fts3EvalNearTrim2(). */
115396
+ while( pIter<pEnd && *pIter==0 ) pIter++;
115397
+
115398
+ pDL->pNextDocid = pIter;
115399
+ assert( *pIter || pIter>=&pDL->aAll[pDL->nAll] );
115400
+ *pbEof = 0;
115401
+ }
115402
+ }
115403
+
115404
+ return rc;
115405
+}
115406
+
115407
+static void fts3EvalStartReaders(
115408
+ Fts3Cursor *pCsr,
115409
+ Fts3Expr *pExpr,
115410
+ int bOptOk,
115411
+ int *pRc
115412
+){
115413
+ if( pExpr && SQLITE_OK==*pRc ){
115414
+ if( pExpr->eType==FTSQUERY_PHRASE ){
115415
+ int i;
115416
+ int nToken = pExpr->pPhrase->nToken;
115417
+ for(i=0; i<nToken; i++){
115418
+ if( pExpr->pPhrase->aToken[i].pDeferred==0 ) break;
115419
+ }
115420
+ pExpr->bDeferred = (i==nToken);
115421
+ *pRc = fts3EvalPhraseStart(pCsr, bOptOk, pExpr->pPhrase);
115422
+ }else{
115423
+ fts3EvalStartReaders(pCsr, pExpr->pLeft, bOptOk, pRc);
115424
+ fts3EvalStartReaders(pCsr, pExpr->pRight, bOptOk, pRc);
115425
+ pExpr->bDeferred = (pExpr->pLeft->bDeferred && pExpr->pRight->bDeferred);
115426
+ }
115427
+ }
115428
+}
115429
+
115430
+
115431
+typedef struct Fts3TokenAndCost Fts3TokenAndCost;
115432
+struct Fts3TokenAndCost {
115433
+ Fts3PhraseToken *pToken;
115434
+ Fts3Expr *pRoot;
115435
+ int nOvfl;
115436
+ int iCol;
115437
+};
115438
+
115439
+static void fts3EvalTokenCosts(
115440
+ Fts3Cursor *pCsr,
115441
+ Fts3Expr *pRoot,
115442
+ Fts3Expr *pExpr,
115443
+ Fts3TokenAndCost **ppTC,
115444
+ Fts3Expr ***ppOr,
115445
+ int *pRc
115446
+){
115447
+ if( *pRc==SQLITE_OK && pExpr ){
115448
+ if( pExpr->eType==FTSQUERY_PHRASE ){
115449
+ Fts3Phrase *pPhrase = pExpr->pPhrase;
115450
+ int i;
115451
+ for(i=0; *pRc==SQLITE_OK && i<pPhrase->nToken; i++){
115452
+ Fts3TokenAndCost *pTC = (*ppTC)++;
115453
+ pTC->pRoot = pRoot;
115454
+ pTC->pToken = &pPhrase->aToken[i];
115455
+ pTC->iCol = pPhrase->iColumn;
115456
+ *pRc = sqlite3Fts3MsrOvfl(pCsr, pTC->pToken->pSegcsr, &pTC->nOvfl);
115457
+ }
115458
+ }else if( pExpr->eType!=FTSQUERY_NOT ){
115459
+ if( pExpr->eType==FTSQUERY_OR ){
115460
+ pRoot = pExpr->pLeft;
115461
+ **ppOr = pRoot;
115462
+ (*ppOr)++;
115463
+ }
115464
+ fts3EvalTokenCosts(pCsr, pRoot, pExpr->pLeft, ppTC, ppOr, pRc);
115465
+ if( pExpr->eType==FTSQUERY_OR ){
115466
+ pRoot = pExpr->pRight;
115467
+ **ppOr = pRoot;
115468
+ (*ppOr)++;
115469
+ }
115470
+ fts3EvalTokenCosts(pCsr, pRoot, pExpr->pRight, ppTC, ppOr, pRc);
115471
+ }
115472
+ }
115473
+}
115474
+
115475
+static int fts3EvalAverageDocsize(Fts3Cursor *pCsr, int *pnPage){
115476
+ if( pCsr->nRowAvg==0 ){
115477
+ /* The average document size, which is required to calculate the cost
115478
+ ** of each doclist, has not yet been determined. Read the required
115479
+ ** data from the %_stat table to calculate it.
115480
+ **
115481
+ ** Entry 0 of the %_stat table is a blob containing (nCol+1) FTS3
115482
+ ** varints, where nCol is the number of columns in the FTS3 table.
115483
+ ** The first varint is the number of documents currently stored in
115484
+ ** the table. The following nCol varints contain the total amount of
115485
+ ** data stored in all rows of each column of the table, from left
115486
+ ** to right.
115487
+ */
115488
+ int rc;
115489
+ Fts3Table *p = (Fts3Table*)pCsr->base.pVtab;
115490
+ sqlite3_stmt *pStmt;
115491
+ sqlite3_int64 nDoc = 0;
115492
+ sqlite3_int64 nByte = 0;
115493
+ const char *pEnd;
115494
+ const char *a;
115495
+
115496
+ rc = sqlite3Fts3SelectDoctotal(p, &pStmt);
115497
+ if( rc!=SQLITE_OK ) return rc;
115498
+ a = sqlite3_column_blob(pStmt, 0);
115499
+ assert( a );
115500
+
115501
+ pEnd = &a[sqlite3_column_bytes(pStmt, 0)];
115502
+ a += sqlite3Fts3GetVarint(a, &nDoc);
115503
+ while( a<pEnd ){
115504
+ a += sqlite3Fts3GetVarint(a, &nByte);
115505
+ }
115506
+ if( nDoc==0 || nByte==0 ){
115507
+ sqlite3_reset(pStmt);
115508
+ return SQLITE_CORRUPT_VTAB;
115509
+ }
115510
+
115511
+ pCsr->nDoc = nDoc;
115512
+ pCsr->nRowAvg = (int)(((nByte / nDoc) + p->nPgsz) / p->nPgsz);
115513
+ assert( pCsr->nRowAvg>0 );
115514
+ rc = sqlite3_reset(pStmt);
115515
+ if( rc!=SQLITE_OK ) return rc;
115516
+ }
115517
+
115518
+ *pnPage = pCsr->nRowAvg;
115519
+ return SQLITE_OK;
115520
+}
115521
+
115522
+static int fts3EvalSelectDeferred(
115523
+ Fts3Cursor *pCsr,
115524
+ Fts3Expr *pRoot,
115525
+ Fts3TokenAndCost *aTC,
115526
+ int nTC
115527
+){
115528
+ int nDocSize = 0;
115529
+ int nDocEst = 0;
115530
+ int rc = SQLITE_OK;
115531
+ Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab;
115532
+ int ii;
115533
+
115534
+ int nOvfl = 0;
115535
+ int nTerm = 0;
115536
+
115537
+ for(ii=0; ii<nTC; ii++){
115538
+ if( aTC[ii].pRoot==pRoot ){
115539
+ nOvfl += aTC[ii].nOvfl;
115540
+ nTerm++;
115541
+ }
115542
+ }
115543
+ if( nOvfl==0 || nTerm<2 ) return SQLITE_OK;
115544
+
115545
+ rc = fts3EvalAverageDocsize(pCsr, &nDocSize);
115546
+
115547
+ for(ii=0; ii<nTerm && rc==SQLITE_OK; ii++){
115548
+ int jj;
115549
+ Fts3TokenAndCost *pTC = 0;
115550
+
115551
+ for(jj=0; jj<nTC; jj++){
115552
+ if( aTC[jj].pToken && aTC[jj].pRoot==pRoot
115553
+ && (!pTC || aTC[jj].nOvfl<pTC->nOvfl)
115554
+ ){
115555
+ pTC = &aTC[jj];
115556
+ }
115557
+ }
115558
+ assert( pTC );
115559
+
115560
+ /* At this point pTC points to the cheapest remaining token. */
115561
+ if( ii==0 ){
115562
+ if( pTC->nOvfl ){
115563
+ nDocEst = (pTC->nOvfl * pTab->nPgsz + pTab->nPgsz) / 10;
115564
+ }else{
115565
+ /* TODO: Fix this so that the doclist need not be read twice. */
115566
+ Fts3PhraseToken *pToken = pTC->pToken;
115567
+ int nList = 0;
115568
+ char *pList = 0;
115569
+ rc = fts3TermSelect(pTab, pToken, pTC->iCol, 1, &nList, &pList);
115570
+ if( rc==SQLITE_OK ){
115571
+ nDocEst = fts3DoclistCountDocids(1, pList, nList);
115572
+ }
115573
+ sqlite3_free(pList);
115574
+ if( rc==SQLITE_OK ){
115575
+ rc = sqlite3Fts3TermSegReaderCursor(pCsr,
115576
+ pToken->z, pToken->n, pToken->isPrefix, &pToken->pSegcsr
115577
+ );
115578
+ }
115579
+ }
115580
+ }else{
115581
+ if( pTC->nOvfl>=(nDocEst*nDocSize) ){
115582
+ Fts3PhraseToken *pToken = pTC->pToken;
115583
+ rc = sqlite3Fts3DeferToken(pCsr, pToken, pTC->iCol);
115584
+ fts3SegReaderCursorFree(pToken->pSegcsr);
115585
+ pToken->pSegcsr = 0;
115586
+ }
115587
+ nDocEst = 1 + (nDocEst/4);
115588
+ }
115589
+ pTC->pToken = 0;
115590
+ }
115591
+
115592
+ return rc;
115593
+}
115594
+
115595
+SQLITE_PRIVATE int sqlite3Fts3EvalStart(Fts3Cursor *pCsr, Fts3Expr *pExpr, int bOptOk){
115596
+ Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab;
115597
+ int rc = SQLITE_OK;
115598
+ int nToken = 0;
115599
+ int nOr = 0;
115600
+
115601
+ /* Allocate a MultiSegReader for each token in the expression. */
115602
+ fts3EvalAllocateReaders(pCsr, pExpr, &nToken, &nOr, &rc);
115603
+
115604
+ /* Call fts3EvalPhraseStart() on all phrases in the expression. TODO:
115605
+ ** This call will eventually also be responsible for determining which
115606
+ ** tokens are 'deferred' until the document text is loaded into memory.
115607
+ **
115608
+ ** Each token in each phrase is dealt with using one of the following
115609
+ ** three strategies:
115610
+ **
115611
+ ** 1. Entire doclist loaded into memory as part of the
115612
+ ** fts3EvalStartReaders() call.
115613
+ **
115614
+ ** 2. Doclist loaded into memory incrementally, as part of each
115615
+ ** sqlite3Fts3EvalNext() call.
115616
+ **
115617
+ ** 3. Token doclist is never loaded. Instead, documents are loaded into
115618
+ ** memory and scanned for the token as part of the sqlite3Fts3EvalNext()
115619
+ ** call. This is known as a "deferred" token.
115620
+ */
115621
+
115622
+ /* If bOptOk is true, check if there are any tokens that should be deferred.
115623
+ */
115624
+ if( rc==SQLITE_OK && bOptOk && nToken>1 && pTab->bHasStat ){
115625
+ Fts3TokenAndCost *aTC;
115626
+ Fts3Expr **apOr;
115627
+ aTC = (Fts3TokenAndCost *)sqlite3_malloc(
115628
+ sizeof(Fts3TokenAndCost) * nToken
115629
+ + sizeof(Fts3Expr *) * nOr * 2
115630
+ );
115631
+ apOr = (Fts3Expr **)&aTC[nToken];
115632
+
115633
+ if( !aTC ){
115634
+ rc = SQLITE_NOMEM;
115635
+ }else{
115636
+ int ii;
115637
+ Fts3TokenAndCost *pTC = aTC;
115638
+ Fts3Expr **ppOr = apOr;
115639
+
115640
+ fts3EvalTokenCosts(pCsr, 0, pExpr, &pTC, &ppOr, &rc);
115641
+ nToken = pTC-aTC;
115642
+ nOr = ppOr-apOr;
115643
+
115644
+ if( rc==SQLITE_OK ){
115645
+ rc = fts3EvalSelectDeferred(pCsr, 0, aTC, nToken);
115646
+ for(ii=0; rc==SQLITE_OK && ii<nOr; ii++){
115647
+ rc = fts3EvalSelectDeferred(pCsr, apOr[ii], aTC, nToken);
115648
+ }
115649
+ }
115650
+
115651
+ sqlite3_free(aTC);
115652
+ }
115653
+ }
115654
+
115655
+ fts3EvalStartReaders(pCsr, pExpr, bOptOk, &rc);
115656
+ return rc;
115657
+}
115658
+
115659
+static void fts3EvalZeroPoslist(Fts3Phrase *pPhrase){
115660
+ if( pPhrase->doclist.bFreeList ){
115661
+ sqlite3_free(pPhrase->doclist.pList);
115662
+ }
115663
+ pPhrase->doclist.pList = 0;
115664
+ pPhrase->doclist.nList = 0;
115665
+ pPhrase->doclist.bFreeList = 0;
115666
+}
115667
+
115668
+static int fts3EvalNearTrim2(
115669
+ int nNear,
115670
+ char *aTmp, /* Temporary space to use */
115671
+ char **paPoslist, /* IN/OUT: Position list */
115672
+ int *pnToken, /* IN/OUT: Tokens in phrase of *paPoslist */
115673
+ Fts3Phrase *pPhrase /* The phrase object to trim the doclist of */
115674
+){
115675
+ int nParam1 = nNear + pPhrase->nToken;
115676
+ int nParam2 = nNear + *pnToken;
115677
+ int nNew;
115678
+ char *p2;
115679
+ char *pOut;
115680
+ int res;
115681
+
115682
+ assert( pPhrase->doclist.pList );
115683
+
115684
+ p2 = pOut = pPhrase->doclist.pList;
115685
+ res = fts3PoslistNearMerge(
115686
+ &pOut, aTmp, nParam1, nParam2, paPoslist, &p2
115687
+ );
115688
+ if( res ){
115689
+ nNew = (pOut - pPhrase->doclist.pList) - 1;
115690
+ assert( pPhrase->doclist.pList[nNew]=='\0' );
115691
+ assert( nNew<=pPhrase->doclist.nList && nNew>0 );
115692
+ memset(&pPhrase->doclist.pList[nNew], 0, pPhrase->doclist.nList - nNew);
115693
+ pPhrase->doclist.nList = nNew;
115694
+ *paPoslist = pPhrase->doclist.pList;
115695
+ *pnToken = pPhrase->nToken;
115696
+ }
115697
+
115698
+ return res;
115699
+}
115700
+
115701
+static int fts3EvalNearTest(Fts3Expr *pExpr, int *pRc){
115702
+ int res = 1;
115703
+
115704
+ /* The following block runs if pExpr is the root of a NEAR query.
115705
+ ** For example, the query:
115706
+ **
115707
+ ** "w" NEAR "x" NEAR "y" NEAR "z"
115708
+ **
115709
+ ** which is represented in tree form as:
115710
+ **
115711
+ ** |
115712
+ ** +--NEAR--+ <-- root of NEAR query
115713
+ ** | |
115714
+ ** +--NEAR--+ "z"
115715
+ ** | |
115716
+ ** +--NEAR--+ "y"
115717
+ ** | |
115718
+ ** "w" "x"
115719
+ **
115720
+ ** The right-hand child of a NEAR node is always a phrase. The
115721
+ ** left-hand child may be either a phrase or a NEAR node. There are
115722
+ ** no exceptions to this.
115723
+ */
115724
+ if( *pRc==SQLITE_OK
115725
+ && pExpr->eType==FTSQUERY_NEAR
115726
+ && pExpr->bEof==0
115727
+ && (pExpr->pParent==0 || pExpr->pParent->eType!=FTSQUERY_NEAR)
115728
+ ){
115729
+ Fts3Expr *p;
115730
+ int nTmp = 0; /* Bytes of temp space */
115731
+ char *aTmp; /* Temp space for PoslistNearMerge() */
115732
+
115733
+ /* Allocate temporary working space. */
115734
+ for(p=pExpr; p->pLeft; p=p->pLeft){
115735
+ nTmp += p->pRight->pPhrase->doclist.nList;
115736
+ }
115737
+ nTmp += p->pPhrase->doclist.nList;
115738
+ aTmp = sqlite3_malloc(nTmp*2);
115739
+ if( !aTmp ){
115740
+ *pRc = SQLITE_NOMEM;
115741
+ res = 0;
115742
+ }else{
115743
+ char *aPoslist = p->pPhrase->doclist.pList;
115744
+ int nToken = p->pPhrase->nToken;
115745
+
115746
+ for(p=p->pParent;res && p && p->eType==FTSQUERY_NEAR; p=p->pParent){
115747
+ Fts3Phrase *pPhrase = p->pRight->pPhrase;
115748
+ int nNear = p->nNear;
115749
+ res = fts3EvalNearTrim2(nNear, aTmp, &aPoslist, &nToken, pPhrase);
115750
+ }
115751
+
115752
+ aPoslist = pExpr->pRight->pPhrase->doclist.pList;
115753
+ nToken = pExpr->pRight->pPhrase->nToken;
115754
+ for(p=pExpr->pLeft; p && res; p=p->pLeft){
115755
+ int nNear = p->pParent->nNear;
115756
+ Fts3Phrase *pPhrase = (
115757
+ p->eType==FTSQUERY_NEAR ? p->pRight->pPhrase : p->pPhrase
115758
+ );
115759
+ res = fts3EvalNearTrim2(nNear, aTmp, &aPoslist, &nToken, pPhrase);
115760
+ }
115761
+ }
115762
+
115763
+ sqlite3_free(aTmp);
115764
+ }
115765
+
115766
+ return res;
115767
+}
115768
+
115769
+/*
115770
+** This macro is used by the fts3EvalNext() function. The two arguments are
115771
+** 64-bit docid values. If the current query is "ORDER BY docid ASC", then
115772
+** the macro returns (i1 - i2). Or if it is "ORDER BY docid DESC", then
115773
+** it returns (i2 - i1). This allows the same code to be used for merging
115774
+** doclists in ascending or descending order.
115775
+*/
115776
+#define DOCID_CMP(i1, i2) ((pCsr->bDesc?-1:1) * (i1-i2))
115777
+
115778
+static void fts3EvalNext(
115779
+ Fts3Cursor *pCsr,
115780
+ Fts3Expr *pExpr,
115781
+ int *pRc
115782
+){
115783
+ if( *pRc==SQLITE_OK ){
115784
+ assert( pExpr->bEof==0 );
115785
+ pExpr->bStart = 1;
115786
+
115787
+ switch( pExpr->eType ){
115788
+ case FTSQUERY_NEAR:
115789
+ case FTSQUERY_AND: {
115790
+ Fts3Expr *pLeft = pExpr->pLeft;
115791
+ Fts3Expr *pRight = pExpr->pRight;
115792
+ assert( !pLeft->bDeferred || !pRight->bDeferred );
115793
+ if( pLeft->bDeferred ){
115794
+ fts3EvalNext(pCsr, pRight, pRc);
115795
+ pExpr->iDocid = pRight->iDocid;
115796
+ pExpr->bEof = pRight->bEof;
115797
+ }else if( pRight->bDeferred ){
115798
+ fts3EvalNext(pCsr, pLeft, pRc);
115799
+ pExpr->iDocid = pLeft->iDocid;
115800
+ pExpr->bEof = pLeft->bEof;
115801
+ }else{
115802
+ fts3EvalNext(pCsr, pLeft, pRc);
115803
+ fts3EvalNext(pCsr, pRight, pRc);
115804
+
115805
+ while( !pLeft->bEof && !pRight->bEof && *pRc==SQLITE_OK ){
115806
+ sqlite3_int64 iDiff = DOCID_CMP(pLeft->iDocid, pRight->iDocid);
115807
+ if( iDiff==0 ) break;
115808
+ if( iDiff<0 ){
115809
+ fts3EvalNext(pCsr, pLeft, pRc);
115810
+ }else{
115811
+ fts3EvalNext(pCsr, pRight, pRc);
115812
+ }
115813
+ }
115814
+
115815
+ pExpr->iDocid = pLeft->iDocid;
115816
+ pExpr->bEof = (pLeft->bEof || pRight->bEof);
115817
+ }
115818
+ break;
115819
+ }
115820
+
115821
+ case FTSQUERY_OR: {
115822
+ Fts3Expr *pLeft = pExpr->pLeft;
115823
+ Fts3Expr *pRight = pExpr->pRight;
115824
+ sqlite3_int64 iCmp = DOCID_CMP(pLeft->iDocid, pRight->iDocid);
115825
+
115826
+ assert( pLeft->bStart || pLeft->iDocid==pRight->iDocid );
115827
+ assert( pRight->bStart || pLeft->iDocid==pRight->iDocid );
115828
+
115829
+ if( pRight->bEof || (pLeft->bEof==0 && iCmp<0) ){
115830
+ fts3EvalNext(pCsr, pLeft, pRc);
115831
+ }else if( pLeft->bEof || (pRight->bEof==0 && iCmp>0) ){
115832
+ fts3EvalNext(pCsr, pRight, pRc);
115833
+ }else{
115834
+ fts3EvalNext(pCsr, pLeft, pRc);
115835
+ fts3EvalNext(pCsr, pRight, pRc);
115836
+ }
115837
+
115838
+ pExpr->bEof = (pLeft->bEof && pRight->bEof);
115839
+ iCmp = DOCID_CMP(pLeft->iDocid, pRight->iDocid);
115840
+ if( pRight->bEof || (pLeft->bEof==0 && iCmp<0) ){
115841
+ pExpr->iDocid = pLeft->iDocid;
115842
+ }else{
115843
+ pExpr->iDocid = pRight->iDocid;
115844
+ }
115845
+
115846
+ break;
115847
+ }
115848
+
115849
+ case FTSQUERY_NOT: {
115850
+ Fts3Expr *pLeft = pExpr->pLeft;
115851
+ Fts3Expr *pRight = pExpr->pRight;
115852
+
115853
+ if( pRight->bStart==0 ){
115854
+ fts3EvalNext(pCsr, pRight, pRc);
115855
+ assert( *pRc!=SQLITE_OK || pRight->bStart );
115856
+ }
115857
+
115858
+ fts3EvalNext(pCsr, pLeft, pRc);
115859
+ if( pLeft->bEof==0 ){
115860
+ while( !*pRc
115861
+ && !pRight->bEof
115862
+ && DOCID_CMP(pLeft->iDocid, pRight->iDocid)>0
115863
+ ){
115864
+ fts3EvalNext(pCsr, pRight, pRc);
115865
+ }
115866
+ }
115867
+ pExpr->iDocid = pLeft->iDocid;
115868
+ pExpr->bEof = pLeft->bEof;
115869
+ break;
115870
+ }
115871
+
115872
+ default: {
115873
+ Fts3Phrase *pPhrase = pExpr->pPhrase;
115874
+ fts3EvalZeroPoslist(pPhrase);
115875
+ *pRc = fts3EvalPhraseNext(pCsr, pPhrase, &pExpr->bEof);
115876
+ pExpr->iDocid = pPhrase->doclist.iDocid;
115877
+ break;
115878
+ }
115879
+ }
115880
+ }
115881
+}
115882
+
115883
+static int fts3EvalDeferredTest(Fts3Cursor *pCsr, Fts3Expr *pExpr, int *pRc){
115884
+ int bHit = 1;
115885
+ if( *pRc==SQLITE_OK ){
115886
+ switch( pExpr->eType ){
115887
+ case FTSQUERY_NEAR:
115888
+ case FTSQUERY_AND:
115889
+ bHit = (
115890
+ fts3EvalDeferredTest(pCsr, pExpr->pLeft, pRc)
115891
+ && fts3EvalDeferredTest(pCsr, pExpr->pRight, pRc)
115892
+ && fts3EvalNearTest(pExpr, pRc)
115893
+ );
115894
+
115895
+ /* If the NEAR expression does not match any rows, zero the doclist for
115896
+ ** all phrases involved in the NEAR. This is because the snippet(),
115897
+ ** offsets() and matchinfo() functions are not supposed to recognize
115898
+ ** any instances of phrases that are part of unmatched NEAR queries.
115899
+ ** For example if this expression:
115900
+ **
115901
+ ** ... MATCH 'a OR (b NEAR c)'
115902
+ **
115903
+ ** is matched against a row containing:
115904
+ **
115905
+ ** 'a b d e'
115906
+ **
115907
+ ** then any snippet() should ony highlight the "a" term, not the "b"
115908
+ ** (as "b" is part of a non-matching NEAR clause).
115909
+ */
115910
+ if( bHit==0
115911
+ && pExpr->eType==FTSQUERY_NEAR
115912
+ && (pExpr->pParent==0 || pExpr->pParent->eType!=FTSQUERY_NEAR)
115913
+ ){
115914
+ Fts3Expr *p;
115915
+ for(p=pExpr; p->pPhrase==0; p=p->pLeft){
115916
+ if( p->pRight->iDocid==pCsr->iPrevId ){
115917
+ fts3EvalZeroPoslist(p->pRight->pPhrase);
115918
+ }
115919
+ }
115920
+ if( p->iDocid==pCsr->iPrevId ){
115921
+ fts3EvalZeroPoslist(p->pPhrase);
115922
+ }
115923
+ }
115924
+
115925
+ break;
115926
+
115927
+ case FTSQUERY_OR: {
115928
+ int bHit1 = fts3EvalDeferredTest(pCsr, pExpr->pLeft, pRc);
115929
+ int bHit2 = fts3EvalDeferredTest(pCsr, pExpr->pRight, pRc);
115930
+ bHit = bHit1 || bHit2;
115931
+ break;
115932
+ }
115933
+
115934
+ case FTSQUERY_NOT:
115935
+ bHit = (
115936
+ fts3EvalDeferredTest(pCsr, pExpr->pLeft, pRc)
115937
+ && !fts3EvalDeferredTest(pCsr, pExpr->pRight, pRc)
115938
+ );
115939
+ break;
115940
+
115941
+ default: {
115942
+ if( pCsr->pDeferred
115943
+ && (pExpr->iDocid==pCsr->iPrevId || pExpr->bDeferred)
115944
+ ){
115945
+ Fts3Phrase *pPhrase = pExpr->pPhrase;
115946
+ assert( pExpr->bDeferred || pPhrase->doclist.bFreeList==0 );
115947
+ if( pExpr->bDeferred ){
115948
+ fts3EvalZeroPoslist(pPhrase);
115949
+ }
115950
+ *pRc = fts3EvalDeferredPhrase(pCsr, pPhrase);
115951
+ bHit = (pPhrase->doclist.pList!=0);
115952
+ pExpr->iDocid = pCsr->iPrevId;
115953
+ }else{
115954
+ bHit = (pExpr->bEof==0 && pExpr->iDocid==pCsr->iPrevId);
115955
+ }
115956
+ break;
115957
+ }
115958
+ }
115959
+ }
115960
+ return bHit;
115961
+}
115962
+
115963
+/*
115964
+** Return 1 if both of the following are true:
115965
+**
115966
+** 1. *pRc is SQLITE_OK when this function returns, and
115967
+**
115968
+** 2. After scanning the current FTS table row for the deferred tokens,
115969
+** it is determined that the row does not match the query.
115970
+**
115971
+** Or, if no error occurs and it seems the current row does match the FTS
115972
+** query, return 0.
115973
+*/
115974
+static int fts3EvalLoadDeferred(Fts3Cursor *pCsr, int *pRc){
115975
+ int rc = *pRc;
115976
+ int bMiss = 0;
115977
+ if( rc==SQLITE_OK ){
115978
+ if( pCsr->pDeferred ){
115979
+ rc = fts3CursorSeek(0, pCsr);
115980
+ if( rc==SQLITE_OK ){
115981
+ rc = sqlite3Fts3CacheDeferredDoclists(pCsr);
115982
+ }
115983
+ }
115984
+ bMiss = (0==fts3EvalDeferredTest(pCsr, pCsr->pExpr, &rc));
115985
+ sqlite3Fts3FreeDeferredDoclists(pCsr);
115986
+ *pRc = rc;
115987
+ }
115988
+ return (rc==SQLITE_OK && bMiss);
115989
+}
115990
+
115991
+/*
115992
+** Advance to the next document that matches the FTS expression in
115993
+** Fts3Cursor.pExpr.
115994
+*/
115995
+SQLITE_PRIVATE int sqlite3Fts3EvalNext(Fts3Cursor *pCsr){
115996
+ int rc = SQLITE_OK; /* Return Code */
115997
+ Fts3Expr *pExpr = pCsr->pExpr;
115998
+ assert( pCsr->isEof==0 );
115999
+ if( pExpr==0 ){
116000
+ pCsr->isEof = 1;
116001
+ }else{
116002
+ do {
116003
+ if( pCsr->isRequireSeek==0 ){
116004
+ sqlite3_reset(pCsr->pStmt);
116005
+ }
116006
+ assert( sqlite3_data_count(pCsr->pStmt)==0 );
116007
+ fts3EvalNext(pCsr, pExpr, &rc);
116008
+ pCsr->isEof = pExpr->bEof;
116009
+ pCsr->isRequireSeek = 1;
116010
+ pCsr->isMatchinfoNeeded = 1;
116011
+ pCsr->iPrevId = pExpr->iDocid;
116012
+ }while( pCsr->isEof==0 && fts3EvalLoadDeferred(pCsr, &rc) );
116013
+ }
116014
+ return rc;
116015
+}
116016
+
116017
+/*
116018
+** Restart interation for expression pExpr so that the next call to
116019
+** sqlite3Fts3EvalNext() visits the first row. Do not allow incremental
116020
+** loading or merging of phrase doclists for this iteration.
116021
+**
116022
+** If *pRc is other than SQLITE_OK when this function is called, it is
116023
+** a no-op. If an error occurs within this function, *pRc is set to an
116024
+** SQLite error code before returning.
116025
+*/
116026
+static void fts3EvalRestart(
116027
+ Fts3Cursor *pCsr,
116028
+ Fts3Expr *pExpr,
116029
+ int *pRc
116030
+){
116031
+ if( pExpr && *pRc==SQLITE_OK ){
116032
+ Fts3Phrase *pPhrase = pExpr->pPhrase;
116033
+
116034
+ if( pPhrase ){
116035
+ fts3EvalZeroPoslist(pPhrase);
116036
+ if( pPhrase->bIncr ){
116037
+ sqlite3Fts3EvalPhraseCleanup(pPhrase);
116038
+ memset(&pPhrase->doclist, 0, sizeof(Fts3Doclist));
116039
+ *pRc = sqlite3Fts3EvalStart(pCsr, pExpr, 0);
116040
+ }else{
116041
+ pPhrase->doclist.pNextDocid = 0;
116042
+ pPhrase->doclist.iDocid = 0;
116043
+ }
116044
+ }
116045
+
116046
+ pExpr->iDocid = 0;
116047
+ pExpr->bEof = 0;
116048
+ pExpr->bStart = 0;
116049
+
116050
+ fts3EvalRestart(pCsr, pExpr->pLeft, pRc);
116051
+ fts3EvalRestart(pCsr, pExpr->pRight, pRc);
116052
+ }
116053
+}
116054
+
116055
+/*
116056
+** After allocating the Fts3Expr.aMI[] array for each phrase in the
116057
+** expression rooted at pExpr, the cursor iterates through all rows matched
116058
+** by pExpr, calling this function for each row. This function increments
116059
+** the values in Fts3Expr.aMI[] according to the position-list currently
116060
+** found in Fts3Expr.pPhrase->doclist.pList for each of the phrase
116061
+** expression nodes.
116062
+*/
116063
+static void fts3EvalUpdateCounts(Fts3Expr *pExpr){
116064
+ if( pExpr ){
116065
+ Fts3Phrase *pPhrase = pExpr->pPhrase;
116066
+ if( pPhrase && pPhrase->doclist.pList ){
116067
+ int iCol = 0;
116068
+ char *p = pPhrase->doclist.pList;
116069
+
116070
+ assert( *p );
116071
+ while( 1 ){
116072
+ u8 c = 0;
116073
+ int iCnt = 0;
116074
+ while( 0xFE & (*p | c) ){
116075
+ if( (c&0x80)==0 ) iCnt++;
116076
+ c = *p++ & 0x80;
116077
+ }
116078
+
116079
+ /* aMI[iCol*3 + 1] = Number of occurrences
116080
+ ** aMI[iCol*3 + 2] = Number of rows containing at least one instance
116081
+ */
116082
+ pExpr->aMI[iCol*3 + 1] += iCnt;
116083
+ pExpr->aMI[iCol*3 + 2] += (iCnt>0);
116084
+ if( *p==0x00 ) break;
116085
+ p++;
116086
+ p += sqlite3Fts3GetVarint32(p, &iCol);
116087
+ }
116088
+ }
116089
+
116090
+ fts3EvalUpdateCounts(pExpr->pLeft);
116091
+ fts3EvalUpdateCounts(pExpr->pRight);
116092
+ }
116093
+}
116094
+
116095
+/*
116096
+** Expression pExpr must be of type FTSQUERY_PHRASE.
116097
+**
116098
+** If it is not already allocated and populated, this function allocates and
116099
+** populates the Fts3Expr.aMI[] array for expression pExpr. If pExpr is part
116100
+** of a NEAR expression, then it also allocates and populates the same array
116101
+** for all other phrases that are part of the NEAR expression.
116102
+**
116103
+** SQLITE_OK is returned if the aMI[] array is successfully allocated and
116104
+** populated. Otherwise, if an error occurs, an SQLite error code is returned.
116105
+*/
116106
+static int fts3EvalGatherStats(
116107
+ Fts3Cursor *pCsr, /* Cursor object */
116108
+ Fts3Expr *pExpr /* FTSQUERY_PHRASE expression */
116109
+){
116110
+ int rc = SQLITE_OK; /* Return code */
116111
+
116112
+ assert( pExpr->eType==FTSQUERY_PHRASE );
116113
+ if( pExpr->aMI==0 ){
116114
+ Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab;
116115
+ Fts3Expr *pRoot; /* Root of NEAR expression */
116116
+ Fts3Expr *p; /* Iterator used for several purposes */
116117
+
116118
+ sqlite3_int64 iPrevId = pCsr->iPrevId;
116119
+ sqlite3_int64 iDocid;
116120
+ u8 bEof;
116121
+
116122
+ /* Find the root of the NEAR expression */
116123
+ pRoot = pExpr;
116124
+ while( pRoot->pParent && pRoot->pParent->eType==FTSQUERY_NEAR ){
116125
+ pRoot = pRoot->pParent;
116126
+ }
116127
+ iDocid = pRoot->iDocid;
116128
+ bEof = pRoot->bEof;
116129
+ assert( pRoot->bStart );
116130
+
116131
+ /* Allocate space for the aMSI[] array of each FTSQUERY_PHRASE node */
116132
+ for(p=pRoot; p; p=p->pLeft){
116133
+ Fts3Expr *pE = (p->eType==FTSQUERY_PHRASE?p:p->pRight);
116134
+ assert( pE->aMI==0 );
116135
+ pE->aMI = (u32 *)sqlite3_malloc(pTab->nColumn * 3 * sizeof(u32));
116136
+ if( !pE->aMI ) return SQLITE_NOMEM;
116137
+ memset(pE->aMI, 0, pTab->nColumn * 3 * sizeof(u32));
116138
+ }
116139
+
116140
+ fts3EvalRestart(pCsr, pRoot, &rc);
116141
+
116142
+ while( pCsr->isEof==0 && rc==SQLITE_OK ){
116143
+
116144
+ do {
116145
+ /* Ensure the %_content statement is reset. */
116146
+ if( pCsr->isRequireSeek==0 ) sqlite3_reset(pCsr->pStmt);
116147
+ assert( sqlite3_data_count(pCsr->pStmt)==0 );
116148
+
116149
+ /* Advance to the next document */
116150
+ fts3EvalNext(pCsr, pRoot, &rc);
116151
+ pCsr->isEof = pRoot->bEof;
116152
+ pCsr->isRequireSeek = 1;
116153
+ pCsr->isMatchinfoNeeded = 1;
116154
+ pCsr->iPrevId = pRoot->iDocid;
116155
+ }while( pCsr->isEof==0
116156
+ && pRoot->eType==FTSQUERY_NEAR
116157
+ && fts3EvalLoadDeferred(pCsr, &rc)
116158
+ );
116159
+
116160
+ if( rc==SQLITE_OK && pCsr->isEof==0 ){
116161
+ fts3EvalUpdateCounts(pRoot);
116162
+ }
116163
+ }
116164
+
116165
+ pCsr->isEof = 0;
116166
+ pCsr->iPrevId = iPrevId;
116167
+
116168
+ if( bEof ){
116169
+ pRoot->bEof = bEof;
116170
+ }else{
116171
+ /* Caution: pRoot may iterate through docids in ascending or descending
116172
+ ** order. For this reason, even though it seems more defensive, the
116173
+ ** do loop can not be written:
116174
+ **
116175
+ ** do {...} while( pRoot->iDocid<iDocid && rc==SQLITE_OK );
116176
+ */
116177
+ fts3EvalRestart(pCsr, pRoot, &rc);
116178
+ do {
116179
+ fts3EvalNext(pCsr, pRoot, &rc);
116180
+ assert( pRoot->bEof==0 );
116181
+ }while( pRoot->iDocid!=iDocid && rc==SQLITE_OK );
116182
+ fts3EvalLoadDeferred(pCsr, &rc);
116183
+ }
116184
+ }
116185
+ return rc;
116186
+}
116187
+
116188
+/*
116189
+** This function is used by the matchinfo() module to query a phrase
116190
+** expression node for the following information:
116191
+**
116192
+** 1. The total number of occurrences of the phrase in each column of
116193
+** the FTS table (considering all rows), and
116194
+**
116195
+** 2. For each column, the number of rows in the table for which the
116196
+** column contains at least one instance of the phrase.
116197
+**
116198
+** If no error occurs, SQLITE_OK is returned and the values for each column
116199
+** written into the array aiOut as follows:
116200
+**
116201
+** aiOut[iCol*3 + 1] = Number of occurrences
116202
+** aiOut[iCol*3 + 2] = Number of rows containing at least one instance
116203
+**
116204
+** Caveats:
116205
+**
116206
+** * If a phrase consists entirely of deferred tokens, then all output
116207
+** values are set to the number of documents in the table. In other
116208
+** words we assume that very common tokens occur exactly once in each
116209
+** column of each row of the table.
116210
+**
116211
+** * If a phrase contains some deferred tokens (and some non-deferred
116212
+** tokens), count the potential occurrence identified by considering
116213
+** the non-deferred tokens instead of actual phrase occurrences.
116214
+**
116215
+** * If the phrase is part of a NEAR expression, then only phrase instances
116216
+** that meet the NEAR constraint are included in the counts.
116217
+*/
116218
+SQLITE_PRIVATE int sqlite3Fts3EvalPhraseStats(
116219
+ Fts3Cursor *pCsr, /* FTS cursor handle */
116220
+ Fts3Expr *pExpr, /* Phrase expression */
116221
+ u32 *aiOut /* Array to write results into (see above) */
116222
+){
116223
+ Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab;
116224
+ int rc = SQLITE_OK;
116225
+ int iCol;
116226
+
116227
+ if( pExpr->bDeferred && pExpr->pParent->eType!=FTSQUERY_NEAR ){
116228
+ assert( pCsr->nDoc>0 );
116229
+ for(iCol=0; iCol<pTab->nColumn; iCol++){
116230
+ aiOut[iCol*3 + 1] = pCsr->nDoc;
116231
+ aiOut[iCol*3 + 2] = pCsr->nDoc;
116232
+ }
116233
+ }else{
116234
+ rc = fts3EvalGatherStats(pCsr, pExpr);
116235
+ if( rc==SQLITE_OK ){
116236
+ assert( pExpr->aMI );
116237
+ for(iCol=0; iCol<pTab->nColumn; iCol++){
116238
+ aiOut[iCol*3 + 1] = pExpr->aMI[iCol*3 + 1];
116239
+ aiOut[iCol*3 + 2] = pExpr->aMI[iCol*3 + 2];
116240
+ }
116241
+ }
116242
+ }
116243
+
116244
+ return rc;
116245
+}
116246
+
116247
+/*
116248
+** The expression pExpr passed as the second argument to this function
116249
+** must be of type FTSQUERY_PHRASE.
116250
+**
116251
+** The returned value is either NULL or a pointer to a buffer containing
116252
+** a position-list indicating the occurrences of the phrase in column iCol
116253
+** of the current row.
116254
+**
116255
+** More specifically, the returned buffer contains 1 varint for each
116256
+** occurence of the phrase in the column, stored using the normal (delta+2)
116257
+** compression and is terminated by either an 0x01 or 0x00 byte. For example,
116258
+** if the requested column contains "a b X c d X X" and the position-list
116259
+** for 'X' is requested, the buffer returned may contain:
116260
+**
116261
+** 0x04 0x05 0x03 0x01 or 0x04 0x05 0x03 0x00
116262
+**
116263
+** This function works regardless of whether or not the phrase is deferred,
116264
+** incremental, or neither.
116265
+*/
116266
+SQLITE_PRIVATE char *sqlite3Fts3EvalPhrasePoslist(
116267
+ Fts3Cursor *pCsr, /* FTS3 cursor object */
116268
+ Fts3Expr *pExpr, /* Phrase to return doclist for */
116269
+ int iCol /* Column to return position list for */
116270
+){
116271
+ Fts3Phrase *pPhrase = pExpr->pPhrase;
116272
+ Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab;
116273
+ char *pIter = pPhrase->doclist.pList;
116274
+ int iThis;
116275
+
116276
+ assert( iCol>=0 && iCol<pTab->nColumn );
116277
+ if( !pIter
116278
+ || pExpr->bEof
116279
+ || pExpr->iDocid!=pCsr->iPrevId
116280
+ || (pPhrase->iColumn<pTab->nColumn && pPhrase->iColumn!=iCol)
116281
+ ){
116282
+ return 0;
116283
+ }
116284
+
116285
+ assert( pPhrase->doclist.nList>0 );
116286
+ if( *pIter==0x01 ){
116287
+ pIter++;
116288
+ pIter += sqlite3Fts3GetVarint32(pIter, &iThis);
116289
+ }else{
116290
+ iThis = 0;
116291
+ }
116292
+ while( iThis<iCol ){
116293
+ fts3ColumnlistCopy(0, &pIter);
116294
+ if( *pIter==0x00 ) return 0;
116295
+ pIter++;
116296
+ pIter += sqlite3Fts3GetVarint32(pIter, &iThis);
116297
+ }
116298
+
116299
+ return ((iCol==iThis)?pIter:0);
116300
+}
116301
+
116302
+/*
116303
+** Free all components of the Fts3Phrase structure that were allocated by
116304
+** the eval module. Specifically, this means to free:
116305
+**
116306
+** * the contents of pPhrase->doclist, and
116307
+** * any Fts3MultiSegReader objects held by phrase tokens.
116308
+*/
116309
+SQLITE_PRIVATE void sqlite3Fts3EvalPhraseCleanup(Fts3Phrase *pPhrase){
116310
+ if( pPhrase ){
116311
+ int i;
116312
+ sqlite3_free(pPhrase->doclist.aAll);
116313
+ fts3EvalZeroPoslist(pPhrase);
116314
+ memset(&pPhrase->doclist, 0, sizeof(Fts3Doclist));
116315
+ for(i=0; i<pPhrase->nToken; i++){
116316
+ fts3SegReaderCursorFree(pPhrase->aToken[i].pSegcsr);
116317
+ pPhrase->aToken[i].pSegcsr = 0;
116318
+ }
116319
+ }
116320
+}
116321
+
115622116322
 #endif
115623116323
115624116324
 /************** End of fts3.c ************************************************/
115625116325
 /************** Begin file fts3_aux.c ****************************************/
115626116326
 /*
@@ -115648,11 +116348,11 @@
115648116348
 Fts3Table *pFts3Tab;
115649116349
 };
115650116350
115651116351
 struct Fts3auxCursor {
115652116352
 sqlite3_vtab_cursor base; /* Base class used by SQLite core */
115653
- Fts3SegReaderCursor csr; /* Must be right after "base" */
116353
+ Fts3MultiSegReader csr; /* Must be right after "base" */
115654116354
 Fts3SegFilter filter;
115655116355
 char *zStop;
115656116356
 int nStop; /* Byte-length of string zStop */
115657116357
 int isEof; /* True if cursor is at EOF */
115658116358
 sqlite3_int64 iRowid; /* Current rowid */
@@ -115716,10 +116416,11 @@
115716116416
115717116417
 p->pFts3Tab = (Fts3Table *)&p[1];
115718116418
 p->pFts3Tab->zDb = (char *)&p->pFts3Tab[1];
115719116419
 p->pFts3Tab->zName = &p->pFts3Tab->zDb[nDb+1];
115720116420
 p->pFts3Tab->db = db;
116421
+ p->pFts3Tab->nIndex = 1;
115721116422
115722116423
 memcpy((char *)p->pFts3Tab->zDb, zDb, nDb);
115723116424
 memcpy((char *)p->pFts3Tab->zName, zFts3, nFts3);
115724116425
 sqlite3Fts3Dequote((char *)p->pFts3Tab->zName);
115725116426
@@ -115996,11 +116697,11 @@
115996116697
 pCsr->zStop = sqlite3_mprintf("%s", sqlite3_value_text(apVal[iIdx]));
115997116698
 pCsr->nStop = sqlite3_value_bytes(apVal[iIdx]);
115998116699
 if( pCsr->zStop==0 ) return SQLITE_NOMEM;
115999116700
 }
116000116701
116001
- rc = sqlite3Fts3SegReaderCursor(pFts3, FTS3_SEGCURSOR_ALL,
116702
+ rc = sqlite3Fts3SegReaderCursor(pFts3, 0, FTS3_SEGCURSOR_ALL,
116002116703
 pCsr->filter.zTerm, pCsr->filter.nTerm, 0, isScan, &pCsr->csr
116003116704
 );
116004116705
 if( rc==SQLITE_OK ){
116005116706
 rc = sqlite3Fts3SegReaderStart(pFts3, &pCsr->csr, &pCsr->filter);
116006116707
 }
@@ -116175,16 +116876,25 @@
116175116876
 ** Default span for NEAR operators.
116176116877
 */
116177116878
 #define SQLITE_FTS3_DEFAULT_NEAR_PARAM 10
116178116879
116179116880
116881
+/*
116882
+** isNot:
116883
+** This variable is used by function getNextNode(). When getNextNode() is
116884
+** called, it sets ParseContext.isNot to true if the 'next node' is a
116885
+** FTSQUERY_PHRASE with a unary "-" attached to it. i.e. "mysql" in the
116886
+** FTS3 query "sqlite -mysql". Otherwise, ParseContext.isNot is set to
116887
+** zero.
116888
+*/
116180116889
 typedef struct ParseContext ParseContext;
116181116890
 struct ParseContext {
116182116891
 sqlite3_tokenizer *pTokenizer; /* Tokenizer module */
116183116892
 const char **azCol; /* Array of column names for fts3 table */
116184116893
 int nCol; /* Number of entries in azCol[] */
116185116894
 int iDefaultCol; /* Default column to query */
116895
+ int isNot; /* True if getNextNode() sees a unary - */
116186116896
 sqlite3_context *pCtx; /* Write error message here */
116187116897
 int nNest; /* Number of nested brackets */
116188116898
 };
116189116899
116190116900
 /*
@@ -116266,11 +116976,11 @@
116266116976
 if( iEnd<n && z[iEnd]=='*' ){
116267116977
 pRet->pPhrase->aToken[0].isPrefix = 1;
116268116978
 iEnd++;
116269116979
 }
116270116980
 if( !sqlite3_fts3_enable_parentheses && iStart>0 && z[iStart-1]=='-' ){
116271
- pRet->pPhrase->isNot = 1;
116981
+ pParse->isNot = 1;
116272116982
 }
116273116983
 }
116274116984
 nConsumed = iEnd;
116275116985
 }
116276116986
@@ -116318,71 +117028,86 @@
116318117028
 Fts3Expr *p = 0;
116319117029
 sqlite3_tokenizer_cursor *pCursor = 0;
116320117030
 char *zTemp = 0;
116321117031
 int nTemp = 0;
116322117032
117033
+ const int nSpace = sizeof(Fts3Expr) + sizeof(Fts3Phrase);
117034
+ int nToken = 0;
117035
+
117036
+ /* The final Fts3Expr data structure, including the Fts3Phrase,
117037
+ ** Fts3PhraseToken structures token buffers are all stored as a single
117038
+ ** allocation so that the expression can be freed with a single call to
117039
+ ** sqlite3_free(). Setting this up requires a two pass approach.
117040
+ **
117041
+ ** The first pass, in the block below, uses a tokenizer cursor to iterate
117042
+ ** through the tokens in the expression. This pass uses fts3ReallocOrFree()
117043
+ ** to assemble data in two dynamic buffers:
117044
+ **
117045
+ ** Buffer p: Points to the Fts3Expr structure, followed by the Fts3Phrase
117046
+ ** structure, followed by the array of Fts3PhraseToken
117047
+ ** structures. This pass only populates the Fts3PhraseToken array.
117048
+ **
117049
+ ** Buffer zTemp: Contains copies of all tokens.
117050
+ **
117051
+ ** The second pass, in the block that begins "if( rc==SQLITE_DONE )" below,
117052
+ ** appends buffer zTemp to buffer p, and fills in the Fts3Expr and Fts3Phrase
117053
+ ** structures.
117054
+ */
116323117055
 rc = pModule->xOpen(pTokenizer, zInput, nInput, &pCursor);
116324117056
 if( rc==SQLITE_OK ){
116325117057
 int ii;
116326117058
 pCursor->pTokenizer = pTokenizer;
116327117059
 for(ii=0; rc==SQLITE_OK; ii++){
116328
- const char *zToken;
116329
- int nToken, iBegin, iEnd, iPos;
116330
- rc = pModule->xNext(pCursor, &zToken, &nToken, &iBegin, &iEnd, &iPos);
117060
+ const char *zByte;
117061
+ int nByte, iBegin, iEnd, iPos;
117062
+ rc = pModule->xNext(pCursor, &zByte, &nByte, &iBegin, &iEnd, &iPos);
116331117063
 if( rc==SQLITE_OK ){
116332
- int nByte = sizeof(Fts3Expr) + sizeof(Fts3Phrase);
116333
- p = fts3ReallocOrFree(p, nByte+ii*sizeof(Fts3PhraseToken));
116334
- zTemp = fts3ReallocOrFree(zTemp, nTemp + nToken);
116335
- if( !p || !zTemp ){
116336
- goto no_mem;
116337
- }
116338
- if( ii==0 ){
116339
- memset(p, 0, nByte);
116340
- p->pPhrase = (Fts3Phrase *)&p[1];
116341
- }
116342
- p->pPhrase = (Fts3Phrase *)&p[1];
116343
- memset(&p->pPhrase->aToken[ii], 0, sizeof(Fts3PhraseToken));
116344
- p->pPhrase->nToken = ii+1;
116345
- p->pPhrase->aToken[ii].n = nToken;
116346
- memcpy(&zTemp[nTemp], zToken, nToken);
116347
- nTemp += nToken;
116348
- if( iEnd<nInput && zInput[iEnd]=='*' ){
116349
- p->pPhrase->aToken[ii].isPrefix = 1;
116350
- }else{
116351
- p->pPhrase->aToken[ii].isPrefix = 0;
116352
- }
117064
+ Fts3PhraseToken *pToken;
117065
+
117066
+ p = fts3ReallocOrFree(p, nSpace + ii*sizeof(Fts3PhraseToken));
117067
+ if( !p ) goto no_mem;
117068
+
117069
+ zTemp = fts3ReallocOrFree(zTemp, nTemp + nByte);
117070
+ if( !zTemp ) goto no_mem;
117071
+
117072
+ assert( nToken==ii );
117073
+ pToken = &((Fts3Phrase *)(&p[1]))->aToken[ii];
117074
+ memset(pToken, 0, sizeof(Fts3PhraseToken));
117075
+
117076
+ memcpy(&zTemp[nTemp], zByte, nByte);
117077
+ nTemp += nByte;
117078
+
117079
+ pToken->n = nByte;
117080
+ pToken->isPrefix = (iEnd<nInput && zInput[iEnd]=='*');
117081
+ nToken = ii+1;
116353117082
 }
116354117083
 }
116355117084
116356117085
 pModule->xClose(pCursor);
116357117086
 pCursor = 0;
116358117087
 }
116359117088
116360117089
 if( rc==SQLITE_DONE ){
116361117090
 int jj;
116362
- char *zNew = NULL;
116363
- int nNew = 0;
116364
- int nByte = sizeof(Fts3Expr) + sizeof(Fts3Phrase);
116365
- nByte += (p?(p->pPhrase->nToken-1):0) * sizeof(Fts3PhraseToken);
116366
- p = fts3ReallocOrFree(p, nByte + nTemp);
116367
- if( !p ){
116368
- goto no_mem;
116369
- }
116370
- if( zTemp ){
116371
- zNew = &(((char *)p)[nByte]);
116372
- memcpy(zNew, zTemp, nTemp);
116373
- }else{
116374
- memset(p, 0, nByte+nTemp);
116375
- }
117091
+ char *zBuf = 0;
117092
+
117093
+ p = fts3ReallocOrFree(p, nSpace + nToken*sizeof(Fts3PhraseToken) + nTemp);
117094
+ if( !p ) goto no_mem;
117095
+ memset(p, 0, (char *)&(((Fts3Phrase *)&p[1])->aToken[0])-(char *)p);
117096
+ p->eType = FTSQUERY_PHRASE;
116376117097
 p->pPhrase = (Fts3Phrase *)&p[1];
117098
+ p->pPhrase->iColumn = pParse->iDefaultCol;
117099
+ p->pPhrase->nToken = nToken;
117100
+
117101
+ zBuf = (char *)&p->pPhrase->aToken[nToken];
117102
+ memcpy(zBuf, zTemp, nTemp);
117103
+ sqlite3_free(zTemp);
117104
+
116377117105
 for(jj=0; jj<p->pPhrase->nToken; jj++){
116378
- p->pPhrase->aToken[jj].z = &zNew[nNew];
116379
- nNew += p->pPhrase->aToken[jj].n;
117106
+ p->pPhrase->aToken[jj].z = zBuf;
117107
+ zBuf += p->pPhrase->aToken[jj].n;
116380117108
 }
116381
- sqlite3_free(zTemp);
116382
- p->eType = FTSQUERY_PHRASE;
116383
- p->pPhrase->iColumn = pParse->iDefaultCol;
116384117109
 rc = SQLITE_OK;
116385117110
 }
116386117111
116387117112
 *ppExpr = p;
116388117113
 return rc;
@@ -116434,10 +117159,12 @@
116434117159
 int rc;
116435117160
 Fts3Expr *pRet = 0;
116436117161
116437117162
 const char *zInput = z;
116438117163
 int nInput = n;
117164
+
117165
+ pParse->isNot = 0;
116439117166
116440117167
 /* Skip over any whitespace before checking for a keyword, an open or
116441117168
 ** close bracket, or a quoted string.
116442117169
 */
116443117170
 while( nInput>0 && fts3isspace(*zInput) ){
@@ -116653,11 +117380,11 @@
116653117380
 rc = getNextNode(pParse, zIn, nIn, &p, &nByte);
116654117381
 if( rc==SQLITE_OK ){
116655117382
 int isPhrase;
116656117383
116657117384
 if( !sqlite3_fts3_enable_parentheses
116658
- && p->eType==FTSQUERY_PHRASE && p->pPhrase->isNot
117385
+ && p->eType==FTSQUERY_PHRASE && pParse->isNot
116659117386
 ){
116660117387
 /* Create an implicit NOT operator. */
116661117388
 Fts3Expr *pNot = fts3MallocZero(sizeof(Fts3Expr));
116662117389
 if( !pNot ){
116663117390
 sqlite3Fts3ExprFree(p);
@@ -116671,11 +117398,10 @@
116671117398
 }
116672117399
 pNotBranch = pNot;
116673117400
 p = pPrev;
116674117401
 }else{
116675117402
 int eType = p->eType;
116676
- assert( eType!=FTSQUERY_PHRASE || !p->pPhrase->isNot );
116677117403
 isPhrase = (eType==FTSQUERY_PHRASE || p->pLeft);
116678117404
116679117405
 /* The isRequirePhrase variable is set to true if a phrase or
116680117406
 ** an expression contained in parenthesis is required. If a
116681117407
 ** binary operator (AND, OR, NOT or NEAR) is encounted when
@@ -116834,13 +117560,15 @@
116834117560
 /*
116835117561
 ** Free a parsed fts3 query expression allocated by sqlite3Fts3ExprParse().
116836117562
 */
116837117563
 SQLITE_PRIVATE void sqlite3Fts3ExprFree(Fts3Expr *p){
116838117564
 if( p ){
117565
+ assert( p->eType==FTSQUERY_PHRASE || p->pPhrase==0 );
116839117566
 sqlite3Fts3ExprFree(p->pLeft);
116840117567
 sqlite3Fts3ExprFree(p->pRight);
116841
- sqlite3_free(p->aDoclist);
117568
+ sqlite3Fts3EvalPhraseCleanup(p->pPhrase);
117569
+ sqlite3_free(p->aMI);
116842117570
 sqlite3_free(p);
116843117571
 }
116844117572
 }
116845117573
116846117574
 /****************************************************************************
@@ -116893,11 +117621,11 @@
116893117621
 switch( pExpr->eType ){
116894117622
 case FTSQUERY_PHRASE: {
116895117623
 Fts3Phrase *pPhrase = pExpr->pPhrase;
116896117624
 int i;
116897117625
 zBuf = sqlite3_mprintf(
116898
- "%zPHRASE %d %d", zBuf, pPhrase->iColumn, pPhrase->isNot);
117626
+ "%zPHRASE %d 0", zBuf, pPhrase->iColumn);
116899117627
 for(i=0; zBuf && i<pPhrase->nToken; i++){
116900117628
 zBuf = sqlite3_mprintf("%z %.*s%s", zBuf,
116901117629
 pPhrase->aToken[i].n, pPhrase->aToken[i].z,
116902117630
 (pPhrase->aToken[i].isPrefix?"+":"")
116903117631
 );
@@ -118811,18 +119539,44 @@
118811119539
 ** it is always safe to read up to two varints from it without risking an
118812119540
 ** overread, even if the node data is corrupted.
118813119541
 */
118814119542
 #define FTS3_NODE_PADDING (FTS3_VARINT_MAX*2)
118815119543
119544
+/*
119545
+** Under certain circumstances, b-tree nodes (doclists) can be loaded into
119546
+** memory incrementally instead of all at once. This can be a big performance
119547
+** win (reduced IO and CPU) if SQLite stops calling the virtual table xNext()
119548
+** method before retrieving all query results (as may happen, for example,
119549
+** if a query has a LIMIT clause).
119550
+**
119551
+** Incremental loading is used for b-tree nodes FTS3_NODE_CHUNK_THRESHOLD
119552
+** bytes and larger. Nodes are loaded in chunks of FTS3_NODE_CHUNKSIZE bytes.
119553
+** The code is written so that the hard lower-limit for each of these values
119554
+** is 1. Clearly such small values would be inefficient, but can be useful
119555
+** for testing purposes.
119556
+**
119557
+** If this module is built with SQLITE_TEST defined, these constants may
119558
+** be overridden at runtime for testing purposes. File fts3_test.c contains
119559
+** a Tcl interface to read and write the values.
119560
+*/
119561
+#ifdef SQLITE_TEST
119562
+int test_fts3_node_chunksize = (4*1024);
119563
+int test_fts3_node_chunk_threshold = (4*1024)*4;
119564
+# define FTS3_NODE_CHUNKSIZE test_fts3_node_chunksize
119565
+# define FTS3_NODE_CHUNK_THRESHOLD test_fts3_node_chunk_threshold
119566
+#else
119567
+# define FTS3_NODE_CHUNKSIZE (4*1024)
119568
+# define FTS3_NODE_CHUNK_THRESHOLD (FTS3_NODE_CHUNKSIZE*4)
119569
+#endif
119570
+
118816119571
 typedef struct PendingList PendingList;
118817119572
 typedef struct SegmentNode SegmentNode;
118818119573
 typedef struct SegmentWriter SegmentWriter;
118819119574
118820119575
 /*
118821
-** Data structure used while accumulating terms in the pending-terms hash
118822
-** table. The hash table entry maps from term (a string) to a malloc'd
118823
-** instance of this structure.
119576
+** An instance of the following data structure is used to build doclists
119577
+** incrementally. See function fts3PendingListAppend() for details.
118824119578
 */
118825119579
 struct PendingList {
118826119580
 int nData;
118827119581
 char *aData;
118828119582
 int nSpace;
@@ -118849,11 +119603,10 @@
118849119603
 ** of type Fts3SegReader* are also used by code in fts3.c to iterate through
118850119604
 ** terms when querying the full-text index. See functions:
118851119605
 **
118852119606
 ** sqlite3Fts3SegReaderNew()
118853119607
 ** sqlite3Fts3SegReaderFree()
118854
-** sqlite3Fts3SegReaderCost()
118855119608
 ** sqlite3Fts3SegReaderIterate()
118856119609
 **
118857119610
 ** Methods used to manipulate Fts3SegReader structures:
118858119611
 **
118859119612
 ** fts3SegReaderNext()
@@ -118868,10 +119621,13 @@
118868119621
 sqlite3_int64 iEndBlock; /* Rowid of final block in segment (or 0) */
118869119622
 sqlite3_int64 iCurrentBlock; /* Current leaf block (or 0) */
118870119623
118871119624
 char *aNode; /* Pointer to node data (or NULL) */
118872119625
 int nNode; /* Size of buffer at aNode (or 0) */
119626
+ int nPopulate; /* If >0, bytes of buffer aNode[] loaded */
119627
+ sqlite3_blob *pBlob; /* If not NULL, blob handle to read node */
119628
+
118873119629
 Fts3HashElem **ppNextElem;
118874119630
118875119631
 /* Variables set by fts3SegReaderNext(). These may be read directly
118876119632
 ** by the caller. They are valid from the time SegmentReaderNew() returns
118877119633
 ** until SegmentReaderNext() returns something other than SQLITE_OK
@@ -118881,12 +119637,15 @@
118881119637
 char *zTerm; /* Pointer to current term */
118882119638
 int nTermAlloc; /* Allocated size of zTerm buffer */
118883119639
 char *aDoclist; /* Pointer to doclist of current entry */
118884119640
 int nDoclist; /* Size of doclist in current entry */
118885119641
118886
- /* The following variables are used to iterate through the current doclist */
119642
+ /* The following variables are used by fts3SegReaderNextDocid() to iterate
119643
+ ** through the current doclist (aDoclist/nDoclist).
119644
+ */
118887119645
 char *pOffsetList;
119646
+ int nOffsetList; /* For descending pending seg-readers only */
118888119647
 sqlite3_int64 iDocid;
118889119648
 };
118890119649
118891119650
 #define fts3SegReaderIsPending(p) ((p)->ppNextElem!=0)
118892119651
 #define fts3SegReaderIsRootOnly(p) ((p)->aNode==(char *)&(p)[1])
@@ -118920,10 +119679,18 @@
118920119679
 ** within the fts3SegWriterXXX() family of functions described above.
118921119680
 **
118922119681
 ** fts3NodeAddTerm()
118923119682
 ** fts3NodeWrite()
118924119683
 ** fts3NodeFree()
119684
+**
119685
+** When a b+tree is written to the database (either as a result of a merge
119686
+** or the pending-terms table being flushed), leaves are written into the
119687
+** database file as soon as they are completely populated. The interior of
119688
+** the tree is assembled in memory and written out only once all leaves have
119689
+** been populated and stored. This is Ok, as the b+-tree fanout is usually
119690
+** very large, meaning that the interior of the tree consumes relatively
119691
+** little memory.
118925119692
 */
118926119693
 struct SegmentNode {
118927119694
 SegmentNode *pParent; /* Parent node (or NULL for root node) */
118928119695
 SegmentNode *pRight; /* Pointer to right-sibling */
118929119696
 SegmentNode *pLeftmost; /* Pointer to left-most node of this depth */
@@ -118950,21 +119717,26 @@
118950119717
 #define SQL_NEXT_SEGMENT_INDEX 8
118951119718
 #define SQL_INSERT_SEGMENTS 9
118952119719
 #define SQL_NEXT_SEGMENTS_ID 10
118953119720
 #define SQL_INSERT_SEGDIR 11
118954119721
 #define SQL_SELECT_LEVEL 12
118955
-#define SQL_SELECT_ALL_LEVEL 13
119722
+#define SQL_SELECT_LEVEL_RANGE 13
118956119723
 #define SQL_SELECT_LEVEL_COUNT 14
118957
-#define SQL_SELECT_SEGDIR_COUNT_MAX 15
118958
-#define SQL_DELETE_SEGDIR_BY_LEVEL 16
119724
+#define SQL_SELECT_SEGDIR_MAX_LEVEL 15
119725
+#define SQL_DELETE_SEGDIR_LEVEL 16
118959119726
 #define SQL_DELETE_SEGMENTS_RANGE 17
118960119727
 #define SQL_CONTENT_INSERT 18
118961119728
 #define SQL_DELETE_DOCSIZE 19
118962119729
 #define SQL_REPLACE_DOCSIZE 20
118963119730
 #define SQL_SELECT_DOCSIZE 21
118964119731
 #define SQL_SELECT_DOCTOTAL 22
118965119732
 #define SQL_REPLACE_DOCTOTAL 23
119733
+
119734
+#define SQL_SELECT_ALL_PREFIX_LEVEL 24
119735
+#define SQL_DELETE_ALL_TERMS_SEGDIR 25
119736
+
119737
+#define SQL_DELETE_SEGDIR_RANGE 26
118966119738
118967119739
 /*
118968119740
 ** This function is used to obtain an SQLite prepared statement handle
118969119741
 ** for the statement identified by the second argument. If successful,
118970119742
 ** *pp is set to the requested statement handle and SQLITE_OK returned.
@@ -118997,23 +119769,29 @@
118997119769
118998119770
 /* Return segments in order from oldest to newest.*/
118999119771
 /* 12 */ "SELECT idx, start_block, leaves_end_block, end_block, root "
119000119772
 "FROM %Q.'%q_segdir' WHERE level = ? ORDER BY idx ASC",
119001119773
 /* 13 */ "SELECT idx, start_block, leaves_end_block, end_block, root "
119002
- "FROM %Q.'%q_segdir' ORDER BY level DESC, idx ASC",
119774
+ "FROM %Q.'%q_segdir' WHERE level BETWEEN ? AND ?"
119775
+ "ORDER BY level DESC, idx ASC",
119003119776
119004119777
 /* 14 */ "SELECT count(*) FROM %Q.'%q_segdir' WHERE level = ?",
119005
-/* 15 */ "SELECT count(*), max(level) FROM %Q.'%q_segdir'",
119778
+/* 15 */ "SELECT max(level) FROM %Q.'%q_segdir' WHERE level BETWEEN ? AND ?",
119006119779
119007119780
 /* 16 */ "DELETE FROM %Q.'%q_segdir' WHERE level = ?",
119008119781
 /* 17 */ "DELETE FROM %Q.'%q_segments' WHERE blockid BETWEEN ? AND ?",
119009119782
 /* 18 */ "INSERT INTO %Q.'%q_content' VALUES(%s)",
119010119783
 /* 19 */ "DELETE FROM %Q.'%q_docsize' WHERE docid = ?",
119011119784
 /* 20 */ "REPLACE INTO %Q.'%q_docsize' VALUES(?,?)",
119012119785
 /* 21 */ "SELECT size FROM %Q.'%q_docsize' WHERE docid=?",
119013119786
 /* 22 */ "SELECT value FROM %Q.'%q_stat' WHERE id=0",
119014119787
 /* 23 */ "REPLACE INTO %Q.'%q_stat' VALUES(0,?)",
119788
+/* 24 */ "",
119789
+/* 25 */ "",
119790
+
119791
+/* 26 */ "DELETE FROM %Q.'%q_segdir' WHERE level BETWEEN ? AND ?",
119792
+
119015119793
 };
119016119794
 int rc = SQLITE_OK;
119017119795
 sqlite3_stmt *pStmt;
119018119796
119019119797
 assert( SizeofArray(azSql)==SizeofArray(p->aStmt) );
@@ -119165,18 +119943,36 @@
119165119943
 ** 1: start_block
119166119944
 ** 2: leaves_end_block
119167119945
 ** 3: end_block
119168119946
 ** 4: root
119169119947
 */
119170
-SQLITE_PRIVATE int sqlite3Fts3AllSegdirs(Fts3Table *p, int iLevel, sqlite3_stmt **ppStmt){
119948
+SQLITE_PRIVATE int sqlite3Fts3AllSegdirs(
119949
+ Fts3Table *p, /* FTS3 table */
119950
+ int iIndex, /* Index for p->aIndex[] */
119951
+ int iLevel, /* Level to select */
119952
+ sqlite3_stmt **ppStmt /* OUT: Compiled statement */
119953
+){
119171119954
 int rc;
119172119955
 sqlite3_stmt *pStmt = 0;
119956
+
119957
+ assert( iLevel==FTS3_SEGCURSOR_ALL || iLevel>=0 );
119958
+ assert( iLevel<FTS3_SEGDIR_MAXLEVEL );
119959
+ assert( iIndex>=0 && iIndex<p->nIndex );
119960
+
119173119961
 if( iLevel<0 ){
119174
- rc = fts3SqlStmt(p, SQL_SELECT_ALL_LEVEL, &pStmt, 0);
119962
+ /* "SELECT * FROM %_segdir WHERE level BETWEEN ? AND ? ORDER BY ..." */
119963
+ rc = fts3SqlStmt(p, SQL_SELECT_LEVEL_RANGE, &pStmt, 0);
119964
+ if( rc==SQLITE_OK ){
119965
+ sqlite3_bind_int(pStmt, 1, iIndex*FTS3_SEGDIR_MAXLEVEL);
119966
+ sqlite3_bind_int(pStmt, 2, (iIndex+1)*FTS3_SEGDIR_MAXLEVEL-1);
119967
+ }
119175119968
 }else{
119969
+ /* "SELECT * FROM %_segdir WHERE level = ? ORDER BY ..." */
119176119970
 rc = fts3SqlStmt(p, SQL_SELECT_LEVEL, &pStmt, 0);
119177
- if( rc==SQLITE_OK ) sqlite3_bind_int(pStmt, 1, iLevel);
119971
+ if( rc==SQLITE_OK ){
119972
+ sqlite3_bind_int(pStmt, 1, iLevel+iIndex*FTS3_SEGDIR_MAXLEVEL);
119973
+ }
119178119974
 }
119179119975
 *ppStmt = pStmt;
119180119976
 return rc;
119181119977
 }
119182119978
@@ -119286,10 +120082,51 @@
119286120082
 *pp = p;
119287120083
 return 1;
119288120084
 }
119289120085
 return 0;
119290120086
 }
120087
+
120088
+/*
120089
+** Free a PendingList object allocated by fts3PendingListAppend().
120090
+*/
120091
+static void fts3PendingListDelete(PendingList *pList){
120092
+ sqlite3_free(pList);
120093
+}
120094
+
120095
+/*
120096
+** Add an entry to one of the pending-terms hash tables.
120097
+*/
120098
+static int fts3PendingTermsAddOne(
120099
+ Fts3Table *p,
120100
+ int iCol,
120101
+ int iPos,
120102
+ Fts3Hash *pHash, /* Pending terms hash table to add entry to */
120103
+ const char *zToken,
120104
+ int nToken
120105
+){
120106
+ PendingList *pList;
120107
+ int rc = SQLITE_OK;
120108
+
120109
+ pList = (PendingList *)fts3HashFind(pHash, zToken, nToken);
120110
+ if( pList ){
120111
+ p->nPendingData -= (pList->nData + nToken + sizeof(Fts3HashElem));
120112
+ }
120113
+ if( fts3PendingListAppend(&pList, p->iPrevDocid, iCol, iPos, &rc) ){
120114
+ if( pList==fts3HashInsert(pHash, zToken, nToken, pList) ){
120115
+ /* Malloc failed while inserting the new entry. This can only
120116
+ ** happen if there was no previous entry for this token.
120117
+ */
120118
+ assert( 0==fts3HashFind(pHash, zToken, nToken) );
120119
+ sqlite3_free(pList);
120120
+ rc = SQLITE_NOMEM;
120121
+ }
120122
+ }
120123
+ if( rc==SQLITE_OK ){
120124
+ p->nPendingData += (pList->nData + nToken + sizeof(Fts3HashElem));
120125
+ }
120126
+ return rc;
120127
+}
119291120128
119292120129
 /*
119293120130
 ** Tokenize the nul-terminated string zText and add all tokens to the
119294120131
 ** pending-terms hash-table. The docid used is that currently stored in
119295120132
 ** p->iPrevDocid, and the column is specified by argument iCol.
@@ -119335,12 +120172,11 @@
119335120172
119336120173
 xNext = pModule->xNext;
119337120174
 while( SQLITE_OK==rc
119338120175
 && SQLITE_OK==(rc = xNext(pCsr, &zToken, &nToken, &iStart, &iEnd, &iPos))
119339120176
 ){
119340
- PendingList *pList;
119341
-
120177
+ int i;
119342120178
 if( iPos>=nWord ) nWord = iPos+1;
119343120179
119344120180
 /* Positions cannot be negative; we use -1 as a terminator internally.
119345120181
 ** Tokens must have a non-zero length.
119346120182
 */
@@ -119347,26 +120183,23 @@
119347120183
 if( iPos<0 || !zToken || nToken<=0 ){
119348120184
 rc = SQLITE_ERROR;
119349120185
 break;
119350120186
 }
119351120187
119352
- pList = (PendingList *)fts3HashFind(&p->pendingTerms, zToken, nToken);
119353
- if( pList ){
119354
- p->nPendingData -= (pList->nData + nToken + sizeof(Fts3HashElem));
119355
- }
119356
- if( fts3PendingListAppend(&pList, p->iPrevDocid, iCol, iPos, &rc) ){
119357
- if( pList==fts3HashInsert(&p->pendingTerms, zToken, nToken, pList) ){
119358
- /* Malloc failed while inserting the new entry. This can only
119359
- ** happen if there was no previous entry for this token.
119360
- */
119361
- assert( 0==fts3HashFind(&p->pendingTerms, zToken, nToken) );
119362
- sqlite3_free(pList);
119363
- rc = SQLITE_NOMEM;
119364
- }
119365
- }
119366
- if( rc==SQLITE_OK ){
119367
- p->nPendingData += (pList->nData + nToken + sizeof(Fts3HashElem));
120188
+ /* Add the term to the terms index */
120189
+ rc = fts3PendingTermsAddOne(
120190
+ p, iCol, iPos, &p->aIndex[0].hPending, zToken, nToken
120191
+ );
120192
+
120193
+ /* Add the term to each of the prefix indexes that it is not too
120194
+ ** short for. */
120195
+ for(i=1; rc==SQLITE_OK && i<p->nIndex; i++){
120196
+ struct Fts3Index *pIndex = &p->aIndex[i];
120197
+ if( nToken<pIndex->nPrefix ) continue;
120198
+ rc = fts3PendingTermsAddOne(
120199
+ p, iCol, iPos, &pIndex->hPending, zToken, pIndex->nPrefix
120200
+ );
119368120201
 }
119369120202
 }
119370120203
119371120204
 pModule->xClose(pCsr);
119372120205
 *pnWord = nWord;
@@ -119392,18 +120225,23 @@
119392120225
 p->iPrevDocid = iDocid;
119393120226
 return SQLITE_OK;
119394120227
 }
119395120228
119396120229
 /*
119397
-** Discard the contents of the pending-terms hash table.
120230
+** Discard the contents of the pending-terms hash tables.
119398120231
 */
119399120232
 SQLITE_PRIVATE void sqlite3Fts3PendingTermsClear(Fts3Table *p){
119400
- Fts3HashElem *pElem;
119401
- for(pElem=fts3HashFirst(&p->pendingTerms); pElem; pElem=fts3HashNext(pElem)){
119402
- sqlite3_free(fts3HashData(pElem));
120233
+ int i;
120234
+ for(i=0; i<p->nIndex; i++){
120235
+ Fts3HashElem *pElem;
120236
+ Fts3Hash *pHash = &p->aIndex[i].hPending;
120237
+ for(pElem=fts3HashFirst(pHash); pElem; pElem=fts3HashNext(pElem)){
120238
+ PendingList *pList = (PendingList *)fts3HashData(pElem);
120239
+ fts3PendingListDelete(pList);
120240
+ }
120241
+ fts3HashClear(pHash);
119403120242
 }
119404
- fts3HashClear(&p->pendingTerms);
119405120243
 p->nPendingData = 0;
119406120244
 }
119407120245
119408120246
 /*
119409120247
 ** This function is called by the xUpdate() method as part of an INSERT
@@ -119555,11 +120393,11 @@
119555120393
119556120394
 /*
119557120395
 ** Forward declaration to account for the circular dependency between
119558120396
 ** functions fts3SegmentMerge() and fts3AllocateSegdirIdx().
119559120397
 */
119560
-static int fts3SegmentMerge(Fts3Table *, int);
120398
+static int fts3SegmentMerge(Fts3Table *, int, int);
119561120399
119562120400
 /*
119563120401
 ** This function allocates a new level iLevel index in the segdir table.
119564120402
 ** Usually, indexes are allocated within a level sequentially starting
119565120403
 ** with 0, so the allocated index is one greater than the value returned
@@ -119572,19 +120410,24 @@
119572120410
 ** allocated index is 0.
119573120411
 **
119574120412
 ** If successful, *piIdx is set to the allocated index slot and SQLITE_OK
119575120413
 ** returned. Otherwise, an SQLite error code is returned.
119576120414
 */
119577
-static int fts3AllocateSegdirIdx(Fts3Table *p, int iLevel, int *piIdx){
120415
+static int fts3AllocateSegdirIdx(
120416
+ Fts3Table *p,
120417
+ int iIndex, /* Index for p->aIndex */
120418
+ int iLevel,
120419
+ int *piIdx
120420
+){
119578120421
 int rc; /* Return Code */
119579120422
 sqlite3_stmt *pNextIdx; /* Query for next idx at level iLevel */
119580120423
 int iNext = 0; /* Result of query pNextIdx */
119581120424
119582120425
 /* Set variable iNext to the next available segdir index at level iLevel. */
119583120426
 rc = fts3SqlStmt(p, SQL_NEXT_SEGMENT_INDEX, &pNextIdx, 0);
119584120427
 if( rc==SQLITE_OK ){
119585
- sqlite3_bind_int(pNextIdx, 1, iLevel);
120428
+ sqlite3_bind_int(pNextIdx, 1, iIndex*FTS3_SEGDIR_MAXLEVEL + iLevel);
119586120429
 if( SQLITE_ROW==sqlite3_step(pNextIdx) ){
119587120430
 iNext = sqlite3_column_int(pNextIdx, 0);
119588120431
 }
119589120432
 rc = sqlite3_reset(pNextIdx);
119590120433
 }
@@ -119594,11 +120437,11 @@
119594120437
 ** full, merge all segments in level iLevel into a single iLevel+1
119595120438
 ** segment and allocate (newly freed) index 0 at level iLevel. Otherwise,
119596120439
 ** if iNext is less than FTS3_MERGE_COUNT, allocate index iNext.
119597120440
 */
119598120441
 if( iNext>=FTS3_MERGE_COUNT ){
119599
- rc = fts3SegmentMerge(p, iLevel);
120442
+ rc = fts3SegmentMerge(p, iIndex, iLevel);
119600120443
 *piIdx = 0;
119601120444
 }else{
119602120445
 *piIdx = iNext;
119603120446
 }
119604120447
 }
@@ -119635,11 +120478,12 @@
119635120478
 */
119636120479
 SQLITE_PRIVATE int sqlite3Fts3ReadBlock(
119637120480
 Fts3Table *p, /* FTS3 table handle */
119638120481
 sqlite3_int64 iBlockid, /* Access the row with blockid=$iBlockid */
119639120482
 char **paBlob, /* OUT: Blob data in malloc'd buffer */
119640
- int *pnBlob /* OUT: Size of blob data */
120483
+ int *pnBlob, /* OUT: Size of blob data */
120484
+ int *pnLoad /* OUT: Bytes actually loaded */
119641120485
 ){
119642120486
 int rc; /* Return code */
119643120487
119644120488
 /* pnBlob must be non-NULL. paBlob may be NULL or non-NULL. */
119645120489
 assert( pnBlob);
@@ -119656,25 +120500,29 @@
119656120500
 );
119657120501
 }
119658120502
119659120503
 if( rc==SQLITE_OK ){
119660120504
 int nByte = sqlite3_blob_bytes(p->pSegments);
120505
+ *pnBlob = nByte;
119661120506
 if( paBlob ){
119662120507
 char *aByte = sqlite3_malloc(nByte + FTS3_NODE_PADDING);
119663120508
 if( !aByte ){
119664120509
 rc = SQLITE_NOMEM;
119665120510
 }else{
120511
+ if( pnLoad && nByte>(FTS3_NODE_CHUNK_THRESHOLD) ){
120512
+ nByte = FTS3_NODE_CHUNKSIZE;
120513
+ *pnLoad = nByte;
120514
+ }
119666120515
 rc = sqlite3_blob_read(p->pSegments, aByte, nByte, 0);
119667120516
 memset(&aByte[nByte], 0, FTS3_NODE_PADDING);
119668120517
 if( rc!=SQLITE_OK ){
119669120518
 sqlite3_free(aByte);
119670120519
 aByte = 0;
119671120520
 }
119672120521
 }
119673120522
 *paBlob = aByte;
119674120523
 }
119675
- *pnBlob = nByte;
119676120524
 }
119677120525
119678120526
 return rc;
119679120527
 }
119680120528
@@ -119684,17 +120532,59 @@
119684120532
 */
119685120533
 SQLITE_PRIVATE void sqlite3Fts3SegmentsClose(Fts3Table *p){
119686120534
 sqlite3_blob_close(p->pSegments);
119687120535
 p->pSegments = 0;
119688120536
 }
120537
+
120538
+static int fts3SegReaderIncrRead(Fts3SegReader *pReader){
120539
+ int nRead; /* Number of bytes to read */
120540
+ int rc; /* Return code */
120541
+
120542
+ nRead = MIN(pReader->nNode - pReader->nPopulate, FTS3_NODE_CHUNKSIZE);
120543
+ rc = sqlite3_blob_read(
120544
+ pReader->pBlob,
120545
+ &pReader->aNode[pReader->nPopulate],
120546
+ nRead,
120547
+ pReader->nPopulate
120548
+ );
120549
+
120550
+ if( rc==SQLITE_OK ){
120551
+ pReader->nPopulate += nRead;
120552
+ memset(&pReader->aNode[pReader->nPopulate], 0, FTS3_NODE_PADDING);
120553
+ if( pReader->nPopulate==pReader->nNode ){
120554
+ sqlite3_blob_close(pReader->pBlob);
120555
+ pReader->pBlob = 0;
120556
+ pReader->nPopulate = 0;
120557
+ }
120558
+ }
120559
+ return rc;
120560
+}
120561
+
120562
+static int fts3SegReaderRequire(Fts3SegReader *pReader, char *pFrom, int nByte){
120563
+ int rc = SQLITE_OK;
120564
+ assert( !pReader->pBlob
120565
+ || (pFrom>=pReader->aNode && pFrom<&pReader->aNode[pReader->nNode])
120566
+ );
120567
+ while( pReader->pBlob && rc==SQLITE_OK
120568
+ && (pFrom - pReader->aNode + nByte)>pReader->nPopulate
120569
+ ){
120570
+ rc = fts3SegReaderIncrRead(pReader);
120571
+ }
120572
+ return rc;
120573
+}
119689120574
119690120575
 /*
119691120576
 ** Move the iterator passed as the first argument to the next term in the
119692120577
 ** segment. If successful, SQLITE_OK is returned. If there is no next term,
119693120578
 ** SQLITE_DONE. Otherwise, an SQLite error code.
119694120579
 */
119695
-static int fts3SegReaderNext(Fts3Table *p, Fts3SegReader *pReader){
120580
+static int fts3SegReaderNext(
120581
+ Fts3Table *p,
120582
+ Fts3SegReader *pReader,
120583
+ int bIncr
120584
+){
120585
+ int rc; /* Return code of various sub-routines */
119696120586
 char *pNext; /* Cursor variable */
119697120587
 int nPrefix; /* Number of bytes in term prefix */
119698120588
 int nSuffix; /* Number of bytes in term suffix */
119699120589
119700120590
 if( !pReader->aDoclist ){
@@ -119702,11 +120592,10 @@
119702120592
 }else{
119703120593
 pNext = &pReader->aDoclist[pReader->nDoclist];
119704120594
 }
119705120595
119706120596
 if( !pNext || pNext>=&pReader->aNode[pReader->nNode] ){
119707
- int rc; /* Return code from Fts3ReadBlock() */
119708120597
119709120598
 if( fts3SegReaderIsPending(pReader) ){
119710120599
 Fts3HashElem *pElem = *(pReader->ppNextElem);
119711120600
 if( pElem==0 ){
119712120601
 pReader->aNode = 0;
@@ -119722,10 +120611,12 @@
119722120611
 return SQLITE_OK;
119723120612
 }
119724120613
119725120614
 if( !fts3SegReaderIsRootOnly(pReader) ){
119726120615
 sqlite3_free(pReader->aNode);
120616
+ sqlite3_blob_close(pReader->pBlob);
120617
+ pReader->pBlob = 0;
119727120618
 }
119728120619
 pReader->aNode = 0;
119729120620
119730120621
 /* If iCurrentBlock>=iLeafEndBlock, this is an EOF condition. All leaf
119731120622
 ** blocks have already been traversed. */
@@ -119733,19 +120624,29 @@
119733120624
 if( pReader->iCurrentBlock>=pReader->iLeafEndBlock ){
119734120625
 return SQLITE_OK;
119735120626
 }
119736120627
119737120628
 rc = sqlite3Fts3ReadBlock(
119738
- p, ++pReader->iCurrentBlock, &pReader->aNode, &pReader->nNode
120629
+ p, ++pReader->iCurrentBlock, &pReader->aNode, &pReader->nNode,
120630
+ (bIncr ? &pReader->nPopulate : 0)
119739120631
 );
119740120632
 if( rc!=SQLITE_OK ) return rc;
120633
+ assert( pReader->pBlob==0 );
120634
+ if( bIncr && pReader->nPopulate<pReader->nNode ){
120635
+ pReader->pBlob = p->pSegments;
120636
+ p->pSegments = 0;
120637
+ }
119741120638
 pNext = pReader->aNode;
119742120639
 }
120640
+
120641
+ assert( !fts3SegReaderIsPending(pReader) );
120642
+
120643
+ rc = fts3SegReaderRequire(pReader, pNext, FTS3_VARINT_MAX*2);
120644
+ if( rc!=SQLITE_OK ) return rc;
119743120645
119744120646
 /* Because of the FTS3_NODE_PADDING bytes of padding, the following is
119745
- ** safe (no risk of overread) even if the node data is corrupted.
119746
- */
120647
+ ** safe (no risk of overread) even if the node data is corrupted. */
119747120648
 pNext += sqlite3Fts3GetVarint32(pNext, &nPrefix);
119748120649
 pNext += sqlite3Fts3GetVarint32(pNext, &nSuffix);
119749120650
 if( nPrefix<0 || nSuffix<=0
119750120651
 || &pNext[nSuffix]>&pReader->aNode[pReader->nNode]
119751120652
 ){
@@ -119759,10 +120660,14 @@
119759120660
 return SQLITE_NOMEM;
119760120661
 }
119761120662
 pReader->zTerm = zNew;
119762120663
 pReader->nTermAlloc = nNew;
119763120664
 }
120665
+
120666
+ rc = fts3SegReaderRequire(pReader, pNext, nSuffix+FTS3_VARINT_MAX);
120667
+ if( rc!=SQLITE_OK ) return rc;
120668
+
119764120669
 memcpy(&pReader->zTerm[nPrefix], pNext, nSuffix);
119765120670
 pReader->nTerm = nPrefix+nSuffix;
119766120671
 pNext += nSuffix;
119767120672
 pNext += sqlite3Fts3GetVarint32(pNext, &pReader->nDoclist);
119768120673
 pReader->aDoclist = pNext;
@@ -119771,11 +120676,11 @@
119771120676
 /* Check that the doclist does not appear to extend past the end of the
119772120677
 ** b-tree node. And that the final byte of the doclist is 0x00. If either
119773120678
 ** of these statements is untrue, then the data structure is corrupt.
119774120679
 */
119775120680
 if( &pReader->aDoclist[pReader->nDoclist]>&pReader->aNode[pReader->nNode]
119776
- || pReader->aDoclist[pReader->nDoclist-1]
120681
+ || (pReader->nPopulate==0 && pReader->aDoclist[pReader->nDoclist-1])
119777120682
 ){
119778120683
 return SQLITE_CORRUPT_VTAB;
119779120684
 }
119780120685
 return SQLITE_OK;
119781120686
 }
@@ -119782,16 +120687,30 @@
119782120687
119783120688
 /*
119784120689
 ** Set the SegReader to point to the first docid in the doclist associated
119785120690
 ** with the current term.
119786120691
 */
119787
-static void fts3SegReaderFirstDocid(Fts3SegReader *pReader){
119788
- int n;
120692
+static int fts3SegReaderFirstDocid(Fts3Table *pTab, Fts3SegReader *pReader){
120693
+ int rc = SQLITE_OK;
119789120694
 assert( pReader->aDoclist );
119790120695
 assert( !pReader->pOffsetList );
119791
- n = sqlite3Fts3GetVarint(pReader->aDoclist, &pReader->iDocid);
119792
- pReader->pOffsetList = &pReader->aDoclist[n];
120696
+ if( pTab->bDescIdx && fts3SegReaderIsPending(pReader) ){
120697
+ u8 bEof = 0;
120698
+ pReader->iDocid = 0;
120699
+ pReader->nOffsetList = 0;
120700
+ sqlite3Fts3DoclistPrev(0,
120701
+ pReader->aDoclist, pReader->nDoclist, &pReader->pOffsetList,
120702
+ &pReader->iDocid, &pReader->nOffsetList, &bEof
120703
+ );
120704
+ }else{
120705
+ rc = fts3SegReaderRequire(pReader, pReader->aDoclist, FTS3_VARINT_MAX);
120706
+ if( rc==SQLITE_OK ){
120707
+ int n = sqlite3Fts3GetVarint(pReader->aDoclist, &pReader->iDocid);
120708
+ pReader->pOffsetList = &pReader->aDoclist[n];
120709
+ }
120710
+ }
120711
+ return rc;
119793120712
 }
119794120713
119795120714
 /*
119796120715
 ** Advance the SegReader to point to the next docid in the doclist
119797120716
 ** associated with the current term.
@@ -119800,132 +120719,126 @@
119800120719
 ** *ppOffsetList is set to point to the first column-offset list
119801120720
 ** in the doclist entry (i.e. immediately past the docid varint).
119802120721
 ** *pnOffsetList is set to the length of the set of column-offset
119803120722
 ** lists, not including the nul-terminator byte. For example:
119804120723
 */
119805
-static void fts3SegReaderNextDocid(
119806
- Fts3SegReader *pReader,
119807
- char **ppOffsetList,
119808
- int *pnOffsetList
120724
+static int fts3SegReaderNextDocid(
120725
+ Fts3Table *pTab,
120726
+ Fts3SegReader *pReader, /* Reader to advance to next docid */
120727
+ char **ppOffsetList, /* OUT: Pointer to current position-list */
120728
+ int *pnOffsetList /* OUT: Length of *ppOffsetList in bytes */
119809120729
 ){
120730
+ int rc = SQLITE_OK;
119810120731
 char *p = pReader->pOffsetList;
119811120732
 char c = 0;
119812120733
119813
- /* Pointer p currently points at the first byte of an offset list. The
119814
- ** following two lines advance it to point one byte past the end of
119815
- ** the same offset list.
119816
- */
119817
- while( *p | c ) c = *p++ & 0x80;
119818
- p++;
119819
-
119820
- /* If required, populate the output variables with a pointer to and the
119821
- ** size of the previous offset-list.
119822
- */
119823
- if( ppOffsetList ){
119824
- *ppOffsetList = pReader->pOffsetList;
119825
- *pnOffsetList = (int)(p - pReader->pOffsetList - 1);
119826
- }
119827
-
119828
- /* If there are no more entries in the doclist, set pOffsetList to
119829
- ** NULL. Otherwise, set Fts3SegReader.iDocid to the next docid and
119830
- ** Fts3SegReader.pOffsetList to point to the next offset list before
119831
- ** returning.
119832
- */
119833
- if( p>=&pReader->aDoclist[pReader->nDoclist] ){
119834
- pReader->pOffsetList = 0;
120734
+ assert( p );
120735
+
120736
+ if( pTab->bDescIdx && fts3SegReaderIsPending(pReader) ){
120737
+ /* A pending-terms seg-reader for an FTS4 table that uses order=desc.
120738
+ ** Pending-terms doclists are always built up in ascending order, so
120739
+ ** we have to iterate through them backwards here. */
120740
+ u8 bEof = 0;
120741
+ if( ppOffsetList ){
120742
+ *ppOffsetList = pReader->pOffsetList;
120743
+ *pnOffsetList = pReader->nOffsetList - 1;
120744
+ }
120745
+ sqlite3Fts3DoclistPrev(0,
120746
+ pReader->aDoclist, pReader->nDoclist, &p, &pReader->iDocid,
120747
+ &pReader->nOffsetList, &bEof
120748
+ );
120749
+ if( bEof ){
120750
+ pReader->pOffsetList = 0;
120751
+ }else{
120752
+ pReader->pOffsetList = p;
120753
+ }
119835120754
 }else{
119836
- sqlite3_int64 iDelta;
119837
- pReader->pOffsetList = p + sqlite3Fts3GetVarint(p, &iDelta);
119838
- pReader->iDocid += iDelta;
119839
- }
119840
-}
119841
-
119842
-/*
119843
-** This function is called to estimate the amount of data that will be
119844
-** loaded from the disk If SegReaderIterate() is called on this seg-reader,
119845
-** in units of average document size.
119846
-**
119847
-** This can be used as follows: If the caller has a small doclist that
119848
-** contains references to N documents, and is considering merging it with
119849
-** a large doclist (size X "average documents"), it may opt not to load
119850
-** the large doclist if X>N.
119851
-*/
119852
-SQLITE_PRIVATE int sqlite3Fts3SegReaderCost(
119853
- Fts3Cursor *pCsr, /* FTS3 cursor handle */
119854
- Fts3SegReader *pReader, /* Segment-reader handle */
119855
- int *pnCost /* IN/OUT: Number of bytes read */
120755
+
120756
+ /* Pointer p currently points at the first byte of an offset list. The
120757
+ ** following block advances it to point one byte past the end of
120758
+ ** the same offset list. */
120759
+ while( 1 ){
120760
+
120761
+ /* The following line of code (and the "p++" below the while() loop) is
120762
+ ** normally all that is required to move pointer p to the desired
120763
+ ** position. The exception is if this node is being loaded from disk
120764
+ ** incrementally and pointer "p" now points to the first byte passed
120765
+ ** the populated part of pReader->aNode[].
120766
+ */
120767
+ while( *p | c ) c = *p++ & 0x80;
120768
+ assert( *p==0 );
120769
+
120770
+ if( pReader->pBlob==0 || p<&pReader->aNode[pReader->nPopulate] ) break;
120771
+ rc = fts3SegReaderIncrRead(pReader);
120772
+ if( rc!=SQLITE_OK ) return rc;
120773
+ }
120774
+ p++;
120775
+
120776
+ /* If required, populate the output variables with a pointer to and the
120777
+ ** size of the previous offset-list.
120778
+ */
120779
+ if( ppOffsetList ){
120780
+ *ppOffsetList = pReader->pOffsetList;
120781
+ *pnOffsetList = (int)(p - pReader->pOffsetList - 1);
120782
+ }
120783
+
120784
+ /* If there are no more entries in the doclist, set pOffsetList to
120785
+ ** NULL. Otherwise, set Fts3SegReader.iDocid to the next docid and
120786
+ ** Fts3SegReader.pOffsetList to point to the next offset list before
120787
+ ** returning.
120788
+ */
120789
+ if( p>=&pReader->aDoclist[pReader->nDoclist] ){
120790
+ pReader->pOffsetList = 0;
120791
+ }else{
120792
+ rc = fts3SegReaderRequire(pReader, p, FTS3_VARINT_MAX);
120793
+ if( rc==SQLITE_OK ){
120794
+ sqlite3_int64 iDelta;
120795
+ pReader->pOffsetList = p + sqlite3Fts3GetVarint(p, &iDelta);
120796
+ if( pTab->bDescIdx ){
120797
+ pReader->iDocid -= iDelta;
120798
+ }else{
120799
+ pReader->iDocid += iDelta;
120800
+ }
120801
+ }
120802
+ }
120803
+ }
120804
+
120805
+ return SQLITE_OK;
120806
+}
120807
+
120808
+
120809
+SQLITE_PRIVATE int sqlite3Fts3MsrOvfl(
120810
+ Fts3Cursor *pCsr,
120811
+ Fts3MultiSegReader *pMsr,
120812
+ int *pnOvfl
119856120813
 ){
119857120814
 Fts3Table *p = (Fts3Table*)pCsr->base.pVtab;
119858
- int rc = SQLITE_OK; /* Return code */
119859
- int nCost = 0; /* Cost in bytes to return */
119860
- int pgsz = p->nPgsz; /* Database page size */
119861
-
119862
- /* If this seg-reader is reading the pending-terms table, or if all data
119863
- ** for the segment is stored on the root page of the b-tree, then the cost
119864
- ** is zero. In this case all required data is already in main memory.
119865
- */
119866
- if( p->bHasStat
119867
- && !fts3SegReaderIsPending(pReader)
119868
- && !fts3SegReaderIsRootOnly(pReader)
119869
- ){
119870
- int nBlob = 0;
119871
- sqlite3_int64 iBlock;
119872
-
119873
- if( pCsr->nRowAvg==0 ){
119874
- /* The average document size, which is required to calculate the cost
119875
- ** of each doclist, has not yet been determined. Read the required
119876
- ** data from the %_stat table to calculate it.
119877
- **
119878
- ** Entry 0 of the %_stat table is a blob containing (nCol+1) FTS3
119879
- ** varints, where nCol is the number of columns in the FTS3 table.
119880
- ** The first varint is the number of documents currently stored in
119881
- ** the table. The following nCol varints contain the total amount of
119882
- ** data stored in all rows of each column of the table, from left
119883
- ** to right.
119884
- */
119885
- sqlite3_stmt *pStmt;
119886
- sqlite3_int64 nDoc = 0;
119887
- sqlite3_int64 nByte = 0;
119888
- const char *pEnd;
119889
- const char *a;
119890
-
119891
- rc = sqlite3Fts3SelectDoctotal(p, &pStmt);
119892
- if( rc!=SQLITE_OK ) return rc;
119893
- a = sqlite3_column_blob(pStmt, 0);
119894
- assert( a );
119895
-
119896
- pEnd = &a[sqlite3_column_bytes(pStmt, 0)];
119897
- a += sqlite3Fts3GetVarint(a, &nDoc);
119898
- while( a<pEnd ){
119899
- a += sqlite3Fts3GetVarint(a, &nByte);
119900
- }
119901
- if( nDoc==0 || nByte==0 ){
119902
- sqlite3_reset(pStmt);
119903
- return SQLITE_CORRUPT_VTAB;
119904
- }
119905
-
119906
- pCsr->nRowAvg = (int)(((nByte / nDoc) + pgsz) / pgsz);
119907
- assert( pCsr->nRowAvg>0 );
119908
- rc = sqlite3_reset(pStmt);
119909
- if( rc!=SQLITE_OK ) return rc;
119910
- }
119911
-
119912
- /* Assume that a blob flows over onto overflow pages if it is larger
119913
- ** than (pgsz-35) bytes in size (the file-format documentation
119914
- ** confirms this).
119915
- */
119916
- for(iBlock=pReader->iStartBlock; iBlock<=pReader->iLeafEndBlock; iBlock++){
119917
- rc = sqlite3Fts3ReadBlock(p, iBlock, 0, &nBlob);
119918
- if( rc!=SQLITE_OK ) break;
119919
- if( (nBlob+35)>pgsz ){
119920
- int nOvfl = (nBlob + 34)/pgsz;
119921
- nCost += ((nOvfl + pCsr->nRowAvg - 1)/pCsr->nRowAvg);
119922
- }
119923
- }
119924
- }
119925
-
119926
- *pnCost += nCost;
120815
+ int nOvfl = 0;
120816
+ int ii;
120817
+ int rc = SQLITE_OK;
120818
+ int pgsz = p->nPgsz;
120819
+
120820
+ assert( p->bHasStat );
120821
+ assert( pgsz>0 );
120822
+
120823
+ for(ii=0; rc==SQLITE_OK && ii<pMsr->nSegment; ii++){
120824
+ Fts3SegReader *pReader = pMsr->apSegment[ii];
120825
+ if( !fts3SegReaderIsPending(pReader)
120826
+ && !fts3SegReaderIsRootOnly(pReader)
120827
+ ){
120828
+ int jj;
120829
+ for(jj=pReader->iStartBlock; jj<=pReader->iLeafEndBlock; jj++){
120830
+ int nBlob;
120831
+ rc = sqlite3Fts3ReadBlock(p, jj, 0, &nBlob, 0);
120832
+ if( rc!=SQLITE_OK ) break;
120833
+ if( (nBlob+35)>pgsz ){
120834
+ nOvfl += (nBlob + 34)/pgsz;
120835
+ }
120836
+ }
120837
+ }
120838
+ }
120839
+ *pnOvfl = nOvfl;
119927120840
 return rc;
119928120841
 }
119929120842
119930120843
 /*
119931120844
 ** Free all allocations associated with the iterator passed as the
@@ -119934,10 +120847,11 @@
119934120847
 SQLITE_PRIVATE void sqlite3Fts3SegReaderFree(Fts3SegReader *pReader){
119935120848
 if( pReader && !fts3SegReaderIsPending(pReader) ){
119936120849
 sqlite3_free(pReader->zTerm);
119937120850
 if( !fts3SegReaderIsRootOnly(pReader) ){
119938120851
 sqlite3_free(pReader->aNode);
120852
+ sqlite3_blob_close(pReader->pBlob);
119939120853
 }
119940120854
 }
119941120855
 sqlite3_free(pReader);
119942120856
 }
119943120857
@@ -120010,28 +120924,46 @@
120010120924
 }
120011120925
120012120926
 /*
120013120927
 ** This function is used to allocate an Fts3SegReader that iterates through
120014120928
 ** a subset of the terms stored in the Fts3Table.pendingTerms array.
120929
+**
120930
+** If the isPrefixIter parameter is zero, then the returned SegReader iterates
120931
+** through each term in the pending-terms table. Or, if isPrefixIter is
120932
+** non-zero, it iterates through each term and its prefixes. For example, if
120933
+** the pending terms hash table contains the terms "sqlite", "mysql" and
120934
+** "firebird", then the iterator visits the following 'terms' (in the order
120935
+** shown):
120936
+**
120937
+** f fi fir fire fireb firebi firebir firebird
120938
+** m my mys mysq mysql
120939
+** s sq sql sqli sqlit sqlite
120940
+**
120941
+** Whereas if isPrefixIter is zero, the terms visited are:
120942
+**
120943
+** firebird mysql sqlite
120015120944
 */
120016120945
 SQLITE_PRIVATE int sqlite3Fts3SegReaderPending(
120017120946
 Fts3Table *p, /* Virtual table handle */
120947
+ int iIndex, /* Index for p->aIndex */
120018120948
 const char *zTerm, /* Term to search for */
120019120949
 int nTerm, /* Size of buffer zTerm */
120020
- int isPrefix, /* True for a term-prefix query */
120950
+ int bPrefix, /* True for a prefix iterator */
120021120951
 Fts3SegReader **ppReader /* OUT: SegReader for pending-terms */
120022120952
 ){
120023120953
 Fts3SegReader *pReader = 0; /* Fts3SegReader object to return */
120024120954
 Fts3HashElem **aElem = 0; /* Array of term hash entries to scan */
120025120955
 int nElem = 0; /* Size of array at aElem */
120026120956
 int rc = SQLITE_OK; /* Return Code */
120957
+ Fts3Hash *pHash;
120027120958
120028
- if( isPrefix ){
120959
+ pHash = &p->aIndex[iIndex].hPending;
120960
+ if( bPrefix ){
120029120961
 int nAlloc = 0; /* Size of allocated array at aElem */
120030120962
 Fts3HashElem *pE = 0; /* Iterator variable */
120031120963
120032
- for(pE=fts3HashFirst(&p->pendingTerms); pE; pE=fts3HashNext(pE)){
120964
+ for(pE=fts3HashFirst(pHash); pE; pE=fts3HashNext(pE)){
120033120965
 char *zKey = (char *)fts3HashKey(pE);
120034120966
 int nKey = fts3HashKeysize(pE);
120035120967
 if( nTerm==0 || (nKey>=nTerm && 0==memcmp(zKey, zTerm, nTerm)) ){
120036120968
 if( nElem==nAlloc ){
120037120969
 Fts3HashElem **aElem2;
@@ -120044,10 +120976,11 @@
120044120976
 nElem = 0;
120045120977
 break;
120046120978
 }
120047120979
 aElem = aElem2;
120048120980
 }
120981
+
120049120982
 aElem[nElem++] = pE;
120050120983
 }
120051120984
 }
120052120985
120053120986
 /* If more than one term matches the prefix, sort the Fts3HashElem
@@ -120057,11 +120990,13 @@
120057120990
 if( nElem>1 ){
120058120991
 qsort(aElem, nElem, sizeof(Fts3HashElem *), fts3CompareElemByTerm);
120059120992
 }
120060120993
120061120994
 }else{
120062
- Fts3HashElem *pE = fts3HashFindElem(&p->pendingTerms, zTerm, nTerm);
120995
+ /* The query is a simple term lookup that matches at most one term in
120996
+ ** the index. All that is required is a straight hash-lookup. */
120997
+ Fts3HashElem *pE = fts3HashFindElem(pHash, zTerm, nTerm);
120063120998
 if( pE ){
120064120999
 aElem = &pE;
120065121000
 nElem = 1;
120066121001
 }
120067121002
 }
@@ -120077,11 +121012,11 @@
120077121012
 pReader->ppNextElem = (Fts3HashElem **)&pReader[1];
120078121013
 memcpy(pReader->ppNextElem, aElem, nElem*sizeof(Fts3HashElem *));
120079121014
 }
120080121015
 }
120081121016
120082
- if( isPrefix ){
121017
+ if( bPrefix ){
120083121018
 sqlite3_free(aElem);
120084121019
 }
120085121020
 *ppReader = pReader;
120086121021
 return rc;
120087121022
 }
@@ -120137,10 +121072,22 @@
120137121072
 if( pLhs->iDocid==pRhs->iDocid ){
120138121073
 rc = pRhs->iIdx - pLhs->iIdx;
120139121074
 }else{
120140121075
 rc = (pLhs->iDocid > pRhs->iDocid) ? 1 : -1;
120141121076
 }
121077
+ }
121078
+ assert( pLhs->aNode && pRhs->aNode );
121079
+ return rc;
121080
+}
121081
+static int fts3SegReaderDoclistCmpRev(Fts3SegReader *pLhs, Fts3SegReader *pRhs){
121082
+ int rc = (pLhs->pOffsetList==0)-(pRhs->pOffsetList==0);
121083
+ if( rc==0 ){
121084
+ if( pLhs->iDocid==pRhs->iDocid ){
121085
+ rc = pRhs->iIdx - pLhs->iIdx;
121086
+ }else{
121087
+ rc = (pLhs->iDocid < pRhs->iDocid) ? 1 : -1;
121088
+ }
120142121089
 }
120143121090
 assert( pLhs->aNode && pRhs->aNode );
120144121091
 return rc;
120145121092
 }
120146121093
@@ -120689,25 +121636,34 @@
120689121636
 }
120690121637
 return rc;
120691121638
 }
120692121639
120693121640
 /*
120694
-** Set *pnSegment to the total number of segments in the database. Set
120695
-** *pnMax to the largest segment level in the database (segment levels
120696
-** are stored in the 'level' column of the %_segdir table).
121641
+** Set *pnMax to the largest segment level in the database for the index
121642
+** iIndex.
121643
+**
121644
+** Segment levels are stored in the 'level' column of the %_segdir table.
120697121645
 **
120698121646
 ** Return SQLITE_OK if successful, or an SQLite error code if not.
120699121647
 */
120700
-static int fts3SegmentCountMax(Fts3Table *p, int *pnSegment, int *pnMax){
121648
+static int fts3SegmentMaxLevel(Fts3Table *p, int iIndex, int *pnMax){
120701121649
 sqlite3_stmt *pStmt;
120702121650
 int rc;
121651
+ assert( iIndex>=0 && iIndex<p->nIndex );
120703121652
120704
- rc = fts3SqlStmt(p, SQL_SELECT_SEGDIR_COUNT_MAX, &pStmt, 0);
121653
+ /* Set pStmt to the compiled version of:
121654
+ **
121655
+ ** SELECT max(level) FROM %Q.'%q_segdir' WHERE level BETWEEN ? AND ?
121656
+ **
121657
+ ** (1024 is actually the value of macro FTS3_SEGDIR_PREFIXLEVEL_STR).
121658
+ */
121659
+ rc = fts3SqlStmt(p, SQL_SELECT_SEGDIR_MAX_LEVEL, &pStmt, 0);
120705121660
 if( rc!=SQLITE_OK ) return rc;
121661
+ sqlite3_bind_int(pStmt, 1, iIndex*FTS3_SEGDIR_MAXLEVEL);
121662
+ sqlite3_bind_int(pStmt, 2, (iIndex+1)*FTS3_SEGDIR_MAXLEVEL - 1);
120706121663
 if( SQLITE_ROW==sqlite3_step(pStmt) ){
120707
- *pnSegment = sqlite3_column_int(pStmt, 0);
120708
- *pnMax = sqlite3_column_int(pStmt, 1);
121664
+ *pnMax = sqlite3_column_int(pStmt, 0);
120709121665
 }
120710121666
 return sqlite3_reset(pStmt);
120711121667
 }
120712121668
120713121669
 /*
@@ -120724,10 +121680,11 @@
120724121680
 **
120725121681
 ** SQLITE_OK is returned if successful, otherwise an SQLite error code.
120726121682
 */
120727121683
 static int fts3DeleteSegdir(
120728121684
 Fts3Table *p, /* Virtual table handle */
121685
+ int iIndex, /* Index for p->aIndex */
120729121686
 int iLevel, /* Level of %_segdir entries to delete */
120730121687
 Fts3SegReader **apSegment, /* Array of SegReader objects */
120731121688
 int nReader /* Size of array apSegment */
120732121689
 ){
120733121690
 int rc; /* Return Code */
@@ -120746,23 +121703,28 @@
120746121703
 }
120747121704
 if( rc!=SQLITE_OK ){
120748121705
 return rc;
120749121706
 }
120750121707
121708
+ assert( iLevel>=0 || iLevel==FTS3_SEGCURSOR_ALL );
120751121709
 if( iLevel==FTS3_SEGCURSOR_ALL ){
120752
- fts3SqlExec(&rc, p, SQL_DELETE_ALL_SEGDIR, 0);
120753
- }else if( iLevel==FTS3_SEGCURSOR_PENDING ){
120754
- sqlite3Fts3PendingTermsClear(p);
121710
+ rc = fts3SqlStmt(p, SQL_DELETE_SEGDIR_RANGE, &pDelete, 0);
121711
+ if( rc==SQLITE_OK ){
121712
+ sqlite3_bind_int(pDelete, 1, iIndex*FTS3_SEGDIR_MAXLEVEL);
121713
+ sqlite3_bind_int(pDelete, 2, (iIndex+1) * FTS3_SEGDIR_MAXLEVEL - 1);
121714
+ }
120755121715
 }else{
120756
- assert( iLevel>=0 );
120757
- rc = fts3SqlStmt(p, SQL_DELETE_SEGDIR_BY_LEVEL, &pDelete, 0);
121716
+ rc = fts3SqlStmt(p, SQL_DELETE_SEGDIR_LEVEL, &pDelete, 0);
120758121717
 if( rc==SQLITE_OK ){
120759
- sqlite3_bind_int(pDelete, 1, iLevel);
120760
- sqlite3_step(pDelete);
120761
- rc = sqlite3_reset(pDelete);
121718
+ sqlite3_bind_int(pDelete, 1, iIndex*FTS3_SEGDIR_MAXLEVEL + iLevel);
120762121719
 }
120763121720
 }
121721
+
121722
+ if( rc==SQLITE_OK ){
121723
+ sqlite3_step(pDelete);
121724
+ rc = sqlite3_reset(pDelete);
121725
+ }
120764121726
120765121727
 return rc;
120766121728
 }
120767121729
120768121730
 /*
@@ -120805,14 +121767,124 @@
120805121767
 }
120806121768
120807121769
 *ppList = pList;
120808121770
 *pnList = nList;
120809121771
 }
121772
+
121773
+SQLITE_PRIVATE int sqlite3Fts3MsrIncrStart(
121774
+ Fts3Table *p, /* Virtual table handle */
121775
+ Fts3MultiSegReader *pCsr, /* Cursor object */
121776
+ int iCol, /* Column to match on. */
121777
+ const char *zTerm, /* Term to iterate through a doclist for */
121778
+ int nTerm /* Number of bytes in zTerm */
121779
+){
121780
+ int i;
121781
+ int nSegment = pCsr->nSegment;
121782
+ int (*xCmp)(Fts3SegReader *, Fts3SegReader *) = (
121783
+ p->bDescIdx ? fts3SegReaderDoclistCmpRev : fts3SegReaderDoclistCmp
121784
+ );
121785
+
121786
+ assert( pCsr->pFilter==0 );
121787
+ assert( zTerm && nTerm>0 );
121788
+
121789
+ /* Advance each segment iterator until it points to the term zTerm/nTerm. */
121790
+ for(i=0; i<nSegment; i++){
121791
+ Fts3SegReader *pSeg = pCsr->apSegment[i];
121792
+ do {
121793
+ int rc = fts3SegReaderNext(p, pSeg, 1);
121794
+ if( rc!=SQLITE_OK ) return rc;
121795
+ }while( fts3SegReaderTermCmp(pSeg, zTerm, nTerm)<0 );
121796
+ }
121797
+ fts3SegReaderSort(pCsr->apSegment, nSegment, nSegment, fts3SegReaderCmp);
121798
+
121799
+ /* Determine how many of the segments actually point to zTerm/nTerm. */
121800
+ for(i=0; i<nSegment; i++){
121801
+ Fts3SegReader *pSeg = pCsr->apSegment[i];
121802
+ if( !pSeg->aNode || fts3SegReaderTermCmp(pSeg, zTerm, nTerm) ){
121803
+ break;
121804
+ }
121805
+ }
121806
+ pCsr->nAdvance = i;
121807
+
121808
+ /* Advance each of the segments to point to the first docid. */
121809
+ for(i=0; i<pCsr->nAdvance; i++){
121810
+ int rc = fts3SegReaderFirstDocid(p, pCsr->apSegment[i]);
121811
+ if( rc!=SQLITE_OK ) return rc;
121812
+ }
121813
+ fts3SegReaderSort(pCsr->apSegment, i, i, xCmp);
121814
+
121815
+ assert( iCol<0 || iCol<p->nColumn );
121816
+ pCsr->iColFilter = iCol;
121817
+
121818
+ return SQLITE_OK;
121819
+}
121820
+
121821
+SQLITE_PRIVATE int sqlite3Fts3MsrIncrNext(
121822
+ Fts3Table *p, /* Virtual table handle */
121823
+ Fts3MultiSegReader *pMsr, /* Multi-segment-reader handle */
121824
+ sqlite3_int64 *piDocid, /* OUT: Docid value */
121825
+ char **paPoslist, /* OUT: Pointer to position list */
121826
+ int *pnPoslist /* OUT: Size of position list in bytes */
121827
+){
121828
+ int nMerge = pMsr->nAdvance;
121829
+ Fts3SegReader **apSegment = pMsr->apSegment;
121830
+ int (*xCmp)(Fts3SegReader *, Fts3SegReader *) = (
121831
+ p->bDescIdx ? fts3SegReaderDoclistCmpRev : fts3SegReaderDoclistCmp
121832
+ );
121833
+
121834
+ if( nMerge==0 ){
121835
+ *paPoslist = 0;
121836
+ return SQLITE_OK;
121837
+ }
121838
+
121839
+ while( 1 ){
121840
+ Fts3SegReader *pSeg;
121841
+ pSeg = pMsr->apSegment[0];
121842
+
121843
+ if( pSeg->pOffsetList==0 ){
121844
+ *paPoslist = 0;
121845
+ break;
121846
+ }else{
121847
+ int rc;
121848
+ char *pList;
121849
+ int nList;
121850
+ int j;
121851
+ sqlite3_int64 iDocid = apSegment[0]->iDocid;
121852
+
121853
+ rc = fts3SegReaderNextDocid(p, apSegment[0], &pList, &nList);
121854
+ j = 1;
121855
+ while( rc==SQLITE_OK
121856
+ && j<nMerge
121857
+ && apSegment[j]->pOffsetList
121858
+ && apSegment[j]->iDocid==iDocid
121859
+ ){
121860
+ rc = fts3SegReaderNextDocid(p, apSegment[j], 0, 0);
121861
+ j++;
121862
+ }
121863
+ if( rc!=SQLITE_OK ) return rc;
121864
+ fts3SegReaderSort(pMsr->apSegment, nMerge, j, xCmp);
121865
+
121866
+ if( pMsr->iColFilter>=0 ){
121867
+ fts3ColumnFilter(pMsr->iColFilter, &pList, &nList);
121868
+ }
121869
+
121870
+ if( nList>0 ){
121871
+ *piDocid = iDocid;
121872
+ *paPoslist = pList;
121873
+ *pnPoslist = nList;
121874
+ break;
121875
+ }
121876
+ }
121877
+
121878
+ }
121879
+
121880
+ return SQLITE_OK;
121881
+}
120810121882
120811121883
 SQLITE_PRIVATE int sqlite3Fts3SegReaderStart(
120812121884
 Fts3Table *p, /* Virtual table handle */
120813
- Fts3SegReaderCursor *pCsr, /* Cursor object */
121885
+ Fts3MultiSegReader *pCsr, /* Cursor object */
120814121886
 Fts3SegFilter *pFilter /* Restrictions on range of iteration */
120815121887
 ){
120816121888
 int i;
120817121889
120818121890
 /* Initialize the cursor object */
@@ -120827,11 +121899,11 @@
120827121899
 for(i=0; i<pCsr->nSegment; i++){
120828121900
 int nTerm = pFilter->nTerm;
120829121901
 const char *zTerm = pFilter->zTerm;
120830121902
 Fts3SegReader *pSeg = pCsr->apSegment[i];
120831121903
 do {
120832
- int rc = fts3SegReaderNext(p, pSeg);
121904
+ int rc = fts3SegReaderNext(p, pSeg, 0);
120833121905
 if( rc!=SQLITE_OK ) return rc;
120834121906
 }while( zTerm && fts3SegReaderTermCmp(pSeg, zTerm, nTerm)<0 );
120835121907
 }
120836121908
 fts3SegReaderSort(
120837121909
 pCsr->apSegment, pCsr->nSegment, pCsr->nSegment, fts3SegReaderCmp);
@@ -120839,11 +121911,11 @@
120839121911
 return SQLITE_OK;
120840121912
 }
120841121913
120842121914
 SQLITE_PRIVATE int sqlite3Fts3SegReaderStep(
120843121915
 Fts3Table *p, /* Virtual table handle */
120844
- Fts3SegReaderCursor *pCsr /* Cursor object */
121916
+ Fts3MultiSegReader *pCsr /* Cursor object */
120845121917
 ){
120846121918
 int rc = SQLITE_OK;
120847121919
120848121920
 int isIgnoreEmpty = (pCsr->pFilter->flags & FTS3_SEGMENT_IGNORE_EMPTY);
120849121921
 int isRequirePos = (pCsr->pFilter->flags & FTS3_SEGMENT_REQUIRE_POS);
@@ -120852,10 +121924,13 @@
120852121924
 int isScan = (pCsr->pFilter->flags & FTS3_SEGMENT_SCAN);
120853121925
120854121926
 Fts3SegReader **apSegment = pCsr->apSegment;
120855121927
 int nSegment = pCsr->nSegment;
120856121928
 Fts3SegFilter *pFilter = pCsr->pFilter;
121929
+ int (*xCmp)(Fts3SegReader *, Fts3SegReader *) = (
121930
+ p->bDescIdx ? fts3SegReaderDoclistCmpRev : fts3SegReaderDoclistCmp
121931
+ );
120857121932
120858121933
 if( pCsr->nSegment==0 ) return SQLITE_OK;
120859121934
120860121935
 do {
120861121936
 int nMerge;
@@ -120863,11 +121938,11 @@
120863121938
120864121939
 /* Advance the first pCsr->nAdvance entries in the apSegment[] array
120865121940
 ** forward. Then sort the list in order of current term again.
120866121941
 */
120867121942
 for(i=0; i<pCsr->nAdvance; i++){
120868
- rc = fts3SegReaderNext(p, apSegment[i]);
121943
+ rc = fts3SegReaderNext(p, apSegment[i], 0);
120869121944
 if( rc!=SQLITE_OK ) return rc;
120870121945
 }
120871121946
 fts3SegReaderSort(apSegment, nSegment, pCsr->nAdvance, fts3SegReaderCmp);
120872121947
 pCsr->nAdvance = 0;
120873121948
@@ -120902,11 +121977,14 @@
120902121977
 ){
120903121978
 nMerge++;
120904121979
 }
120905121980
120906121981
 assert( isIgnoreEmpty || (isRequirePos && !isColFilter) );
120907
- if( nMerge==1 && !isIgnoreEmpty ){
121982
+ if( nMerge==1
121983
+ && !isIgnoreEmpty
121984
+ && (p->bDescIdx==0 || fts3SegReaderIsPending(apSegment[0])==0)
121985
+ ){
120908121986
 pCsr->aDoclist = apSegment[0]->aDoclist;
120909121987
 pCsr->nDoclist = apSegment[0]->nDoclist;
120910121988
 rc = SQLITE_ROW;
120911121989
 }else{
120912121990
 int nDoclist = 0; /* Size of doclist */
@@ -120915,56 +121993,66 @@
120915121993
 /* The current term of the first nMerge entries in the array
120916121994
 ** of Fts3SegReader objects is the same. The doclists must be merged
120917121995
 ** and a single term returned with the merged doclist.
120918121996
 */
120919121997
 for(i=0; i<nMerge; i++){
120920
- fts3SegReaderFirstDocid(apSegment[i]);
121998
+ fts3SegReaderFirstDocid(p, apSegment[i]);
120921121999
 }
120922
- fts3SegReaderSort(apSegment, nMerge, nMerge, fts3SegReaderDoclistCmp);
122000
+ fts3SegReaderSort(apSegment, nMerge, nMerge, xCmp);
120923122001
 while( apSegment[0]->pOffsetList ){
120924122002
 int j; /* Number of segments that share a docid */
120925122003
 char *pList;
120926122004
 int nList;
120927122005
 int nByte;
120928122006
 sqlite3_int64 iDocid = apSegment[0]->iDocid;
120929
- fts3SegReaderNextDocid(apSegment[0], &pList, &nList);
122007
+ fts3SegReaderNextDocid(p, apSegment[0], &pList, &nList);
120930122008
 j = 1;
120931122009
 while( j<nMerge
120932122010
 && apSegment[j]->pOffsetList
120933122011
 && apSegment[j]->iDocid==iDocid
120934122012
 ){
120935
- fts3SegReaderNextDocid(apSegment[j], 0, 0);
122013
+ fts3SegReaderNextDocid(p, apSegment[j], 0, 0);
120936122014
 j++;
120937122015
 }
120938122016
120939122017
 if( isColFilter ){
120940122018
 fts3ColumnFilter(pFilter->iCol, &pList, &nList);
120941122019
 }
120942122020
120943122021
 if( !isIgnoreEmpty || nList>0 ){
120944
- nByte = sqlite3Fts3VarintLen(iDocid-iPrev) + (isRequirePos?nList+1:0);
122022
+
122023
+ /* Calculate the 'docid' delta value to write into the merged
122024
+ ** doclist. */
122025
+ sqlite3_int64 iDelta;
122026
+ if( p->bDescIdx && nDoclist>0 ){
122027
+ iDelta = iPrev - iDocid;
122028
+ }else{
122029
+ iDelta = iDocid - iPrev;
122030
+ }
122031
+ assert( iDelta>0 || (nDoclist==0 && iDelta==iDocid) );
122032
+ assert( nDoclist>0 || iDelta==iDocid );
122033
+
122034
+ nByte = sqlite3Fts3VarintLen(iDelta) + (isRequirePos?nList+1:0);
120945122035
 if( nDoclist+nByte>pCsr->nBuffer ){
120946122036
 char *aNew;
120947122037
 pCsr->nBuffer = (nDoclist+nByte)*2;
120948122038
 aNew = sqlite3_realloc(pCsr->aBuffer, pCsr->nBuffer);
120949122039
 if( !aNew ){
120950122040
 return SQLITE_NOMEM;
120951122041
 }
120952122042
 pCsr->aBuffer = aNew;
120953122043
 }
120954
- nDoclist += sqlite3Fts3PutVarint(
120955
- &pCsr->aBuffer[nDoclist], iDocid-iPrev
120956
- );
122044
+ nDoclist += sqlite3Fts3PutVarint(&pCsr->aBuffer[nDoclist], iDelta);
120957122045
 iPrev = iDocid;
120958122046
 if( isRequirePos ){
120959122047
 memcpy(&pCsr->aBuffer[nDoclist], pList, nList);
120960122048
 nDoclist += nList;
120961122049
 pCsr->aBuffer[nDoclist++] = '\0';
120962122050
 }
120963122051
 }
120964122052
120965
- fts3SegReaderSort(apSegment, nMerge, j, fts3SegReaderDoclistCmp);
122053
+ fts3SegReaderSort(apSegment, nMerge, j, xCmp);
120966122054
 }
120967122055
 if( nDoclist>0 ){
120968122056
 pCsr->aDoclist = pCsr->aBuffer;
120969122057
 pCsr->nDoclist = nDoclist;
120970122058
 rc = SQLITE_ROW;
@@ -120973,13 +122061,14 @@
120973122061
 pCsr->nAdvance = nMerge;
120974122062
 }while( rc==SQLITE_OK );
120975122063
120976122064
 return rc;
120977122065
 }
122066
+
120978122067
120979122068
 SQLITE_PRIVATE void sqlite3Fts3SegReaderFinish(
120980
- Fts3SegReaderCursor *pCsr /* Cursor object */
122069
+ Fts3MultiSegReader *pCsr /* Cursor object */
120981122070
 ){
120982122071
 if( pCsr ){
120983122072
 int i;
120984122073
 for(i=0; i<pCsr->nSegment; i++){
120985122074
 sqlite3Fts3SegReaderFree(pCsr->apSegment[i]);
@@ -121002,47 +122091,60 @@
121002122091
 ** If this function is called with iLevel<0, but there is only one
121003122092
 ** segment in the database, SQLITE_DONE is returned immediately.
121004122093
 ** Otherwise, if successful, SQLITE_OK is returned. If an error occurs,
121005122094
 ** an SQLite error code is returned.
121006122095
 */
121007
-static int fts3SegmentMerge(Fts3Table *p, int iLevel){
122096
+static int fts3SegmentMerge(Fts3Table *p, int iIndex, int iLevel){
121008122097
 int rc; /* Return code */
121009122098
 int iIdx = 0; /* Index of new segment */
121010
- int iNewLevel = 0; /* Level to create new segment at */
122099
+ int iNewLevel = 0; /* Level/index to create new segment at */
121011122100
 SegmentWriter *pWriter = 0; /* Used to write the new, merged, segment */
121012122101
 Fts3SegFilter filter; /* Segment term filter condition */
121013
- Fts3SegReaderCursor csr; /* Cursor to iterate through level(s) */
122102
+ Fts3MultiSegReader csr; /* Cursor to iterate through level(s) */
122103
+ int bIgnoreEmpty = 0; /* True to ignore empty segments */
121014122104
121015
- rc = sqlite3Fts3SegReaderCursor(p, iLevel, 0, 0, 1, 0, &csr);
122105
+ assert( iLevel==FTS3_SEGCURSOR_ALL
122106
+ || iLevel==FTS3_SEGCURSOR_PENDING
122107
+ || iLevel>=0
122108
+ );
122109
+ assert( iLevel<FTS3_SEGDIR_MAXLEVEL );
122110
+ assert( iIndex>=0 && iIndex<p->nIndex );
122111
+
122112
+ rc = sqlite3Fts3SegReaderCursor(p, iIndex, iLevel, 0, 0, 1, 0, &csr);
121016122113
 if( rc!=SQLITE_OK || csr.nSegment==0 ) goto finished;
121017122114
121018122115
 if( iLevel==FTS3_SEGCURSOR_ALL ){
121019122116
 /* This call is to merge all segments in the database to a single
121020122117
 ** segment. The level of the new segment is equal to the the numerically
121021
- ** greatest segment level currently present in the database. The index
121022
- ** of the new segment is always 0. */
121023
- int nDummy; /* TODO: Remove this */
122118
+ ** greatest segment level currently present in the database for this
122119
+ ** index. The idx of the new segment is always 0. */
121024122120
 if( csr.nSegment==1 ){
121025122121
 rc = SQLITE_DONE;
121026122122
 goto finished;
121027122123
 }
121028
- rc = fts3SegmentCountMax(p, &nDummy, &iNewLevel);
122124
+ rc = fts3SegmentMaxLevel(p, iIndex, &iNewLevel);
122125
+ bIgnoreEmpty = 1;
122126
+
122127
+ }else if( iLevel==FTS3_SEGCURSOR_PENDING ){
122128
+ iNewLevel = iIndex * FTS3_SEGDIR_MAXLEVEL;
122129
+ rc = fts3AllocateSegdirIdx(p, iIndex, 0, &iIdx);
121029122130
 }else{
121030
- /* This call is to merge all segments at level iLevel. Find the next
122131
+ /* This call is to merge all segments at level iLevel. find the next
121031122132
 ** available segment index at level iLevel+1. The call to
121032122133
 ** fts3AllocateSegdirIdx() will merge the segments at level iLevel+1 to
121033122134
 ** a single iLevel+2 segment if necessary. */
121034
- iNewLevel = iLevel+1;
121035
- rc = fts3AllocateSegdirIdx(p, iNewLevel, &iIdx);
122135
+ rc = fts3AllocateSegdirIdx(p, iIndex, iLevel+1, &iIdx);
122136
+ iNewLevel = iIndex * FTS3_SEGDIR_MAXLEVEL + iLevel+1;
121036122137
 }
121037122138
 if( rc!=SQLITE_OK ) goto finished;
121038122139
 assert( csr.nSegment>0 );
121039
- assert( iNewLevel>=0 );
122140
+ assert( iNewLevel>=(iIndex*FTS3_SEGDIR_MAXLEVEL) );
122141
+ assert( iNewLevel<((iIndex+1)*FTS3_SEGDIR_MAXLEVEL) );
121040122142
121041122143
 memset(&filter, 0, sizeof(Fts3SegFilter));
121042122144
 filter.flags = FTS3_SEGMENT_REQUIRE_POS;
121043
- filter.flags |= (iLevel==FTS3_SEGCURSOR_ALL ? FTS3_SEGMENT_IGNORE_EMPTY : 0);
122145
+ filter.flags |= (bIgnoreEmpty ? FTS3_SEGMENT_IGNORE_EMPTY : 0);
121044122146
121045122147
 rc = sqlite3Fts3SegReaderStart(p, &csr, &filter);
121046122148
 while( SQLITE_OK==rc ){
121047122149
 rc = sqlite3Fts3SegReaderStep(p, &csr);
121048122150
 if( rc!=SQLITE_ROW ) break;
@@ -121050,12 +122152,14 @@
121050122152
 csr.zTerm, csr.nTerm, csr.aDoclist, csr.nDoclist);
121051122153
 }
121052122154
 if( rc!=SQLITE_OK ) goto finished;
121053122155
 assert( pWriter );
121054122156
121055
- rc = fts3DeleteSegdir(p, iLevel, csr.apSegment, csr.nSegment);
121056
- if( rc!=SQLITE_OK ) goto finished;
122157
+ if( iLevel!=FTS3_SEGCURSOR_PENDING ){
122158
+ rc = fts3DeleteSegdir(p, iIndex, iLevel, csr.apSegment, csr.nSegment);
122159
+ if( rc!=SQLITE_OK ) goto finished;
122160
+ }
121057122161
 rc = fts3SegWriterFlush(p, pWriter, iNewLevel, iIdx);
121058122162
121059122163
 finished:
121060122164
 fts3SegWriterFree(pWriter);
121061122165
 sqlite3Fts3SegReaderFinish(&csr);
@@ -121062,14 +122166,21 @@
121062122166
 return rc;
121063122167
 }
121064122168
121065122169
121066122170
 /*
121067
-** Flush the contents of pendingTerms to a level 0 segment.
122171
+** Flush the contents of pendingTerms to level 0 segments.
121068122172
 */
121069122173
 SQLITE_PRIVATE int sqlite3Fts3PendingTermsFlush(Fts3Table *p){
121070
- return fts3SegmentMerge(p, FTS3_SEGCURSOR_PENDING);
122174
+ int rc = SQLITE_OK;
122175
+ int i;
122176
+ for(i=0; rc==SQLITE_OK && i<p->nIndex; i++){
122177
+ rc = fts3SegmentMerge(p, i, FTS3_SEGCURSOR_PENDING);
122178
+ if( rc==SQLITE_DONE ) rc = SQLITE_OK;
122179
+ }
122180
+ sqlite3Fts3PendingTermsClear(p);
122181
+ return rc;
121071122182
 }
121072122183
121073122184
 /*
121074122185
 ** Encode N integers as varints into a blob.
121075122186
 */
@@ -121215,10 +122326,27 @@
121215122326
 sqlite3_bind_blob(pStmt, 1, pBlob, nBlob, SQLITE_STATIC);
121216122327
 sqlite3_step(pStmt);
121217122328
 *pRC = sqlite3_reset(pStmt);
121218122329
 sqlite3_free(a);
121219122330
 }
122331
+
122332
+static int fts3DoOptimize(Fts3Table *p, int bReturnDone){
122333
+ int i;
122334
+ int bSeenDone = 0;
122335
+ int rc = SQLITE_OK;
122336
+ for(i=0; rc==SQLITE_OK && i<p->nIndex; i++){
122337
+ rc = fts3SegmentMerge(p, i, FTS3_SEGCURSOR_ALL);
122338
+ if( rc==SQLITE_DONE ){
122339
+ bSeenDone = 1;
122340
+ rc = SQLITE_OK;
122341
+ }
122342
+ }
122343
+ sqlite3Fts3SegmentsClose(p);
122344
+ sqlite3Fts3PendingTermsClear(p);
122345
+
122346
+ return (rc==SQLITE_OK && bReturnDone && bSeenDone) ? SQLITE_DONE : rc;
122347
+}
121220122348
121221122349
 /*
121222122350
 ** Handle a 'special' INSERT of the form:
121223122351
 **
121224122352
 ** "INSERT INTO tbl(tbl) VALUES(<expr>)"
@@ -121232,16 +122360,11 @@
121232122360
 int nVal = sqlite3_value_bytes(pVal);
121233122361
121234122362
 if( !zVal ){
121235122363
 return SQLITE_NOMEM;
121236122364
 }else if( nVal==8 && 0==sqlite3_strnicmp(zVal, "optimize", 8) ){
121237
- rc = fts3SegmentMerge(p, FTS3_SEGCURSOR_ALL);
121238
- if( rc==SQLITE_DONE ){
121239
- rc = SQLITE_OK;
121240
- }else{
121241
- sqlite3Fts3PendingTermsClear(p);
121242
- }
122365
+ rc = fts3DoOptimize(p, 0);
121243122366
 #ifdef SQLITE_TEST
121244122367
 }else if( nVal>9 && 0==sqlite3_strnicmp(zVal, "nodesize=", 9) ){
121245122368
 p->nNodeSize = atoi(&zVal[9]);
121246122369
 rc = SQLITE_OK;
121247122370
 }else if( nVal>11 && 0==sqlite3_strnicmp(zVal, "maxpending=", 9) ){
@@ -121250,61 +122373,23 @@
121250122373
 #endif
121251122374
 }else{
121252122375
 rc = SQLITE_ERROR;
121253122376
 }
121254122377
121255
- sqlite3Fts3SegmentsClose(p);
121256122378
 return rc;
121257122379
 }
121258122380
121259
-/*
121260
-** Return the deferred doclist associated with deferred token pDeferred.
121261
-** This function assumes that sqlite3Fts3CacheDeferredDoclists() has already
121262
-** been called to allocate and populate the doclist.
121263
-*/
121264
-SQLITE_PRIVATE char *sqlite3Fts3DeferredDoclist(Fts3DeferredToken *pDeferred, int *pnByte){
121265
- if( pDeferred->pList ){
121266
- *pnByte = pDeferred->pList->nData;
121267
- return pDeferred->pList->aData;
121268
- }
121269
- *pnByte = 0;
121270
- return 0;
121271
-}
121272
-
121273
-/*
121274
-** Helper fucntion for FreeDeferredDoclists(). This function removes all
121275
-** references to deferred doclists from within the tree of Fts3Expr
121276
-** structures headed by
121277
-*/
121278
-static void fts3DeferredDoclistClear(Fts3Expr *pExpr){
121279
- if( pExpr ){
121280
- fts3DeferredDoclistClear(pExpr->pLeft);
121281
- fts3DeferredDoclistClear(pExpr->pRight);
121282
- if( pExpr->isLoaded ){
121283
- sqlite3_free(pExpr->aDoclist);
121284
- pExpr->isLoaded = 0;
121285
- pExpr->aDoclist = 0;
121286
- pExpr->nDoclist = 0;
121287
- pExpr->pCurrent = 0;
121288
- pExpr->iCurrent = 0;
121289
- }
121290
- }
121291
-}
121292
-
121293122381
 /*
121294122382
 ** Delete all cached deferred doclists. Deferred doclists are cached
121295122383
 ** (allocated) by the sqlite3Fts3CacheDeferredDoclists() function.
121296122384
 */
121297122385
 SQLITE_PRIVATE void sqlite3Fts3FreeDeferredDoclists(Fts3Cursor *pCsr){
121298122386
 Fts3DeferredToken *pDef;
121299122387
 for(pDef=pCsr->pDeferred; pDef; pDef=pDef->pNext){
121300
- sqlite3_free(pDef->pList);
122388
+ fts3PendingListDelete(pDef->pList);
121301122389
 pDef->pList = 0;
121302122390
 }
121303
- if( pCsr->pDeferred ){
121304
- fts3DeferredDoclistClear(pCsr->pExpr);
121305
- }
121306122391
 }
121307122392
121308122393
 /*
121309122394
 ** Free all entries in the pCsr->pDeffered list. Entries are added to
121310122395
 ** this list using sqlite3Fts3DeferToken().
@@ -121312,11 +122397,11 @@
121312122397
 SQLITE_PRIVATE void sqlite3Fts3FreeDeferredTokens(Fts3Cursor *pCsr){
121313122398
 Fts3DeferredToken *pDef;
121314122399
 Fts3DeferredToken *pNext;
121315122400
 for(pDef=pCsr->pDeferred; pDef; pDef=pNext){
121316122401
 pNext = pDef->pNext;
121317
- sqlite3_free(pDef->pList);
122402
+ fts3PendingListDelete(pDef->pList);
121318122403
 sqlite3_free(pDef);
121319122404
 }
121320122405
 pCsr->pDeferred = 0;
121321122406
 }
121322122407
@@ -121376,10 +122461,37 @@
121376122461
 }
121377122462
 }
121378122463
121379122464
 return rc;
121380122465
 }
122466
+
122467
+SQLITE_PRIVATE int sqlite3Fts3DeferredTokenList(
122468
+ Fts3DeferredToken *p,
122469
+ char **ppData,
122470
+ int *pnData
122471
+){
122472
+ char *pRet;
122473
+ int nSkip;
122474
+ sqlite3_int64 dummy;
122475
+
122476
+ *ppData = 0;
122477
+ *pnData = 0;
122478
+
122479
+ if( p->pList==0 ){
122480
+ return SQLITE_OK;
122481
+ }
122482
+
122483
+ pRet = (char *)sqlite3_malloc(p->pList->nData);
122484
+ if( !pRet ) return SQLITE_NOMEM;
122485
+
122486
+ nSkip = sqlite3Fts3GetVarint(p->pList->aData, &dummy);
122487
+ *pnData = p->pList->nData - nSkip;
122488
+ *ppData = pRet;
122489
+
122490
+ memcpy(pRet, &p->pList->aData[nSkip], *pnData);
122491
+ return SQLITE_OK;
122492
+}
121381122493
121382122494
 /*
121383122495
 ** Add an entry for token pToken to the pCsr->pDeferred list.
121384122496
 */
121385122497
 SQLITE_PRIVATE int sqlite3Fts3DeferToken(
@@ -121451,11 +122563,11 @@
121451122563
 ){
121452122564
 Fts3Table *p = (Fts3Table *)pVtab;
121453122565
 int rc = SQLITE_OK; /* Return Code */
121454122566
 int isRemove = 0; /* True for an UPDATE or DELETE */
121455122567
 sqlite3_int64 iRemove = 0; /* Rowid removed by UPDATE or DELETE */
121456
- u32 *aSzIns; /* Sizes of inserted documents */
122568
+ u32 *aSzIns = 0; /* Sizes of inserted documents */
121457122569
 u32 *aSzDel; /* Sizes of deleted documents */
121458122570
 int nChng = 0; /* Net change in number of documents */
121459122571
 int bInsertDone = 0;
121460122572
121461122573
 assert( p->pSegments==0 );
@@ -121466,16 +122578,20 @@
121466122578
 */
121467122579
 if( nArg>1
121468122580
 && sqlite3_value_type(apVal[0])==SQLITE_NULL
121469122581
 && sqlite3_value_type(apVal[p->nColumn+2])!=SQLITE_NULL
121470122582
 ){
121471
- return fts3SpecialInsert(p, apVal[p->nColumn+2]);
122583
+ rc = fts3SpecialInsert(p, apVal[p->nColumn+2]);
122584
+ goto update_out;
121472122585
 }
121473122586
121474122587
 /* Allocate space to hold the change in document sizes */
121475122588
 aSzIns = sqlite3_malloc( sizeof(aSzIns[0])*(p->nColumn+1)*2 );
121476
- if( aSzIns==0 ) return SQLITE_NOMEM;
122589
+ if( aSzIns==0 ){
122590
+ rc = SQLITE_NOMEM;
122591
+ goto update_out;
122592
+ }
121477122593
 aSzDel = &aSzIns[p->nColumn+1];
121478122594
 memset(aSzIns, 0, sizeof(aSzIns[0])*(p->nColumn+1)*2);
121479122595
121480122596
 /* If this is an INSERT operation, or an UPDATE that modifies the rowid
121481122597
 ** value, then this operation requires constraint handling.
@@ -121521,12 +122637,11 @@
121521122637
 bInsertDone = 1;
121522122638
 }
121523122639
 }
121524122640
 }
121525122641
 if( rc!=SQLITE_OK ){
121526
- sqlite3_free(aSzIns);
121527
- return rc;
122642
+ goto update_out;
121528122643
 }
121529122644
121530122645
 /* If this is a DELETE or UPDATE operation, remove the old record. */
121531122646
 if( sqlite3_value_type(apVal[0])!=SQLITE_NULL ){
121532122647
 assert( sqlite3_value_type(apVal[0])==SQLITE_INTEGER );
@@ -121555,10 +122670,11 @@
121555122670
121556122671
 if( p->bHasStat ){
121557122672
 fts3UpdateDocTotals(&rc, p, aSzIns, aSzDel, nChng);
121558122673
 }
121559122674
122675
+ update_out:
121560122676
 sqlite3_free(aSzIns);
121561122677
 sqlite3Fts3SegmentsClose(p);
121562122678
 return rc;
121563122679
 }
121564122680
@@ -121569,16 +122685,14 @@
121569122685
 */
121570122686
 SQLITE_PRIVATE int sqlite3Fts3Optimize(Fts3Table *p){
121571122687
 int rc;
121572122688
 rc = sqlite3_exec(p->db, "SAVEPOINT fts3", 0, 0, 0);
121573122689
 if( rc==SQLITE_OK ){
121574
- rc = fts3SegmentMerge(p, FTS3_SEGCURSOR_ALL);
121575
- if( rc==SQLITE_OK ){
121576
- rc = sqlite3_exec(p->db, "RELEASE fts3", 0, 0, 0);
121577
- if( rc==SQLITE_OK ){
121578
- sqlite3Fts3PendingTermsClear(p);
121579
- }
122690
+ rc = fts3DoOptimize(p, 1);
122691
+ if( rc==SQLITE_OK || rc==SQLITE_DONE ){
122692
+ int rc2 = sqlite3_exec(p->db, "RELEASE fts3", 0, 0, 0);
122693
+ if( rc2!=SQLITE_OK ) rc = rc2;
121580122694
 }else{
121581122695
 sqlite3_exec(p->db, "ROLLBACK TO fts3", 0, 0, 0);
121582122696
 sqlite3_exec(p->db, "RELEASE fts3", 0, 0, 0);
121583122697
 }
121584122698
 }
@@ -121763,76 +122877,24 @@
121763122877
 ){
121764122878
 int iPhrase = 0; /* Variable used as the phrase counter */
121765122879
 return fts3ExprIterate2(pExpr, &iPhrase, x, pCtx);
121766122880
 }
121767122881
121768
-/*
121769
-** The argument to this function is always a phrase node. Its doclist
121770
-** (Fts3Expr.aDoclist[]) and the doclists associated with all phrase nodes
121771
-** to the left of this one in the query tree have already been loaded.
121772
-**
121773
-** If this phrase node is part of a series of phrase nodes joined by
121774
-** NEAR operators (and is not the left-most of said series), then elements are
121775
-** removed from the phrases doclist consistent with the NEAR restriction. If
121776
-** required, elements may be removed from the doclists of phrases to the
121777
-** left of this one that are part of the same series of NEAR operator
121778
-** connected phrases.
121779
-**
121780
-** If an OOM error occurs, SQLITE_NOMEM is returned. Otherwise, SQLITE_OK.
121781
-*/
121782
-static int fts3ExprNearTrim(Fts3Expr *pExpr){
121783
- int rc = SQLITE_OK;
121784
- Fts3Expr *pParent = pExpr->pParent;
121785
-
121786
- assert( pExpr->eType==FTSQUERY_PHRASE );
121787
- while( rc==SQLITE_OK
121788
- && pParent
121789
- && pParent->eType==FTSQUERY_NEAR
121790
- && pParent->pRight==pExpr
121791
- ){
121792
- /* This expression (pExpr) is the right-hand-side of a NEAR operator.
121793
- ** Find the expression to the left of the same operator.
121794
- */
121795
- int nNear = pParent->nNear;
121796
- Fts3Expr *pLeft = pParent->pLeft;
121797
-
121798
- if( pLeft->eType!=FTSQUERY_PHRASE ){
121799
- assert( pLeft->eType==FTSQUERY_NEAR );
121800
- assert( pLeft->pRight->eType==FTSQUERY_PHRASE );
121801
- pLeft = pLeft->pRight;
121802
- }
121803
-
121804
- rc = sqlite3Fts3ExprNearTrim(pLeft, pExpr, nNear);
121805
-
121806
- pExpr = pLeft;
121807
- pParent = pExpr->pParent;
121808
- }
121809
-
121810
- return rc;
121811
-}
121812
-
121813122882
 /*
121814122883
 ** This is an fts3ExprIterate() callback used while loading the doclists
121815122884
 ** for each phrase into Fts3Expr.aDoclist[]/nDoclist. See also
121816122885
 ** fts3ExprLoadDoclists().
121817122886
 */
121818122887
 static int fts3ExprLoadDoclistsCb(Fts3Expr *pExpr, int iPhrase, void *ctx){
121819122888
 int rc = SQLITE_OK;
122889
+ Fts3Phrase *pPhrase = pExpr->pPhrase;
121820122890
 LoadDoclistCtx *p = (LoadDoclistCtx *)ctx;
121821122891
121822122892
 UNUSED_PARAMETER(iPhrase);
121823122893
121824122894
 p->nPhrase++;
121825
- p->nToken += pExpr->pPhrase->nToken;
121826
-
121827
- if( pExpr->isLoaded==0 ){
121828
- rc = sqlite3Fts3ExprLoadDoclist(p->pCsr, pExpr);
121829
- pExpr->isLoaded = 1;
121830
- if( rc==SQLITE_OK ){
121831
- rc = fts3ExprNearTrim(pExpr);
121832
- }
121833
- }
122895
+ p->nToken += pPhrase->nToken;
121834122896
121835122897
 return rc;
121836122898
 }
121837122899
121838122900
 /*
@@ -122002,11 +123064,11 @@
122002123064
 SnippetPhrase *pPhrase = &p->aPhrase[iPhrase];
122003123065
 char *pCsr;
122004123066
122005123067
 pPhrase->nToken = pExpr->pPhrase->nToken;
122006123068
122007
- pCsr = sqlite3Fts3FindPositions(p->pCsr, pExpr, p->pCsr->iPrevId, p->iCol);
123069
+ pCsr = sqlite3Fts3EvalPhrasePoslist(p->pCsr, pExpr, p->iCol);
122008123070
 if( pCsr ){
122009123071
 int iFirst = 0;
122010123072
 pPhrase->pList = pCsr;
122011123073
 fts3GetDeltaPosition(&pCsr, &iFirst);
122012123074
 pPhrase->pHead = pCsr;
@@ -122359,30 +123421,10 @@
122359123421
122360123422
 *ppCollist = pEnd;
122361123423
 return nEntry;
122362123424
 }
122363123425
122364
-static void fts3LoadColumnlistCounts(char **pp, u32 *aOut, int isGlobal){
122365
- char *pCsr = *pp;
122366
- while( *pCsr ){
122367
- int nHit;
122368
- sqlite3_int64 iCol = 0;
122369
- if( *pCsr==0x01 ){
122370
- pCsr++;
122371
- pCsr += sqlite3Fts3GetVarint(pCsr, &iCol);
122372
- }
122373
- nHit = fts3ColumnlistCount(&pCsr);
122374
- assert( nHit>0 );
122375
- if( isGlobal ){
122376
- aOut[iCol*3+1]++;
122377
- }
122378
- aOut[iCol*3] += nHit;
122379
- }
122380
- pCsr++;
122381
- *pp = pCsr;
122382
-}
122383
-
122384123426
 /*
122385123427
 ** fts3ExprIterate() callback used to collect the "global" matchinfo stats
122386123428
 ** for a single query.
122387123429
 **
122388123430
 ** fts3ExprIterate() callback to load the 'global' elements of a
@@ -122412,52 +123454,13 @@
122412123454
 Fts3Expr *pExpr, /* Phrase expression node */
122413123455
 int iPhrase, /* Phrase number (numbered from zero) */
122414123456
 void *pCtx /* Pointer to MatchInfo structure */
122415123457
 ){
122416123458
 MatchInfo *p = (MatchInfo *)pCtx;
122417
- Fts3Cursor *pCsr = p->pCursor;
122418
- char *pIter;
122419
- char *pEnd;
122420
- char *pFree = 0;
122421
- u32 *aOut = &p->aMatchinfo[3*iPhrase*p->nCol];
122422
-
122423
- assert( pExpr->isLoaded );
122424
- assert( pExpr->eType==FTSQUERY_PHRASE );
122425
-
122426
- if( pCsr->pDeferred ){
122427
- Fts3Phrase *pPhrase = pExpr->pPhrase;
122428
- int ii;
122429
- for(ii=0; ii<pPhrase->nToken; ii++){
122430
- if( pPhrase->aToken[ii].bFulltext ) break;
122431
- }
122432
- if( ii<pPhrase->nToken ){
122433
- int nFree = 0;
122434
- int rc = sqlite3Fts3ExprLoadFtDoclist(pCsr, pExpr, &pFree, &nFree);
122435
- if( rc!=SQLITE_OK ) return rc;
122436
- pIter = pFree;
122437
- pEnd = &pFree[nFree];
122438
- }else{
122439
- int iCol; /* Column index */
122440
- for(iCol=0; iCol<p->nCol; iCol++){
122441
- aOut[iCol*3 + 1] = (u32)p->nDoc;
122442
- aOut[iCol*3 + 2] = (u32)p->nDoc;
122443
- }
122444
- return SQLITE_OK;
122445
- }
122446
- }else{
122447
- pIter = pExpr->aDoclist;
122448
- pEnd = &pExpr->aDoclist[pExpr->nDoclist];
122449
- }
122450
-
122451
- /* Fill in the global hit count matrix row for this phrase. */
122452
- while( pIter<pEnd ){
122453
- while( *pIter++ & 0x80 ); /* Skip past docid. */
122454
- fts3LoadColumnlistCounts(&pIter, &aOut[1], 1);
122455
- }
122456
-
122457
- sqlite3_free(pFree);
122458
- return SQLITE_OK;
123459
+ return sqlite3Fts3EvalPhraseStats(
123460
+ p->pCursor, pExpr, &p->aMatchinfo[3*iPhrase*p->nCol]
123461
+ );
122459123462
 }
122460123463
122461123464
 /*
122462123465
 ** fts3ExprIterate() callback used to collect the "local" part of the
122463123466
 ** FTS3_MATCHINFO_HITS array. The local stats are those elements of the
@@ -122470,18 +123473,17 @@
122470123473
 ){
122471123474
 MatchInfo *p = (MatchInfo *)pCtx;
122472123475
 int iStart = iPhrase * p->nCol * 3;
122473123476
 int i;
122474123477
122475
- for(i=0; i<p->nCol; i++) p->aMatchinfo[iStart+i*3] = 0;
122476
-
122477
- if( pExpr->aDoclist ){
123478
+ for(i=0; i<p->nCol; i++){
122478123479
 char *pCsr;
122479
-
122480
- pCsr = sqlite3Fts3FindPositions(p->pCursor, pExpr, p->pCursor->iPrevId, -1);
123480
+ pCsr = sqlite3Fts3EvalPhrasePoslist(p->pCursor, pExpr, i);
122481123481
 if( pCsr ){
122482
- fts3LoadColumnlistCounts(&pCsr, &p->aMatchinfo[iStart], 0);
123482
+ p->aMatchinfo[iStart+i*3] = fts3ColumnlistCount(&pCsr);
123483
+ }else{
123484
+ p->aMatchinfo[iStart+i*3] = 0;
122483123485
 }
122484123486
 }
122485123487
122486123488
 return SQLITE_OK;
122487123489
 }
@@ -122563,13 +123565,12 @@
122563123565
 ** values for a matchinfo() FTS3_MATCHINFO_LCS request.
122564123566
 */
122565123567
 typedef struct LcsIterator LcsIterator;
122566123568
 struct LcsIterator {
122567123569
 Fts3Expr *pExpr; /* Pointer to phrase expression */
122568
- char *pRead; /* Cursor used to iterate through aDoclist */
122569123570
 int iPosOffset; /* Tokens count up to end of this phrase */
122570
- int iCol; /* Current column number */
123571
+ char *pRead; /* Cursor used to iterate through aDoclist */
122571123572
 int iPos; /* Current position */
122572123573
 };
122573123574
122574123575
 /*
122575123576
 ** If LcsIterator.iCol is set to the following value, the iterator has
@@ -122596,21 +123597,14 @@
122596123597
 char *pRead = pIter->pRead;
122597123598
 sqlite3_int64 iRead;
122598123599
 int rc = 0;
122599123600
122600123601
 pRead += sqlite3Fts3GetVarint(pRead, &iRead);
122601
- if( iRead==0 ){
122602
- pIter->iCol = LCS_ITERATOR_FINISHED;
123602
+ if( iRead==0 || iRead==1 ){
123603
+ pRead = 0;
122603123604
 rc = 1;
122604123605
 }else{
122605
- if( iRead==1 ){
122606
- pRead += sqlite3Fts3GetVarint(pRead, &iRead);
122607
- pIter->iCol = (int)iRead;
122608
- pIter->iPos = pIter->iPosOffset;
122609
- pRead += sqlite3Fts3GetVarint(pRead, &iRead);
122610
- rc = 1;
122611
- }
122612123606
 pIter->iPos += (int)(iRead-2);
122613123607
 }
122614123608
122615123609
 pIter->pRead = pRead;
122616123610
 return rc;
@@ -122638,46 +123632,38 @@
122638123632
 **/
122639123633
 aIter = sqlite3_malloc(sizeof(LcsIterator) * pCsr->nPhrase);
122640123634
 if( !aIter ) return SQLITE_NOMEM;
122641123635
 memset(aIter, 0, sizeof(LcsIterator) * pCsr->nPhrase);
122642123636
 (void)fts3ExprIterate(pCsr->pExpr, fts3MatchinfoLcsCb, (void*)aIter);
123637
+
122643123638
 for(i=0; i<pInfo->nPhrase; i++){
122644123639
 LcsIterator *pIter = &aIter[i];
122645123640
 nToken -= pIter->pExpr->pPhrase->nToken;
122646123641
 pIter->iPosOffset = nToken;
122647
- pIter->pRead = sqlite3Fts3FindPositions(pCsr,pIter->pExpr,pCsr->iPrevId,-1);
122648
- if( pIter->pRead ){
122649
- pIter->iPos = pIter->iPosOffset;
122650
- fts3LcsIteratorAdvance(&aIter[i]);
122651
- }else{
122652
- pIter->iCol = LCS_ITERATOR_FINISHED;
122653
- }
122654123642
 }
122655123643
122656123644
 for(iCol=0; iCol<pInfo->nCol; iCol++){
122657123645
 int nLcs = 0; /* LCS value for this column */
122658123646
 int nLive = 0; /* Number of iterators in aIter not at EOF */
122659123647
122660
- /* Loop through the iterators in aIter[]. Set nLive to the number of
122661
- ** iterators that point to a position-list corresponding to column iCol.
122662
- */
122663123648
 for(i=0; i<pInfo->nPhrase; i++){
122664
- assert( aIter[i].iCol>=iCol );
122665
- if( aIter[i].iCol==iCol ) nLive++;
123649
+ LcsIterator *pIt = &aIter[i];
123650
+ pIt->pRead = sqlite3Fts3EvalPhrasePoslist(pCsr, pIt->pExpr, iCol);
123651
+ if( pIt->pRead ){
123652
+ pIt->iPos = pIt->iPosOffset;
123653
+ fts3LcsIteratorAdvance(&aIter[i]);
123654
+ nLive++;
123655
+ }
122666123656
 }
122667123657
122668
- /* The following loop runs until all iterators in aIter[] have finished
122669
- ** iterating through positions in column iCol. Exactly one of the
122670
- ** iterators is advanced each time the body of the loop is run.
122671
- */
122672123658
 while( nLive>0 ){
122673123659
 LcsIterator *pAdv = 0; /* The iterator to advance by one position */
122674123660
 int nThisLcs = 0; /* LCS for the current iterator positions */
122675123661
122676123662
 for(i=0; i<pInfo->nPhrase; i++){
122677123663
 LcsIterator *pIter = &aIter[i];
122678
- if( iCol!=pIter->iCol ){
123664
+ if( pIter->pRead==0 ){
122679123665
 /* This iterator is already at EOF for this column. */
122680123666
 nThisLcs = 0;
122681123667
 }else{
122682123668
 if( pAdv==0 || pIter->iPos<pAdv->iPos ){
122683123669
 pAdv = pIter;
@@ -123013,11 +123999,11 @@
123013123999
 int iTerm; /* For looping through nTerm phrase terms */
123014124000
 char *pList; /* Pointer to position list for phrase */
123015124001
 int iPos = 0; /* First position in position-list */
123016124002
123017124003
 UNUSED_PARAMETER(iPhrase);
123018
- pList = sqlite3Fts3FindPositions(p->pCsr, pExpr, p->iDocid, p->iCol);
124004
+ pList = sqlite3Fts3EvalPhrasePoslist(p->pCsr, pExpr, p->iCol);
123019124005
 nTerm = pExpr->pPhrase->nToken;
123020124006
 if( pList ){
123021124007
 fts3GetDeltaPosition(&pList, &iPos);
123022124008
 assert( iPos>=0 );
123023124009
 }
123024124010
--- src/sqlite3.c
+++ src/sqlite3.c
@@ -650,11 +650,11 @@
650 ** [sqlite3_libversion_number()], [sqlite3_sourceid()],
651 ** [sqlite_version()] and [sqlite_source_id()].
652 */
653 #define SQLITE_VERSION "3.7.7"
654 #define SQLITE_VERSION_NUMBER 3007007
655 #define SQLITE_SOURCE_ID "2011-06-03 14:19:10 0206bc6f87bb9393218a380fc5b18039d334a8d8"
656
657 /*
658 ** CAPI3REF: Run-Time Library Version Numbers
659 ** KEYWORDS: sqlite3_version, sqlite3_sourceid
660 **
@@ -8483,10 +8483,11 @@
8483 SQLITE_PRIVATE int sqlite3VdbeAddOp2(Vdbe*,int,int,int);
8484 SQLITE_PRIVATE int sqlite3VdbeAddOp3(Vdbe*,int,int,int,int);
8485 SQLITE_PRIVATE int sqlite3VdbeAddOp4(Vdbe*,int,int,int,int,const char *zP4,int);
8486 SQLITE_PRIVATE int sqlite3VdbeAddOp4Int(Vdbe*,int,int,int,int,int);
8487 SQLITE_PRIVATE int sqlite3VdbeAddOpList(Vdbe*, int nOp, VdbeOpList const *aOp);
 
8488 SQLITE_PRIVATE void sqlite3VdbeChangeP1(Vdbe*, int addr, int P1);
8489 SQLITE_PRIVATE void sqlite3VdbeChangeP2(Vdbe*, int addr, int P2);
8490 SQLITE_PRIVATE void sqlite3VdbeChangeP3(Vdbe*, int addr, int P3);
8491 SQLITE_PRIVATE void sqlite3VdbeChangeP5(Vdbe*, u8 P5);
8492 SQLITE_PRIVATE void sqlite3VdbeJumpHere(Vdbe*, int addr);
@@ -9282,11 +9283,11 @@
9282 ** sqlite3_close().
9283 *
9284 ** A thread must be holding a mutex on the corresponding Btree in order
9285 ** to access Schema content. This implies that the thread must also be
9286 ** holding a mutex on the sqlite3 connection pointer that owns the Btree.
9287 ** For a TEMP Schema, on the connection mutex is required.
9288 */
9289 struct Schema {
9290 int schema_cookie; /* Database schema version number for this file */
9291 int iGeneration; /* Generation counter. Incremented with each change */
9292 Hash tblHash; /* All tables indexed by name */
@@ -11479,11 +11480,11 @@
11479 SQLITE_PRIVATE int sqlite3AtoF(const char *z, double*, int, u8);
11480 SQLITE_PRIVATE int sqlite3GetInt32(const char *, int*);
11481 SQLITE_PRIVATE int sqlite3Atoi(const char*);
11482 SQLITE_PRIVATE int sqlite3Utf16ByteLen(const void *pData, int nChar);
11483 SQLITE_PRIVATE int sqlite3Utf8CharLen(const char *pData, int nByte);
11484 SQLITE_PRIVATE int sqlite3Utf8Read(const u8*, const u8**);
11485
11486 /*
11487 ** Routines to read and write variable-length integers. These used to
11488 ** be defined locally, but now we use the varint routines in the util.c
11489 ** file. Code should use the MACRO forms below, as the Varint32 versions
@@ -20086,11 +20087,11 @@
20086 } \
20087 if( c<0x80 \
20088 || (c&0xFFFFF800)==0xD800 \
20089 || (c&0xFFFFFFFE)==0xFFFE ){ c = 0xFFFD; } \
20090 }
20091 SQLITE_PRIVATE int sqlite3Utf8Read(
20092 const unsigned char *zIn, /* First byte of UTF-8 character */
20093 const unsigned char **pzNext /* Write first byte past UTF-8 char here */
20094 ){
20095 unsigned int c;
20096
@@ -35805,11 +35806,11 @@
35805 memset(pCache, 0, sz);
35806 if( separateCache ){
35807 pGroup = (PGroup*)&pCache[1];
35808 pGroup->mxPinned = 10;
35809 }else{
35810 pGroup = &pcache1_g.grp;
35811 }
35812 pCache->pGroup = pGroup;
35813 pCache->szPage = szPage;
35814 pCache->bPurgeable = (bPurgeable ? 1 : 0);
35815 if( bPurgeable ){
@@ -48324,10 +48325,12 @@
48324 **
48325 ** This routine works only for pages that do not contain overflow cells.
48326 */
48327 #define findCell(P,I) \
48328 ((P)->aData + ((P)->maskPage & get2byte(&(P)->aData[(P)->cellOffset+2*(I)])))
 
 
48329
48330 /*
48331 ** This a more complex version of findCell() that works for
48332 ** pages that do contain overflow cells.
48333 */
@@ -51918,11 +51921,11 @@
51918 assert( pCur->apPage[pCur->iPage]->nCell==0 );
51919 return SQLITE_OK;
51920 }
51921 assert( pCur->apPage[0]->intKey || pIdxKey );
51922 for(;;){
51923 int lwr, upr;
51924 Pgno chldPg;
51925 MemPage *pPage = pCur->apPage[pCur->iPage];
51926 int c;
51927
51928 /* pPage->nCell must be greater than zero. If this is the root-page
@@ -51934,18 +51937,18 @@
51934 assert( pPage->nCell>0 );
51935 assert( pPage->intKey==(pIdxKey==0) );
51936 lwr = 0;
51937 upr = pPage->nCell-1;
51938 if( biasRight ){
51939 pCur->aiIdx[pCur->iPage] = (u16)upr;
51940 }else{
51941 pCur->aiIdx[pCur->iPage] = (u16)((upr+lwr)/2);
51942 }
51943 for(;;){
51944 int idx = pCur->aiIdx[pCur->iPage]; /* Index of current cell in pPage */
51945 u8 *pCell; /* Pointer to current cell in pPage */
51946
 
51947 pCur->info.nSize = 0;
51948 pCell = findCell(pPage, idx) + pPage->childPtrSize;
51949 if( pPage->intKey ){
51950 i64 nCellKey;
51951 if( pPage->hasData ){
@@ -52024,11 +52027,11 @@
52024 upr = idx-1;
52025 }
52026 if( lwr>upr ){
52027 break;
52028 }
52029 pCur->aiIdx[pCur->iPage] = (u16)((lwr+upr)/2);
52030 }
52031 assert( lwr==upr+1 );
52032 assert( pPage->isInit );
52033 if( pPage->leaf ){
52034 chldPg = 0;
@@ -52886,13 +52889,13 @@
52886 if( rc ){
52887 *pRC = rc;
52888 return;
52889 }
52890 endPtr = &data[pPage->cellOffset + 2*pPage->nCell - 2];
 
52891 while( ptr<endPtr ){
52892 ptr[0] = ptr[2];
52893 ptr[1] = ptr[3];
52894 ptr += 2;
52895 }
52896 pPage->nCell--;
52897 put2byte(&data[hdr+3], pPage->nCell);
52898 pPage->nFree += 2;
@@ -52929,10 +52932,11 @@
52929 int end; /* First byte past the last cell pointer in data[] */
52930 int ins; /* Index in data[] where new cell pointer is inserted */
52931 int cellOffset; /* Address of first cell pointer in data[] */
52932 u8 *data; /* The content of the whole page */
52933 u8 *ptr; /* Used for moving information around in data[] */
 
52934
52935 int nSkip = (iChild ? 4 : 0);
52936
52937 if( *pRC ) return;
52938
@@ -52979,13 +52983,16 @@
52979 pPage->nFree -= (u16)(2 + sz);
52980 memcpy(&data[idx+nSkip], pCell+nSkip, sz-nSkip);
52981 if( iChild ){
52982 put4byte(&data[idx], iChild);
52983 }
52984 for(j=end, ptr=&data[j]; j>ins; j-=2, ptr-=2){
52985 ptr[0] = ptr[-2];
52986 ptr[1] = ptr[-1];
 
 
 
52987 }
52988 put2byte(&data[ins], idx);
52989 put2byte(&data[pPage->hdrOffset+3], pPage->nCell);
52990 #ifndef SQLITE_OMIT_AUTOVACUUM
52991 if( pPage->pBt->autoVacuum ){
@@ -53026,14 +53033,15 @@
53026 assert( get2byteNotZero(&data[hdr+5])==nUsable );
53027
53028 pCellptr = &data[pPage->cellOffset + nCell*2];
53029 cellbody = nUsable;
53030 for(i=nCell-1; i>=0; i--){
 
53031 pCellptr -= 2;
53032 cellbody -= aSize[i];
53033 put2byte(pCellptr, cellbody);
53034 memcpy(&data[cellbody], apCell[i], aSize[i]);
53035 }
53036 put2byte(&data[hdr+3], nCell);
53037 put2byte(&data[hdr+5], cellbody);
53038 pPage->nFree -= (nCell*2 + nUsable - cellbody);
53039 pPage->nCell = (u16)nCell;
@@ -53483,16 +53491,28 @@
53483 memcpy(pOld, apOld[i], sizeof(MemPage));
53484 pOld->aData = (void*)&pOld[1];
53485 memcpy(pOld->aData, apOld[i]->aData, pBt->pageSize);
53486
53487 limit = pOld->nCell+pOld->nOverflow;
53488 for(j=0; j<limit; j++){
53489 assert( nCell<nMaxCells );
53490 apCell[nCell] = findOverflowCell(pOld, j);
53491 szCell[nCell] = cellSizePtr(pOld, apCell[nCell]);
53492 nCell++;
53493 }
 
 
 
 
 
 
 
 
 
 
 
 
53494 if( i<nOld-1 && !leafData){
53495 u16 sz = (u16)szNew[i];
53496 u8 *pTemp;
53497 assert( nCell<nMaxCells );
53498 szCell[nCell] = sz;
@@ -57637,17 +57657,10 @@
57637 pOp->p1 = p1;
57638 pOp->p2 = p2;
57639 pOp->p3 = p3;
57640 pOp->p4.p = 0;
57641 pOp->p4type = P4_NOTUSED;
57642 p->expired = 0;
57643 if( op==OP_ParseSchema ){
57644 /* Any program that uses the OP_ParseSchema opcode needs to lock
57645 ** all btrees. */
57646 int j;
57647 for(j=0; j<p->db->nDb; j++) sqlite3VdbeUsesBtree(p, j);
57648 }
57649 #ifdef SQLITE_DEBUG
57650 pOp->zComment = 0;
57651 if( sqlite3VdbeAddopTrace ) sqlite3VdbePrintOp(0, i, &p->aOp[i]);
57652 #endif
57653 #ifdef VDBE_PROFILE
@@ -57681,10 +57694,24 @@
57681 ){
57682 int addr = sqlite3VdbeAddOp3(p, op, p1, p2, p3);
57683 sqlite3VdbeChangeP4(p, addr, zP4, p4type);
57684 return addr;
57685 }
 
 
 
 
 
 
 
 
 
 
 
 
 
 
57686
57687 /*
57688 ** Add an opcode that includes the p4 value as an integer.
57689 */
57690 SQLITE_PRIVATE int sqlite3VdbeAddOp4Int(
@@ -64083,10 +64110,20 @@
64083 u.ag.ctx.pColl = pOp[-1].p4.pColl;
64084 }
64085 db->lastRowid = lastRowid;
64086 (*u.ag.ctx.pFunc->xFunc)(&u.ag.ctx, u.ag.n, u.ag.apVal); /* IMP: R-24505-23230 */
64087 lastRowid = db->lastRowid;
 
 
 
 
 
 
 
 
 
 
64088 if( db->mallocFailed ){
64089 /* Even though a malloc() has failed, the implementation of the
64090 ** user function may have called an sqlite3_result_XXX() function
64091 ** to return a value. The following call releases any resources
64092 ** associated with such a value.
@@ -64093,19 +64130,10 @@
64093 */
64094 sqlite3VdbeMemRelease(&u.ag.ctx.s);
64095 goto no_mem;
64096 }
64097
64098 /* If any auxiliary data functions have been called by this user function,
64099 ** immediately call the destructor for any non-static values.
64100 */
64101 if( u.ag.ctx.pVdbeFunc ){
64102 sqlite3VdbeDeleteAuxData(u.ag.ctx.pVdbeFunc, pOp->p1);
64103 pOp->p4.pVdbeFunc = u.ag.ctx.pVdbeFunc;
64104 pOp->p4type = P4_VDBEFUNC;
64105 }
64106
64107 /* If the function returned an error, throw an exception */
64108 if( u.ag.ctx.isError ){
64109 sqlite3SetString(&p->zErrMsg, db, "%s", sqlite3_value_text(&u.ag.ctx.s));
64110 rc = u.ag.ctx.isError;
64111 }
@@ -75255,18 +75283,18 @@
75255 sqlite3VdbeAddOp4(v, OP_DropTable, iDb, 0, 0, pTab->zName, 0);
75256
75257 /* Reload the table, index and permanent trigger schemas. */
75258 zWhere = sqlite3MPrintf(pParse->db, "tbl_name=%Q", zName);
75259 if( !zWhere ) return;
75260 sqlite3VdbeAddOp4(v, OP_ParseSchema, iDb, 0, 0, zWhere, P4_DYNAMIC);
75261
75262 #ifndef SQLITE_OMIT_TRIGGER
75263 /* Now, if the table is not stored in the temp database, reload any temp
75264 ** triggers. Don't use IN(...) in case SQLITE_OMIT_SUBQUERY is defined.
75265 */
75266 if( (zWhere=whereTempTriggers(pParse, pTab))!=0 ){
75267 sqlite3VdbeAddOp4(v, OP_ParseSchema, 1, 0, 0, zWhere, P4_DYNAMIC);
75268 }
75269 #endif
75270 }
75271
75272 /*
@@ -78875,12 +78903,12 @@
78875 }
78876 }
78877 #endif
78878
78879 /* Reparse everything to update our internal data structures */
78880 sqlite3VdbeAddOp4(v, OP_ParseSchema, iDb, 0, 0,
78881 sqlite3MPrintf(db, "tbl_name='%q'",p->zName), P4_DYNAMIC);
78882 }
78883
78884
78885 /* Add the table to the in-memory representation of the database.
78886 */
@@ -80073,13 +80101,12 @@
80073 ** to invalidate all pre-compiled statements.
80074 */
80075 if( pTblName ){
80076 sqlite3RefillIndex(pParse, pIndex, iMem);
80077 sqlite3ChangeCookie(pParse, iDb);
80078 sqlite3VdbeAddOp4(v, OP_ParseSchema, iDb, 0, 0,
80079 sqlite3MPrintf(db, "name='%q' AND type='index'", pIndex->zName),
80080 P4_DYNAMIC);
80081 sqlite3VdbeAddOp1(v, OP_Expire, 0);
80082 }
80083 }
80084
80085 /* When adding an index to the list of indices for a table, make
@@ -82621,14 +82648,14 @@
82621 ** character is exactly one byte in size. Also, all characters are
82622 ** able to participate in upper-case-to-lower-case mappings in EBCDIC
82623 ** whereas only characters less than 0x80 do in ASCII.
82624 */
82625 #if defined(SQLITE_EBCDIC)
82626 # define sqlite3Utf8Read(A,C) (*(A++))
82627 # define GlogUpperToLower(A) A = sqlite3UpperToLower[A]
82628 #else
82629 # define GlogUpperToLower(A) if( A<0x80 ){ A = sqlite3UpperToLower[A]; }
82630 #endif
82631
82632 static const struct compareInfo globInfo = { '*', '?', '[', 0 };
82633 /* The correct SQL-92 behavior is for the LIKE operator to ignore
82634 ** case. Thus 'a' LIKE 'A' would be true. */
@@ -82667,13 +82694,13 @@
82667 */
82668 static int patternCompare(
82669 const u8 *zPattern, /* The glob pattern */
82670 const u8 *zString, /* The string to compare against the glob */
82671 const struct compareInfo *pInfo, /* Information about how to do the compare */
82672 const int esc /* The escape character */
82673 ){
82674 int c, c2;
82675 int invert;
82676 int seen;
82677 u8 matchOne = pInfo->matchOne;
82678 u8 matchAll = pInfo->matchAll;
82679 u8 matchSet = pInfo->matchSet;
@@ -82723,11 +82750,11 @@
82723 }else if( !prevEscape && c==matchOne ){
82724 if( sqlite3Utf8Read(zString, &zString)==0 ){
82725 return 0;
82726 }
82727 }else if( c==matchSet ){
82728 int prior_c = 0;
82729 assert( esc==0 ); /* This only occurs for GLOB, not LIKE */
82730 seen = 0;
82731 invert = 0;
82732 c = sqlite3Utf8Read(zString, &zString);
82733 if( c==0 ) return 0;
@@ -82799,11 +82826,11 @@
82799 sqlite3_context *context,
82800 int argc,
82801 sqlite3_value **argv
82802 ){
82803 const unsigned char *zA, *zB;
82804 int escape = 0;
82805 int nPat;
82806 sqlite3 *db = sqlite3_context_db_handle(context);
82807
82808 zB = sqlite3_value_text(argv[0]);
82809 zA = sqlite3_value_text(argv[1]);
@@ -84110,17 +84137,29 @@
84110 }
84111
84112 /* If the parent table is the same as the child table, and we are about
84113 ** to increment the constraint-counter (i.e. this is an INSERT operation),
84114 ** then check if the row being inserted matches itself. If so, do not
84115 ** increment the constraint-counter. */
 
 
 
 
 
 
84116 if( pTab==pFKey->pFrom && nIncr==1 ){
84117 int iJump = sqlite3VdbeCurrentAddr(v) + nCol + 1;
84118 for(i=0; i<nCol; i++){
84119 int iChild = aiCol[i]+1+regData;
84120 int iParent = pIdx->aiColumn[i]+1+regData;
 
 
 
 
 
84121 sqlite3VdbeAddOp3(v, OP_Ne, iChild, iJump, iParent);
 
84122 }
84123 sqlite3VdbeAddOp2(v, OP_Goto, 0, iOk);
84124 }
84125
84126 sqlite3VdbeAddOp3(v, OP_MakeRecord, regTemp, nCol, regRec);
@@ -95336,13 +95375,12 @@
95336 "INSERT INTO %Q.%s VALUES('trigger',%Q,%Q,0,'CREATE TRIGGER %q')",
95337 db->aDb[iDb].zName, SCHEMA_TABLE(iDb), zName,
95338 pTrig->table, z);
95339 sqlite3DbFree(db, z);
95340 sqlite3ChangeCookie(pParse, iDb);
95341 sqlite3VdbeAddOp4(v, OP_ParseSchema, iDb, 0, 0, sqlite3MPrintf(
95342 db, "type='trigger' AND name='%q'", zName), P4_DYNAMIC
95343 );
95344 }
95345
95346 if( db->init.busy ){
95347 Trigger *pLink = pTrig;
95348 Hash *pHash = &db->aDb[iDb].pSchema->trigHash;
@@ -96392,11 +96430,11 @@
96392 aRegIdx = sqlite3DbMallocRaw(db, sizeof(Index*) * nIdx );
96393 if( aRegIdx==0 ) goto update_cleanup;
96394 }
96395 for(j=0, pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext, j++){
96396 int reg;
96397 if( chngRowid ){
96398 reg = ++pParse->nMem;
96399 }else{
96400 reg = 0;
96401 for(i=0; i<pIdx->nColumn; i++){
96402 if( aXRef[pIdx->aiColumn[i]]>=0 ){
@@ -97547,11 +97585,11 @@
97547 v = sqlite3GetVdbe(pParse);
97548 sqlite3ChangeCookie(pParse, iDb);
97549
97550 sqlite3VdbeAddOp2(v, OP_Expire, 0, 0);
97551 zWhere = sqlite3MPrintf(db, "name='%q' AND type='table'", pTab->zName);
97552 sqlite3VdbeAddOp4(v, OP_ParseSchema, iDb, 0, 0, zWhere, P4_DYNAMIC);
97553 sqlite3VdbeAddOp4(v, OP_VCreate, iDb, 0, 0,
97554 pTab->zName, sqlite3Strlen30(pTab->zName) + 1);
97555 }
97556
97557 /* If we are rereading the sqlite_master table create the in-memory
@@ -107274,17 +107312,16 @@
107274 #ifndef SQLITE_OMIT_BLOB_LITERAL
107275 case 'x': case 'X': {
107276 testcase( z[0]=='x' ); testcase( z[0]=='X' );
107277 if( z[1]=='\'' ){
107278 *tokenType = TK_BLOB;
107279 for(i=2; (c=z[i])!=0 && c!='\''; i++){
107280 if( !sqlite3Isxdigit(c) ){
107281 *tokenType = TK_ILLEGAL;
107282 }
107283 }
107284 if( i%2 || !c ) *tokenType = TK_ILLEGAL;
107285 if( c ) i++;
107286 return i;
107287 }
107288 /* Otherwise fall through to the next case */
107289 }
107290 #endif
@@ -111710,16 +111747,39 @@
111710 ** similar macro called ArraySize(). Use a different name to avoid
111711 ** a collision when building an amalgamation with built-in FTS3.
111712 */
111713 #define SizeofArray(X) ((int)(sizeof(X)/sizeof(X[0])))
111714
 
 
 
 
 
111715 /*
111716 ** Maximum length of a varint encoded integer. The varint format is different
111717 ** from that used by SQLite, so the maximum length is 10, not 9.
111718 */
111719 #define FTS3_VARINT_MAX 10
111720
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
111721 /*
111722 ** The testcase() macro is only used by the amalgamation. If undefined,
111723 ** make it a no-op.
111724 */
111725 #ifndef testcase
@@ -111787,14 +111847,15 @@
111787 typedef struct Fts3Cursor Fts3Cursor;
111788 typedef struct Fts3Expr Fts3Expr;
111789 typedef struct Fts3Phrase Fts3Phrase;
111790 typedef struct Fts3PhraseToken Fts3PhraseToken;
111791
 
111792 typedef struct Fts3SegFilter Fts3SegFilter;
111793 typedef struct Fts3DeferredToken Fts3DeferredToken;
111794 typedef struct Fts3SegReader Fts3SegReader;
111795 typedef struct Fts3SegReaderCursor Fts3SegReaderCursor;
111796
111797 /*
111798 ** A connection to a fulltext index is an instance of the following
111799 ** structure. The xCreate and xConnect methods create an instance
111800 ** of this structure and xDestroy and xDisconnect free that instance.
@@ -111811,33 +111872,45 @@
111811 sqlite3_tokenizer *pTokenizer; /* tokenizer for inserts and queries */
111812
111813 /* Precompiled statements used by the implementation. Each of these
111814 ** statements is run and reset within a single virtual table API call.
111815 */
111816 sqlite3_stmt *aStmt[24];
111817
111818 char *zReadExprlist;
111819 char *zWriteExprlist;
111820
111821 int nNodeSize; /* Soft limit for node size */
111822 u8 bHasStat; /* True if %_stat table exists */
111823 u8 bHasDocsize; /* True if %_docsize table exists */
 
111824 int nPgsz; /* Page size for host database */
111825 char *zSegmentsTbl; /* Name of %_segments table */
111826 sqlite3_blob *pSegments; /* Blob handle open on %_segments table */
111827
111828 /* The following hash table is used to buffer pending index updates during
 
 
111829 ** transactions. Variable nPendingData estimates the memory size of the
111830 ** pending data, including hash table overhead, but not malloc overhead.
111831 ** When nPendingData exceeds nMaxPendingData, the buffer is flushed
111832 ** automatically. Variable iPrevDocid is the docid of the most recently
111833 ** inserted record.
 
 
 
 
 
111834 */
111835 int nMaxPendingData;
111836 int nPendingData;
111837 sqlite_int64 iPrevDocid;
111838 Fts3Hash pendingTerms;
 
 
 
 
111839
111840 #if defined(SQLITE_DEBUG)
111841 /* State variables used for validating that the transaction control
111842 ** methods of the virtual table are called at appropriate times. These
111843 ** values do not contribution to the FTS computation; they are used for
@@ -111864,13 +111937,14 @@
111864 Fts3DeferredToken *pDeferred; /* Deferred search tokens, if any */
111865 sqlite3_int64 iPrevId; /* Previous id read from aDoclist */
111866 char *pNextId; /* Pointer into the body of aDoclist */
111867 char *aDoclist; /* List of docids for full-text queries */
111868 int nDoclist; /* Size of buffer at aDoclist */
111869 int desc; /* True to sort in descending order */
111870 int eEvalmode; /* An FTS3_EVAL_XX constant */
111871 int nRowAvg; /* Average size of database rows, in pages */
 
111872
111873 int isMatchinfoNeeded; /* True when aMatchinfo[] needs filling in */
111874 u32 *aMatchinfo; /* Information about most recent match */
111875 int nMatchinfo; /* Number of elements in aMatchinfo[] */
111876 char *zMatchinfo; /* Matchinfo specification */
@@ -111897,66 +111971,90 @@
111897 */
111898 #define FTS3_FULLSCAN_SEARCH 0 /* Linear scan of %_content table */
111899 #define FTS3_DOCID_SEARCH 1 /* Lookup by rowid on %_content table */
111900 #define FTS3_FULLTEXT_SEARCH 2 /* Full-text index search */
111901
 
 
 
 
 
 
 
 
 
 
 
 
111902 /*
111903 ** A "phrase" is a sequence of one or more tokens that must match in
111904 ** sequence. A single token is the base case and the most common case.
111905 ** For a sequence of tokens contained in double-quotes (i.e. "one two three")
111906 ** nToken will be the number of tokens in the string.
111907 **
111908 ** The nDocMatch and nMatch variables contain data that may be used by the
111909 ** matchinfo() function. They are populated when the full-text index is
111910 ** queried for hits on the phrase. If one or more tokens in the phrase
111911 ** are deferred, the nDocMatch and nMatch variables are populated based
111912 ** on the assumption that the
111913 */
111914 struct Fts3PhraseToken {
111915 char *z; /* Text of the token */
111916 int n; /* Number of bytes in buffer z */
111917 int isPrefix; /* True if token ends with a "*" character */
 
 
 
 
111918 int bFulltext; /* True if full-text index was used */
111919 Fts3SegReaderCursor *pSegcsr; /* Segment-reader for this token */
111920 Fts3DeferredToken *pDeferred; /* Deferred token object for this token */
 
111921 };
111922
111923 struct Fts3Phrase {
111924 /* Variables populated by fts3_expr.c when parsing a MATCH expression */
 
 
 
 
 
 
111925 int nToken; /* Number of tokens in the phrase */
111926 int iColumn; /* Index of column this phrase must match */
111927 int isNot; /* Phrase prefixed by unary not (-) operator */
111928 Fts3PhraseToken aToken[1]; /* One entry for each token in the phrase */
111929 };
111930
111931 /*
111932 ** A tree of these objects forms the RHS of a MATCH operator.
111933 **
111934 ** If Fts3Expr.eType is either FTSQUERY_NEAR or FTSQUERY_PHRASE and isLoaded
111935 ** is true, then aDoclist points to a malloced buffer, size nDoclist bytes,
111936 ** containing the results of the NEAR or phrase query in FTS3 doclist
111937 ** format. As usual, the initial "Length" field found in doclists stored
111938 ** on disk is omitted from this buffer.
111939 **
111940 ** Variable pCurrent always points to the start of a docid field within
111941 ** aDoclist. Since the doclist is usually scanned in docid order, this can
111942 ** be used to accelerate seeking to the required docid within the doclist.
 
 
 
 
 
 
 
 
111943 */
111944 struct Fts3Expr {
111945 int eType; /* One of the FTSQUERY_XXX values defined below */
111946 int nNear; /* Valid if eType==FTSQUERY_NEAR */
111947 Fts3Expr *pParent; /* pParent->pLeft==this or pParent->pRight==this */
111948 Fts3Expr *pLeft; /* Left operand */
111949 Fts3Expr *pRight; /* Right operand */
111950 Fts3Phrase *pPhrase; /* Valid if eType==FTSQUERY_PHRASE */
111951
111952 int isLoaded; /* True if aDoclist/nDoclist are initialized. */
111953 char *aDoclist; /* Buffer containing doclist */
111954 int nDoclist; /* Size of aDoclist in bytes */
 
 
111955
111956 sqlite3_int64 iCurrent;
111957 char *pCurrent;
111958 };
111959
111960 /*
111961 ** Candidate values for Fts3Query.eType. Note that the order of the first
111962 ** four values is in order of precedence when parsing expressions. For
@@ -111980,35 +112078,36 @@
111980 SQLITE_PRIVATE int sqlite3Fts3PendingTermsFlush(Fts3Table *);
111981 SQLITE_PRIVATE void sqlite3Fts3PendingTermsClear(Fts3Table *);
111982 SQLITE_PRIVATE int sqlite3Fts3Optimize(Fts3Table *);
111983 SQLITE_PRIVATE int sqlite3Fts3SegReaderNew(int, sqlite3_int64,
111984 sqlite3_int64, sqlite3_int64, const char *, int, Fts3SegReader**);
111985 SQLITE_PRIVATE int sqlite3Fts3SegReaderPending(Fts3Table*,const char*,int,int,Fts3SegReader**);
 
111986 SQLITE_PRIVATE void sqlite3Fts3SegReaderFree(Fts3SegReader *);
111987 SQLITE_PRIVATE int sqlite3Fts3SegReaderCost(Fts3Cursor *, Fts3SegReader *, int *);
111988 SQLITE_PRIVATE int sqlite3Fts3AllSegdirs(Fts3Table*, int, sqlite3_stmt **);
111989 SQLITE_PRIVATE int sqlite3Fts3ReadLock(Fts3Table *);
111990 SQLITE_PRIVATE int sqlite3Fts3ReadBlock(Fts3Table*, sqlite3_int64, char **, int*);
111991
111992 SQLITE_PRIVATE int sqlite3Fts3SelectDoctotal(Fts3Table *, sqlite3_stmt **);
111993 SQLITE_PRIVATE int sqlite3Fts3SelectDocsize(Fts3Table *, sqlite3_int64, sqlite3_stmt **);
111994
111995 SQLITE_PRIVATE void sqlite3Fts3FreeDeferredTokens(Fts3Cursor *);
111996 SQLITE_PRIVATE int sqlite3Fts3DeferToken(Fts3Cursor *, Fts3PhraseToken *, int);
111997 SQLITE_PRIVATE int sqlite3Fts3CacheDeferredDoclists(Fts3Cursor *);
111998 SQLITE_PRIVATE void sqlite3Fts3FreeDeferredDoclists(Fts3Cursor *);
111999 SQLITE_PRIVATE char *sqlite3Fts3DeferredDoclist(Fts3DeferredToken *, int *);
112000 SQLITE_PRIVATE void sqlite3Fts3SegmentsClose(Fts3Table *);
112001
112002 #define FTS3_SEGCURSOR_PENDING -1
112003 #define FTS3_SEGCURSOR_ALL -2
 
112004
112005 SQLITE_PRIVATE int sqlite3Fts3SegReaderStart(Fts3Table*, Fts3SegReaderCursor*, Fts3SegFilter*);
112006 SQLITE_PRIVATE int sqlite3Fts3SegReaderStep(Fts3Table *, Fts3SegReaderCursor *);
112007 SQLITE_PRIVATE void sqlite3Fts3SegReaderFinish(Fts3SegReaderCursor *);
 
112008 SQLITE_PRIVATE int sqlite3Fts3SegReaderCursor(
112009 Fts3Table *, int, const char *, int, int, int, Fts3SegReaderCursor *);
112010
112011 /* Flags allowed as part of the 4th argument to SegmentReaderIterate() */
112012 #define FTS3_SEGMENT_REQUIRE_POS 0x00000001
112013 #define FTS3_SEGMENT_IGNORE_EMPTY 0x00000002
112014 #define FTS3_SEGMENT_COLUMN_FILTER 0x00000004
@@ -112021,21 +112120,24 @@
112021 int nTerm;
112022 int iCol;
112023 int flags;
112024 };
112025
112026 struct Fts3SegReaderCursor {
112027 /* Used internally by sqlite3Fts3SegReaderXXX() calls */
112028 Fts3SegReader **apSegment; /* Array of Fts3SegReader objects */
112029 int nSegment; /* Size of apSegment array */
112030 int nAdvance; /* How many seg-readers to advance */
112031 Fts3SegFilter *pFilter; /* Pointer to filter object */
112032 char *aBuffer; /* Buffer to merge doclists in */
112033 int nBuffer; /* Allocated size of aBuffer[] in bytes */
112034
112035 /* Cost of running this iterator. Used by fts3.c only. */
112036 int nCost;
 
 
 
112037
112038 /* Output values. Valid only after Fts3SegReaderStep() returns SQLITE_ROW. */
112039 char *zTerm; /* Pointer to term buffer */
112040 int nTerm; /* Size of zTerm in bytes */
112041 char *aDoclist; /* Pointer to doclist buffer */
@@ -112046,15 +112148,13 @@
112046 SQLITE_PRIVATE int sqlite3Fts3PutVarint(char *, sqlite3_int64);
112047 SQLITE_PRIVATE int sqlite3Fts3GetVarint(const char *, sqlite_int64 *);
112048 SQLITE_PRIVATE int sqlite3Fts3GetVarint32(const char *, int *);
112049 SQLITE_PRIVATE int sqlite3Fts3VarintLen(sqlite3_uint64);
112050 SQLITE_PRIVATE void sqlite3Fts3Dequote(char *);
 
112051
112052 SQLITE_PRIVATE char *sqlite3Fts3FindPositions(Fts3Cursor *, Fts3Expr *, sqlite3_int64, int);
112053 SQLITE_PRIVATE int sqlite3Fts3ExprLoadDoclist(Fts3Cursor *, Fts3Expr *);
112054 SQLITE_PRIVATE int sqlite3Fts3ExprLoadFtDoclist(Fts3Cursor *, Fts3Expr *, char **, int *);
112055 SQLITE_PRIVATE int sqlite3Fts3ExprNearTrim(Fts3Expr *, Fts3Expr *, int);
112056
112057 /* fts3_tokenizer.c */
112058 SQLITE_PRIVATE const char *sqlite3Fts3NextToken(const char *, int *);
112059 SQLITE_PRIVATE int sqlite3Fts3InitHashTable(sqlite3 *, Fts3Hash *, const char *);
112060 SQLITE_PRIVATE int sqlite3Fts3InitTokenizer(Fts3Hash *pHash, const char *,
@@ -112079,10 +112179,33 @@
112079 SQLITE_PRIVATE int sqlite3Fts3InitTerm(sqlite3 *db);
112080 #endif
112081
112082 /* fts3_aux.c */
112083 SQLITE_PRIVATE int sqlite3Fts3InitAux(sqlite3 *db);
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
112084
112085 #endif /* _FTSINT_H */
112086
112087 /************** End of fts3Int.h *********************************************/
112088 /************** Continuing where we left off in fts3.c ***********************/
@@ -112200,16 +112323,16 @@
112200
112201 /*
112202 ** When this function is called, *pp points to the first byte following a
112203 ** varint that is part of a doclist (or position-list, or any other list
112204 ** of varints). This function moves *pp to point to the start of that varint,
112205 ** and decrements the value stored in *pVal by the varint value.
112206 **
112207 ** Argument pStart points to the first byte of the doclist that the
112208 ** varint is part of.
112209 */
112210 static void fts3GetReverseDeltaVarint(
112211 char **pp,
112212 char *pStart,
112213 sqlite3_int64 *pVal
112214 ){
112215 sqlite3_int64 iVal;
@@ -112221,25 +112344,11 @@
112221 for(p = (*pp)-2; p>=pStart && *p&0x80; p--);
112222 p++;
112223 *pp = p;
112224
112225 sqlite3Fts3GetVarint(p, &iVal);
112226 *pVal -= iVal;
112227 }
112228
112229 /*
112230 ** As long as *pp has not reached its end (pEnd), then do the same
112231 ** as fts3GetDeltaVarint(): read a single varint and add it to *pVal.
112232 ** But if we have reached the end of the varint, just set *pp=0 and
112233 ** leave *pVal unchanged.
112234 */
112235 static void fts3GetDeltaVarint2(char **pp, char *pEnd, sqlite3_int64 *pVal){
112236 if( *pp>=pEnd ){
112237 *pp = 0;
112238 }else{
112239 fts3GetDeltaVarint(pp, pVal);
112240 }
112241 }
112242
112243 /*
112244 ** The xDisconnect() virtual table method.
112245 */
@@ -112608,10 +112717,62 @@
112608 fts3Appendf(pRc, &zRet, ",%s(?)", zFunction);
112609 }
112610 sqlite3_free(zFree);
112611 return zRet;
112612 }
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
112613
112614 /*
112615 ** This function is the implementation of both the xConnect and xCreate
112616 ** methods of the FTS3 virtual table.
112617 **
@@ -112641,16 +112802,23 @@
112641 int nCol = 0; /* Number of columns in the FTS table */
112642 char *zCsr; /* Space for holding column names */
112643 int nDb; /* Bytes required to hold database name */
112644 int nName; /* Bytes required to hold table name */
112645 int isFts4 = (argv[0][3]=='4'); /* True for FTS4, false for FTS3 */
112646 int bNoDocsize = 0; /* True to omit %_docsize table */
112647 const char **aCol; /* Array of column names */
112648 sqlite3_tokenizer *pTokenizer = 0; /* Tokenizer for this table */
112649
112650 char *zCompress = 0;
112651 char *zUncompress = 0;
 
 
 
 
 
 
 
 
112652
112653 assert( strlen(argv[0])==4 );
112654 assert( (sqlite3_strnicmp(argv[0], "fts4", 4)==0 && isFts4)
112655 || (sqlite3_strnicmp(argv[0], "fts3", 4)==0 && !isFts4)
112656 );
@@ -112687,32 +112855,76 @@
112687 rc = sqlite3Fts3InitTokenizer(pHash, &z[9], &pTokenizer, pzErr);
112688 }
112689
112690 /* Check if it is an FTS4 special argument. */
112691 else if( isFts4 && fts3IsSpecialColumn(z, &nKey, &zVal) ){
 
 
 
 
 
 
 
 
 
 
 
 
 
112692 if( !zVal ){
112693 rc = SQLITE_NOMEM;
112694 goto fts3_init_out;
112695 }
112696 if( nKey==9 && 0==sqlite3_strnicmp(z, "matchinfo", 9) ){
112697 if( strlen(zVal)==4 && 0==sqlite3_strnicmp(zVal, "fts3", 4) ){
112698 bNoDocsize = 1;
112699 }else{
112700 *pzErr = sqlite3_mprintf("unrecognized matchinfo: %s", zVal);
112701 rc = SQLITE_ERROR;
112702 }
112703 }else if( nKey==8 && 0==sqlite3_strnicmp(z, "compress", 8) ){
112704 zCompress = zVal;
112705 zVal = 0;
112706 }else if( nKey==10 && 0==sqlite3_strnicmp(z, "uncompress", 10) ){
112707 zUncompress = zVal;
112708 zVal = 0;
112709 }else{
112710 *pzErr = sqlite3_mprintf("unrecognized parameter: %s", z);
112711 rc = SQLITE_ERROR;
112712 }
112713 sqlite3_free(zVal);
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
112714 }
112715
112716 /* Otherwise, the argument is a column name. */
112717 else {
112718 nString += (int)(strlen(z) + 1);
@@ -112732,14 +112944,21 @@
112732 rc = sqlite3Fts3InitTokenizer(pHash, "simple", &pTokenizer, pzErr);
112733 if( rc!=SQLITE_OK ) goto fts3_init_out;
112734 }
112735 assert( pTokenizer );
112736
 
 
 
 
 
 
112737
112738 /* Allocate and populate the Fts3Table structure. */
112739 nByte = sizeof(Fts3Table) + /* Fts3Table */
112740 nCol * sizeof(char *) + /* azColumn */
 
112741 nName + /* zName */
112742 nDb + /* zDb */
112743 nString; /* Space for azColumn strings */
112744 p = (Fts3Table*)sqlite3_malloc(nByte);
112745 if( p==0 ){
@@ -112750,20 +112969,26 @@
112750 p->db = db;
112751 p->nColumn = nCol;
112752 p->nPendingData = 0;
112753 p->azColumn = (char **)&p[1];
112754 p->pTokenizer = pTokenizer;
112755 p->nNodeSize = 1000;
112756 p->nMaxPendingData = FTS3_MAX_PENDING_DATA;
112757 p->bHasDocsize = (isFts4 && bNoDocsize==0);
112758 p->bHasStat = isFts4;
 
112759 TESTONLY( p->inTransaction = -1 );
112760 TESTONLY( p->mxSavepoint = -1 );
112761 fts3HashInit(&p->pendingTerms, FTS3_HASH_STRING, 1);
 
 
 
 
 
 
112762
112763 /* Fill in the zName and zDb fields of the vtab structure. */
112764 zCsr = (char *)&p->azColumn[nCol];
112765 p->zName = zCsr;
112766 memcpy(zCsr, argv[2], nName);
112767 zCsr += nName;
112768 p->zDb = zCsr;
112769 memcpy(zCsr, argv[1], nDb);
@@ -112797,19 +113022,20 @@
112797 if( isCreate ){
112798 rc = fts3CreateTables(p);
112799 }
112800
112801 /* Figure out the page-size for the database. This is required in order to
112802 ** estimate the cost of loading large doclists from the database (see
112803 ** function sqlite3Fts3SegReaderCost() for details).
112804 */
112805 fts3DatabasePageSize(&rc, p);
 
112806
112807 /* Declare the table schema to SQLite. */
112808 fts3DeclareVtab(&rc, p);
112809
112810 fts3_init_out:
 
 
112811 sqlite3_free(zCompress);
112812 sqlite3_free(zUncompress);
112813 sqlite3_free((void *)aCol);
112814 if( rc!=SQLITE_OK ){
112815 if( p ){
@@ -112816,10 +113042,11 @@
112816 fts3DisconnectMethod((sqlite3_vtab *)p);
112817 }else if( pTokenizer ){
112818 pTokenizer->pModule->xDestroy(pTokenizer);
112819 }
112820 }else{
 
112821 *ppVTab = &p->base;
112822 }
112823 return rc;
112824 }
112825
@@ -112913,14 +113140,15 @@
112913 if( pOrder->desc ){
112914 pInfo->idxStr = "DESC";
112915 }else{
112916 pInfo->idxStr = "ASC";
112917 }
 
112918 }
112919 pInfo->orderByConsumed = 1;
112920 }
112921
 
112922 return SQLITE_OK;
112923 }
112924
112925 /*
112926 ** Implementation of xOpen method.
@@ -112952,10 +113180,11 @@
112952 sqlite3_finalize(pCsr->pStmt);
112953 sqlite3Fts3ExprFree(pCsr->pExpr);
112954 sqlite3Fts3FreeDeferredTokens(pCsr);
112955 sqlite3_free(pCsr->aDoclist);
112956 sqlite3_free(pCsr->aMatchinfo);
 
112957 sqlite3_free(pCsr);
112958 return SQLITE_OK;
112959 }
112960
112961 /*
@@ -112963,12 +113192,12 @@
112963 ** of the %_content table that contains the last match. Return
112964 ** SQLITE_OK on success.
112965 */
112966 static int fts3CursorSeek(sqlite3_context *pContext, Fts3Cursor *pCsr){
112967 if( pCsr->isRequireSeek ){
112968 pCsr->isRequireSeek = 0;
112969 sqlite3_bind_int64(pCsr->pStmt, 1, pCsr->iPrevId);
 
112970 if( SQLITE_ROW==sqlite3_step(pCsr->pStmt) ){
112971 return SQLITE_OK;
112972 }else{
112973 int rc = sqlite3_reset(pCsr->pStmt);
112974 if( rc==SQLITE_OK ){
@@ -113145,21 +113374,21 @@
113145 if( rc==SQLITE_OK && iHeight>1 ){
113146 char *zBlob = 0; /* Blob read from %_segments table */
113147 int nBlob; /* Size of zBlob in bytes */
113148
113149 if( piLeaf && piLeaf2 && (*piLeaf!=*piLeaf2) ){
113150 rc = sqlite3Fts3ReadBlock(p, *piLeaf, &zBlob, &nBlob);
113151 if( rc==SQLITE_OK ){
113152 rc = fts3SelectLeaf(p, zTerm, nTerm, zBlob, nBlob, piLeaf, 0);
113153 }
113154 sqlite3_free(zBlob);
113155 piLeaf = 0;
113156 zBlob = 0;
113157 }
113158
113159 if( rc==SQLITE_OK ){
113160 rc = sqlite3Fts3ReadBlock(p, piLeaf ? *piLeaf : *piLeaf2, &zBlob, &nBlob);
113161 }
113162 if( rc==SQLITE_OK ){
113163 rc = fts3SelectLeaf(p, zTerm, nTerm, zBlob, nBlob, piLeaf, piLeaf2);
113164 }
113165 sqlite3_free(zBlob);
@@ -113531,11 +113760,23 @@
113531 *pp = p;
113532 return 1;
113533 }
113534
113535 /*
113536 ** Merge two position-lists as required by the NEAR operator.
 
 
 
 
 
 
 
 
 
 
 
 
113537 */
113538 static int fts3PoslistNearMerge(
113539 char **pp, /* Output buffer */
113540 char *aTmp, /* Temporary buffer space */
113541 int nRight, /* Maximum difference in token positions */
@@ -113544,218 +113785,31 @@
113544 char **pp2 /* IN/OUT: Right input list */
113545 ){
113546 char *p1 = *pp1;
113547 char *p2 = *pp2;
113548
113549 if( !pp ){
113550 if( fts3PoslistPhraseMerge(0, nRight, 0, 0, pp1, pp2) ) return 1;
113551 *pp1 = p1;
113552 *pp2 = p2;
113553 return fts3PoslistPhraseMerge(0, nLeft, 0, 0, pp2, pp1);
113554 }else{
113555 char *pTmp1 = aTmp;
113556 char *pTmp2;
113557 char *aTmp2;
113558 int res = 1;
113559
113560 fts3PoslistPhraseMerge(&pTmp1, nRight, 0, 0, pp1, pp2);
113561 aTmp2 = pTmp2 = pTmp1;
113562 *pp1 = p1;
113563 *pp2 = p2;
113564 fts3PoslistPhraseMerge(&pTmp2, nLeft, 1, 0, pp2, pp1);
113565 if( pTmp1!=aTmp && pTmp2!=aTmp2 ){
113566 fts3PoslistMerge(pp, &aTmp, &aTmp2);
113567 }else if( pTmp1!=aTmp ){
113568 fts3PoslistCopy(pp, &aTmp);
113569 }else if( pTmp2!=aTmp2 ){
113570 fts3PoslistCopy(pp, &aTmp2);
113571 }else{
113572 res = 0;
113573 }
113574
113575 return res;
113576 }
113577 }
113578
113579 /*
113580 ** Values that may be used as the first parameter to fts3DoclistMerge().
113581 */
113582 #define MERGE_NOT 2 /* D + D -> D */
113583 #define MERGE_AND 3 /* D + D -> D */
113584 #define MERGE_OR 4 /* D + D -> D */
113585 #define MERGE_POS_OR 5 /* P + P -> P */
113586 #define MERGE_PHRASE 6 /* P + P -> D */
113587 #define MERGE_POS_PHRASE 7 /* P + P -> P */
113588 #define MERGE_NEAR 8 /* P + P -> D */
113589 #define MERGE_POS_NEAR 9 /* P + P -> P */
113590
113591 /*
113592 ** Merge the two doclists passed in buffer a1 (size n1 bytes) and a2
113593 ** (size n2 bytes). The output is written to pre-allocated buffer aBuffer,
113594 ** which is guaranteed to be large enough to hold the results. The number
113595 ** of bytes written to aBuffer is stored in *pnBuffer before returning.
113596 **
113597 ** If successful, SQLITE_OK is returned. Otherwise, if a malloc error
113598 ** occurs while allocating a temporary buffer as part of the merge operation,
113599 ** SQLITE_NOMEM is returned.
113600 */
113601 static int fts3DoclistMerge(
113602 int mergetype, /* One of the MERGE_XXX constants */
113603 int nParam1, /* Used by MERGE_NEAR and MERGE_POS_NEAR */
113604 int nParam2, /* Used by MERGE_NEAR and MERGE_POS_NEAR */
113605 char *aBuffer, /* Pre-allocated output buffer */
113606 int *pnBuffer, /* OUT: Bytes written to aBuffer */
113607 char *a1, /* Buffer containing first doclist */
113608 int n1, /* Size of buffer a1 */
113609 char *a2, /* Buffer containing second doclist */
113610 int n2, /* Size of buffer a2 */
113611 int *pnDoc /* OUT: Number of docids in output */
113612 ){
113613 sqlite3_int64 i1 = 0;
113614 sqlite3_int64 i2 = 0;
113615 sqlite3_int64 iPrev = 0;
113616
113617 char *p = aBuffer;
113618 char *p1 = a1;
113619 char *p2 = a2;
113620 char *pEnd1 = &a1[n1];
113621 char *pEnd2 = &a2[n2];
113622 int nDoc = 0;
113623
113624 assert( mergetype==MERGE_OR || mergetype==MERGE_POS_OR
113625 || mergetype==MERGE_AND || mergetype==MERGE_NOT
113626 || mergetype==MERGE_PHRASE || mergetype==MERGE_POS_PHRASE
113627 || mergetype==MERGE_NEAR || mergetype==MERGE_POS_NEAR
113628 );
113629
113630 if( !aBuffer ){
113631 *pnBuffer = 0;
113632 return SQLITE_NOMEM;
113633 }
113634
113635 /* Read the first docid from each doclist */
113636 fts3GetDeltaVarint2(&p1, pEnd1, &i1);
113637 fts3GetDeltaVarint2(&p2, pEnd2, &i2);
113638
113639 switch( mergetype ){
113640 case MERGE_OR:
113641 case MERGE_POS_OR:
113642 while( p1 || p2 ){
113643 if( p2 && p1 && i1==i2 ){
113644 fts3PutDeltaVarint(&p, &iPrev, i1);
113645 if( mergetype==MERGE_POS_OR ) fts3PoslistMerge(&p, &p1, &p2);
113646 fts3GetDeltaVarint2(&p1, pEnd1, &i1);
113647 fts3GetDeltaVarint2(&p2, pEnd2, &i2);
113648 }else if( !p2 || (p1 && i1<i2) ){
113649 fts3PutDeltaVarint(&p, &iPrev, i1);
113650 if( mergetype==MERGE_POS_OR ) fts3PoslistCopy(&p, &p1);
113651 fts3GetDeltaVarint2(&p1, pEnd1, &i1);
113652 }else{
113653 fts3PutDeltaVarint(&p, &iPrev, i2);
113654 if( mergetype==MERGE_POS_OR ) fts3PoslistCopy(&p, &p2);
113655 fts3GetDeltaVarint2(&p2, pEnd2, &i2);
113656 }
113657 }
113658 break;
113659
113660 case MERGE_AND:
113661 while( p1 && p2 ){
113662 if( i1==i2 ){
113663 fts3PutDeltaVarint(&p, &iPrev, i1);
113664 fts3GetDeltaVarint2(&p1, pEnd1, &i1);
113665 fts3GetDeltaVarint2(&p2, pEnd2, &i2);
113666 nDoc++;
113667 }else if( i1<i2 ){
113668 fts3GetDeltaVarint2(&p1, pEnd1, &i1);
113669 }else{
113670 fts3GetDeltaVarint2(&p2, pEnd2, &i2);
113671 }
113672 }
113673 break;
113674
113675 case MERGE_NOT:
113676 while( p1 ){
113677 if( p2 && i1==i2 ){
113678 fts3GetDeltaVarint2(&p1, pEnd1, &i1);
113679 fts3GetDeltaVarint2(&p2, pEnd2, &i2);
113680 }else if( !p2 || i1<i2 ){
113681 fts3PutDeltaVarint(&p, &iPrev, i1);
113682 fts3GetDeltaVarint2(&p1, pEnd1, &i1);
113683 }else{
113684 fts3GetDeltaVarint2(&p2, pEnd2, &i2);
113685 }
113686 }
113687 break;
113688
113689 case MERGE_POS_PHRASE:
113690 case MERGE_PHRASE: {
113691 char **ppPos = (mergetype==MERGE_PHRASE ? 0 : &p);
113692 while( p1 && p2 ){
113693 if( i1==i2 ){
113694 char *pSave = p;
113695 sqlite3_int64 iPrevSave = iPrev;
113696 fts3PutDeltaVarint(&p, &iPrev, i1);
113697 if( 0==fts3PoslistPhraseMerge(ppPos, nParam1, 0, 1, &p1, &p2) ){
113698 p = pSave;
113699 iPrev = iPrevSave;
113700 }else{
113701 nDoc++;
113702 }
113703 fts3GetDeltaVarint2(&p1, pEnd1, &i1);
113704 fts3GetDeltaVarint2(&p2, pEnd2, &i2);
113705 }else if( i1<i2 ){
113706 fts3PoslistCopy(0, &p1);
113707 fts3GetDeltaVarint2(&p1, pEnd1, &i1);
113708 }else{
113709 fts3PoslistCopy(0, &p2);
113710 fts3GetDeltaVarint2(&p2, pEnd2, &i2);
113711 }
113712 }
113713 break;
113714 }
113715
113716 default: assert( mergetype==MERGE_POS_NEAR || mergetype==MERGE_NEAR ); {
113717 char *aTmp = 0;
113718 char **ppPos = 0;
113719
113720 if( mergetype==MERGE_POS_NEAR ){
113721 ppPos = &p;
113722 aTmp = sqlite3_malloc(2*(n1+n2+1));
113723 if( !aTmp ){
113724 return SQLITE_NOMEM;
113725 }
113726 }
113727
113728 while( p1 && p2 ){
113729 if( i1==i2 ){
113730 char *pSave = p;
113731 sqlite3_int64 iPrevSave = iPrev;
113732 fts3PutDeltaVarint(&p, &iPrev, i1);
113733
113734 if( !fts3PoslistNearMerge(ppPos, aTmp, nParam1, nParam2, &p1, &p2) ){
113735 iPrev = iPrevSave;
113736 p = pSave;
113737 }
113738
113739 fts3GetDeltaVarint2(&p1, pEnd1, &i1);
113740 fts3GetDeltaVarint2(&p2, pEnd2, &i2);
113741 }else if( i1<i2 ){
113742 fts3PoslistCopy(0, &p1);
113743 fts3GetDeltaVarint2(&p1, pEnd1, &i1);
113744 }else{
113745 fts3PoslistCopy(0, &p2);
113746 fts3GetDeltaVarint2(&p2, pEnd2, &i2);
113747 }
113748 }
113749 sqlite3_free(aTmp);
113750 break;
113751 }
113752 }
113753
113754 if( pnDoc ) *pnDoc = nDoc;
113755 *pnBuffer = (int)(p-aBuffer);
113756 return SQLITE_OK;
113757 }
113758
113759 /*
113760 ** A pointer to an instance of this structure is used as the context
113761 ** argument to sqlite3Fts3SegReaderIterate()
@@ -113764,10 +113818,152 @@
113764 struct TermSelect {
113765 int isReqPos;
113766 char *aaOutput[16]; /* Malloc'd output buffer */
113767 int anOutput[16]; /* Size of output in bytes */
113768 };
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
113769
113770 /*
113771 ** Merge all doclists in the TermSelect.aaOutput[] array into a single
113772 ** doclist stored in TermSelect.aaOutput[0]. If successful, delete all
113773 ** other doclists (except the aaOutput[0] one) and return SQLITE_OK.
@@ -113774,12 +113970,11 @@
113774 **
113775 ** If an OOM error occurs, return SQLITE_NOMEM. In this case it is
113776 ** the responsibility of the caller to free any doclists left in the
113777 ** TermSelect.aaOutput[] array.
113778 */
113779 static int fts3TermSelectMerge(TermSelect *pTS){
113780 int mergetype = (pTS->isReqPos ? MERGE_POS_OR : MERGE_OR);
113781 char *aOut = 0;
113782 int nOut = 0;
113783 int i;
113784
113785 /* Loop through the doclists in the aaOutput[] array. Merge them all
@@ -113790,19 +113985,21 @@
113790 if( !aOut ){
113791 aOut = pTS->aaOutput[i];
113792 nOut = pTS->anOutput[i];
113793 pTS->aaOutput[i] = 0;
113794 }else{
113795 int nNew = nOut + pTS->anOutput[i];
113796 char *aNew = sqlite3_malloc(nNew);
113797 if( !aNew ){
 
 
 
 
113798 sqlite3_free(aOut);
113799 return SQLITE_NOMEM;
113800 }
113801 fts3DoclistMerge(mergetype, 0, 0,
113802 aNew, &nNew, pTS->aaOutput[i], pTS->anOutput[i], aOut, nOut, 0
113803 );
113804 sqlite3_free(pTS->aaOutput[i]);
113805 sqlite3_free(aOut);
113806 pTS->aaOutput[i] = 0;
113807 aOut = aNew;
113808 nOut = nNew;
@@ -113834,217 +114031,241 @@
113834 UNUSED_PARAMETER(zTerm);
113835 UNUSED_PARAMETER(nTerm);
113836
113837 if( pTS->aaOutput[0]==0 ){
113838 /* If this is the first term selected, copy the doclist to the output
113839 ** buffer using memcpy(). TODO: Add a way to transfer control of the
113840 ** aDoclist buffer from the caller so as to avoid the memcpy().
113841 */
113842 pTS->aaOutput[0] = sqlite3_malloc(nDoclist);
113843 pTS->anOutput[0] = nDoclist;
113844 if( pTS->aaOutput[0] ){
113845 memcpy(pTS->aaOutput[0], aDoclist, nDoclist);
113846 }else{
113847 return SQLITE_NOMEM;
113848 }
113849 }else{
113850 int mergetype = (pTS->isReqPos ? MERGE_POS_OR : MERGE_OR);
113851 char *aMerge = aDoclist;
113852 int nMerge = nDoclist;
113853 int iOut;
113854
113855 for(iOut=0; iOut<SizeofArray(pTS->aaOutput); iOut++){
113856 char *aNew;
113857 int nNew;
113858 if( pTS->aaOutput[iOut]==0 ){
113859 assert( iOut>0 );
113860 pTS->aaOutput[iOut] = aMerge;
113861 pTS->anOutput[iOut] = nMerge;
113862 break;
113863 }
113864
113865 nNew = nMerge + pTS->anOutput[iOut];
113866 aNew = sqlite3_malloc(nNew);
113867 if( !aNew ){
113868 if( aMerge!=aDoclist ){
113869 sqlite3_free(aMerge);
113870 }
113871 return SQLITE_NOMEM;
113872 }
113873 fts3DoclistMerge(mergetype, 0, 0, aNew, &nNew,
113874 pTS->aaOutput[iOut], pTS->anOutput[iOut], aMerge, nMerge, 0
113875 );
113876
113877 if( iOut>0 ) sqlite3_free(aMerge);
113878 sqlite3_free(pTS->aaOutput[iOut]);
113879 pTS->aaOutput[iOut] = 0;
113880
113881 aMerge = aNew;
113882 nMerge = nNew;
113883 if( (iOut+1)==SizeofArray(pTS->aaOutput) ){
113884 pTS->aaOutput[iOut] = aMerge;
113885 pTS->anOutput[iOut] = nMerge;
113886 }
113887 }
113888 }
113889 return SQLITE_OK;
113890 }
113891
113892 static int fts3DeferredTermSelect(
113893 Fts3DeferredToken *pToken, /* Phrase token */
113894 int isTermPos, /* True to include positions */
113895 int *pnOut, /* OUT: Size of list */
113896 char **ppOut /* OUT: Body of list */
113897 ){
113898 char *aSource;
113899 int nSource;
113900
113901 aSource = sqlite3Fts3DeferredDoclist(pToken, &nSource);
113902 if( !aSource ){
113903 *pnOut = 0;
113904 *ppOut = 0;
113905 }else if( isTermPos ){
113906 *ppOut = sqlite3_malloc(nSource);
113907 if( !*ppOut ) return SQLITE_NOMEM;
113908 memcpy(*ppOut, aSource, nSource);
113909 *pnOut = nSource;
113910 }else{
113911 sqlite3_int64 docid;
113912 *pnOut = sqlite3Fts3GetVarint(aSource, &docid);
113913 *ppOut = sqlite3_malloc(*pnOut);
113914 if( !*ppOut ) return SQLITE_NOMEM;
113915 sqlite3Fts3PutVarint(*ppOut, docid);
113916 }
113917
113918 return SQLITE_OK;
113919 }
113920
113921 SQLITE_PRIVATE int sqlite3Fts3SegReaderCursor(
113922 Fts3Table *p, /* FTS3 table handle */
 
113923 int iLevel, /* Level of segments to scan */
113924 const char *zTerm, /* Term to query for */
113925 int nTerm, /* Size of zTerm in bytes */
113926 int isPrefix, /* True for a prefix search */
113927 int isScan, /* True to scan from zTerm to EOF */
113928 Fts3SegReaderCursor *pCsr /* Cursor object to populate */
113929 ){
113930 int rc = SQLITE_OK;
113931 int rc2;
113932 int iAge = 0;
113933 sqlite3_stmt *pStmt = 0;
113934 Fts3SegReader *pPending = 0;
113935
113936 assert( iLevel==FTS3_SEGCURSOR_ALL
113937 || iLevel==FTS3_SEGCURSOR_PENDING
113938 || iLevel>=0
113939 );
113940 assert( FTS3_SEGCURSOR_PENDING<0 );
113941 assert( FTS3_SEGCURSOR_ALL<0 );
113942 assert( iLevel==FTS3_SEGCURSOR_ALL || (zTerm==0 && isPrefix==1) );
113943 assert( isPrefix==0 || isScan==0 );
113944
113945
113946 memset(pCsr, 0, sizeof(Fts3SegReaderCursor));
113947
113948 /* If iLevel is less than 0, include a seg-reader for the pending-terms. */
113949 assert( isScan==0 || fts3HashCount(&p->pendingTerms)==0 );
113950 if( iLevel<0 && isScan==0 ){
113951 rc = sqlite3Fts3SegReaderPending(p, zTerm, nTerm, isPrefix, &pPending);
113952 if( rc==SQLITE_OK && pPending ){
113953 int nByte = (sizeof(Fts3SegReader *) * 16);
113954 pCsr->apSegment = (Fts3SegReader **)sqlite3_malloc(nByte);
113955 if( pCsr->apSegment==0 ){
113956 rc = SQLITE_NOMEM;
113957 }else{
113958 pCsr->apSegment[0] = pPending;
113959 pCsr->nSegment = 1;
113960 pPending = 0;
113961 }
113962 }
113963 }
113964
113965 if( iLevel!=FTS3_SEGCURSOR_PENDING ){
113966 if( rc==SQLITE_OK ){
113967 rc = sqlite3Fts3AllSegdirs(p, iLevel, &pStmt);
113968 }
 
113969 while( rc==SQLITE_OK && SQLITE_ROW==(rc = sqlite3_step(pStmt)) ){
 
113970
113971 /* Read the values returned by the SELECT into local variables. */
113972 sqlite3_int64 iStartBlock = sqlite3_column_int64(pStmt, 1);
113973 sqlite3_int64 iLeavesEndBlock = sqlite3_column_int64(pStmt, 2);
113974 sqlite3_int64 iEndBlock = sqlite3_column_int64(pStmt, 3);
113975 int nRoot = sqlite3_column_bytes(pStmt, 4);
113976 char const *zRoot = sqlite3_column_blob(pStmt, 4);
113977
113978 /* If nSegment is a multiple of 16 the array needs to be extended. */
113979 if( (pCsr->nSegment%16)==0 ){
113980 Fts3SegReader **apNew;
113981 int nByte = (pCsr->nSegment + 16)*sizeof(Fts3SegReader*);
113982 apNew = (Fts3SegReader **)sqlite3_realloc(pCsr->apSegment, nByte);
113983 if( !apNew ){
113984 rc = SQLITE_NOMEM;
113985 goto finished;
113986 }
113987 pCsr->apSegment = apNew;
113988 }
113989
113990 /* If zTerm is not NULL, and this segment is not stored entirely on its
113991 ** root node, the range of leaves scanned can be reduced. Do this. */
113992 if( iStartBlock && zTerm ){
113993 sqlite3_int64 *pi = (isPrefix ? &iLeavesEndBlock : 0);
113994 rc = fts3SelectLeaf(p, zTerm, nTerm, zRoot, nRoot, &iStartBlock, pi);
113995 if( rc!=SQLITE_OK ) goto finished;
113996 if( isPrefix==0 && isScan==0 ) iLeavesEndBlock = iStartBlock;
113997 }
113998
113999 rc = sqlite3Fts3SegReaderNew(iAge, iStartBlock, iLeavesEndBlock,
114000 iEndBlock, zRoot, nRoot, &pCsr->apSegment[pCsr->nSegment]
114001 );
114002 if( rc!=SQLITE_OK ) goto finished;
114003 pCsr->nSegment++;
114004 iAge++;
114005 }
114006 }
114007
114008 finished:
114009 rc2 = sqlite3_reset(pStmt);
114010 if( rc==SQLITE_DONE ) rc = rc2;
114011 sqlite3Fts3SegReaderFree(pPending);
114012
114013 return rc;
114014 }
114015
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
114016
114017 static int fts3TermSegReaderCursor(
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
114018 Fts3Cursor *pCsr, /* Virtual table cursor handle */
114019 const char *zTerm, /* Term to query for */
114020 int nTerm, /* Size of zTerm in bytes */
114021 int isPrefix, /* True for a prefix search */
114022 Fts3SegReaderCursor **ppSegcsr /* OUT: Allocated seg-reader cursor */
114023 ){
114024 Fts3SegReaderCursor *pSegcsr; /* Object to allocate and return */
114025 int rc = SQLITE_NOMEM; /* Return code */
114026
114027 pSegcsr = sqlite3_malloc(sizeof(Fts3SegReaderCursor));
114028 if( pSegcsr ){
 
 
114029 Fts3Table *p = (Fts3Table *)pCsr->base.pVtab;
114030 int i;
114031 int nCost = 0;
114032 rc = sqlite3Fts3SegReaderCursor(
114033 p, FTS3_SEGCURSOR_ALL, zTerm, nTerm, isPrefix, 0, pSegcsr);
114034
114035 for(i=0; rc==SQLITE_OK && i<pSegcsr->nSegment; i++){
114036 rc = sqlite3Fts3SegReaderCost(pCsr, pSegcsr->apSegment[i], &nCost);
114037 }
114038 pSegcsr->nCost = nCost;
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
114039 }
114040
114041 *ppSegcsr = pSegcsr;
114042 return rc;
114043 }
114044
114045 static void fts3SegReaderCursorFree(Fts3SegReaderCursor *pSegcsr){
114046 sqlite3Fts3SegReaderFinish(pSegcsr);
114047 sqlite3_free(pSegcsr);
114048 }
114049
114050 /*
@@ -114065,11 +114286,11 @@
114065 int isReqPos, /* True to include position lists in output */
114066 int *pnOut, /* OUT: Size of buffer at *ppOut */
114067 char **ppOut /* OUT: Malloced result buffer */
114068 ){
114069 int rc; /* Return code */
114070 Fts3SegReaderCursor *pSegcsr; /* Seg-reader cursor for this term */
114071 TermSelect tsc; /* Context object for fts3TermSelectCb() */
114072 Fts3SegFilter filter; /* Segment term filter configuration */
114073
114074 pSegcsr = pTok->pSegcsr;
114075 memset(&tsc, 0, sizeof(TermSelect));
@@ -114091,11 +114312,11 @@
114091 pSegcsr->zTerm, pSegcsr->nTerm, pSegcsr->aDoclist, pSegcsr->nDoclist
114092 );
114093 }
114094
114095 if( rc==SQLITE_OK ){
114096 rc = fts3TermSelectMerge(&tsc);
114097 }
114098 if( rc==SQLITE_OK ){
114099 *ppOut = tsc.aaOutput[0];
114100 *pnOut = tsc.anOutput[0];
114101 }else{
@@ -114141,664 +114362,10 @@
114141 }
114142
114143 return nDoc;
114144 }
114145
114146 /*
114147 ** Call sqlite3Fts3DeferToken() for each token in the expression pExpr.
114148 */
114149 static int fts3DeferExpression(Fts3Cursor *pCsr, Fts3Expr *pExpr){
114150 int rc = SQLITE_OK;
114151 if( pExpr ){
114152 rc = fts3DeferExpression(pCsr, pExpr->pLeft);
114153 if( rc==SQLITE_OK ){
114154 rc = fts3DeferExpression(pCsr, pExpr->pRight);
114155 }
114156 if( pExpr->eType==FTSQUERY_PHRASE ){
114157 int iCol = pExpr->pPhrase->iColumn;
114158 int i;
114159 for(i=0; rc==SQLITE_OK && i<pExpr->pPhrase->nToken; i++){
114160 Fts3PhraseToken *pToken = &pExpr->pPhrase->aToken[i];
114161 if( pToken->pDeferred==0 ){
114162 rc = sqlite3Fts3DeferToken(pCsr, pToken, iCol);
114163 }
114164 }
114165 }
114166 }
114167 return rc;
114168 }
114169
114170 /*
114171 ** This function removes the position information from a doclist. When
114172 ** called, buffer aList (size *pnList bytes) contains a doclist that includes
114173 ** position information. This function removes the position information so
114174 ** that aList contains only docids, and adjusts *pnList to reflect the new
114175 ** (possibly reduced) size of the doclist.
114176 */
114177 static void fts3DoclistStripPositions(
114178 char *aList, /* IN/OUT: Buffer containing doclist */
114179 int *pnList /* IN/OUT: Size of doclist in bytes */
114180 ){
114181 if( aList ){
114182 char *aEnd = &aList[*pnList]; /* Pointer to one byte after EOF */
114183 char *p = aList; /* Input cursor */
114184 char *pOut = aList; /* Output cursor */
114185
114186 while( p<aEnd ){
114187 sqlite3_int64 delta;
114188 p += sqlite3Fts3GetVarint(p, &delta);
114189 fts3PoslistCopy(0, &p);
114190 pOut += sqlite3Fts3PutVarint(pOut, delta);
114191 }
114192
114193 *pnList = (int)(pOut - aList);
114194 }
114195 }
114196
114197 /*
114198 ** Return a DocList corresponding to the phrase *pPhrase.
114199 **
114200 ** If this function returns SQLITE_OK, but *pnOut is set to a negative value,
114201 ** then no tokens in the phrase were looked up in the full-text index. This
114202 ** is only possible when this function is called from within xFilter(). The
114203 ** caller should assume that all documents match the phrase. The actual
114204 ** filtering will take place in xNext().
114205 */
114206 static int fts3PhraseSelect(
114207 Fts3Cursor *pCsr, /* Virtual table cursor handle */
114208 Fts3Phrase *pPhrase, /* Phrase to return a doclist for */
114209 int isReqPos, /* True if output should contain positions */
114210 char **paOut, /* OUT: Pointer to malloc'd result buffer */
114211 int *pnOut /* OUT: Size of buffer at *paOut */
114212 ){
114213 char *pOut = 0;
114214 int nOut = 0;
114215 int rc = SQLITE_OK;
114216 int ii;
114217 int iCol = pPhrase->iColumn;
114218 int isTermPos = (pPhrase->nToken>1 || isReqPos);
114219 Fts3Table *p = (Fts3Table *)pCsr->base.pVtab;
114220 int isFirst = 1;
114221
114222 int iPrevTok = 0;
114223 int nDoc = 0;
114224
114225 /* If this is an xFilter() evaluation, create a segment-reader for each
114226 ** phrase token. Or, if this is an xNext() or snippet/offsets/matchinfo
114227 ** evaluation, only create segment-readers if there are no Fts3DeferredToken
114228 ** objects attached to the phrase-tokens.
114229 */
114230 for(ii=0; ii<pPhrase->nToken; ii++){
114231 Fts3PhraseToken *pTok = &pPhrase->aToken[ii];
114232 if( pTok->pSegcsr==0 ){
114233 if( (pCsr->eEvalmode==FTS3_EVAL_FILTER)
114234 || (pCsr->eEvalmode==FTS3_EVAL_NEXT && pCsr->pDeferred==0)
114235 || (pCsr->eEvalmode==FTS3_EVAL_MATCHINFO && pTok->bFulltext)
114236 ){
114237 rc = fts3TermSegReaderCursor(
114238 pCsr, pTok->z, pTok->n, pTok->isPrefix, &pTok->pSegcsr
114239 );
114240 if( rc!=SQLITE_OK ) return rc;
114241 }
114242 }
114243 }
114244
114245 for(ii=0; ii<pPhrase->nToken; ii++){
114246 Fts3PhraseToken *pTok; /* Token to find doclist for */
114247 int iTok = 0; /* The token being queried this iteration */
114248 char *pList = 0; /* Pointer to token doclist */
114249 int nList = 0; /* Size of buffer at pList */
114250
114251 /* Select a token to process. If this is an xFilter() call, then tokens
114252 ** are processed in order from least to most costly. Otherwise, tokens
114253 ** are processed in the order in which they occur in the phrase.
114254 */
114255 if( pCsr->eEvalmode==FTS3_EVAL_MATCHINFO ){
114256 assert( isReqPos );
114257 iTok = ii;
114258 pTok = &pPhrase->aToken[iTok];
114259 if( pTok->bFulltext==0 ) continue;
114260 }else if( pCsr->eEvalmode==FTS3_EVAL_NEXT || isReqPos ){
114261 iTok = ii;
114262 pTok = &pPhrase->aToken[iTok];
114263 }else{
114264 int nMinCost = 0x7FFFFFFF;
114265 int jj;
114266
114267 /* Find the remaining token with the lowest cost. */
114268 for(jj=0; jj<pPhrase->nToken; jj++){
114269 Fts3SegReaderCursor *pSegcsr = pPhrase->aToken[jj].pSegcsr;
114270 if( pSegcsr && pSegcsr->nCost<nMinCost ){
114271 iTok = jj;
114272 nMinCost = pSegcsr->nCost;
114273 }
114274 }
114275 pTok = &pPhrase->aToken[iTok];
114276
114277 /* This branch is taken if it is determined that loading the doclist
114278 ** for the next token would require more IO than loading all documents
114279 ** currently identified by doclist pOut/nOut. No further doclists will
114280 ** be loaded from the full-text index for this phrase.
114281 */
114282 if( nMinCost>nDoc && ii>0 ){
114283 rc = fts3DeferExpression(pCsr, pCsr->pExpr);
114284 break;
114285 }
114286 }
114287
114288 if( pCsr->eEvalmode==FTS3_EVAL_NEXT && pTok->pDeferred ){
114289 rc = fts3DeferredTermSelect(pTok->pDeferred, isTermPos, &nList, &pList);
114290 }else{
114291 if( pTok->pSegcsr ){
114292 rc = fts3TermSelect(p, pTok, iCol, isTermPos, &nList, &pList);
114293 }
114294 pTok->bFulltext = 1;
114295 }
114296 assert( rc!=SQLITE_OK || pCsr->eEvalmode || pTok->pSegcsr==0 );
114297 if( rc!=SQLITE_OK ) break;
114298
114299 if( isFirst ){
114300 pOut = pList;
114301 nOut = nList;
114302 if( pCsr->eEvalmode==FTS3_EVAL_FILTER && pPhrase->nToken>1 ){
114303 nDoc = fts3DoclistCountDocids(1, pOut, nOut);
114304 }
114305 isFirst = 0;
114306 iPrevTok = iTok;
114307 }else{
114308 /* Merge the new term list and the current output. */
114309 char *aLeft, *aRight;
114310 int nLeft, nRight;
114311 int nDist;
114312 int mt;
114313
114314 /* If this is the final token of the phrase, and positions were not
114315 ** requested by the caller, use MERGE_PHRASE instead of POS_PHRASE.
114316 ** This drops the position information from the output list.
114317 */
114318 mt = MERGE_POS_PHRASE;
114319 if( ii==pPhrase->nToken-1 && !isReqPos ) mt = MERGE_PHRASE;
114320
114321 assert( iPrevTok!=iTok );
114322 if( iPrevTok<iTok ){
114323 aLeft = pOut;
114324 nLeft = nOut;
114325 aRight = pList;
114326 nRight = nList;
114327 nDist = iTok-iPrevTok;
114328 iPrevTok = iTok;
114329 }else{
114330 aRight = pOut;
114331 nRight = nOut;
114332 aLeft = pList;
114333 nLeft = nList;
114334 nDist = iPrevTok-iTok;
114335 }
114336 pOut = aRight;
114337 fts3DoclistMerge(
114338 mt, nDist, 0, pOut, &nOut, aLeft, nLeft, aRight, nRight, &nDoc
114339 );
114340 sqlite3_free(aLeft);
114341 }
114342 assert( nOut==0 || pOut!=0 );
114343 }
114344
114345 if( rc==SQLITE_OK ){
114346 if( ii!=pPhrase->nToken ){
114347 assert( pCsr->eEvalmode==FTS3_EVAL_FILTER && isReqPos==0 );
114348 fts3DoclistStripPositions(pOut, &nOut);
114349 }
114350 *paOut = pOut;
114351 *pnOut = nOut;
114352 }else{
114353 sqlite3_free(pOut);
114354 }
114355 return rc;
114356 }
114357
114358 /*
114359 ** This function merges two doclists according to the requirements of a
114360 ** NEAR operator.
114361 **
114362 ** Both input doclists must include position information. The output doclist
114363 ** includes position information if the first argument to this function
114364 ** is MERGE_POS_NEAR, or does not if it is MERGE_NEAR.
114365 */
114366 static int fts3NearMerge(
114367 int mergetype, /* MERGE_POS_NEAR or MERGE_NEAR */
114368 int nNear, /* Parameter to NEAR operator */
114369 int nTokenLeft, /* Number of tokens in LHS phrase arg */
114370 char *aLeft, /* Doclist for LHS (incl. positions) */
114371 int nLeft, /* Size of LHS doclist in bytes */
114372 int nTokenRight, /* As nTokenLeft */
114373 char *aRight, /* As aLeft */
114374 int nRight, /* As nRight */
114375 char **paOut, /* OUT: Results of merge (malloced) */
114376 int *pnOut /* OUT: Sized of output buffer */
114377 ){
114378 char *aOut; /* Buffer to write output doclist to */
114379 int rc; /* Return code */
114380
114381 assert( mergetype==MERGE_POS_NEAR || MERGE_NEAR );
114382
114383 aOut = sqlite3_malloc(nLeft+nRight+1);
114384 if( aOut==0 ){
114385 rc = SQLITE_NOMEM;
114386 }else{
114387 rc = fts3DoclistMerge(mergetype, nNear+nTokenRight, nNear+nTokenLeft,
114388 aOut, pnOut, aLeft, nLeft, aRight, nRight, 0
114389 );
114390 if( rc!=SQLITE_OK ){
114391 sqlite3_free(aOut);
114392 aOut = 0;
114393 }
114394 }
114395
114396 *paOut = aOut;
114397 return rc;
114398 }
114399
114400 /*
114401 ** This function is used as part of the processing for the snippet() and
114402 ** offsets() functions.
114403 **
114404 ** Both pLeft and pRight are expression nodes of type FTSQUERY_PHRASE. Both
114405 ** have their respective doclists (including position information) loaded
114406 ** in Fts3Expr.aDoclist/nDoclist. This function removes all entries from
114407 ** each doclist that are not within nNear tokens of a corresponding entry
114408 ** in the other doclist.
114409 */
114410 SQLITE_PRIVATE int sqlite3Fts3ExprNearTrim(Fts3Expr *pLeft, Fts3Expr *pRight, int nNear){
114411 int rc; /* Return code */
114412
114413 assert( pLeft->eType==FTSQUERY_PHRASE );
114414 assert( pRight->eType==FTSQUERY_PHRASE );
114415 assert( pLeft->isLoaded && pRight->isLoaded );
114416
114417 if( pLeft->aDoclist==0 || pRight->aDoclist==0 ){
114418 sqlite3_free(pLeft->aDoclist);
114419 sqlite3_free(pRight->aDoclist);
114420 pRight->aDoclist = 0;
114421 pLeft->aDoclist = 0;
114422 rc = SQLITE_OK;
114423 }else{
114424 char *aOut; /* Buffer in which to assemble new doclist */
114425 int nOut; /* Size of buffer aOut in bytes */
114426
114427 rc = fts3NearMerge(MERGE_POS_NEAR, nNear,
114428 pLeft->pPhrase->nToken, pLeft->aDoclist, pLeft->nDoclist,
114429 pRight->pPhrase->nToken, pRight->aDoclist, pRight->nDoclist,
114430 &aOut, &nOut
114431 );
114432 if( rc!=SQLITE_OK ) return rc;
114433 sqlite3_free(pRight->aDoclist);
114434 pRight->aDoclist = aOut;
114435 pRight->nDoclist = nOut;
114436
114437 rc = fts3NearMerge(MERGE_POS_NEAR, nNear,
114438 pRight->pPhrase->nToken, pRight->aDoclist, pRight->nDoclist,
114439 pLeft->pPhrase->nToken, pLeft->aDoclist, pLeft->nDoclist,
114440 &aOut, &nOut
114441 );
114442 sqlite3_free(pLeft->aDoclist);
114443 pLeft->aDoclist = aOut;
114444 pLeft->nDoclist = nOut;
114445 }
114446 return rc;
114447 }
114448
114449
114450 /*
114451 ** Allocate an Fts3SegReaderArray for each token in the expression pExpr.
114452 ** The allocated objects are stored in the Fts3PhraseToken.pArray member
114453 ** variables of each token structure.
114454 */
114455 static int fts3ExprAllocateSegReaders(
114456 Fts3Cursor *pCsr, /* FTS3 table */
114457 Fts3Expr *pExpr, /* Expression to create seg-readers for */
114458 int *pnExpr /* OUT: Number of AND'd expressions */
114459 ){
114460 int rc = SQLITE_OK; /* Return code */
114461
114462 assert( pCsr->eEvalmode==FTS3_EVAL_FILTER );
114463 if( pnExpr && pExpr->eType!=FTSQUERY_AND ){
114464 (*pnExpr)++;
114465 pnExpr = 0;
114466 }
114467
114468 if( pExpr->eType==FTSQUERY_PHRASE ){
114469 Fts3Phrase *pPhrase = pExpr->pPhrase;
114470 int ii;
114471
114472 for(ii=0; rc==SQLITE_OK && ii<pPhrase->nToken; ii++){
114473 Fts3PhraseToken *pTok = &pPhrase->aToken[ii];
114474 if( pTok->pSegcsr==0 ){
114475 rc = fts3TermSegReaderCursor(
114476 pCsr, pTok->z, pTok->n, pTok->isPrefix, &pTok->pSegcsr
114477 );
114478 }
114479 }
114480 }else{
114481 rc = fts3ExprAllocateSegReaders(pCsr, pExpr->pLeft, pnExpr);
114482 if( rc==SQLITE_OK ){
114483 rc = fts3ExprAllocateSegReaders(pCsr, pExpr->pRight, pnExpr);
114484 }
114485 }
114486 return rc;
114487 }
114488
114489 /*
114490 ** Free the Fts3SegReaderArray objects associated with each token in the
114491 ** expression pExpr. In other words, this function frees the resources
114492 ** allocated by fts3ExprAllocateSegReaders().
114493 */
114494 static void fts3ExprFreeSegReaders(Fts3Expr *pExpr){
114495 if( pExpr ){
114496 Fts3Phrase *pPhrase = pExpr->pPhrase;
114497 if( pPhrase ){
114498 int kk;
114499 for(kk=0; kk<pPhrase->nToken; kk++){
114500 fts3SegReaderCursorFree(pPhrase->aToken[kk].pSegcsr);
114501 pPhrase->aToken[kk].pSegcsr = 0;
114502 }
114503 }
114504 fts3ExprFreeSegReaders(pExpr->pLeft);
114505 fts3ExprFreeSegReaders(pExpr->pRight);
114506 }
114507 }
114508
114509 /*
114510 ** Return the sum of the costs of all tokens in the expression pExpr. This
114511 ** function must be called after Fts3SegReaderArrays have been allocated
114512 ** for all tokens using fts3ExprAllocateSegReaders().
114513 */
114514 static int fts3ExprCost(Fts3Expr *pExpr){
114515 int nCost; /* Return value */
114516 if( pExpr->eType==FTSQUERY_PHRASE ){
114517 Fts3Phrase *pPhrase = pExpr->pPhrase;
114518 int ii;
114519 nCost = 0;
114520 for(ii=0; ii<pPhrase->nToken; ii++){
114521 Fts3SegReaderCursor *pSegcsr = pPhrase->aToken[ii].pSegcsr;
114522 if( pSegcsr ) nCost += pSegcsr->nCost;
114523 }
114524 }else{
114525 nCost = fts3ExprCost(pExpr->pLeft) + fts3ExprCost(pExpr->pRight);
114526 }
114527 return nCost;
114528 }
114529
114530 /*
114531 ** The following is a helper function (and type) for fts3EvalExpr(). It
114532 ** must be called after Fts3SegReaders have been allocated for every token
114533 ** in the expression. See the context it is called from in fts3EvalExpr()
114534 ** for further explanation.
114535 */
114536 typedef struct ExprAndCost ExprAndCost;
114537 struct ExprAndCost {
114538 Fts3Expr *pExpr;
114539 int nCost;
114540 };
114541 static void fts3ExprAssignCosts(
114542 Fts3Expr *pExpr, /* Expression to create seg-readers for */
114543 ExprAndCost **ppExprCost /* OUT: Write to *ppExprCost */
114544 ){
114545 if( pExpr->eType==FTSQUERY_AND ){
114546 fts3ExprAssignCosts(pExpr->pLeft, ppExprCost);
114547 fts3ExprAssignCosts(pExpr->pRight, ppExprCost);
114548 }else{
114549 (*ppExprCost)->pExpr = pExpr;
114550 (*ppExprCost)->nCost = fts3ExprCost(pExpr);
114551 (*ppExprCost)++;
114552 }
114553 }
114554
114555 /*
114556 ** Evaluate the full-text expression pExpr against FTS3 table pTab. Store
114557 ** the resulting doclist in *paOut and *pnOut. This routine mallocs for
114558 ** the space needed to store the output. The caller is responsible for
114559 ** freeing the space when it has finished.
114560 **
114561 ** This function is called in two distinct contexts:
114562 **
114563 ** * From within the virtual table xFilter() method. In this case, the
114564 ** output doclist contains entries for all rows in the table, based on
114565 ** data read from the full-text index.
114566 **
114567 ** In this case, if the query expression contains one or more tokens that
114568 ** are very common, then the returned doclist may contain a superset of
114569 ** the documents that actually match the expression.
114570 **
114571 ** * From within the virtual table xNext() method. This call is only made
114572 ** if the call from within xFilter() found that there were very common
114573 ** tokens in the query expression and did return a superset of the
114574 ** matching documents. In this case the returned doclist contains only
114575 ** entries that correspond to the current row of the table. Instead of
114576 ** reading the data for each token from the full-text index, the data is
114577 ** already available in-memory in the Fts3PhraseToken.pDeferred structures.
114578 ** See fts3EvalDeferred() for how it gets there.
114579 **
114580 ** In the first case above, Fts3Cursor.doDeferred==0. In the second (if it is
114581 ** required) Fts3Cursor.doDeferred==1.
114582 **
114583 ** If the SQLite invokes the snippet(), offsets() or matchinfo() function
114584 ** as part of a SELECT on an FTS3 table, this function is called on each
114585 ** individual phrase expression in the query. If there were very common tokens
114586 ** found in the xFilter() call, then this function is called once for phrase
114587 ** for each row visited, and the returned doclist contains entries for the
114588 ** current row only. Otherwise, if there were no very common tokens, then this
114589 ** function is called once only for each phrase in the query and the returned
114590 ** doclist contains entries for all rows of the table.
114591 **
114592 ** Fts3Cursor.doDeferred==1 when this function is called on phrases as a
114593 ** result of a snippet(), offsets() or matchinfo() invocation.
114594 */
114595 static int fts3EvalExpr(
114596 Fts3Cursor *p, /* Virtual table cursor handle */
114597 Fts3Expr *pExpr, /* Parsed fts3 expression */
114598 char **paOut, /* OUT: Pointer to malloc'd result buffer */
114599 int *pnOut, /* OUT: Size of buffer at *paOut */
114600 int isReqPos /* Require positions in output buffer */
114601 ){
114602 int rc = SQLITE_OK; /* Return code */
114603
114604 /* Zero the output parameters. */
114605 *paOut = 0;
114606 *pnOut = 0;
114607
114608 if( pExpr ){
114609 assert( pExpr->eType==FTSQUERY_NEAR || pExpr->eType==FTSQUERY_OR
114610 || pExpr->eType==FTSQUERY_AND || pExpr->eType==FTSQUERY_NOT
114611 || pExpr->eType==FTSQUERY_PHRASE
114612 );
114613 assert( pExpr->eType==FTSQUERY_PHRASE || isReqPos==0 );
114614
114615 if( pExpr->eType==FTSQUERY_PHRASE ){
114616 rc = fts3PhraseSelect(p, pExpr->pPhrase,
114617 isReqPos || (pExpr->pParent && pExpr->pParent->eType==FTSQUERY_NEAR),
114618 paOut, pnOut
114619 );
114620 fts3ExprFreeSegReaders(pExpr);
114621 }else if( p->eEvalmode==FTS3_EVAL_FILTER && pExpr->eType==FTSQUERY_AND ){
114622 ExprAndCost *aExpr = 0; /* Array of AND'd expressions and costs */
114623 int nExpr = 0; /* Size of aExpr[] */
114624 char *aRet = 0; /* Doclist to return to caller */
114625 int nRet = 0; /* Length of aRet[] in bytes */
114626 int nDoc = 0x7FFFFFFF;
114627
114628 assert( !isReqPos );
114629
114630 rc = fts3ExprAllocateSegReaders(p, pExpr, &nExpr);
114631 if( rc==SQLITE_OK ){
114632 assert( nExpr>1 );
114633 aExpr = sqlite3_malloc(sizeof(ExprAndCost) * nExpr);
114634 if( !aExpr ) rc = SQLITE_NOMEM;
114635 }
114636 if( rc==SQLITE_OK ){
114637 int ii; /* Used to iterate through expressions */
114638
114639 fts3ExprAssignCosts(pExpr, &aExpr);
114640 aExpr -= nExpr;
114641 for(ii=0; ii<nExpr; ii++){
114642 char *aNew;
114643 int nNew;
114644 int jj;
114645 ExprAndCost *pBest = 0;
114646
114647 for(jj=0; jj<nExpr; jj++){
114648 ExprAndCost *pCand = &aExpr[jj];
114649 if( pCand->pExpr && (pBest==0 || pCand->nCost<pBest->nCost) ){
114650 pBest = pCand;
114651 }
114652 }
114653
114654 if( pBest->nCost>nDoc ){
114655 rc = fts3DeferExpression(p, p->pExpr);
114656 break;
114657 }else{
114658 rc = fts3EvalExpr(p, pBest->pExpr, &aNew, &nNew, 0);
114659 if( rc!=SQLITE_OK ) break;
114660 pBest->pExpr = 0;
114661 if( ii==0 ){
114662 aRet = aNew;
114663 nRet = nNew;
114664 nDoc = fts3DoclistCountDocids(0, aRet, nRet);
114665 }else{
114666 fts3DoclistMerge(
114667 MERGE_AND, 0, 0, aRet, &nRet, aRet, nRet, aNew, nNew, &nDoc
114668 );
114669 sqlite3_free(aNew);
114670 }
114671 }
114672 }
114673 }
114674
114675 if( rc==SQLITE_OK ){
114676 *paOut = aRet;
114677 *pnOut = nRet;
114678 }else{
114679 assert( *paOut==0 );
114680 sqlite3_free(aRet);
114681 }
114682 sqlite3_free(aExpr);
114683 fts3ExprFreeSegReaders(pExpr);
114684
114685 }else{
114686 char *aLeft;
114687 char *aRight;
114688 int nLeft;
114689 int nRight;
114690
114691 assert( pExpr->eType==FTSQUERY_NEAR
114692 || pExpr->eType==FTSQUERY_OR
114693 || pExpr->eType==FTSQUERY_NOT
114694 || (pExpr->eType==FTSQUERY_AND && p->eEvalmode==FTS3_EVAL_NEXT)
114695 );
114696
114697 if( 0==(rc = fts3EvalExpr(p, pExpr->pRight, &aRight, &nRight, isReqPos))
114698 && 0==(rc = fts3EvalExpr(p, pExpr->pLeft, &aLeft, &nLeft, isReqPos))
114699 ){
114700 switch( pExpr->eType ){
114701 case FTSQUERY_NEAR: {
114702 Fts3Expr *pLeft;
114703 Fts3Expr *pRight;
114704 int mergetype = MERGE_NEAR;
114705 if( pExpr->pParent && pExpr->pParent->eType==FTSQUERY_NEAR ){
114706 mergetype = MERGE_POS_NEAR;
114707 }
114708 pLeft = pExpr->pLeft;
114709 while( pLeft->eType==FTSQUERY_NEAR ){
114710 pLeft=pLeft->pRight;
114711 }
114712 pRight = pExpr->pRight;
114713 assert( pRight->eType==FTSQUERY_PHRASE );
114714 assert( pLeft->eType==FTSQUERY_PHRASE );
114715
114716 rc = fts3NearMerge(mergetype, pExpr->nNear,
114717 pLeft->pPhrase->nToken, aLeft, nLeft,
114718 pRight->pPhrase->nToken, aRight, nRight,
114719 paOut, pnOut
114720 );
114721 sqlite3_free(aLeft);
114722 break;
114723 }
114724
114725 case FTSQUERY_OR: {
114726 /* Allocate a buffer for the output. The maximum size is the
114727 ** sum of the sizes of the two input buffers. The +1 term is
114728 ** so that a buffer of zero bytes is never allocated - this can
114729 ** cause fts3DoclistMerge() to incorrectly return SQLITE_NOMEM.
114730 */
114731 char *aBuffer = sqlite3_malloc(nRight+nLeft+1);
114732 rc = fts3DoclistMerge(MERGE_OR, 0, 0, aBuffer, pnOut,
114733 aLeft, nLeft, aRight, nRight, 0
114734 );
114735 *paOut = aBuffer;
114736 sqlite3_free(aLeft);
114737 break;
114738 }
114739
114740 default: {
114741 assert( FTSQUERY_NOT==MERGE_NOT && FTSQUERY_AND==MERGE_AND );
114742 fts3DoclistMerge(pExpr->eType, 0, 0, aLeft, pnOut,
114743 aLeft, nLeft, aRight, nRight, 0
114744 );
114745 *paOut = aLeft;
114746 break;
114747 }
114748 }
114749 }
114750 sqlite3_free(aRight);
114751 }
114752 }
114753
114754 assert( rc==SQLITE_OK || *paOut==0 );
114755 return rc;
114756 }
114757
114758 /*
114759 ** This function is called from within xNext() for each row visited by
114760 ** an FTS3 query. If evaluating the FTS3 query expression within xFilter()
114761 ** was able to determine the exact set of matching rows, this function sets
114762 ** *pbRes to true and returns SQLITE_IO immediately.
114763 **
114764 ** Otherwise, if evaluating the query expression within xFilter() returned a
114765 ** superset of the matching documents instead of an exact set (this happens
114766 ** when the query includes very common tokens and it is deemed too expensive to
114767 ** load their doclists from disk), this function tests if the current row
114768 ** really does match the FTS3 query.
114769 **
114770 ** If an error occurs, an SQLite error code is returned. Otherwise, SQLITE_OK
114771 ** is returned and *pbRes is set to true if the current row matches the
114772 ** FTS3 query (and should be included in the results returned to SQLite), or
114773 ** false otherwise.
114774 */
114775 static int fts3EvalDeferred(
114776 Fts3Cursor *pCsr, /* FTS3 cursor pointing at row to test */
114777 int *pbRes /* OUT: Set to true if row is a match */
114778 ){
114779 int rc = SQLITE_OK;
114780 if( pCsr->pDeferred==0 ){
114781 *pbRes = 1;
114782 }else{
114783 rc = fts3CursorSeek(0, pCsr);
114784 if( rc==SQLITE_OK ){
114785 sqlite3Fts3FreeDeferredDoclists(pCsr);
114786 rc = sqlite3Fts3CacheDeferredDoclists(pCsr);
114787 }
114788 if( rc==SQLITE_OK ){
114789 char *a = 0;
114790 int n = 0;
114791 rc = fts3EvalExpr(pCsr, pCsr->pExpr, &a, &n, 0);
114792 assert( n>=0 );
114793 *pbRes = (n>0);
114794 sqlite3_free(a);
114795 }
114796 }
114797 return rc;
114798 }
114799
114800 /*
114801 ** Advance the cursor to the next row in the %_content table that
114802 ** matches the search criteria. For a MATCH search, this will be
114803 ** the next row that matches. For a full-table scan, this will be
114804 ** simply the next row in the %_content table. For a docid lookup,
@@ -114807,43 +114374,24 @@
114807 ** Return SQLITE_OK if nothing goes wrong. SQLITE_OK is returned
114808 ** even if we reach end-of-file. The fts3EofMethod() will be called
114809 ** subsequently to determine whether or not an EOF was hit.
114810 */
114811 static int fts3NextMethod(sqlite3_vtab_cursor *pCursor){
114812 int res;
114813 int rc = SQLITE_OK; /* Return code */
114814 Fts3Cursor *pCsr = (Fts3Cursor *)pCursor;
114815
114816 pCsr->eEvalmode = FTS3_EVAL_NEXT;
114817 do {
114818 if( pCsr->aDoclist==0 ){
114819 if( SQLITE_ROW!=sqlite3_step(pCsr->pStmt) ){
114820 pCsr->isEof = 1;
114821 rc = sqlite3_reset(pCsr->pStmt);
114822 break;
114823 }
114824 pCsr->iPrevId = sqlite3_column_int64(pCsr->pStmt, 0);
114825 }else{
114826 if( pCsr->desc==0 ){
114827 if( pCsr->pNextId>=&pCsr->aDoclist[pCsr->nDoclist] ){
114828 pCsr->isEof = 1;
114829 break;
114830 }
114831 fts3GetDeltaVarint(&pCsr->pNextId, &pCsr->iPrevId);
114832 }else{
114833 fts3GetReverseDeltaVarint(&pCsr->pNextId,pCsr->aDoclist,&pCsr->iPrevId);
114834 if( pCsr->pNextId<=pCsr->aDoclist ){
114835 pCsr->isEof = 1;
114836 break;
114837 }
114838 }
114839 sqlite3_reset(pCsr->pStmt);
114840 pCsr->isRequireSeek = 1;
114841 pCsr->isMatchinfoNeeded = 1;
114842 }
114843 }while( SQLITE_OK==(rc = fts3EvalDeferred(pCsr, &res)) && res==0 );
114844
114845 return rc;
114846 }
114847
114848 /*
114849 ** This is the xFilter interface for the virtual table. See
@@ -114866,15 +114414,11 @@
114866 int idxNum, /* Strategy index */
114867 const char *idxStr, /* Unused */
114868 int nVal, /* Number of elements in apVal */
114869 sqlite3_value **apVal /* Arguments for the indexing scheme */
114870 ){
114871 const char *azSql[] = {
114872 "SELECT %s FROM %Q.'%q_content' AS x WHERE docid = ?", /* non-full-scan */
114873 "SELECT %s FROM %Q.'%q_content' AS x ORDER BY docid %s", /* full-scan */
114874 };
114875 int rc; /* Return code */
114876 char *zSql; /* SQL statement used to access %_content */
114877 Fts3Table *p = (Fts3Table *)pCursor->pVtab;
114878 Fts3Cursor *pCsr = (Fts3Cursor *)pCursor;
114879
114880 UNUSED_PARAMETER(idxStr);
@@ -114889,10 +114433,17 @@
114889 sqlite3_finalize(pCsr->pStmt);
114890 sqlite3_free(pCsr->aDoclist);
114891 sqlite3Fts3ExprFree(pCsr->pExpr);
114892 memset(&pCursor[1], 0, sizeof(Fts3Cursor)-sizeof(sqlite3_vtab_cursor));
114893
 
 
 
 
 
 
 
114894 if( idxNum!=FTS3_DOCID_SEARCH && idxNum!=FTS3_FULLSCAN_SEARCH ){
114895 int iCol = idxNum-FTS3_FULLTEXT_SEARCH;
114896 const char *zQuery = (const char *)sqlite3_value_text(apVal[0]);
114897
114898 if( zQuery==0 && sqlite3_value_type(apVal[0])!=SQLITE_NULL ){
@@ -114902,20 +114453,21 @@
114902 rc = sqlite3Fts3ExprParse(p->pTokenizer, p->azColumn, p->nColumn,
114903 iCol, zQuery, -1, &pCsr->pExpr
114904 );
114905 if( rc!=SQLITE_OK ){
114906 if( rc==SQLITE_ERROR ){
114907 p->base.zErrMsg = sqlite3_mprintf("malformed MATCH expression: [%s]",
114908 zQuery);
114909 }
114910 return rc;
114911 }
114912
114913 rc = sqlite3Fts3ReadLock(p);
114914 if( rc!=SQLITE_OK ) return rc;
114915
114916 rc = fts3EvalExpr(pCsr, pCsr->pExpr, &pCsr->aDoclist, &pCsr->nDoclist, 0);
 
114917 sqlite3Fts3SegmentsClose(p);
114918 if( rc!=SQLITE_OK ) return rc;
114919 pCsr->pNextId = pCsr->aDoclist;
114920 pCsr->iPrevId = 0;
114921 }
@@ -114923,41 +114475,28 @@
114923 /* Compile a SELECT statement for this cursor. For a full-table-scan, the
114924 ** statement loops through all rows of the %_content table. For a
114925 ** full-text query or docid lookup, the statement retrieves a single
114926 ** row by docid.
114927 */
114928 zSql = (char *)azSql[idxNum==FTS3_FULLSCAN_SEARCH];
114929 zSql = sqlite3_mprintf(
114930 zSql, p->zReadExprlist, p->zDb, p->zName, (idxStr ? idxStr : "ASC")
114931 );
114932 if( !zSql ){
114933 rc = SQLITE_NOMEM;
114934 }else{
114935 rc = sqlite3_prepare_v2(p->db, zSql, -1, &pCsr->pStmt, 0);
114936 sqlite3_free(zSql);
114937 }
114938 if( rc==SQLITE_OK && idxNum==FTS3_DOCID_SEARCH ){
 
 
 
 
 
114939 rc = sqlite3_bind_value(pCsr->pStmt, 1, apVal[0]);
114940 }
114941 pCsr->eSearch = (i16)idxNum;
114942
114943 assert( pCsr->desc==0 );
114944 if( rc!=SQLITE_OK ) return rc;
114945 if( rc==SQLITE_OK && pCsr->nDoclist>0 && idxStr && idxStr[0]=='D' ){
114946 sqlite3_int64 iDocid = 0;
114947 char *csr = pCsr->aDoclist;
114948 while( csr<&pCsr->aDoclist[pCsr->nDoclist] ){
114949 fts3GetDeltaVarint(&csr, &iDocid);
114950 }
114951 pCsr->pNextId = csr;
114952 pCsr->iPrevId = iDocid;
114953 pCsr->desc = 1;
114954 pCsr->isRequireSeek = 1;
114955 pCsr->isMatchinfoNeeded = 1;
114956 pCsr->eEvalmode = FTS3_EVAL_NEXT;
114957 return SQLITE_OK;
114958 }
114959 return fts3NextMethod(pCursor);
114960 }
114961
114962 /*
114963 ** This is the xEof method of the virtual table. SQLite calls this
@@ -114973,20 +114512,11 @@
114973 ** exposes %_content.docid as the rowid for the virtual table. The
114974 ** rowid should be written to *pRowid.
114975 */
114976 static int fts3RowidMethod(sqlite3_vtab_cursor *pCursor, sqlite_int64 *pRowid){
114977 Fts3Cursor *pCsr = (Fts3Cursor *) pCursor;
114978 if( pCsr->aDoclist ){
114979 *pRowid = pCsr->iPrevId;
114980 }else{
114981 /* This branch runs if the query is implemented using a full-table scan
114982 ** (not using the full-text index). In this case grab the rowid from the
114983 ** SELECT statement.
114984 */
114985 assert( pCsr->isRequireSeek==0 );
114986 *pRowid = sqlite3_column_int64(pCsr->pStmt, 0);
114987 }
114988 return SQLITE_OK;
114989 }
114990
114991 /*
114992 ** This is the xColumn method, called by SQLite to request a value from
@@ -114995,11 +114525,11 @@
114995 static int fts3ColumnMethod(
114996 sqlite3_vtab_cursor *pCursor, /* Cursor to retrieve value from */
114997 sqlite3_context *pContext, /* Context for sqlite3_result_xxx() calls */
114998 int iCol /* Index of column to read value from */
114999 ){
115000 int rc; /* Return Code */
115001 Fts3Cursor *pCsr = (Fts3Cursor *) pCursor;
115002 Fts3Table *p = (Fts3Table *)pCursor->pVtab;
115003
115004 /* The column value supplied by SQLite must be in range. */
115005 assert( iCol>=0 && iCol<=p->nColumn+1 );
@@ -115006,25 +114536,24 @@
115006
115007 if( iCol==p->nColumn+1 ){
115008 /* This call is a request for the "docid" column. Since "docid" is an
115009 ** alias for "rowid", use the xRowid() method to obtain the value.
115010 */
115011 sqlite3_int64 iRowid;
115012 rc = fts3RowidMethod(pCursor, &iRowid);
115013 sqlite3_result_int64(pContext, iRowid);
115014 }else if( iCol==p->nColumn ){
115015 /* The extra column whose name is the same as the table.
115016 ** Return a blob which is a pointer to the cursor.
115017 */
115018 sqlite3_result_blob(pContext, &pCsr, sizeof(pCsr), SQLITE_TRANSIENT);
115019 rc = SQLITE_OK;
115020 }else{
115021 rc = fts3CursorSeek(0, pCsr);
115022 if( rc==SQLITE_OK ){
115023 sqlite3_result_value(pContext, sqlite3_column_value(pCsr->pStmt, iCol+1));
115024 }
115025 }
 
 
115026 return rc;
115027 }
115028
115029 /*
115030 ** This function is the implementation of the xUpdate callback used by
@@ -115054,10 +114583,11 @@
115054 ** Implementation of xBegin() method. This is a no-op.
115055 */
115056 static int fts3BeginMethod(sqlite3_vtab *pVtab){
115057 UNUSED_PARAMETER(pVtab);
115058 TESTONLY( Fts3Table *p = (Fts3Table*)pVtab );
 
115059 assert( p->nPendingData==0 );
115060 assert( p->inTransaction!=1 );
115061 TESTONLY( p->inTransaction = 1 );
115062 TESTONLY( p->mxSavepoint = -1; );
115063 return SQLITE_OK;
@@ -115071,10 +114601,11 @@
115071 static int fts3CommitMethod(sqlite3_vtab *pVtab){
115072 UNUSED_PARAMETER(pVtab);
115073 TESTONLY( Fts3Table *p = (Fts3Table*)pVtab );
115074 assert( p->nPendingData==0 );
115075 assert( p->inTransaction!=0 );
 
115076 TESTONLY( p->inTransaction = 0 );
115077 TESTONLY( p->mxSavepoint = -1; );
115078 return SQLITE_OK;
115079 }
115080
@@ -115088,132 +114619,30 @@
115088 assert( p->inTransaction!=0 );
115089 TESTONLY( p->inTransaction = 0 );
115090 TESTONLY( p->mxSavepoint = -1; );
115091 return SQLITE_OK;
115092 }
115093
115094 /*
115095 ** Load the doclist associated with expression pExpr to pExpr->aDoclist.
115096 ** The loaded doclist contains positions as well as the document ids.
115097 ** This is used by the matchinfo(), snippet() and offsets() auxillary
115098 ** functions.
115099 */
115100 SQLITE_PRIVATE int sqlite3Fts3ExprLoadDoclist(Fts3Cursor *pCsr, Fts3Expr *pExpr){
115101 int rc;
115102 assert( pExpr->eType==FTSQUERY_PHRASE && pExpr->pPhrase );
115103 assert( pCsr->eEvalmode==FTS3_EVAL_NEXT );
115104 rc = fts3EvalExpr(pCsr, pExpr, &pExpr->aDoclist, &pExpr->nDoclist, 1);
115105 return rc;
115106 }
115107
115108 SQLITE_PRIVATE int sqlite3Fts3ExprLoadFtDoclist(
115109 Fts3Cursor *pCsr,
115110 Fts3Expr *pExpr,
115111 char **paDoclist,
115112 int *pnDoclist
115113 ){
115114 int rc;
115115 assert( pCsr->eEvalmode==FTS3_EVAL_NEXT );
115116 assert( pExpr->eType==FTSQUERY_PHRASE && pExpr->pPhrase );
115117 pCsr->eEvalmode = FTS3_EVAL_MATCHINFO;
115118 rc = fts3EvalExpr(pCsr, pExpr, paDoclist, pnDoclist, 1);
115119 pCsr->eEvalmode = FTS3_EVAL_NEXT;
115120 return rc;
115121 }
115122
115123
115124 /*
115125 ** When called, *ppPoslist must point to the byte immediately following the
115126 ** end of a position-list. i.e. ( (*ppPoslist)[-1]==POS_END ). This function
115127 ** moves *ppPoslist so that it instead points to the first byte of the
115128 ** same position list.
115129 */
115130 static void fts3ReversePoslist(char *pStart, char **ppPoslist){
115131 char *p = &(*ppPoslist)[-3];
115132 char c = p[1];
 
 
115133 while( p>pStart && (*p & 0x80) | c ){
115134 c = *p--;
115135 }
115136 if( p>pStart ){ p = &p[2]; }
115137 while( *p++&0x80 );
115138 *ppPoslist = p;
115139 }
115140
115141
115142 /*
115143 ** After ExprLoadDoclist() (see above) has been called, this function is
115144 ** used to iterate/search through the position lists that make up the doclist
115145 ** stored in pExpr->aDoclist.
115146 */
115147 SQLITE_PRIVATE char *sqlite3Fts3FindPositions(
115148 Fts3Cursor *pCursor, /* Associate FTS3 cursor */
115149 Fts3Expr *pExpr, /* Access this expressions doclist */
115150 sqlite3_int64 iDocid, /* Docid associated with requested pos-list */
115151 int iCol /* Column of requested pos-list */
115152 ){
115153 assert( pExpr->isLoaded );
115154 if( pExpr->aDoclist ){
115155 char *pEnd = &pExpr->aDoclist[pExpr->nDoclist];
115156 char *pCsr;
115157
115158 if( pExpr->pCurrent==0 ){
115159 if( pCursor->desc==0 ){
115160 pExpr->pCurrent = pExpr->aDoclist;
115161 pExpr->iCurrent = 0;
115162 fts3GetDeltaVarint(&pExpr->pCurrent, &pExpr->iCurrent);
115163 }else{
115164 pCsr = pExpr->aDoclist;
115165 while( pCsr<pEnd ){
115166 fts3GetDeltaVarint(&pCsr, &pExpr->iCurrent);
115167 fts3PoslistCopy(0, &pCsr);
115168 }
115169 fts3ReversePoslist(pExpr->aDoclist, &pCsr);
115170 pExpr->pCurrent = pCsr;
115171 }
115172 }
115173 pCsr = pExpr->pCurrent;
115174 assert( pCsr );
115175
115176 while( (pCursor->desc==0 && pCsr<pEnd)
115177 || (pCursor->desc && pCsr>pExpr->aDoclist)
115178 ){
115179 if( pCursor->desc==0 && pExpr->iCurrent<iDocid ){
115180 fts3PoslistCopy(0, &pCsr);
115181 if( pCsr<pEnd ){
115182 fts3GetDeltaVarint(&pCsr, &pExpr->iCurrent);
115183 }
115184 pExpr->pCurrent = pCsr;
115185 }else if( pCursor->desc && pExpr->iCurrent>iDocid ){
115186 fts3GetReverseDeltaVarint(&pCsr, pExpr->aDoclist, &pExpr->iCurrent);
115187 fts3ReversePoslist(pExpr->aDoclist, &pCsr);
115188 pExpr->pCurrent = pCsr;
115189 }else{
115190 if( pExpr->iCurrent==iDocid ){
115191 int iThis = 0;
115192 if( iCol<0 ){
115193 /* If iCol is negative, return a pointer to the start of the
115194 ** position-list (instead of a pointer to the start of a list
115195 ** of offsets associated with a specific column).
115196 */
115197 return pCsr;
115198 }
115199 while( iThis<iCol ){
115200 fts3ColumnlistCopy(0, &pCsr);
115201 if( *pCsr==0x00 ) return 0;
115202 pCsr++;
115203 pCsr += sqlite3Fts3GetVarint32(pCsr, &iThis);
115204 }
115205 if( iCol==iThis && (*pCsr&0xFE) ) return pCsr;
115206 }
115207 return 0;
115208 }
115209 }
115210 }
115211
115212 return 0;
115213 }
115214
115215 /*
115216 ** Helper function used by the implementation of the overloaded snippet(),
115217 ** offsets() and optimize() SQL functions.
115218 **
115219 ** If the value passed as the third argument is a blob of size
@@ -115441,16 +114870,15 @@
115441 );
115442 return rc;
115443 }
115444
115445 static int fts3SavepointMethod(sqlite3_vtab *pVtab, int iSavepoint){
115446 Fts3Table *p = (Fts3Table*)pVtab;
115447 UNUSED_PARAMETER(iSavepoint);
115448 assert( p->inTransaction );
115449 assert( p->mxSavepoint < iSavepoint );
115450 TESTONLY( p->mxSavepoint = iSavepoint );
115451 return sqlite3Fts3PendingTermsFlush(p);
115452 }
115453 static int fts3ReleaseMethod(sqlite3_vtab *pVtab, int iSavepoint){
115454 TESTONLY( Fts3Table *p = (Fts3Table*)pVtab );
115455 UNUSED_PARAMETER(iSavepoint);
115456 UNUSED_PARAMETER(pVtab);
@@ -115617,10 +115045,1282 @@
115617 SQLITE_EXTENSION_INIT2(pApi)
115618 return sqlite3Fts3Init(db);
115619 }
115620 #endif
115621
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
115622 #endif
115623
115624 /************** End of fts3.c ************************************************/
115625 /************** Begin file fts3_aux.c ****************************************/
115626 /*
@@ -115648,11 +116348,11 @@
115648 Fts3Table *pFts3Tab;
115649 };
115650
115651 struct Fts3auxCursor {
115652 sqlite3_vtab_cursor base; /* Base class used by SQLite core */
115653 Fts3SegReaderCursor csr; /* Must be right after "base" */
115654 Fts3SegFilter filter;
115655 char *zStop;
115656 int nStop; /* Byte-length of string zStop */
115657 int isEof; /* True if cursor is at EOF */
115658 sqlite3_int64 iRowid; /* Current rowid */
@@ -115716,10 +116416,11 @@
115716
115717 p->pFts3Tab = (Fts3Table *)&p[1];
115718 p->pFts3Tab->zDb = (char *)&p->pFts3Tab[1];
115719 p->pFts3Tab->zName = &p->pFts3Tab->zDb[nDb+1];
115720 p->pFts3Tab->db = db;
 
115721
115722 memcpy((char *)p->pFts3Tab->zDb, zDb, nDb);
115723 memcpy((char *)p->pFts3Tab->zName, zFts3, nFts3);
115724 sqlite3Fts3Dequote((char *)p->pFts3Tab->zName);
115725
@@ -115996,11 +116697,11 @@
115996 pCsr->zStop = sqlite3_mprintf("%s", sqlite3_value_text(apVal[iIdx]));
115997 pCsr->nStop = sqlite3_value_bytes(apVal[iIdx]);
115998 if( pCsr->zStop==0 ) return SQLITE_NOMEM;
115999 }
116000
116001 rc = sqlite3Fts3SegReaderCursor(pFts3, FTS3_SEGCURSOR_ALL,
116002 pCsr->filter.zTerm, pCsr->filter.nTerm, 0, isScan, &pCsr->csr
116003 );
116004 if( rc==SQLITE_OK ){
116005 rc = sqlite3Fts3SegReaderStart(pFts3, &pCsr->csr, &pCsr->filter);
116006 }
@@ -116175,16 +116876,25 @@
116175 ** Default span for NEAR operators.
116176 */
116177 #define SQLITE_FTS3_DEFAULT_NEAR_PARAM 10
116178
116179
 
 
 
 
 
 
 
 
116180 typedef struct ParseContext ParseContext;
116181 struct ParseContext {
116182 sqlite3_tokenizer *pTokenizer; /* Tokenizer module */
116183 const char **azCol; /* Array of column names for fts3 table */
116184 int nCol; /* Number of entries in azCol[] */
116185 int iDefaultCol; /* Default column to query */
 
116186 sqlite3_context *pCtx; /* Write error message here */
116187 int nNest; /* Number of nested brackets */
116188 };
116189
116190 /*
@@ -116266,11 +116976,11 @@
116266 if( iEnd<n && z[iEnd]=='*' ){
116267 pRet->pPhrase->aToken[0].isPrefix = 1;
116268 iEnd++;
116269 }
116270 if( !sqlite3_fts3_enable_parentheses && iStart>0 && z[iStart-1]=='-' ){
116271 pRet->pPhrase->isNot = 1;
116272 }
116273 }
116274 nConsumed = iEnd;
116275 }
116276
@@ -116318,71 +117028,86 @@
116318 Fts3Expr *p = 0;
116319 sqlite3_tokenizer_cursor *pCursor = 0;
116320 char *zTemp = 0;
116321 int nTemp = 0;
116322
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
116323 rc = pModule->xOpen(pTokenizer, zInput, nInput, &pCursor);
116324 if( rc==SQLITE_OK ){
116325 int ii;
116326 pCursor->pTokenizer = pTokenizer;
116327 for(ii=0; rc==SQLITE_OK; ii++){
116328 const char *zToken;
116329 int nToken, iBegin, iEnd, iPos;
116330 rc = pModule->xNext(pCursor, &zToken, &nToken, &iBegin, &iEnd, &iPos);
116331 if( rc==SQLITE_OK ){
116332 int nByte = sizeof(Fts3Expr) + sizeof(Fts3Phrase);
116333 p = fts3ReallocOrFree(p, nByte+ii*sizeof(Fts3PhraseToken));
116334 zTemp = fts3ReallocOrFree(zTemp, nTemp + nToken);
116335 if( !p || !zTemp ){
116336 goto no_mem;
116337 }
116338 if( ii==0 ){
116339 memset(p, 0, nByte);
116340 p->pPhrase = (Fts3Phrase *)&p[1];
116341 }
116342 p->pPhrase = (Fts3Phrase *)&p[1];
116343 memset(&p->pPhrase->aToken[ii], 0, sizeof(Fts3PhraseToken));
116344 p->pPhrase->nToken = ii+1;
116345 p->pPhrase->aToken[ii].n = nToken;
116346 memcpy(&zTemp[nTemp], zToken, nToken);
116347 nTemp += nToken;
116348 if( iEnd<nInput && zInput[iEnd]=='*' ){
116349 p->pPhrase->aToken[ii].isPrefix = 1;
116350 }else{
116351 p->pPhrase->aToken[ii].isPrefix = 0;
116352 }
116353 }
116354 }
116355
116356 pModule->xClose(pCursor);
116357 pCursor = 0;
116358 }
116359
116360 if( rc==SQLITE_DONE ){
116361 int jj;
116362 char *zNew = NULL;
116363 int nNew = 0;
116364 int nByte = sizeof(Fts3Expr) + sizeof(Fts3Phrase);
116365 nByte += (p?(p->pPhrase->nToken-1):0) * sizeof(Fts3PhraseToken);
116366 p = fts3ReallocOrFree(p, nByte + nTemp);
116367 if( !p ){
116368 goto no_mem;
116369 }
116370 if( zTemp ){
116371 zNew = &(((char *)p)[nByte]);
116372 memcpy(zNew, zTemp, nTemp);
116373 }else{
116374 memset(p, 0, nByte+nTemp);
116375 }
116376 p->pPhrase = (Fts3Phrase *)&p[1];
 
 
 
 
 
 
 
116377 for(jj=0; jj<p->pPhrase->nToken; jj++){
116378 p->pPhrase->aToken[jj].z = &zNew[nNew];
116379 nNew += p->pPhrase->aToken[jj].n;
116380 }
116381 sqlite3_free(zTemp);
116382 p->eType = FTSQUERY_PHRASE;
116383 p->pPhrase->iColumn = pParse->iDefaultCol;
116384 rc = SQLITE_OK;
116385 }
116386
116387 *ppExpr = p;
116388 return rc;
@@ -116434,10 +117159,12 @@
116434 int rc;
116435 Fts3Expr *pRet = 0;
116436
116437 const char *zInput = z;
116438 int nInput = n;
 
 
116439
116440 /* Skip over any whitespace before checking for a keyword, an open or
116441 ** close bracket, or a quoted string.
116442 */
116443 while( nInput>0 && fts3isspace(*zInput) ){
@@ -116653,11 +117380,11 @@
116653 rc = getNextNode(pParse, zIn, nIn, &p, &nByte);
116654 if( rc==SQLITE_OK ){
116655 int isPhrase;
116656
116657 if( !sqlite3_fts3_enable_parentheses
116658 && p->eType==FTSQUERY_PHRASE && p->pPhrase->isNot
116659 ){
116660 /* Create an implicit NOT operator. */
116661 Fts3Expr *pNot = fts3MallocZero(sizeof(Fts3Expr));
116662 if( !pNot ){
116663 sqlite3Fts3ExprFree(p);
@@ -116671,11 +117398,10 @@
116671 }
116672 pNotBranch = pNot;
116673 p = pPrev;
116674 }else{
116675 int eType = p->eType;
116676 assert( eType!=FTSQUERY_PHRASE || !p->pPhrase->isNot );
116677 isPhrase = (eType==FTSQUERY_PHRASE || p->pLeft);
116678
116679 /* The isRequirePhrase variable is set to true if a phrase or
116680 ** an expression contained in parenthesis is required. If a
116681 ** binary operator (AND, OR, NOT or NEAR) is encounted when
@@ -116834,13 +117560,15 @@
116834 /*
116835 ** Free a parsed fts3 query expression allocated by sqlite3Fts3ExprParse().
116836 */
116837 SQLITE_PRIVATE void sqlite3Fts3ExprFree(Fts3Expr *p){
116838 if( p ){
 
116839 sqlite3Fts3ExprFree(p->pLeft);
116840 sqlite3Fts3ExprFree(p->pRight);
116841 sqlite3_free(p->aDoclist);
 
116842 sqlite3_free(p);
116843 }
116844 }
116845
116846 /****************************************************************************
@@ -116893,11 +117621,11 @@
116893 switch( pExpr->eType ){
116894 case FTSQUERY_PHRASE: {
116895 Fts3Phrase *pPhrase = pExpr->pPhrase;
116896 int i;
116897 zBuf = sqlite3_mprintf(
116898 "%zPHRASE %d %d", zBuf, pPhrase->iColumn, pPhrase->isNot);
116899 for(i=0; zBuf && i<pPhrase->nToken; i++){
116900 zBuf = sqlite3_mprintf("%z %.*s%s", zBuf,
116901 pPhrase->aToken[i].n, pPhrase->aToken[i].z,
116902 (pPhrase->aToken[i].isPrefix?"+":"")
116903 );
@@ -118811,18 +119539,44 @@
118811 ** it is always safe to read up to two varints from it without risking an
118812 ** overread, even if the node data is corrupted.
118813 */
118814 #define FTS3_NODE_PADDING (FTS3_VARINT_MAX*2)
118815
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
118816 typedef struct PendingList PendingList;
118817 typedef struct SegmentNode SegmentNode;
118818 typedef struct SegmentWriter SegmentWriter;
118819
118820 /*
118821 ** Data structure used while accumulating terms in the pending-terms hash
118822 ** table. The hash table entry maps from term (a string) to a malloc'd
118823 ** instance of this structure.
118824 */
118825 struct PendingList {
118826 int nData;
118827 char *aData;
118828 int nSpace;
@@ -118849,11 +119603,10 @@
118849 ** of type Fts3SegReader* are also used by code in fts3.c to iterate through
118850 ** terms when querying the full-text index. See functions:
118851 **
118852 ** sqlite3Fts3SegReaderNew()
118853 ** sqlite3Fts3SegReaderFree()
118854 ** sqlite3Fts3SegReaderCost()
118855 ** sqlite3Fts3SegReaderIterate()
118856 **
118857 ** Methods used to manipulate Fts3SegReader structures:
118858 **
118859 ** fts3SegReaderNext()
@@ -118868,10 +119621,13 @@
118868 sqlite3_int64 iEndBlock; /* Rowid of final block in segment (or 0) */
118869 sqlite3_int64 iCurrentBlock; /* Current leaf block (or 0) */
118870
118871 char *aNode; /* Pointer to node data (or NULL) */
118872 int nNode; /* Size of buffer at aNode (or 0) */
 
 
 
118873 Fts3HashElem **ppNextElem;
118874
118875 /* Variables set by fts3SegReaderNext(). These may be read directly
118876 ** by the caller. They are valid from the time SegmentReaderNew() returns
118877 ** until SegmentReaderNext() returns something other than SQLITE_OK
@@ -118881,12 +119637,15 @@
118881 char *zTerm; /* Pointer to current term */
118882 int nTermAlloc; /* Allocated size of zTerm buffer */
118883 char *aDoclist; /* Pointer to doclist of current entry */
118884 int nDoclist; /* Size of doclist in current entry */
118885
118886 /* The following variables are used to iterate through the current doclist */
 
 
118887 char *pOffsetList;
 
118888 sqlite3_int64 iDocid;
118889 };
118890
118891 #define fts3SegReaderIsPending(p) ((p)->ppNextElem!=0)
118892 #define fts3SegReaderIsRootOnly(p) ((p)->aNode==(char *)&(p)[1])
@@ -118920,10 +119679,18 @@
118920 ** within the fts3SegWriterXXX() family of functions described above.
118921 **
118922 ** fts3NodeAddTerm()
118923 ** fts3NodeWrite()
118924 ** fts3NodeFree()
 
 
 
 
 
 
 
 
118925 */
118926 struct SegmentNode {
118927 SegmentNode *pParent; /* Parent node (or NULL for root node) */
118928 SegmentNode *pRight; /* Pointer to right-sibling */
118929 SegmentNode *pLeftmost; /* Pointer to left-most node of this depth */
@@ -118950,21 +119717,26 @@
118950 #define SQL_NEXT_SEGMENT_INDEX 8
118951 #define SQL_INSERT_SEGMENTS 9
118952 #define SQL_NEXT_SEGMENTS_ID 10
118953 #define SQL_INSERT_SEGDIR 11
118954 #define SQL_SELECT_LEVEL 12
118955 #define SQL_SELECT_ALL_LEVEL 13
118956 #define SQL_SELECT_LEVEL_COUNT 14
118957 #define SQL_SELECT_SEGDIR_COUNT_MAX 15
118958 #define SQL_DELETE_SEGDIR_BY_LEVEL 16
118959 #define SQL_DELETE_SEGMENTS_RANGE 17
118960 #define SQL_CONTENT_INSERT 18
118961 #define SQL_DELETE_DOCSIZE 19
118962 #define SQL_REPLACE_DOCSIZE 20
118963 #define SQL_SELECT_DOCSIZE 21
118964 #define SQL_SELECT_DOCTOTAL 22
118965 #define SQL_REPLACE_DOCTOTAL 23
 
 
 
 
 
118966
118967 /*
118968 ** This function is used to obtain an SQLite prepared statement handle
118969 ** for the statement identified by the second argument. If successful,
118970 ** *pp is set to the requested statement handle and SQLITE_OK returned.
@@ -118997,23 +119769,29 @@
118997
118998 /* Return segments in order from oldest to newest.*/
118999 /* 12 */ "SELECT idx, start_block, leaves_end_block, end_block, root "
119000 "FROM %Q.'%q_segdir' WHERE level = ? ORDER BY idx ASC",
119001 /* 13 */ "SELECT idx, start_block, leaves_end_block, end_block, root "
119002 "FROM %Q.'%q_segdir' ORDER BY level DESC, idx ASC",
 
119003
119004 /* 14 */ "SELECT count(*) FROM %Q.'%q_segdir' WHERE level = ?",
119005 /* 15 */ "SELECT count(*), max(level) FROM %Q.'%q_segdir'",
119006
119007 /* 16 */ "DELETE FROM %Q.'%q_segdir' WHERE level = ?",
119008 /* 17 */ "DELETE FROM %Q.'%q_segments' WHERE blockid BETWEEN ? AND ?",
119009 /* 18 */ "INSERT INTO %Q.'%q_content' VALUES(%s)",
119010 /* 19 */ "DELETE FROM %Q.'%q_docsize' WHERE docid = ?",
119011 /* 20 */ "REPLACE INTO %Q.'%q_docsize' VALUES(?,?)",
119012 /* 21 */ "SELECT size FROM %Q.'%q_docsize' WHERE docid=?",
119013 /* 22 */ "SELECT value FROM %Q.'%q_stat' WHERE id=0",
119014 /* 23 */ "REPLACE INTO %Q.'%q_stat' VALUES(0,?)",
 
 
 
 
 
119015 };
119016 int rc = SQLITE_OK;
119017 sqlite3_stmt *pStmt;
119018
119019 assert( SizeofArray(azSql)==SizeofArray(p->aStmt) );
@@ -119165,18 +119943,36 @@
119165 ** 1: start_block
119166 ** 2: leaves_end_block
119167 ** 3: end_block
119168 ** 4: root
119169 */
119170 SQLITE_PRIVATE int sqlite3Fts3AllSegdirs(Fts3Table *p, int iLevel, sqlite3_stmt **ppStmt){
 
 
 
 
 
119171 int rc;
119172 sqlite3_stmt *pStmt = 0;
 
 
 
 
 
119173 if( iLevel<0 ){
119174 rc = fts3SqlStmt(p, SQL_SELECT_ALL_LEVEL, &pStmt, 0);
 
 
 
 
 
119175 }else{
 
119176 rc = fts3SqlStmt(p, SQL_SELECT_LEVEL, &pStmt, 0);
119177 if( rc==SQLITE_OK ) sqlite3_bind_int(pStmt, 1, iLevel);
 
 
119178 }
119179 *ppStmt = pStmt;
119180 return rc;
119181 }
119182
@@ -119286,10 +120082,51 @@
119286 *pp = p;
119287 return 1;
119288 }
119289 return 0;
119290 }
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
119291
119292 /*
119293 ** Tokenize the nul-terminated string zText and add all tokens to the
119294 ** pending-terms hash-table. The docid used is that currently stored in
119295 ** p->iPrevDocid, and the column is specified by argument iCol.
@@ -119335,12 +120172,11 @@
119335
119336 xNext = pModule->xNext;
119337 while( SQLITE_OK==rc
119338 && SQLITE_OK==(rc = xNext(pCsr, &zToken, &nToken, &iStart, &iEnd, &iPos))
119339 ){
119340 PendingList *pList;
119341
119342 if( iPos>=nWord ) nWord = iPos+1;
119343
119344 /* Positions cannot be negative; we use -1 as a terminator internally.
119345 ** Tokens must have a non-zero length.
119346 */
@@ -119347,26 +120183,23 @@
119347 if( iPos<0 || !zToken || nToken<=0 ){
119348 rc = SQLITE_ERROR;
119349 break;
119350 }
119351
119352 pList = (PendingList *)fts3HashFind(&p->pendingTerms, zToken, nToken);
119353 if( pList ){
119354 p->nPendingData -= (pList->nData + nToken + sizeof(Fts3HashElem));
119355 }
119356 if( fts3PendingListAppend(&pList, p->iPrevDocid, iCol, iPos, &rc) ){
119357 if( pList==fts3HashInsert(&p->pendingTerms, zToken, nToken, pList) ){
119358 /* Malloc failed while inserting the new entry. This can only
119359 ** happen if there was no previous entry for this token.
119360 */
119361 assert( 0==fts3HashFind(&p->pendingTerms, zToken, nToken) );
119362 sqlite3_free(pList);
119363 rc = SQLITE_NOMEM;
119364 }
119365 }
119366 if( rc==SQLITE_OK ){
119367 p->nPendingData += (pList->nData + nToken + sizeof(Fts3HashElem));
119368 }
119369 }
119370
119371 pModule->xClose(pCsr);
119372 *pnWord = nWord;
@@ -119392,18 +120225,23 @@
119392 p->iPrevDocid = iDocid;
119393 return SQLITE_OK;
119394 }
119395
119396 /*
119397 ** Discard the contents of the pending-terms hash table.
119398 */
119399 SQLITE_PRIVATE void sqlite3Fts3PendingTermsClear(Fts3Table *p){
119400 Fts3HashElem *pElem;
119401 for(pElem=fts3HashFirst(&p->pendingTerms); pElem; pElem=fts3HashNext(pElem)){
119402 sqlite3_free(fts3HashData(pElem));
 
 
 
 
 
 
119403 }
119404 fts3HashClear(&p->pendingTerms);
119405 p->nPendingData = 0;
119406 }
119407
119408 /*
119409 ** This function is called by the xUpdate() method as part of an INSERT
@@ -119555,11 +120393,11 @@
119555
119556 /*
119557 ** Forward declaration to account for the circular dependency between
119558 ** functions fts3SegmentMerge() and fts3AllocateSegdirIdx().
119559 */
119560 static int fts3SegmentMerge(Fts3Table *, int);
119561
119562 /*
119563 ** This function allocates a new level iLevel index in the segdir table.
119564 ** Usually, indexes are allocated within a level sequentially starting
119565 ** with 0, so the allocated index is one greater than the value returned
@@ -119572,19 +120410,24 @@
119572 ** allocated index is 0.
119573 **
119574 ** If successful, *piIdx is set to the allocated index slot and SQLITE_OK
119575 ** returned. Otherwise, an SQLite error code is returned.
119576 */
119577 static int fts3AllocateSegdirIdx(Fts3Table *p, int iLevel, int *piIdx){
 
 
 
 
 
119578 int rc; /* Return Code */
119579 sqlite3_stmt *pNextIdx; /* Query for next idx at level iLevel */
119580 int iNext = 0; /* Result of query pNextIdx */
119581
119582 /* Set variable iNext to the next available segdir index at level iLevel. */
119583 rc = fts3SqlStmt(p, SQL_NEXT_SEGMENT_INDEX, &pNextIdx, 0);
119584 if( rc==SQLITE_OK ){
119585 sqlite3_bind_int(pNextIdx, 1, iLevel);
119586 if( SQLITE_ROW==sqlite3_step(pNextIdx) ){
119587 iNext = sqlite3_column_int(pNextIdx, 0);
119588 }
119589 rc = sqlite3_reset(pNextIdx);
119590 }
@@ -119594,11 +120437,11 @@
119594 ** full, merge all segments in level iLevel into a single iLevel+1
119595 ** segment and allocate (newly freed) index 0 at level iLevel. Otherwise,
119596 ** if iNext is less than FTS3_MERGE_COUNT, allocate index iNext.
119597 */
119598 if( iNext>=FTS3_MERGE_COUNT ){
119599 rc = fts3SegmentMerge(p, iLevel);
119600 *piIdx = 0;
119601 }else{
119602 *piIdx = iNext;
119603 }
119604 }
@@ -119635,11 +120478,12 @@
119635 */
119636 SQLITE_PRIVATE int sqlite3Fts3ReadBlock(
119637 Fts3Table *p, /* FTS3 table handle */
119638 sqlite3_int64 iBlockid, /* Access the row with blockid=$iBlockid */
119639 char **paBlob, /* OUT: Blob data in malloc'd buffer */
119640 int *pnBlob /* OUT: Size of blob data */
 
119641 ){
119642 int rc; /* Return code */
119643
119644 /* pnBlob must be non-NULL. paBlob may be NULL or non-NULL. */
119645 assert( pnBlob);
@@ -119656,25 +120500,29 @@
119656 );
119657 }
119658
119659 if( rc==SQLITE_OK ){
119660 int nByte = sqlite3_blob_bytes(p->pSegments);
 
119661 if( paBlob ){
119662 char *aByte = sqlite3_malloc(nByte + FTS3_NODE_PADDING);
119663 if( !aByte ){
119664 rc = SQLITE_NOMEM;
119665 }else{
 
 
 
 
119666 rc = sqlite3_blob_read(p->pSegments, aByte, nByte, 0);
119667 memset(&aByte[nByte], 0, FTS3_NODE_PADDING);
119668 if( rc!=SQLITE_OK ){
119669 sqlite3_free(aByte);
119670 aByte = 0;
119671 }
119672 }
119673 *paBlob = aByte;
119674 }
119675 *pnBlob = nByte;
119676 }
119677
119678 return rc;
119679 }
119680
@@ -119684,17 +120532,59 @@
119684 */
119685 SQLITE_PRIVATE void sqlite3Fts3SegmentsClose(Fts3Table *p){
119686 sqlite3_blob_close(p->pSegments);
119687 p->pSegments = 0;
119688 }
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
119689
119690 /*
119691 ** Move the iterator passed as the first argument to the next term in the
119692 ** segment. If successful, SQLITE_OK is returned. If there is no next term,
119693 ** SQLITE_DONE. Otherwise, an SQLite error code.
119694 */
119695 static int fts3SegReaderNext(Fts3Table *p, Fts3SegReader *pReader){
 
 
 
 
 
119696 char *pNext; /* Cursor variable */
119697 int nPrefix; /* Number of bytes in term prefix */
119698 int nSuffix; /* Number of bytes in term suffix */
119699
119700 if( !pReader->aDoclist ){
@@ -119702,11 +120592,10 @@
119702 }else{
119703 pNext = &pReader->aDoclist[pReader->nDoclist];
119704 }
119705
119706 if( !pNext || pNext>=&pReader->aNode[pReader->nNode] ){
119707 int rc; /* Return code from Fts3ReadBlock() */
119708
119709 if( fts3SegReaderIsPending(pReader) ){
119710 Fts3HashElem *pElem = *(pReader->ppNextElem);
119711 if( pElem==0 ){
119712 pReader->aNode = 0;
@@ -119722,10 +120611,12 @@
119722 return SQLITE_OK;
119723 }
119724
119725 if( !fts3SegReaderIsRootOnly(pReader) ){
119726 sqlite3_free(pReader->aNode);
 
 
119727 }
119728 pReader->aNode = 0;
119729
119730 /* If iCurrentBlock>=iLeafEndBlock, this is an EOF condition. All leaf
119731 ** blocks have already been traversed. */
@@ -119733,19 +120624,29 @@
119733 if( pReader->iCurrentBlock>=pReader->iLeafEndBlock ){
119734 return SQLITE_OK;
119735 }
119736
119737 rc = sqlite3Fts3ReadBlock(
119738 p, ++pReader->iCurrentBlock, &pReader->aNode, &pReader->nNode
 
119739 );
119740 if( rc!=SQLITE_OK ) return rc;
 
 
 
 
 
119741 pNext = pReader->aNode;
119742 }
 
 
 
 
 
119743
119744 /* Because of the FTS3_NODE_PADDING bytes of padding, the following is
119745 ** safe (no risk of overread) even if the node data is corrupted.
119746 */
119747 pNext += sqlite3Fts3GetVarint32(pNext, &nPrefix);
119748 pNext += sqlite3Fts3GetVarint32(pNext, &nSuffix);
119749 if( nPrefix<0 || nSuffix<=0
119750 || &pNext[nSuffix]>&pReader->aNode[pReader->nNode]
119751 ){
@@ -119759,10 +120660,14 @@
119759 return SQLITE_NOMEM;
119760 }
119761 pReader->zTerm = zNew;
119762 pReader->nTermAlloc = nNew;
119763 }
 
 
 
 
119764 memcpy(&pReader->zTerm[nPrefix], pNext, nSuffix);
119765 pReader->nTerm = nPrefix+nSuffix;
119766 pNext += nSuffix;
119767 pNext += sqlite3Fts3GetVarint32(pNext, &pReader->nDoclist);
119768 pReader->aDoclist = pNext;
@@ -119771,11 +120676,11 @@
119771 /* Check that the doclist does not appear to extend past the end of the
119772 ** b-tree node. And that the final byte of the doclist is 0x00. If either
119773 ** of these statements is untrue, then the data structure is corrupt.
119774 */
119775 if( &pReader->aDoclist[pReader->nDoclist]>&pReader->aNode[pReader->nNode]
119776 || pReader->aDoclist[pReader->nDoclist-1]
119777 ){
119778 return SQLITE_CORRUPT_VTAB;
119779 }
119780 return SQLITE_OK;
119781 }
@@ -119782,16 +120687,30 @@
119782
119783 /*
119784 ** Set the SegReader to point to the first docid in the doclist associated
119785 ** with the current term.
119786 */
119787 static void fts3SegReaderFirstDocid(Fts3SegReader *pReader){
119788 int n;
119789 assert( pReader->aDoclist );
119790 assert( !pReader->pOffsetList );
119791 n = sqlite3Fts3GetVarint(pReader->aDoclist, &pReader->iDocid);
119792 pReader->pOffsetList = &pReader->aDoclist[n];
 
 
 
 
 
 
 
 
 
 
 
 
 
 
119793 }
119794
119795 /*
119796 ** Advance the SegReader to point to the next docid in the doclist
119797 ** associated with the current term.
@@ -119800,132 +120719,126 @@
119800 ** *ppOffsetList is set to point to the first column-offset list
119801 ** in the doclist entry (i.e. immediately past the docid varint).
119802 ** *pnOffsetList is set to the length of the set of column-offset
119803 ** lists, not including the nul-terminator byte. For example:
119804 */
119805 static void fts3SegReaderNextDocid(
119806 Fts3SegReader *pReader,
119807 char **ppOffsetList,
119808 int *pnOffsetList
 
119809 ){
 
119810 char *p = pReader->pOffsetList;
119811 char c = 0;
119812
119813 /* Pointer p currently points at the first byte of an offset list. The
119814 ** following two lines advance it to point one byte past the end of
119815 ** the same offset list.
119816 */
119817 while( *p | c ) c = *p++ & 0x80;
119818 p++;
119819
119820 /* If required, populate the output variables with a pointer to and the
119821 ** size of the previous offset-list.
119822 */
119823 if( ppOffsetList ){
119824 *ppOffsetList = pReader->pOffsetList;
119825 *pnOffsetList = (int)(p - pReader->pOffsetList - 1);
119826 }
119827
119828 /* If there are no more entries in the doclist, set pOffsetList to
119829 ** NULL. Otherwise, set Fts3SegReader.iDocid to the next docid and
119830 ** Fts3SegReader.pOffsetList to point to the next offset list before
119831 ** returning.
119832 */
119833 if( p>=&pReader->aDoclist[pReader->nDoclist] ){
119834 pReader->pOffsetList = 0;
119835 }else{
119836 sqlite3_int64 iDelta;
119837 pReader->pOffsetList = p + sqlite3Fts3GetVarint(p, &iDelta);
119838 pReader->iDocid += iDelta;
119839 }
119840 }
119841
119842 /*
119843 ** This function is called to estimate the amount of data that will be
119844 ** loaded from the disk If SegReaderIterate() is called on this seg-reader,
119845 ** in units of average document size.
119846 **
119847 ** This can be used as follows: If the caller has a small doclist that
119848 ** contains references to N documents, and is considering merging it with
119849 ** a large doclist (size X "average documents"), it may opt not to load
119850 ** the large doclist if X>N.
119851 */
119852 SQLITE_PRIVATE int sqlite3Fts3SegReaderCost(
119853 Fts3Cursor *pCsr, /* FTS3 cursor handle */
119854 Fts3SegReader *pReader, /* Segment-reader handle */
119855 int *pnCost /* IN/OUT: Number of bytes read */
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
119856 ){
119857 Fts3Table *p = (Fts3Table*)pCsr->base.pVtab;
119858 int rc = SQLITE_OK; /* Return code */
119859 int nCost = 0; /* Cost in bytes to return */
119860 int pgsz = p->nPgsz; /* Database page size */
119861
119862 /* If this seg-reader is reading the pending-terms table, or if all data
119863 ** for the segment is stored on the root page of the b-tree, then the cost
119864 ** is zero. In this case all required data is already in main memory.
119865 */
119866 if( p->bHasStat
119867 && !fts3SegReaderIsPending(pReader)
119868 && !fts3SegReaderIsRootOnly(pReader)
119869 ){
119870 int nBlob = 0;
119871 sqlite3_int64 iBlock;
119872
119873 if( pCsr->nRowAvg==0 ){
119874 /* The average document size, which is required to calculate the cost
119875 ** of each doclist, has not yet been determined. Read the required
119876 ** data from the %_stat table to calculate it.
119877 **
119878 ** Entry 0 of the %_stat table is a blob containing (nCol+1) FTS3
119879 ** varints, where nCol is the number of columns in the FTS3 table.
119880 ** The first varint is the number of documents currently stored in
119881 ** the table. The following nCol varints contain the total amount of
119882 ** data stored in all rows of each column of the table, from left
119883 ** to right.
119884 */
119885 sqlite3_stmt *pStmt;
119886 sqlite3_int64 nDoc = 0;
119887 sqlite3_int64 nByte = 0;
119888 const char *pEnd;
119889 const char *a;
119890
119891 rc = sqlite3Fts3SelectDoctotal(p, &pStmt);
119892 if( rc!=SQLITE_OK ) return rc;
119893 a = sqlite3_column_blob(pStmt, 0);
119894 assert( a );
119895
119896 pEnd = &a[sqlite3_column_bytes(pStmt, 0)];
119897 a += sqlite3Fts3GetVarint(a, &nDoc);
119898 while( a<pEnd ){
119899 a += sqlite3Fts3GetVarint(a, &nByte);
119900 }
119901 if( nDoc==0 || nByte==0 ){
119902 sqlite3_reset(pStmt);
119903 return SQLITE_CORRUPT_VTAB;
119904 }
119905
119906 pCsr->nRowAvg = (int)(((nByte / nDoc) + pgsz) / pgsz);
119907 assert( pCsr->nRowAvg>0 );
119908 rc = sqlite3_reset(pStmt);
119909 if( rc!=SQLITE_OK ) return rc;
119910 }
119911
119912 /* Assume that a blob flows over onto overflow pages if it is larger
119913 ** than (pgsz-35) bytes in size (the file-format documentation
119914 ** confirms this).
119915 */
119916 for(iBlock=pReader->iStartBlock; iBlock<=pReader->iLeafEndBlock; iBlock++){
119917 rc = sqlite3Fts3ReadBlock(p, iBlock, 0, &nBlob);
119918 if( rc!=SQLITE_OK ) break;
119919 if( (nBlob+35)>pgsz ){
119920 int nOvfl = (nBlob + 34)/pgsz;
119921 nCost += ((nOvfl + pCsr->nRowAvg - 1)/pCsr->nRowAvg);
119922 }
119923 }
119924 }
119925
119926 *pnCost += nCost;
119927 return rc;
119928 }
119929
119930 /*
119931 ** Free all allocations associated with the iterator passed as the
@@ -119934,10 +120847,11 @@
119934 SQLITE_PRIVATE void sqlite3Fts3SegReaderFree(Fts3SegReader *pReader){
119935 if( pReader && !fts3SegReaderIsPending(pReader) ){
119936 sqlite3_free(pReader->zTerm);
119937 if( !fts3SegReaderIsRootOnly(pReader) ){
119938 sqlite3_free(pReader->aNode);
 
119939 }
119940 }
119941 sqlite3_free(pReader);
119942 }
119943
@@ -120010,28 +120924,46 @@
120010 }
120011
120012 /*
120013 ** This function is used to allocate an Fts3SegReader that iterates through
120014 ** a subset of the terms stored in the Fts3Table.pendingTerms array.
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
120015 */
120016 SQLITE_PRIVATE int sqlite3Fts3SegReaderPending(
120017 Fts3Table *p, /* Virtual table handle */
 
120018 const char *zTerm, /* Term to search for */
120019 int nTerm, /* Size of buffer zTerm */
120020 int isPrefix, /* True for a term-prefix query */
120021 Fts3SegReader **ppReader /* OUT: SegReader for pending-terms */
120022 ){
120023 Fts3SegReader *pReader = 0; /* Fts3SegReader object to return */
120024 Fts3HashElem **aElem = 0; /* Array of term hash entries to scan */
120025 int nElem = 0; /* Size of array at aElem */
120026 int rc = SQLITE_OK; /* Return Code */
 
120027
120028 if( isPrefix ){
 
120029 int nAlloc = 0; /* Size of allocated array at aElem */
120030 Fts3HashElem *pE = 0; /* Iterator variable */
120031
120032 for(pE=fts3HashFirst(&p->pendingTerms); pE; pE=fts3HashNext(pE)){
120033 char *zKey = (char *)fts3HashKey(pE);
120034 int nKey = fts3HashKeysize(pE);
120035 if( nTerm==0 || (nKey>=nTerm && 0==memcmp(zKey, zTerm, nTerm)) ){
120036 if( nElem==nAlloc ){
120037 Fts3HashElem **aElem2;
@@ -120044,10 +120976,11 @@
120044 nElem = 0;
120045 break;
120046 }
120047 aElem = aElem2;
120048 }
 
120049 aElem[nElem++] = pE;
120050 }
120051 }
120052
120053 /* If more than one term matches the prefix, sort the Fts3HashElem
@@ -120057,11 +120990,13 @@
120057 if( nElem>1 ){
120058 qsort(aElem, nElem, sizeof(Fts3HashElem *), fts3CompareElemByTerm);
120059 }
120060
120061 }else{
120062 Fts3HashElem *pE = fts3HashFindElem(&p->pendingTerms, zTerm, nTerm);
 
 
120063 if( pE ){
120064 aElem = &pE;
120065 nElem = 1;
120066 }
120067 }
@@ -120077,11 +121012,11 @@
120077 pReader->ppNextElem = (Fts3HashElem **)&pReader[1];
120078 memcpy(pReader->ppNextElem, aElem, nElem*sizeof(Fts3HashElem *));
120079 }
120080 }
120081
120082 if( isPrefix ){
120083 sqlite3_free(aElem);
120084 }
120085 *ppReader = pReader;
120086 return rc;
120087 }
@@ -120137,10 +121072,22 @@
120137 if( pLhs->iDocid==pRhs->iDocid ){
120138 rc = pRhs->iIdx - pLhs->iIdx;
120139 }else{
120140 rc = (pLhs->iDocid > pRhs->iDocid) ? 1 : -1;
120141 }
 
 
 
 
 
 
 
 
 
 
 
 
120142 }
120143 assert( pLhs->aNode && pRhs->aNode );
120144 return rc;
120145 }
120146
@@ -120689,25 +121636,34 @@
120689 }
120690 return rc;
120691 }
120692
120693 /*
120694 ** Set *pnSegment to the total number of segments in the database. Set
120695 ** *pnMax to the largest segment level in the database (segment levels
120696 ** are stored in the 'level' column of the %_segdir table).
 
120697 **
120698 ** Return SQLITE_OK if successful, or an SQLite error code if not.
120699 */
120700 static int fts3SegmentCountMax(Fts3Table *p, int *pnSegment, int *pnMax){
120701 sqlite3_stmt *pStmt;
120702 int rc;
 
120703
120704 rc = fts3SqlStmt(p, SQL_SELECT_SEGDIR_COUNT_MAX, &pStmt, 0);
 
 
 
 
 
 
120705 if( rc!=SQLITE_OK ) return rc;
 
 
120706 if( SQLITE_ROW==sqlite3_step(pStmt) ){
120707 *pnSegment = sqlite3_column_int(pStmt, 0);
120708 *pnMax = sqlite3_column_int(pStmt, 1);
120709 }
120710 return sqlite3_reset(pStmt);
120711 }
120712
120713 /*
@@ -120724,10 +121680,11 @@
120724 **
120725 ** SQLITE_OK is returned if successful, otherwise an SQLite error code.
120726 */
120727 static int fts3DeleteSegdir(
120728 Fts3Table *p, /* Virtual table handle */
 
120729 int iLevel, /* Level of %_segdir entries to delete */
120730 Fts3SegReader **apSegment, /* Array of SegReader objects */
120731 int nReader /* Size of array apSegment */
120732 ){
120733 int rc; /* Return Code */
@@ -120746,23 +121703,28 @@
120746 }
120747 if( rc!=SQLITE_OK ){
120748 return rc;
120749 }
120750
 
120751 if( iLevel==FTS3_SEGCURSOR_ALL ){
120752 fts3SqlExec(&rc, p, SQL_DELETE_ALL_SEGDIR, 0);
120753 }else if( iLevel==FTS3_SEGCURSOR_PENDING ){
120754 sqlite3Fts3PendingTermsClear(p);
 
 
120755 }else{
120756 assert( iLevel>=0 );
120757 rc = fts3SqlStmt(p, SQL_DELETE_SEGDIR_BY_LEVEL, &pDelete, 0);
120758 if( rc==SQLITE_OK ){
120759 sqlite3_bind_int(pDelete, 1, iLevel);
120760 sqlite3_step(pDelete);
120761 rc = sqlite3_reset(pDelete);
120762 }
120763 }
 
 
 
 
 
120764
120765 return rc;
120766 }
120767
120768 /*
@@ -120805,14 +121767,124 @@
120805 }
120806
120807 *ppList = pList;
120808 *pnList = nList;
120809 }
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
120810
120811 SQLITE_PRIVATE int sqlite3Fts3SegReaderStart(
120812 Fts3Table *p, /* Virtual table handle */
120813 Fts3SegReaderCursor *pCsr, /* Cursor object */
120814 Fts3SegFilter *pFilter /* Restrictions on range of iteration */
120815 ){
120816 int i;
120817
120818 /* Initialize the cursor object */
@@ -120827,11 +121899,11 @@
120827 for(i=0; i<pCsr->nSegment; i++){
120828 int nTerm = pFilter->nTerm;
120829 const char *zTerm = pFilter->zTerm;
120830 Fts3SegReader *pSeg = pCsr->apSegment[i];
120831 do {
120832 int rc = fts3SegReaderNext(p, pSeg);
120833 if( rc!=SQLITE_OK ) return rc;
120834 }while( zTerm && fts3SegReaderTermCmp(pSeg, zTerm, nTerm)<0 );
120835 }
120836 fts3SegReaderSort(
120837 pCsr->apSegment, pCsr->nSegment, pCsr->nSegment, fts3SegReaderCmp);
@@ -120839,11 +121911,11 @@
120839 return SQLITE_OK;
120840 }
120841
120842 SQLITE_PRIVATE int sqlite3Fts3SegReaderStep(
120843 Fts3Table *p, /* Virtual table handle */
120844 Fts3SegReaderCursor *pCsr /* Cursor object */
120845 ){
120846 int rc = SQLITE_OK;
120847
120848 int isIgnoreEmpty = (pCsr->pFilter->flags & FTS3_SEGMENT_IGNORE_EMPTY);
120849 int isRequirePos = (pCsr->pFilter->flags & FTS3_SEGMENT_REQUIRE_POS);
@@ -120852,10 +121924,13 @@
120852 int isScan = (pCsr->pFilter->flags & FTS3_SEGMENT_SCAN);
120853
120854 Fts3SegReader **apSegment = pCsr->apSegment;
120855 int nSegment = pCsr->nSegment;
120856 Fts3SegFilter *pFilter = pCsr->pFilter;
 
 
 
120857
120858 if( pCsr->nSegment==0 ) return SQLITE_OK;
120859
120860 do {
120861 int nMerge;
@@ -120863,11 +121938,11 @@
120863
120864 /* Advance the first pCsr->nAdvance entries in the apSegment[] array
120865 ** forward. Then sort the list in order of current term again.
120866 */
120867 for(i=0; i<pCsr->nAdvance; i++){
120868 rc = fts3SegReaderNext(p, apSegment[i]);
120869 if( rc!=SQLITE_OK ) return rc;
120870 }
120871 fts3SegReaderSort(apSegment, nSegment, pCsr->nAdvance, fts3SegReaderCmp);
120872 pCsr->nAdvance = 0;
120873
@@ -120902,11 +121977,14 @@
120902 ){
120903 nMerge++;
120904 }
120905
120906 assert( isIgnoreEmpty || (isRequirePos && !isColFilter) );
120907 if( nMerge==1 && !isIgnoreEmpty ){
 
 
 
120908 pCsr->aDoclist = apSegment[0]->aDoclist;
120909 pCsr->nDoclist = apSegment[0]->nDoclist;
120910 rc = SQLITE_ROW;
120911 }else{
120912 int nDoclist = 0; /* Size of doclist */
@@ -120915,56 +121993,66 @@
120915 /* The current term of the first nMerge entries in the array
120916 ** of Fts3SegReader objects is the same. The doclists must be merged
120917 ** and a single term returned with the merged doclist.
120918 */
120919 for(i=0; i<nMerge; i++){
120920 fts3SegReaderFirstDocid(apSegment[i]);
120921 }
120922 fts3SegReaderSort(apSegment, nMerge, nMerge, fts3SegReaderDoclistCmp);
120923 while( apSegment[0]->pOffsetList ){
120924 int j; /* Number of segments that share a docid */
120925 char *pList;
120926 int nList;
120927 int nByte;
120928 sqlite3_int64 iDocid = apSegment[0]->iDocid;
120929 fts3SegReaderNextDocid(apSegment[0], &pList, &nList);
120930 j = 1;
120931 while( j<nMerge
120932 && apSegment[j]->pOffsetList
120933 && apSegment[j]->iDocid==iDocid
120934 ){
120935 fts3SegReaderNextDocid(apSegment[j], 0, 0);
120936 j++;
120937 }
120938
120939 if( isColFilter ){
120940 fts3ColumnFilter(pFilter->iCol, &pList, &nList);
120941 }
120942
120943 if( !isIgnoreEmpty || nList>0 ){
120944 nByte = sqlite3Fts3VarintLen(iDocid-iPrev) + (isRequirePos?nList+1:0);
 
 
 
 
 
 
 
 
 
 
 
 
120945 if( nDoclist+nByte>pCsr->nBuffer ){
120946 char *aNew;
120947 pCsr->nBuffer = (nDoclist+nByte)*2;
120948 aNew = sqlite3_realloc(pCsr->aBuffer, pCsr->nBuffer);
120949 if( !aNew ){
120950 return SQLITE_NOMEM;
120951 }
120952 pCsr->aBuffer = aNew;
120953 }
120954 nDoclist += sqlite3Fts3PutVarint(
120955 &pCsr->aBuffer[nDoclist], iDocid-iPrev
120956 );
120957 iPrev = iDocid;
120958 if( isRequirePos ){
120959 memcpy(&pCsr->aBuffer[nDoclist], pList, nList);
120960 nDoclist += nList;
120961 pCsr->aBuffer[nDoclist++] = '\0';
120962 }
120963 }
120964
120965 fts3SegReaderSort(apSegment, nMerge, j, fts3SegReaderDoclistCmp);
120966 }
120967 if( nDoclist>0 ){
120968 pCsr->aDoclist = pCsr->aBuffer;
120969 pCsr->nDoclist = nDoclist;
120970 rc = SQLITE_ROW;
@@ -120973,13 +122061,14 @@
120973 pCsr->nAdvance = nMerge;
120974 }while( rc==SQLITE_OK );
120975
120976 return rc;
120977 }
 
120978
120979 SQLITE_PRIVATE void sqlite3Fts3SegReaderFinish(
120980 Fts3SegReaderCursor *pCsr /* Cursor object */
120981 ){
120982 if( pCsr ){
120983 int i;
120984 for(i=0; i<pCsr->nSegment; i++){
120985 sqlite3Fts3SegReaderFree(pCsr->apSegment[i]);
@@ -121002,47 +122091,60 @@
121002 ** If this function is called with iLevel<0, but there is only one
121003 ** segment in the database, SQLITE_DONE is returned immediately.
121004 ** Otherwise, if successful, SQLITE_OK is returned. If an error occurs,
121005 ** an SQLite error code is returned.
121006 */
121007 static int fts3SegmentMerge(Fts3Table *p, int iLevel){
121008 int rc; /* Return code */
121009 int iIdx = 0; /* Index of new segment */
121010 int iNewLevel = 0; /* Level to create new segment at */
121011 SegmentWriter *pWriter = 0; /* Used to write the new, merged, segment */
121012 Fts3SegFilter filter; /* Segment term filter condition */
121013 Fts3SegReaderCursor csr; /* Cursor to iterate through level(s) */
 
121014
121015 rc = sqlite3Fts3SegReaderCursor(p, iLevel, 0, 0, 1, 0, &csr);
 
 
 
 
 
 
 
121016 if( rc!=SQLITE_OK || csr.nSegment==0 ) goto finished;
121017
121018 if( iLevel==FTS3_SEGCURSOR_ALL ){
121019 /* This call is to merge all segments in the database to a single
121020 ** segment. The level of the new segment is equal to the the numerically
121021 ** greatest segment level currently present in the database. The index
121022 ** of the new segment is always 0. */
121023 int nDummy; /* TODO: Remove this */
121024 if( csr.nSegment==1 ){
121025 rc = SQLITE_DONE;
121026 goto finished;
121027 }
121028 rc = fts3SegmentCountMax(p, &nDummy, &iNewLevel);
 
 
 
 
 
121029 }else{
121030 /* This call is to merge all segments at level iLevel. Find the next
121031 ** available segment index at level iLevel+1. The call to
121032 ** fts3AllocateSegdirIdx() will merge the segments at level iLevel+1 to
121033 ** a single iLevel+2 segment if necessary. */
121034 iNewLevel = iLevel+1;
121035 rc = fts3AllocateSegdirIdx(p, iNewLevel, &iIdx);
121036 }
121037 if( rc!=SQLITE_OK ) goto finished;
121038 assert( csr.nSegment>0 );
121039 assert( iNewLevel>=0 );
 
121040
121041 memset(&filter, 0, sizeof(Fts3SegFilter));
121042 filter.flags = FTS3_SEGMENT_REQUIRE_POS;
121043 filter.flags |= (iLevel==FTS3_SEGCURSOR_ALL ? FTS3_SEGMENT_IGNORE_EMPTY : 0);
121044
121045 rc = sqlite3Fts3SegReaderStart(p, &csr, &filter);
121046 while( SQLITE_OK==rc ){
121047 rc = sqlite3Fts3SegReaderStep(p, &csr);
121048 if( rc!=SQLITE_ROW ) break;
@@ -121050,12 +122152,14 @@
121050 csr.zTerm, csr.nTerm, csr.aDoclist, csr.nDoclist);
121051 }
121052 if( rc!=SQLITE_OK ) goto finished;
121053 assert( pWriter );
121054
121055 rc = fts3DeleteSegdir(p, iLevel, csr.apSegment, csr.nSegment);
121056 if( rc!=SQLITE_OK ) goto finished;
 
 
121057 rc = fts3SegWriterFlush(p, pWriter, iNewLevel, iIdx);
121058
121059 finished:
121060 fts3SegWriterFree(pWriter);
121061 sqlite3Fts3SegReaderFinish(&csr);
@@ -121062,14 +122166,21 @@
121062 return rc;
121063 }
121064
121065
121066 /*
121067 ** Flush the contents of pendingTerms to a level 0 segment.
121068 */
121069 SQLITE_PRIVATE int sqlite3Fts3PendingTermsFlush(Fts3Table *p){
121070 return fts3SegmentMerge(p, FTS3_SEGCURSOR_PENDING);
 
 
 
 
 
 
 
121071 }
121072
121073 /*
121074 ** Encode N integers as varints into a blob.
121075 */
@@ -121215,10 +122326,27 @@
121215 sqlite3_bind_blob(pStmt, 1, pBlob, nBlob, SQLITE_STATIC);
121216 sqlite3_step(pStmt);
121217 *pRC = sqlite3_reset(pStmt);
121218 sqlite3_free(a);
121219 }
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
121220
121221 /*
121222 ** Handle a 'special' INSERT of the form:
121223 **
121224 ** "INSERT INTO tbl(tbl) VALUES(<expr>)"
@@ -121232,16 +122360,11 @@
121232 int nVal = sqlite3_value_bytes(pVal);
121233
121234 if( !zVal ){
121235 return SQLITE_NOMEM;
121236 }else if( nVal==8 && 0==sqlite3_strnicmp(zVal, "optimize", 8) ){
121237 rc = fts3SegmentMerge(p, FTS3_SEGCURSOR_ALL);
121238 if( rc==SQLITE_DONE ){
121239 rc = SQLITE_OK;
121240 }else{
121241 sqlite3Fts3PendingTermsClear(p);
121242 }
121243 #ifdef SQLITE_TEST
121244 }else if( nVal>9 && 0==sqlite3_strnicmp(zVal, "nodesize=", 9) ){
121245 p->nNodeSize = atoi(&zVal[9]);
121246 rc = SQLITE_OK;
121247 }else if( nVal>11 && 0==sqlite3_strnicmp(zVal, "maxpending=", 9) ){
@@ -121250,61 +122373,23 @@
121250 #endif
121251 }else{
121252 rc = SQLITE_ERROR;
121253 }
121254
121255 sqlite3Fts3SegmentsClose(p);
121256 return rc;
121257 }
121258
121259 /*
121260 ** Return the deferred doclist associated with deferred token pDeferred.
121261 ** This function assumes that sqlite3Fts3CacheDeferredDoclists() has already
121262 ** been called to allocate and populate the doclist.
121263 */
121264 SQLITE_PRIVATE char *sqlite3Fts3DeferredDoclist(Fts3DeferredToken *pDeferred, int *pnByte){
121265 if( pDeferred->pList ){
121266 *pnByte = pDeferred->pList->nData;
121267 return pDeferred->pList->aData;
121268 }
121269 *pnByte = 0;
121270 return 0;
121271 }
121272
121273 /*
121274 ** Helper fucntion for FreeDeferredDoclists(). This function removes all
121275 ** references to deferred doclists from within the tree of Fts3Expr
121276 ** structures headed by
121277 */
121278 static void fts3DeferredDoclistClear(Fts3Expr *pExpr){
121279 if( pExpr ){
121280 fts3DeferredDoclistClear(pExpr->pLeft);
121281 fts3DeferredDoclistClear(pExpr->pRight);
121282 if( pExpr->isLoaded ){
121283 sqlite3_free(pExpr->aDoclist);
121284 pExpr->isLoaded = 0;
121285 pExpr->aDoclist = 0;
121286 pExpr->nDoclist = 0;
121287 pExpr->pCurrent = 0;
121288 pExpr->iCurrent = 0;
121289 }
121290 }
121291 }
121292
121293 /*
121294 ** Delete all cached deferred doclists. Deferred doclists are cached
121295 ** (allocated) by the sqlite3Fts3CacheDeferredDoclists() function.
121296 */
121297 SQLITE_PRIVATE void sqlite3Fts3FreeDeferredDoclists(Fts3Cursor *pCsr){
121298 Fts3DeferredToken *pDef;
121299 for(pDef=pCsr->pDeferred; pDef; pDef=pDef->pNext){
121300 sqlite3_free(pDef->pList);
121301 pDef->pList = 0;
121302 }
121303 if( pCsr->pDeferred ){
121304 fts3DeferredDoclistClear(pCsr->pExpr);
121305 }
121306 }
121307
121308 /*
121309 ** Free all entries in the pCsr->pDeffered list. Entries are added to
121310 ** this list using sqlite3Fts3DeferToken().
@@ -121312,11 +122397,11 @@
121312 SQLITE_PRIVATE void sqlite3Fts3FreeDeferredTokens(Fts3Cursor *pCsr){
121313 Fts3DeferredToken *pDef;
121314 Fts3DeferredToken *pNext;
121315 for(pDef=pCsr->pDeferred; pDef; pDef=pNext){
121316 pNext = pDef->pNext;
121317 sqlite3_free(pDef->pList);
121318 sqlite3_free(pDef);
121319 }
121320 pCsr->pDeferred = 0;
121321 }
121322
@@ -121376,10 +122461,37 @@
121376 }
121377 }
121378
121379 return rc;
121380 }
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
121381
121382 /*
121383 ** Add an entry for token pToken to the pCsr->pDeferred list.
121384 */
121385 SQLITE_PRIVATE int sqlite3Fts3DeferToken(
@@ -121451,11 +122563,11 @@
121451 ){
121452 Fts3Table *p = (Fts3Table *)pVtab;
121453 int rc = SQLITE_OK; /* Return Code */
121454 int isRemove = 0; /* True for an UPDATE or DELETE */
121455 sqlite3_int64 iRemove = 0; /* Rowid removed by UPDATE or DELETE */
121456 u32 *aSzIns; /* Sizes of inserted documents */
121457 u32 *aSzDel; /* Sizes of deleted documents */
121458 int nChng = 0; /* Net change in number of documents */
121459 int bInsertDone = 0;
121460
121461 assert( p->pSegments==0 );
@@ -121466,16 +122578,20 @@
121466 */
121467 if( nArg>1
121468 && sqlite3_value_type(apVal[0])==SQLITE_NULL
121469 && sqlite3_value_type(apVal[p->nColumn+2])!=SQLITE_NULL
121470 ){
121471 return fts3SpecialInsert(p, apVal[p->nColumn+2]);
 
121472 }
121473
121474 /* Allocate space to hold the change in document sizes */
121475 aSzIns = sqlite3_malloc( sizeof(aSzIns[0])*(p->nColumn+1)*2 );
121476 if( aSzIns==0 ) return SQLITE_NOMEM;
 
 
 
121477 aSzDel = &aSzIns[p->nColumn+1];
121478 memset(aSzIns, 0, sizeof(aSzIns[0])*(p->nColumn+1)*2);
121479
121480 /* If this is an INSERT operation, or an UPDATE that modifies the rowid
121481 ** value, then this operation requires constraint handling.
@@ -121521,12 +122637,11 @@
121521 bInsertDone = 1;
121522 }
121523 }
121524 }
121525 if( rc!=SQLITE_OK ){
121526 sqlite3_free(aSzIns);
121527 return rc;
121528 }
121529
121530 /* If this is a DELETE or UPDATE operation, remove the old record. */
121531 if( sqlite3_value_type(apVal[0])!=SQLITE_NULL ){
121532 assert( sqlite3_value_type(apVal[0])==SQLITE_INTEGER );
@@ -121555,10 +122670,11 @@
121555
121556 if( p->bHasStat ){
121557 fts3UpdateDocTotals(&rc, p, aSzIns, aSzDel, nChng);
121558 }
121559
 
121560 sqlite3_free(aSzIns);
121561 sqlite3Fts3SegmentsClose(p);
121562 return rc;
121563 }
121564
@@ -121569,16 +122685,14 @@
121569 */
121570 SQLITE_PRIVATE int sqlite3Fts3Optimize(Fts3Table *p){
121571 int rc;
121572 rc = sqlite3_exec(p->db, "SAVEPOINT fts3", 0, 0, 0);
121573 if( rc==SQLITE_OK ){
121574 rc = fts3SegmentMerge(p, FTS3_SEGCURSOR_ALL);
121575 if( rc==SQLITE_OK ){
121576 rc = sqlite3_exec(p->db, "RELEASE fts3", 0, 0, 0);
121577 if( rc==SQLITE_OK ){
121578 sqlite3Fts3PendingTermsClear(p);
121579 }
121580 }else{
121581 sqlite3_exec(p->db, "ROLLBACK TO fts3", 0, 0, 0);
121582 sqlite3_exec(p->db, "RELEASE fts3", 0, 0, 0);
121583 }
121584 }
@@ -121763,76 +122877,24 @@
121763 ){
121764 int iPhrase = 0; /* Variable used as the phrase counter */
121765 return fts3ExprIterate2(pExpr, &iPhrase, x, pCtx);
121766 }
121767
121768 /*
121769 ** The argument to this function is always a phrase node. Its doclist
121770 ** (Fts3Expr.aDoclist[]) and the doclists associated with all phrase nodes
121771 ** to the left of this one in the query tree have already been loaded.
121772 **
121773 ** If this phrase node is part of a series of phrase nodes joined by
121774 ** NEAR operators (and is not the left-most of said series), then elements are
121775 ** removed from the phrases doclist consistent with the NEAR restriction. If
121776 ** required, elements may be removed from the doclists of phrases to the
121777 ** left of this one that are part of the same series of NEAR operator
121778 ** connected phrases.
121779 **
121780 ** If an OOM error occurs, SQLITE_NOMEM is returned. Otherwise, SQLITE_OK.
121781 */
121782 static int fts3ExprNearTrim(Fts3Expr *pExpr){
121783 int rc = SQLITE_OK;
121784 Fts3Expr *pParent = pExpr->pParent;
121785
121786 assert( pExpr->eType==FTSQUERY_PHRASE );
121787 while( rc==SQLITE_OK
121788 && pParent
121789 && pParent->eType==FTSQUERY_NEAR
121790 && pParent->pRight==pExpr
121791 ){
121792 /* This expression (pExpr) is the right-hand-side of a NEAR operator.
121793 ** Find the expression to the left of the same operator.
121794 */
121795 int nNear = pParent->nNear;
121796 Fts3Expr *pLeft = pParent->pLeft;
121797
121798 if( pLeft->eType!=FTSQUERY_PHRASE ){
121799 assert( pLeft->eType==FTSQUERY_NEAR );
121800 assert( pLeft->pRight->eType==FTSQUERY_PHRASE );
121801 pLeft = pLeft->pRight;
121802 }
121803
121804 rc = sqlite3Fts3ExprNearTrim(pLeft, pExpr, nNear);
121805
121806 pExpr = pLeft;
121807 pParent = pExpr->pParent;
121808 }
121809
121810 return rc;
121811 }
121812
121813 /*
121814 ** This is an fts3ExprIterate() callback used while loading the doclists
121815 ** for each phrase into Fts3Expr.aDoclist[]/nDoclist. See also
121816 ** fts3ExprLoadDoclists().
121817 */
121818 static int fts3ExprLoadDoclistsCb(Fts3Expr *pExpr, int iPhrase, void *ctx){
121819 int rc = SQLITE_OK;
 
121820 LoadDoclistCtx *p = (LoadDoclistCtx *)ctx;
121821
121822 UNUSED_PARAMETER(iPhrase);
121823
121824 p->nPhrase++;
121825 p->nToken += pExpr->pPhrase->nToken;
121826
121827 if( pExpr->isLoaded==0 ){
121828 rc = sqlite3Fts3ExprLoadDoclist(p->pCsr, pExpr);
121829 pExpr->isLoaded = 1;
121830 if( rc==SQLITE_OK ){
121831 rc = fts3ExprNearTrim(pExpr);
121832 }
121833 }
121834
121835 return rc;
121836 }
121837
121838 /*
@@ -122002,11 +123064,11 @@
122002 SnippetPhrase *pPhrase = &p->aPhrase[iPhrase];
122003 char *pCsr;
122004
122005 pPhrase->nToken = pExpr->pPhrase->nToken;
122006
122007 pCsr = sqlite3Fts3FindPositions(p->pCsr, pExpr, p->pCsr->iPrevId, p->iCol);
122008 if( pCsr ){
122009 int iFirst = 0;
122010 pPhrase->pList = pCsr;
122011 fts3GetDeltaPosition(&pCsr, &iFirst);
122012 pPhrase->pHead = pCsr;
@@ -122359,30 +123421,10 @@
122359
122360 *ppCollist = pEnd;
122361 return nEntry;
122362 }
122363
122364 static void fts3LoadColumnlistCounts(char **pp, u32 *aOut, int isGlobal){
122365 char *pCsr = *pp;
122366 while( *pCsr ){
122367 int nHit;
122368 sqlite3_int64 iCol = 0;
122369 if( *pCsr==0x01 ){
122370 pCsr++;
122371 pCsr += sqlite3Fts3GetVarint(pCsr, &iCol);
122372 }
122373 nHit = fts3ColumnlistCount(&pCsr);
122374 assert( nHit>0 );
122375 if( isGlobal ){
122376 aOut[iCol*3+1]++;
122377 }
122378 aOut[iCol*3] += nHit;
122379 }
122380 pCsr++;
122381 *pp = pCsr;
122382 }
122383
122384 /*
122385 ** fts3ExprIterate() callback used to collect the "global" matchinfo stats
122386 ** for a single query.
122387 **
122388 ** fts3ExprIterate() callback to load the 'global' elements of a
@@ -122412,52 +123454,13 @@
122412 Fts3Expr *pExpr, /* Phrase expression node */
122413 int iPhrase, /* Phrase number (numbered from zero) */
122414 void *pCtx /* Pointer to MatchInfo structure */
122415 ){
122416 MatchInfo *p = (MatchInfo *)pCtx;
122417 Fts3Cursor *pCsr = p->pCursor;
122418 char *pIter;
122419 char *pEnd;
122420 char *pFree = 0;
122421 u32 *aOut = &p->aMatchinfo[3*iPhrase*p->nCol];
122422
122423 assert( pExpr->isLoaded );
122424 assert( pExpr->eType==FTSQUERY_PHRASE );
122425
122426 if( pCsr->pDeferred ){
122427 Fts3Phrase *pPhrase = pExpr->pPhrase;
122428 int ii;
122429 for(ii=0; ii<pPhrase->nToken; ii++){
122430 if( pPhrase->aToken[ii].bFulltext ) break;
122431 }
122432 if( ii<pPhrase->nToken ){
122433 int nFree = 0;
122434 int rc = sqlite3Fts3ExprLoadFtDoclist(pCsr, pExpr, &pFree, &nFree);
122435 if( rc!=SQLITE_OK ) return rc;
122436 pIter = pFree;
122437 pEnd = &pFree[nFree];
122438 }else{
122439 int iCol; /* Column index */
122440 for(iCol=0; iCol<p->nCol; iCol++){
122441 aOut[iCol*3 + 1] = (u32)p->nDoc;
122442 aOut[iCol*3 + 2] = (u32)p->nDoc;
122443 }
122444 return SQLITE_OK;
122445 }
122446 }else{
122447 pIter = pExpr->aDoclist;
122448 pEnd = &pExpr->aDoclist[pExpr->nDoclist];
122449 }
122450
122451 /* Fill in the global hit count matrix row for this phrase. */
122452 while( pIter<pEnd ){
122453 while( *pIter++ & 0x80 ); /* Skip past docid. */
122454 fts3LoadColumnlistCounts(&pIter, &aOut[1], 1);
122455 }
122456
122457 sqlite3_free(pFree);
122458 return SQLITE_OK;
122459 }
122460
122461 /*
122462 ** fts3ExprIterate() callback used to collect the "local" part of the
122463 ** FTS3_MATCHINFO_HITS array. The local stats are those elements of the
@@ -122470,18 +123473,17 @@
122470 ){
122471 MatchInfo *p = (MatchInfo *)pCtx;
122472 int iStart = iPhrase * p->nCol * 3;
122473 int i;
122474
122475 for(i=0; i<p->nCol; i++) p->aMatchinfo[iStart+i*3] = 0;
122476
122477 if( pExpr->aDoclist ){
122478 char *pCsr;
122479
122480 pCsr = sqlite3Fts3FindPositions(p->pCursor, pExpr, p->pCursor->iPrevId, -1);
122481 if( pCsr ){
122482 fts3LoadColumnlistCounts(&pCsr, &p->aMatchinfo[iStart], 0);
 
 
122483 }
122484 }
122485
122486 return SQLITE_OK;
122487 }
@@ -122563,13 +123565,12 @@
122563 ** values for a matchinfo() FTS3_MATCHINFO_LCS request.
122564 */
122565 typedef struct LcsIterator LcsIterator;
122566 struct LcsIterator {
122567 Fts3Expr *pExpr; /* Pointer to phrase expression */
122568 char *pRead; /* Cursor used to iterate through aDoclist */
122569 int iPosOffset; /* Tokens count up to end of this phrase */
122570 int iCol; /* Current column number */
122571 int iPos; /* Current position */
122572 };
122573
122574 /*
122575 ** If LcsIterator.iCol is set to the following value, the iterator has
@@ -122596,21 +123597,14 @@
122596 char *pRead = pIter->pRead;
122597 sqlite3_int64 iRead;
122598 int rc = 0;
122599
122600 pRead += sqlite3Fts3GetVarint(pRead, &iRead);
122601 if( iRead==0 ){
122602 pIter->iCol = LCS_ITERATOR_FINISHED;
122603 rc = 1;
122604 }else{
122605 if( iRead==1 ){
122606 pRead += sqlite3Fts3GetVarint(pRead, &iRead);
122607 pIter->iCol = (int)iRead;
122608 pIter->iPos = pIter->iPosOffset;
122609 pRead += sqlite3Fts3GetVarint(pRead, &iRead);
122610 rc = 1;
122611 }
122612 pIter->iPos += (int)(iRead-2);
122613 }
122614
122615 pIter->pRead = pRead;
122616 return rc;
@@ -122638,46 +123632,38 @@
122638 **/
122639 aIter = sqlite3_malloc(sizeof(LcsIterator) * pCsr->nPhrase);
122640 if( !aIter ) return SQLITE_NOMEM;
122641 memset(aIter, 0, sizeof(LcsIterator) * pCsr->nPhrase);
122642 (void)fts3ExprIterate(pCsr->pExpr, fts3MatchinfoLcsCb, (void*)aIter);
 
122643 for(i=0; i<pInfo->nPhrase; i++){
122644 LcsIterator *pIter = &aIter[i];
122645 nToken -= pIter->pExpr->pPhrase->nToken;
122646 pIter->iPosOffset = nToken;
122647 pIter->pRead = sqlite3Fts3FindPositions(pCsr,pIter->pExpr,pCsr->iPrevId,-1);
122648 if( pIter->pRead ){
122649 pIter->iPos = pIter->iPosOffset;
122650 fts3LcsIteratorAdvance(&aIter[i]);
122651 }else{
122652 pIter->iCol = LCS_ITERATOR_FINISHED;
122653 }
122654 }
122655
122656 for(iCol=0; iCol<pInfo->nCol; iCol++){
122657 int nLcs = 0; /* LCS value for this column */
122658 int nLive = 0; /* Number of iterators in aIter not at EOF */
122659
122660 /* Loop through the iterators in aIter[]. Set nLive to the number of
122661 ** iterators that point to a position-list corresponding to column iCol.
122662 */
122663 for(i=0; i<pInfo->nPhrase; i++){
122664 assert( aIter[i].iCol>=iCol );
122665 if( aIter[i].iCol==iCol ) nLive++;
 
 
 
 
 
122666 }
122667
122668 /* The following loop runs until all iterators in aIter[] have finished
122669 ** iterating through positions in column iCol. Exactly one of the
122670 ** iterators is advanced each time the body of the loop is run.
122671 */
122672 while( nLive>0 ){
122673 LcsIterator *pAdv = 0; /* The iterator to advance by one position */
122674 int nThisLcs = 0; /* LCS for the current iterator positions */
122675
122676 for(i=0; i<pInfo->nPhrase; i++){
122677 LcsIterator *pIter = &aIter[i];
122678 if( iCol!=pIter->iCol ){
122679 /* This iterator is already at EOF for this column. */
122680 nThisLcs = 0;
122681 }else{
122682 if( pAdv==0 || pIter->iPos<pAdv->iPos ){
122683 pAdv = pIter;
@@ -123013,11 +123999,11 @@
123013 int iTerm; /* For looping through nTerm phrase terms */
123014 char *pList; /* Pointer to position list for phrase */
123015 int iPos = 0; /* First position in position-list */
123016
123017 UNUSED_PARAMETER(iPhrase);
123018 pList = sqlite3Fts3FindPositions(p->pCsr, pExpr, p->iDocid, p->iCol);
123019 nTerm = pExpr->pPhrase->nToken;
123020 if( pList ){
123021 fts3GetDeltaPosition(&pList, &iPos);
123022 assert( iPos>=0 );
123023 }
123024
--- src/sqlite3.c
+++ src/sqlite3.c
@@ -650,11 +650,11 @@
650 ** [sqlite3_libversion_number()], [sqlite3_sourceid()],
651 ** [sqlite_version()] and [sqlite_source_id()].
652 */
653 #define SQLITE_VERSION "3.7.7"
654 #define SQLITE_VERSION_NUMBER 3007007
655 #define SQLITE_SOURCE_ID "2011-06-15 13:11:06 f9750870ee04935f338e4d808900fee5a8b2b389"
656
657 /*
658 ** CAPI3REF: Run-Time Library Version Numbers
659 ** KEYWORDS: sqlite3_version, sqlite3_sourceid
660 **
@@ -8483,10 +8483,11 @@
8483 SQLITE_PRIVATE int sqlite3VdbeAddOp2(Vdbe*,int,int,int);
8484 SQLITE_PRIVATE int sqlite3VdbeAddOp3(Vdbe*,int,int,int,int);
8485 SQLITE_PRIVATE int sqlite3VdbeAddOp4(Vdbe*,int,int,int,int,const char *zP4,int);
8486 SQLITE_PRIVATE int sqlite3VdbeAddOp4Int(Vdbe*,int,int,int,int,int);
8487 SQLITE_PRIVATE int sqlite3VdbeAddOpList(Vdbe*, int nOp, VdbeOpList const *aOp);
8488 SQLITE_PRIVATE void sqlite3VdbeAddParseSchemaOp(Vdbe*,int,char*);
8489 SQLITE_PRIVATE void sqlite3VdbeChangeP1(Vdbe*, int addr, int P1);
8490 SQLITE_PRIVATE void sqlite3VdbeChangeP2(Vdbe*, int addr, int P2);
8491 SQLITE_PRIVATE void sqlite3VdbeChangeP3(Vdbe*, int addr, int P3);
8492 SQLITE_PRIVATE void sqlite3VdbeChangeP5(Vdbe*, u8 P5);
8493 SQLITE_PRIVATE void sqlite3VdbeJumpHere(Vdbe*, int addr);
@@ -9282,11 +9283,11 @@
9283 ** sqlite3_close().
9284 *
9285 ** A thread must be holding a mutex on the corresponding Btree in order
9286 ** to access Schema content. This implies that the thread must also be
9287 ** holding a mutex on the sqlite3 connection pointer that owns the Btree.
9288 ** For a TEMP Schema, only the connection mutex is required.
9289 */
9290 struct Schema {
9291 int schema_cookie; /* Database schema version number for this file */
9292 int iGeneration; /* Generation counter. Incremented with each change */
9293 Hash tblHash; /* All tables indexed by name */
@@ -11479,11 +11480,11 @@
11480 SQLITE_PRIVATE int sqlite3AtoF(const char *z, double*, int, u8);
11481 SQLITE_PRIVATE int sqlite3GetInt32(const char *, int*);
11482 SQLITE_PRIVATE int sqlite3Atoi(const char*);
11483 SQLITE_PRIVATE int sqlite3Utf16ByteLen(const void *pData, int nChar);
11484 SQLITE_PRIVATE int sqlite3Utf8CharLen(const char *pData, int nByte);
11485 SQLITE_PRIVATE u32 sqlite3Utf8Read(const u8*, const u8**);
11486
11487 /*
11488 ** Routines to read and write variable-length integers. These used to
11489 ** be defined locally, but now we use the varint routines in the util.c
11490 ** file. Code should use the MACRO forms below, as the Varint32 versions
@@ -20086,11 +20087,11 @@
20087 } \
20088 if( c<0x80 \
20089 || (c&0xFFFFF800)==0xD800 \
20090 || (c&0xFFFFFFFE)==0xFFFE ){ c = 0xFFFD; } \
20091 }
20092 SQLITE_PRIVATE u32 sqlite3Utf8Read(
20093 const unsigned char *zIn, /* First byte of UTF-8 character */
20094 const unsigned char **pzNext /* Write first byte past UTF-8 char here */
20095 ){
20096 unsigned int c;
20097
@@ -35805,11 +35806,11 @@
35806 memset(pCache, 0, sz);
35807 if( separateCache ){
35808 pGroup = (PGroup*)&pCache[1];
35809 pGroup->mxPinned = 10;
35810 }else{
35811 pGroup = &pcache1.grp;
35812 }
35813 pCache->pGroup = pGroup;
35814 pCache->szPage = szPage;
35815 pCache->bPurgeable = (bPurgeable ? 1 : 0);
35816 if( bPurgeable ){
@@ -48324,10 +48325,12 @@
48325 **
48326 ** This routine works only for pages that do not contain overflow cells.
48327 */
48328 #define findCell(P,I) \
48329 ((P)->aData + ((P)->maskPage & get2byte(&(P)->aData[(P)->cellOffset+2*(I)])))
48330 #define findCellv2(D,M,O,I) (D+(M&get2byte(D+(O+2*(I)))))
48331
48332
48333 /*
48334 ** This a more complex version of findCell() that works for
48335 ** pages that do contain overflow cells.
48336 */
@@ -51918,11 +51921,11 @@
51921 assert( pCur->apPage[pCur->iPage]->nCell==0 );
51922 return SQLITE_OK;
51923 }
51924 assert( pCur->apPage[0]->intKey || pIdxKey );
51925 for(;;){
51926 int lwr, upr, idx;
51927 Pgno chldPg;
51928 MemPage *pPage = pCur->apPage[pCur->iPage];
51929 int c;
51930
51931 /* pPage->nCell must be greater than zero. If this is the root-page
@@ -51934,18 +51937,18 @@
51937 assert( pPage->nCell>0 );
51938 assert( pPage->intKey==(pIdxKey==0) );
51939 lwr = 0;
51940 upr = pPage->nCell-1;
51941 if( biasRight ){
51942 pCur->aiIdx[pCur->iPage] = (u16)(idx = upr);
51943 }else{
51944 pCur->aiIdx[pCur->iPage] = (u16)(idx = (upr+lwr)/2);
51945 }
51946 for(;;){
 
51947 u8 *pCell; /* Pointer to current cell in pPage */
51948
51949 assert( idx==pCur->aiIdx[pCur->iPage] );
51950 pCur->info.nSize = 0;
51951 pCell = findCell(pPage, idx) + pPage->childPtrSize;
51952 if( pPage->intKey ){
51953 i64 nCellKey;
51954 if( pPage->hasData ){
@@ -52024,11 +52027,11 @@
52027 upr = idx-1;
52028 }
52029 if( lwr>upr ){
52030 break;
52031 }
52032 pCur->aiIdx[pCur->iPage] = (u16)(idx = (lwr+upr)/2);
52033 }
52034 assert( lwr==upr+1 );
52035 assert( pPage->isInit );
52036 if( pPage->leaf ){
52037 chldPg = 0;
@@ -52886,13 +52889,13 @@
52889 if( rc ){
52890 *pRC = rc;
52891 return;
52892 }
52893 endPtr = &data[pPage->cellOffset + 2*pPage->nCell - 2];
52894 assert( (SQLITE_PTR_TO_INT(ptr)&1)==0 ); /* ptr is always 2-byte aligned */
52895 while( ptr<endPtr ){
52896 *(u16*)ptr = *(u16*)&ptr[2];
 
52897 ptr += 2;
52898 }
52899 pPage->nCell--;
52900 put2byte(&data[hdr+3], pPage->nCell);
52901 pPage->nFree += 2;
@@ -52929,10 +52932,11 @@
52932 int end; /* First byte past the last cell pointer in data[] */
52933 int ins; /* Index in data[] where new cell pointer is inserted */
52934 int cellOffset; /* Address of first cell pointer in data[] */
52935 u8 *data; /* The content of the whole page */
52936 u8 *ptr; /* Used for moving information around in data[] */
52937 u8 *endPtr; /* End of the loop */
52938
52939 int nSkip = (iChild ? 4 : 0);
52940
52941 if( *pRC ) return;
52942
@@ -52979,13 +52983,16 @@
52983 pPage->nFree -= (u16)(2 + sz);
52984 memcpy(&data[idx+nSkip], pCell+nSkip, sz-nSkip);
52985 if( iChild ){
52986 put4byte(&data[idx], iChild);
52987 }
52988 ptr = &data[end];
52989 endPtr = &data[ins];
52990 assert( (SQLITE_PTR_TO_INT(ptr)&1)==0 ); /* ptr is always 2-byte aligned */
52991 while( ptr>endPtr ){
52992 *(u16*)ptr = *(u16*)&ptr[-2];
52993 ptr -= 2;
52994 }
52995 put2byte(&data[ins], idx);
52996 put2byte(&data[pPage->hdrOffset+3], pPage->nCell);
52997 #ifndef SQLITE_OMIT_AUTOVACUUM
52998 if( pPage->pBt->autoVacuum ){
@@ -53026,14 +53033,15 @@
53033 assert( get2byteNotZero(&data[hdr+5])==nUsable );
53034
53035 pCellptr = &data[pPage->cellOffset + nCell*2];
53036 cellbody = nUsable;
53037 for(i=nCell-1; i>=0; i--){
53038 u16 sz = aSize[i];
53039 pCellptr -= 2;
53040 cellbody -= sz;
53041 put2byte(pCellptr, cellbody);
53042 memcpy(&data[cellbody], apCell[i], sz);
53043 }
53044 put2byte(&data[hdr+3], nCell);
53045 put2byte(&data[hdr+5], cellbody);
53046 pPage->nFree -= (nCell*2 + nUsable - cellbody);
53047 pPage->nCell = (u16)nCell;
@@ -53483,16 +53491,28 @@
53491 memcpy(pOld, apOld[i], sizeof(MemPage));
53492 pOld->aData = (void*)&pOld[1];
53493 memcpy(pOld->aData, apOld[i]->aData, pBt->pageSize);
53494
53495 limit = pOld->nCell+pOld->nOverflow;
53496 if( pOld->nOverflow>0 ){
53497 for(j=0; j<limit; j++){
53498 assert( nCell<nMaxCells );
53499 apCell[nCell] = findOverflowCell(pOld, j);
53500 szCell[nCell] = cellSizePtr(pOld, apCell[nCell]);
53501 nCell++;
53502 }
53503 }else{
53504 u8 *aData = pOld->aData;
53505 u16 maskPage = pOld->maskPage;
53506 u16 cellOffset = pOld->cellOffset;
53507 for(j=0; j<limit; j++){
53508 assert( nCell<nMaxCells );
53509 apCell[nCell] = findCellv2(aData, maskPage, cellOffset, j);
53510 szCell[nCell] = cellSizePtr(pOld, apCell[nCell]);
53511 nCell++;
53512 }
53513 }
53514 if( i<nOld-1 && !leafData){
53515 u16 sz = (u16)szNew[i];
53516 u8 *pTemp;
53517 assert( nCell<nMaxCells );
53518 szCell[nCell] = sz;
@@ -57637,17 +57657,10 @@
57657 pOp->p1 = p1;
57658 pOp->p2 = p2;
57659 pOp->p3 = p3;
57660 pOp->p4.p = 0;
57661 pOp->p4type = P4_NOTUSED;
 
 
 
 
 
 
 
57662 #ifdef SQLITE_DEBUG
57663 pOp->zComment = 0;
57664 if( sqlite3VdbeAddopTrace ) sqlite3VdbePrintOp(0, i, &p->aOp[i]);
57665 #endif
57666 #ifdef VDBE_PROFILE
@@ -57681,10 +57694,24 @@
57694 ){
57695 int addr = sqlite3VdbeAddOp3(p, op, p1, p2, p3);
57696 sqlite3VdbeChangeP4(p, addr, zP4, p4type);
57697 return addr;
57698 }
57699
57700 /*
57701 ** Add an OP_ParseSchema opcode. This routine is broken out from
57702 ** sqlite3VdbeAddOp4() since it needs to also local all btrees.
57703 **
57704 ** The zWhere string must have been obtained from sqlite3_malloc().
57705 ** This routine will take ownership of the allocated memory.
57706 */
57707 SQLITE_PRIVATE void sqlite3VdbeAddParseSchemaOp(Vdbe *p, int iDb, char *zWhere){
57708 int j;
57709 int addr = sqlite3VdbeAddOp3(p, OP_ParseSchema, iDb, 0, 0);
57710 sqlite3VdbeChangeP4(p, addr, zWhere, P4_DYNAMIC);
57711 for(j=0; j<p->db->nDb; j++) sqlite3VdbeUsesBtree(p, j);
57712 }
57713
57714 /*
57715 ** Add an opcode that includes the p4 value as an integer.
57716 */
57717 SQLITE_PRIVATE int sqlite3VdbeAddOp4Int(
@@ -64083,10 +64110,20 @@
64110 u.ag.ctx.pColl = pOp[-1].p4.pColl;
64111 }
64112 db->lastRowid = lastRowid;
64113 (*u.ag.ctx.pFunc->xFunc)(&u.ag.ctx, u.ag.n, u.ag.apVal); /* IMP: R-24505-23230 */
64114 lastRowid = db->lastRowid;
64115
64116 /* If any auxiliary data functions have been called by this user function,
64117 ** immediately call the destructor for any non-static values.
64118 */
64119 if( u.ag.ctx.pVdbeFunc ){
64120 sqlite3VdbeDeleteAuxData(u.ag.ctx.pVdbeFunc, pOp->p1);
64121 pOp->p4.pVdbeFunc = u.ag.ctx.pVdbeFunc;
64122 pOp->p4type = P4_VDBEFUNC;
64123 }
64124
64125 if( db->mallocFailed ){
64126 /* Even though a malloc() has failed, the implementation of the
64127 ** user function may have called an sqlite3_result_XXX() function
64128 ** to return a value. The following call releases any resources
64129 ** associated with such a value.
@@ -64093,19 +64130,10 @@
64130 */
64131 sqlite3VdbeMemRelease(&u.ag.ctx.s);
64132 goto no_mem;
64133 }
64134
 
 
 
 
 
 
 
 
 
64135 /* If the function returned an error, throw an exception */
64136 if( u.ag.ctx.isError ){
64137 sqlite3SetString(&p->zErrMsg, db, "%s", sqlite3_value_text(&u.ag.ctx.s));
64138 rc = u.ag.ctx.isError;
64139 }
@@ -75255,18 +75283,18 @@
75283 sqlite3VdbeAddOp4(v, OP_DropTable, iDb, 0, 0, pTab->zName, 0);
75284
75285 /* Reload the table, index and permanent trigger schemas. */
75286 zWhere = sqlite3MPrintf(pParse->db, "tbl_name=%Q", zName);
75287 if( !zWhere ) return;
75288 sqlite3VdbeAddParseSchemaOp(v, iDb, zWhere);
75289
75290 #ifndef SQLITE_OMIT_TRIGGER
75291 /* Now, if the table is not stored in the temp database, reload any temp
75292 ** triggers. Don't use IN(...) in case SQLITE_OMIT_SUBQUERY is defined.
75293 */
75294 if( (zWhere=whereTempTriggers(pParse, pTab))!=0 ){
75295 sqlite3VdbeAddParseSchemaOp(v, 1, zWhere);
75296 }
75297 #endif
75298 }
75299
75300 /*
@@ -78875,12 +78903,12 @@
78903 }
78904 }
78905 #endif
78906
78907 /* Reparse everything to update our internal data structures */
78908 sqlite3VdbeAddParseSchemaOp(v, iDb,
78909 sqlite3MPrintf(db, "tbl_name='%q'", p->zName));
78910 }
78911
78912
78913 /* Add the table to the in-memory representation of the database.
78914 */
@@ -80073,13 +80101,12 @@
80101 ** to invalidate all pre-compiled statements.
80102 */
80103 if( pTblName ){
80104 sqlite3RefillIndex(pParse, pIndex, iMem);
80105 sqlite3ChangeCookie(pParse, iDb);
80106 sqlite3VdbeAddParseSchemaOp(v, iDb,
80107 sqlite3MPrintf(db, "name='%q' AND type='index'", pIndex->zName));
 
80108 sqlite3VdbeAddOp1(v, OP_Expire, 0);
80109 }
80110 }
80111
80112 /* When adding an index to the list of indices for a table, make
@@ -82621,14 +82648,14 @@
82648 ** character is exactly one byte in size. Also, all characters are
82649 ** able to participate in upper-case-to-lower-case mappings in EBCDIC
82650 ** whereas only characters less than 0x80 do in ASCII.
82651 */
82652 #if defined(SQLITE_EBCDIC)
82653 # define sqlite3Utf8Read(A,C) (*(A++))
82654 # define GlogUpperToLower(A) A = sqlite3UpperToLower[A]
82655 #else
82656 # define GlogUpperToLower(A) if( !((A)&~0x7f) ){ A = sqlite3UpperToLower[A]; }
82657 #endif
82658
82659 static const struct compareInfo globInfo = { '*', '?', '[', 0 };
82660 /* The correct SQL-92 behavior is for the LIKE operator to ignore
82661 ** case. Thus 'a' LIKE 'A' would be true. */
@@ -82667,13 +82694,13 @@
82694 */
82695 static int patternCompare(
82696 const u8 *zPattern, /* The glob pattern */
82697 const u8 *zString, /* The string to compare against the glob */
82698 const struct compareInfo *pInfo, /* Information about how to do the compare */
82699 u32 esc /* The escape character */
82700 ){
82701 u32 c, c2;
82702 int invert;
82703 int seen;
82704 u8 matchOne = pInfo->matchOne;
82705 u8 matchAll = pInfo->matchAll;
82706 u8 matchSet = pInfo->matchSet;
@@ -82723,11 +82750,11 @@
82750 }else if( !prevEscape && c==matchOne ){
82751 if( sqlite3Utf8Read(zString, &zString)==0 ){
82752 return 0;
82753 }
82754 }else if( c==matchSet ){
82755 u32 prior_c = 0;
82756 assert( esc==0 ); /* This only occurs for GLOB, not LIKE */
82757 seen = 0;
82758 invert = 0;
82759 c = sqlite3Utf8Read(zString, &zString);
82760 if( c==0 ) return 0;
@@ -82799,11 +82826,11 @@
82826 sqlite3_context *context,
82827 int argc,
82828 sqlite3_value **argv
82829 ){
82830 const unsigned char *zA, *zB;
82831 u32 escape = 0;
82832 int nPat;
82833 sqlite3 *db = sqlite3_context_db_handle(context);
82834
82835 zB = sqlite3_value_text(argv[0]);
82836 zA = sqlite3_value_text(argv[1]);
@@ -84110,17 +84137,29 @@
84137 }
84138
84139 /* If the parent table is the same as the child table, and we are about
84140 ** to increment the constraint-counter (i.e. this is an INSERT operation),
84141 ** then check if the row being inserted matches itself. If so, do not
84142 ** increment the constraint-counter.
84143 **
84144 ** If any of the parent-key values are NULL, then the row cannot match
84145 ** itself. So set JUMPIFNULL to make sure we do the OP_Found if any
84146 ** of the parent-key values are NULL (at this point it is known that
84147 ** none of the child key values are).
84148 */
84149 if( pTab==pFKey->pFrom && nIncr==1 ){
84150 int iJump = sqlite3VdbeCurrentAddr(v) + nCol + 1;
84151 for(i=0; i<nCol; i++){
84152 int iChild = aiCol[i]+1+regData;
84153 int iParent = pIdx->aiColumn[i]+1+regData;
84154 assert( aiCol[i]!=pTab->iPKey );
84155 if( pIdx->aiColumn[i]==pTab->iPKey ){
84156 /* The parent key is a composite key that includes the IPK column */
84157 iParent = regData;
84158 }
84159 sqlite3VdbeAddOp3(v, OP_Ne, iChild, iJump, iParent);
84160 sqlite3VdbeChangeP5(v, SQLITE_JUMPIFNULL);
84161 }
84162 sqlite3VdbeAddOp2(v, OP_Goto, 0, iOk);
84163 }
84164
84165 sqlite3VdbeAddOp3(v, OP_MakeRecord, regTemp, nCol, regRec);
@@ -95336,13 +95375,12 @@
95375 "INSERT INTO %Q.%s VALUES('trigger',%Q,%Q,0,'CREATE TRIGGER %q')",
95376 db->aDb[iDb].zName, SCHEMA_TABLE(iDb), zName,
95377 pTrig->table, z);
95378 sqlite3DbFree(db, z);
95379 sqlite3ChangeCookie(pParse, iDb);
95380 sqlite3VdbeAddParseSchemaOp(v, iDb,
95381 sqlite3MPrintf(db, "type='trigger' AND name='%q'", zName));
 
95382 }
95383
95384 if( db->init.busy ){
95385 Trigger *pLink = pTrig;
95386 Hash *pHash = &db->aDb[iDb].pSchema->trigHash;
@@ -96392,11 +96430,11 @@
96430 aRegIdx = sqlite3DbMallocRaw(db, sizeof(Index*) * nIdx );
96431 if( aRegIdx==0 ) goto update_cleanup;
96432 }
96433 for(j=0, pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext, j++){
96434 int reg;
96435 if( hasFK || chngRowid ){
96436 reg = ++pParse->nMem;
96437 }else{
96438 reg = 0;
96439 for(i=0; i<pIdx->nColumn; i++){
96440 if( aXRef[pIdx->aiColumn[i]]>=0 ){
@@ -97547,11 +97585,11 @@
97585 v = sqlite3GetVdbe(pParse);
97586 sqlite3ChangeCookie(pParse, iDb);
97587
97588 sqlite3VdbeAddOp2(v, OP_Expire, 0, 0);
97589 zWhere = sqlite3MPrintf(db, "name='%q' AND type='table'", pTab->zName);
97590 sqlite3VdbeAddParseSchemaOp(v, iDb, zWhere);
97591 sqlite3VdbeAddOp4(v, OP_VCreate, iDb, 0, 0,
97592 pTab->zName, sqlite3Strlen30(pTab->zName) + 1);
97593 }
97594
97595 /* If we are rereading the sqlite_master table create the in-memory
@@ -107274,17 +107312,16 @@
107312 #ifndef SQLITE_OMIT_BLOB_LITERAL
107313 case 'x': case 'X': {
107314 testcase( z[0]=='x' ); testcase( z[0]=='X' );
107315 if( z[1]=='\'' ){
107316 *tokenType = TK_BLOB;
107317 for(i=2; sqlite3Isxdigit(z[i]); i++){}
107318 if( z[i]!='\'' || i%2 ){
107319 *tokenType = TK_ILLEGAL;
107320 while( z[i] && z[i]!='\'' ){ i++; }
107321 }
107322 if( z[i] ) i++;
 
107323 return i;
107324 }
107325 /* Otherwise fall through to the next case */
107326 }
107327 #endif
@@ -111710,16 +111747,39 @@
111747 ** similar macro called ArraySize(). Use a different name to avoid
111748 ** a collision when building an amalgamation with built-in FTS3.
111749 */
111750 #define SizeofArray(X) ((int)(sizeof(X)/sizeof(X[0])))
111751
111752
111753 #ifndef MIN
111754 # define MIN(x,y) ((x)<(y)?(x):(y))
111755 #endif
111756
111757 /*
111758 ** Maximum length of a varint encoded integer. The varint format is different
111759 ** from that used by SQLite, so the maximum length is 10, not 9.
111760 */
111761 #define FTS3_VARINT_MAX 10
111762
111763 /*
111764 ** FTS4 virtual tables may maintain multiple indexes - one index of all terms
111765 ** in the document set and zero or more prefix indexes. All indexes are stored
111766 ** as one or more b+-trees in the %_segments and %_segdir tables.
111767 **
111768 ** It is possible to determine which index a b+-tree belongs to based on the
111769 ** value stored in the "%_segdir.level" column. Given this value L, the index
111770 ** that the b+-tree belongs to is (L<<10). In other words, all b+-trees with
111771 ** level values between 0 and 1023 (inclusive) belong to index 0, all levels
111772 ** between 1024 and 2047 to index 1, and so on.
111773 **
111774 ** It is considered impossible for an index to use more than 1024 levels. In
111775 ** theory though this may happen, but only after at least
111776 ** (FTS3_MERGE_COUNT^1024) separate flushes of the pending-terms tables.
111777 */
111778 #define FTS3_SEGDIR_MAXLEVEL 1024
111779 #define FTS3_SEGDIR_MAXLEVEL_STR "1024"
111780
111781 /*
111782 ** The testcase() macro is only used by the amalgamation. If undefined,
111783 ** make it a no-op.
111784 */
111785 #ifndef testcase
@@ -111787,14 +111847,15 @@
111847 typedef struct Fts3Cursor Fts3Cursor;
111848 typedef struct Fts3Expr Fts3Expr;
111849 typedef struct Fts3Phrase Fts3Phrase;
111850 typedef struct Fts3PhraseToken Fts3PhraseToken;
111851
111852 typedef struct Fts3Doclist Fts3Doclist;
111853 typedef struct Fts3SegFilter Fts3SegFilter;
111854 typedef struct Fts3DeferredToken Fts3DeferredToken;
111855 typedef struct Fts3SegReader Fts3SegReader;
111856 typedef struct Fts3MultiSegReader Fts3MultiSegReader;
111857
111858 /*
111859 ** A connection to a fulltext index is an instance of the following
111860 ** structure. The xCreate and xConnect methods create an instance
111861 ** of this structure and xDestroy and xDisconnect free that instance.
@@ -111811,33 +111872,45 @@
111872 sqlite3_tokenizer *pTokenizer; /* tokenizer for inserts and queries */
111873
111874 /* Precompiled statements used by the implementation. Each of these
111875 ** statements is run and reset within a single virtual table API call.
111876 */
111877 sqlite3_stmt *aStmt[27];
111878
111879 char *zReadExprlist;
111880 char *zWriteExprlist;
111881
111882 int nNodeSize; /* Soft limit for node size */
111883 u8 bHasStat; /* True if %_stat table exists */
111884 u8 bHasDocsize; /* True if %_docsize table exists */
111885 u8 bDescIdx; /* True if doclists are in reverse order */
111886 int nPgsz; /* Page size for host database */
111887 char *zSegmentsTbl; /* Name of %_segments table */
111888 sqlite3_blob *pSegments; /* Blob handle open on %_segments table */
111889
111890 /* TODO: Fix the first paragraph of this comment.
111891 **
111892 ** The following hash table is used to buffer pending index updates during
111893 ** transactions. Variable nPendingData estimates the memory size of the
111894 ** pending data, including hash table overhead, but not malloc overhead.
111895 ** When nPendingData exceeds nMaxPendingData, the buffer is flushed
111896 ** automatically. Variable iPrevDocid is the docid of the most recently
111897 ** inserted record.
111898 **
111899 ** A single FTS4 table may have multiple full-text indexes. For each index
111900 ** there is an entry in the aIndex[] array. Index 0 is an index of all the
111901 ** terms that appear in the document set. Each subsequent index in aIndex[]
111902 ** is an index of prefixes of a specific length.
111903 */
111904 int nIndex; /* Size of aIndex[] */
111905 struct Fts3Index {
111906 int nPrefix; /* Prefix length (0 for main terms index) */
111907 Fts3Hash hPending; /* Pending terms table for this index */
111908 } *aIndex;
111909 int nMaxPendingData; /* Max pending data before flush to disk */
111910 int nPendingData; /* Current bytes of pending data */
111911 sqlite_int64 iPrevDocid; /* Docid of most recently inserted document */
111912
111913 #if defined(SQLITE_DEBUG)
111914 /* State variables used for validating that the transaction control
111915 ** methods of the virtual table are called at appropriate times. These
111916 ** values do not contribution to the FTS computation; they are used for
@@ -111864,13 +111937,14 @@
111937 Fts3DeferredToken *pDeferred; /* Deferred search tokens, if any */
111938 sqlite3_int64 iPrevId; /* Previous id read from aDoclist */
111939 char *pNextId; /* Pointer into the body of aDoclist */
111940 char *aDoclist; /* List of docids for full-text queries */
111941 int nDoclist; /* Size of buffer at aDoclist */
111942 u8 bDesc; /* True to sort in descending order */
111943 int eEvalmode; /* An FTS3_EVAL_XX constant */
111944 int nRowAvg; /* Average size of database rows, in pages */
111945 int nDoc; /* Documents in table */
111946
111947 int isMatchinfoNeeded; /* True when aMatchinfo[] needs filling in */
111948 u32 *aMatchinfo; /* Information about most recent match */
111949 int nMatchinfo; /* Number of elements in aMatchinfo[] */
111950 char *zMatchinfo; /* Matchinfo specification */
@@ -111897,66 +111971,90 @@
111971 */
111972 #define FTS3_FULLSCAN_SEARCH 0 /* Linear scan of %_content table */
111973 #define FTS3_DOCID_SEARCH 1 /* Lookup by rowid on %_content table */
111974 #define FTS3_FULLTEXT_SEARCH 2 /* Full-text index search */
111975
111976
111977 struct Fts3Doclist {
111978 char *aAll; /* Array containing doclist (or NULL) */
111979 int nAll; /* Size of a[] in bytes */
111980 char *pNextDocid; /* Pointer to next docid */
111981
111982 sqlite3_int64 iDocid; /* Current docid (if pList!=0) */
111983 int bFreeList; /* True if pList should be sqlite3_free()d */
111984 char *pList; /* Pointer to position list following iDocid */
111985 int nList; /* Length of position list */
111986 } doclist;
111987
111988 /*
111989 ** A "phrase" is a sequence of one or more tokens that must match in
111990 ** sequence. A single token is the base case and the most common case.
111991 ** For a sequence of tokens contained in double-quotes (i.e. "one two three")
111992 ** nToken will be the number of tokens in the string.
 
 
 
 
 
 
111993 */
111994 struct Fts3PhraseToken {
111995 char *z; /* Text of the token */
111996 int n; /* Number of bytes in buffer z */
111997 int isPrefix; /* True if token ends with a "*" character */
111998
111999 /* Variables above this point are populated when the expression is
112000 ** parsed (by code in fts3_expr.c). Below this point the variables are
112001 ** used when evaluating the expression. */
112002 int bFulltext; /* True if full-text index was used */
 
112003 Fts3DeferredToken *pDeferred; /* Deferred token object for this token */
112004 Fts3MultiSegReader *pSegcsr; /* Segment-reader for this token */
112005 };
112006
112007 struct Fts3Phrase {
112008 /* Cache of doclist for this phrase. */
112009 Fts3Doclist doclist;
112010 int bIncr; /* True if doclist is loaded incrementally */
112011
112012 /* Variables below this point are populated by fts3_expr.c when parsing
112013 ** a MATCH expression. Everything above is part of the evaluation phase.
112014 */
112015 int nToken; /* Number of tokens in the phrase */
112016 int iColumn; /* Index of column this phrase must match */
 
112017 Fts3PhraseToken aToken[1]; /* One entry for each token in the phrase */
112018 };
112019
112020 /*
112021 ** A tree of these objects forms the RHS of a MATCH operator.
112022 **
112023 ** If Fts3Expr.eType is FTSQUERY_PHRASE and isLoaded is true, then aDoclist
112024 ** points to a malloced buffer, size nDoclist bytes, containing the results
112025 ** of this phrase query in FTS3 doclist format. As usual, the initial
112026 ** "Length" field found in doclists stored on disk is omitted from this
112027 ** buffer.
112028 **
112029 ** Variable aMI is used only for FTSQUERY_NEAR nodes to store the global
112030 ** matchinfo data. If it is not NULL, it points to an array of size nCol*3,
112031 ** where nCol is the number of columns in the queried FTS table. The array
112032 ** is populated as follows:
112033 **
112034 ** aMI[iCol*3 + 0] = Undefined
112035 ** aMI[iCol*3 + 1] = Number of occurrences
112036 ** aMI[iCol*3 + 2] = Number of rows containing at least one instance
112037 **
112038 ** The aMI array is allocated using sqlite3_malloc(). It should be freed
112039 ** when the expression node is.
112040 */
112041 struct Fts3Expr {
112042 int eType; /* One of the FTSQUERY_XXX values defined below */
112043 int nNear; /* Valid if eType==FTSQUERY_NEAR */
112044 Fts3Expr *pParent; /* pParent->pLeft==this or pParent->pRight==this */
112045 Fts3Expr *pLeft; /* Left operand */
112046 Fts3Expr *pRight; /* Right operand */
112047 Fts3Phrase *pPhrase; /* Valid if eType==FTSQUERY_PHRASE */
112048
112049 /* The following are used by the fts3_eval.c module. */
112050 sqlite3_int64 iDocid; /* Current docid */
112051 u8 bEof; /* True this expression is at EOF already */
112052 u8 bStart; /* True if iDocid is valid */
112053 u8 bDeferred; /* True if this expression is entirely deferred */
112054
112055 u32 *aMI;
 
112056 };
112057
112058 /*
112059 ** Candidate values for Fts3Query.eType. Note that the order of the first
112060 ** four values is in order of precedence when parsing expressions. For
@@ -111980,35 +112078,36 @@
112078 SQLITE_PRIVATE int sqlite3Fts3PendingTermsFlush(Fts3Table *);
112079 SQLITE_PRIVATE void sqlite3Fts3PendingTermsClear(Fts3Table *);
112080 SQLITE_PRIVATE int sqlite3Fts3Optimize(Fts3Table *);
112081 SQLITE_PRIVATE int sqlite3Fts3SegReaderNew(int, sqlite3_int64,
112082 sqlite3_int64, sqlite3_int64, const char *, int, Fts3SegReader**);
112083 SQLITE_PRIVATE int sqlite3Fts3SegReaderPending(
112084 Fts3Table*,int,const char*,int,int,Fts3SegReader**);
112085 SQLITE_PRIVATE void sqlite3Fts3SegReaderFree(Fts3SegReader *);
112086 SQLITE_PRIVATE int sqlite3Fts3AllSegdirs(Fts3Table*, int, int, sqlite3_stmt **);
 
112087 SQLITE_PRIVATE int sqlite3Fts3ReadLock(Fts3Table *);
112088 SQLITE_PRIVATE int sqlite3Fts3ReadBlock(Fts3Table*, sqlite3_int64, char **, int*, int*);
112089
112090 SQLITE_PRIVATE int sqlite3Fts3SelectDoctotal(Fts3Table *, sqlite3_stmt **);
112091 SQLITE_PRIVATE int sqlite3Fts3SelectDocsize(Fts3Table *, sqlite3_int64, sqlite3_stmt **);
112092
112093 SQLITE_PRIVATE void sqlite3Fts3FreeDeferredTokens(Fts3Cursor *);
112094 SQLITE_PRIVATE int sqlite3Fts3DeferToken(Fts3Cursor *, Fts3PhraseToken *, int);
112095 SQLITE_PRIVATE int sqlite3Fts3CacheDeferredDoclists(Fts3Cursor *);
112096 SQLITE_PRIVATE void sqlite3Fts3FreeDeferredDoclists(Fts3Cursor *);
 
112097 SQLITE_PRIVATE void sqlite3Fts3SegmentsClose(Fts3Table *);
112098
112099 /* Special values interpreted by sqlite3SegReaderCursor() */
112100 #define FTS3_SEGCURSOR_PENDING -1
112101 #define FTS3_SEGCURSOR_ALL -2
112102
112103 SQLITE_PRIVATE int sqlite3Fts3SegReaderStart(Fts3Table*, Fts3MultiSegReader*, Fts3SegFilter*);
112104 SQLITE_PRIVATE int sqlite3Fts3SegReaderStep(Fts3Table *, Fts3MultiSegReader *);
112105 SQLITE_PRIVATE void sqlite3Fts3SegReaderFinish(Fts3MultiSegReader *);
112106
112107 SQLITE_PRIVATE int sqlite3Fts3SegReaderCursor(
112108 Fts3Table *, int, int, const char *, int, int, int, Fts3MultiSegReader *);
112109
112110 /* Flags allowed as part of the 4th argument to SegmentReaderIterate() */
112111 #define FTS3_SEGMENT_REQUIRE_POS 0x00000001
112112 #define FTS3_SEGMENT_IGNORE_EMPTY 0x00000002
112113 #define FTS3_SEGMENT_COLUMN_FILTER 0x00000004
@@ -112021,21 +112120,24 @@
112120 int nTerm;
112121 int iCol;
112122 int flags;
112123 };
112124
112125 struct Fts3MultiSegReader {
112126 /* Used internally by sqlite3Fts3SegReaderXXX() calls */
112127 Fts3SegReader **apSegment; /* Array of Fts3SegReader objects */
112128 int nSegment; /* Size of apSegment array */
112129 int nAdvance; /* How many seg-readers to advance */
112130 Fts3SegFilter *pFilter; /* Pointer to filter object */
112131 char *aBuffer; /* Buffer to merge doclists in */
112132 int nBuffer; /* Allocated size of aBuffer[] in bytes */
112133
112134 int iColFilter; /* If >=0, filter for this column */
112135
112136 /* Used by fts3.c only. */
112137 int nCost; /* Cost of running iterator */
112138 int bLookup; /* True if a lookup of a single entry. */
112139
112140 /* Output values. Valid only after Fts3SegReaderStep() returns SQLITE_ROW. */
112141 char *zTerm; /* Pointer to term buffer */
112142 int nTerm; /* Size of zTerm in bytes */
112143 char *aDoclist; /* Pointer to doclist buffer */
@@ -112046,15 +112148,13 @@
112148 SQLITE_PRIVATE int sqlite3Fts3PutVarint(char *, sqlite3_int64);
112149 SQLITE_PRIVATE int sqlite3Fts3GetVarint(const char *, sqlite_int64 *);
112150 SQLITE_PRIVATE int sqlite3Fts3GetVarint32(const char *, int *);
112151 SQLITE_PRIVATE int sqlite3Fts3VarintLen(sqlite3_uint64);
112152 SQLITE_PRIVATE void sqlite3Fts3Dequote(char *);
112153 SQLITE_PRIVATE void sqlite3Fts3DoclistPrev(int,char*,int,char**,sqlite3_int64*,int*,u8*);
112154
112155 SQLITE_PRIVATE int sqlite3Fts3EvalPhraseStats(Fts3Cursor *, Fts3Expr *, u32 *);
 
 
 
112156
112157 /* fts3_tokenizer.c */
112158 SQLITE_PRIVATE const char *sqlite3Fts3NextToken(const char *, int *);
112159 SQLITE_PRIVATE int sqlite3Fts3InitHashTable(sqlite3 *, Fts3Hash *, const char *);
112160 SQLITE_PRIVATE int sqlite3Fts3InitTokenizer(Fts3Hash *pHash, const char *,
@@ -112079,10 +112179,33 @@
112179 SQLITE_PRIVATE int sqlite3Fts3InitTerm(sqlite3 *db);
112180 #endif
112181
112182 /* fts3_aux.c */
112183 SQLITE_PRIVATE int sqlite3Fts3InitAux(sqlite3 *db);
112184
112185 SQLITE_PRIVATE int sqlite3Fts3TermSegReaderCursor(
112186 Fts3Cursor *pCsr, /* Virtual table cursor handle */
112187 const char *zTerm, /* Term to query for */
112188 int nTerm, /* Size of zTerm in bytes */
112189 int isPrefix, /* True for a prefix search */
112190 Fts3MultiSegReader **ppSegcsr /* OUT: Allocated seg-reader cursor */
112191 );
112192
112193 SQLITE_PRIVATE void sqlite3Fts3EvalPhraseCleanup(Fts3Phrase *);
112194
112195 SQLITE_PRIVATE int sqlite3Fts3EvalStart(Fts3Cursor *, Fts3Expr *, int);
112196 SQLITE_PRIVATE int sqlite3Fts3EvalNext(Fts3Cursor *pCsr);
112197
112198 SQLITE_PRIVATE int sqlite3Fts3MsrIncrStart(
112199 Fts3Table*, Fts3MultiSegReader*, int, const char*, int);
112200 SQLITE_PRIVATE int sqlite3Fts3MsrIncrNext(
112201 Fts3Table *, Fts3MultiSegReader *, sqlite3_int64 *, char **, int *);
112202 SQLITE_PRIVATE char *sqlite3Fts3EvalPhrasePoslist(Fts3Cursor *, Fts3Expr *, int iCol);
112203 SQLITE_PRIVATE int sqlite3Fts3MsrOvfl(Fts3Cursor *, Fts3MultiSegReader *, int *);
112204
112205 SQLITE_PRIVATE int sqlite3Fts3DeferredTokenList(Fts3DeferredToken *, char **, int *);
112206
112207
112208 #endif /* _FTSINT_H */
112209
112210 /************** End of fts3Int.h *********************************************/
112211 /************** Continuing where we left off in fts3.c ***********************/
@@ -112200,16 +112323,16 @@
112323
112324 /*
112325 ** When this function is called, *pp points to the first byte following a
112326 ** varint that is part of a doclist (or position-list, or any other list
112327 ** of varints). This function moves *pp to point to the start of that varint,
112328 ** and sets *pVal by the varint value.
112329 **
112330 ** Argument pStart points to the first byte of the doclist that the
112331 ** varint is part of.
112332 */
112333 static void fts3GetReverseVarint(
112334 char **pp,
112335 char *pStart,
112336 sqlite3_int64 *pVal
112337 ){
112338 sqlite3_int64 iVal;
@@ -112221,25 +112344,11 @@
112344 for(p = (*pp)-2; p>=pStart && *p&0x80; p--);
112345 p++;
112346 *pp = p;
112347
112348 sqlite3Fts3GetVarint(p, &iVal);
112349 *pVal = iVal;
 
 
 
 
 
 
 
 
 
 
 
 
 
 
112350 }
112351
112352 /*
112353 ** The xDisconnect() virtual table method.
112354 */
@@ -112608,10 +112717,62 @@
112717 fts3Appendf(pRc, &zRet, ",%s(?)", zFunction);
112718 }
112719 sqlite3_free(zFree);
112720 return zRet;
112721 }
112722
112723 static int fts3GobbleInt(const char **pp, int *pnOut){
112724 const char *p = *pp;
112725 int nInt = 0;
112726 for(p=*pp; p[0]>='0' && p[0]<='9'; p++){
112727 nInt = nInt * 10 + (p[0] - '0');
112728 }
112729 if( p==*pp ) return SQLITE_ERROR;
112730 *pnOut = nInt;
112731 *pp = p;
112732 return SQLITE_OK;
112733 }
112734
112735
112736 static int fts3PrefixParameter(
112737 const char *zParam, /* ABC in prefix=ABC parameter to parse */
112738 int *pnIndex, /* OUT: size of *apIndex[] array */
112739 struct Fts3Index **apIndex, /* OUT: Array of indexes for this table */
112740 struct Fts3Index **apFree /* OUT: Free this with sqlite3_free() */
112741 ){
112742 struct Fts3Index *aIndex;
112743 int nIndex = 1;
112744
112745 if( zParam && zParam[0] ){
112746 const char *p;
112747 nIndex++;
112748 for(p=zParam; *p; p++){
112749 if( *p==',' ) nIndex++;
112750 }
112751 }
112752
112753 aIndex = sqlite3_malloc(sizeof(struct Fts3Index) * nIndex);
112754 *apIndex = *apFree = aIndex;
112755 *pnIndex = nIndex;
112756 if( !aIndex ){
112757 return SQLITE_NOMEM;
112758 }
112759
112760 memset(aIndex, 0, sizeof(struct Fts3Index) * nIndex);
112761 if( zParam ){
112762 const char *p = zParam;
112763 int i;
112764 for(i=1; i<nIndex; i++){
112765 int nPrefix;
112766 if( fts3GobbleInt(&p, &nPrefix) ) return SQLITE_ERROR;
112767 aIndex[i].nPrefix = nPrefix;
112768 p++;
112769 }
112770 }
112771
112772 return SQLITE_OK;
112773 }
112774
112775 /*
112776 ** This function is the implementation of both the xConnect and xCreate
112777 ** methods of the FTS3 virtual table.
112778 **
@@ -112641,16 +112802,23 @@
112802 int nCol = 0; /* Number of columns in the FTS table */
112803 char *zCsr; /* Space for holding column names */
112804 int nDb; /* Bytes required to hold database name */
112805 int nName; /* Bytes required to hold table name */
112806 int isFts4 = (argv[0][3]=='4'); /* True for FTS4, false for FTS3 */
 
112807 const char **aCol; /* Array of column names */
112808 sqlite3_tokenizer *pTokenizer = 0; /* Tokenizer for this table */
112809
112810 int nIndex; /* Size of aIndex[] array */
112811 struct Fts3Index *aIndex; /* Array of indexes for this table */
112812 struct Fts3Index *aFree = 0; /* Free this before returning */
112813
112814 /* The results of parsing supported FTS4 key=value options: */
112815 int bNoDocsize = 0; /* True to omit %_docsize table */
112816 int bDescIdx = 0; /* True to store descending indexes */
112817 char *zPrefix = 0; /* Prefix parameter value (or NULL) */
112818 char *zCompress = 0; /* compress=? parameter (or NULL) */
112819 char *zUncompress = 0; /* uncompress=? parameter (or NULL) */
112820
112821 assert( strlen(argv[0])==4 );
112822 assert( (sqlite3_strnicmp(argv[0], "fts4", 4)==0 && isFts4)
112823 || (sqlite3_strnicmp(argv[0], "fts3", 4)==0 && !isFts4)
112824 );
@@ -112687,32 +112855,76 @@
112855 rc = sqlite3Fts3InitTokenizer(pHash, &z[9], &pTokenizer, pzErr);
112856 }
112857
112858 /* Check if it is an FTS4 special argument. */
112859 else if( isFts4 && fts3IsSpecialColumn(z, &nKey, &zVal) ){
112860 struct Fts4Option {
112861 const char *zOpt;
112862 int nOpt;
112863 char **pzVar;
112864 } aFts4Opt[] = {
112865 { "matchinfo", 9, 0 }, /* 0 -> MATCHINFO */
112866 { "prefix", 6, 0 }, /* 1 -> PREFIX */
112867 { "compress", 8, 0 }, /* 2 -> COMPRESS */
112868 { "uncompress", 10, 0 }, /* 3 -> UNCOMPRESS */
112869 { "order", 5, 0 } /* 4 -> ORDER */
112870 };
112871
112872 int iOpt;
112873 if( !zVal ){
112874 rc = SQLITE_NOMEM;
112875 }else{
112876 for(iOpt=0; iOpt<SizeofArray(aFts4Opt); iOpt++){
112877 struct Fts4Option *pOp = &aFts4Opt[iOpt];
112878 if( nKey==pOp->nOpt && !sqlite3_strnicmp(z, pOp->zOpt, pOp->nOpt) ){
112879 break;
112880 }
112881 }
112882 if( iOpt==SizeofArray(aFts4Opt) ){
112883 *pzErr = sqlite3_mprintf("unrecognized parameter: %s", z);
112884 rc = SQLITE_ERROR;
112885 }else{
112886 switch( iOpt ){
112887 case 0: /* MATCHINFO */
112888 if( strlen(zVal)!=4 || sqlite3_strnicmp(zVal, "fts3", 4) ){
112889 *pzErr = sqlite3_mprintf("unrecognized matchinfo: %s", zVal);
112890 rc = SQLITE_ERROR;
112891 }
112892 bNoDocsize = 1;
112893 break;
112894
112895 case 1: /* PREFIX */
112896 sqlite3_free(zPrefix);
112897 zPrefix = zVal;
112898 zVal = 0;
112899 break;
112900
112901 case 2: /* COMPRESS */
112902 sqlite3_free(zCompress);
112903 zCompress = zVal;
112904 zVal = 0;
112905 break;
112906
112907 case 3: /* UNCOMPRESS */
112908 sqlite3_free(zUncompress);
112909 zUncompress = zVal;
112910 zVal = 0;
112911 break;
112912
112913 case 4: /* ORDER */
112914 if( (strlen(zVal)!=3 || sqlite3_strnicmp(zVal, "asc", 3))
112915 && (strlen(zVal)!=4 || sqlite3_strnicmp(zVal, "desc", 3))
112916 ){
112917 *pzErr = sqlite3_mprintf("unrecognized order: %s", zVal);
112918 rc = SQLITE_ERROR;
112919 }
112920 bDescIdx = (zVal[0]=='d' || zVal[0]=='D');
112921 break;
112922 }
112923 }
112924 sqlite3_free(zVal);
112925 }
112926 }
112927
112928 /* Otherwise, the argument is a column name. */
112929 else {
112930 nString += (int)(strlen(z) + 1);
@@ -112732,14 +112944,21 @@
112944 rc = sqlite3Fts3InitTokenizer(pHash, "simple", &pTokenizer, pzErr);
112945 if( rc!=SQLITE_OK ) goto fts3_init_out;
112946 }
112947 assert( pTokenizer );
112948
112949 rc = fts3PrefixParameter(zPrefix, &nIndex, &aIndex, &aFree);
112950 if( rc==SQLITE_ERROR ){
112951 assert( zPrefix );
112952 *pzErr = sqlite3_mprintf("error parsing prefix parameter: %s", zPrefix);
112953 }
112954 if( rc!=SQLITE_OK ) goto fts3_init_out;
112955
112956 /* Allocate and populate the Fts3Table structure. */
112957 nByte = sizeof(Fts3Table) + /* Fts3Table */
112958 nCol * sizeof(char *) + /* azColumn */
112959 nIndex * sizeof(struct Fts3Index) + /* aIndex */
112960 nName + /* zName */
112961 nDb + /* zDb */
112962 nString; /* Space for azColumn strings */
112963 p = (Fts3Table*)sqlite3_malloc(nByte);
112964 if( p==0 ){
@@ -112750,20 +112969,26 @@
112969 p->db = db;
112970 p->nColumn = nCol;
112971 p->nPendingData = 0;
112972 p->azColumn = (char **)&p[1];
112973 p->pTokenizer = pTokenizer;
 
112974 p->nMaxPendingData = FTS3_MAX_PENDING_DATA;
112975 p->bHasDocsize = (isFts4 && bNoDocsize==0);
112976 p->bHasStat = isFts4;
112977 p->bDescIdx = bDescIdx;
112978 TESTONLY( p->inTransaction = -1 );
112979 TESTONLY( p->mxSavepoint = -1 );
112980
112981 p->aIndex = (struct Fts3Index *)&p->azColumn[nCol];
112982 memcpy(p->aIndex, aIndex, sizeof(struct Fts3Index) * nIndex);
112983 p->nIndex = nIndex;
112984 for(i=0; i<nIndex; i++){
112985 fts3HashInit(&p->aIndex[i].hPending, FTS3_HASH_STRING, 1);
112986 }
112987
112988 /* Fill in the zName and zDb fields of the vtab structure. */
112989 zCsr = (char *)&p->aIndex[nIndex];
112990 p->zName = zCsr;
112991 memcpy(zCsr, argv[2], nName);
112992 zCsr += nName;
112993 p->zDb = zCsr;
112994 memcpy(zCsr, argv[1], nDb);
@@ -112797,19 +113022,20 @@
113022 if( isCreate ){
113023 rc = fts3CreateTables(p);
113024 }
113025
113026 /* Figure out the page-size for the database. This is required in order to
113027 ** estimate the cost of loading large doclists from the database. */
 
 
113028 fts3DatabasePageSize(&rc, p);
113029 p->nNodeSize = p->nPgsz-35;
113030
113031 /* Declare the table schema to SQLite. */
113032 fts3DeclareVtab(&rc, p);
113033
113034 fts3_init_out:
113035 sqlite3_free(zPrefix);
113036 sqlite3_free(aFree);
113037 sqlite3_free(zCompress);
113038 sqlite3_free(zUncompress);
113039 sqlite3_free((void *)aCol);
113040 if( rc!=SQLITE_OK ){
113041 if( p ){
@@ -112816,10 +113042,11 @@
113042 fts3DisconnectMethod((sqlite3_vtab *)p);
113043 }else if( pTokenizer ){
113044 pTokenizer->pModule->xDestroy(pTokenizer);
113045 }
113046 }else{
113047 assert( p->pSegments==0 );
113048 *ppVTab = &p->base;
113049 }
113050 return rc;
113051 }
113052
@@ -112913,14 +113140,15 @@
113140 if( pOrder->desc ){
113141 pInfo->idxStr = "DESC";
113142 }else{
113143 pInfo->idxStr = "ASC";
113144 }
113145 pInfo->orderByConsumed = 1;
113146 }
 
113147 }
113148
113149 assert( p->pSegments==0 );
113150 return SQLITE_OK;
113151 }
113152
113153 /*
113154 ** Implementation of xOpen method.
@@ -112952,10 +113180,11 @@
113180 sqlite3_finalize(pCsr->pStmt);
113181 sqlite3Fts3ExprFree(pCsr->pExpr);
113182 sqlite3Fts3FreeDeferredTokens(pCsr);
113183 sqlite3_free(pCsr->aDoclist);
113184 sqlite3_free(pCsr->aMatchinfo);
113185 assert( ((Fts3Table *)pCsr->base.pVtab)->pSegments==0 );
113186 sqlite3_free(pCsr);
113187 return SQLITE_OK;
113188 }
113189
113190 /*
@@ -112963,12 +113192,12 @@
113192 ** of the %_content table that contains the last match. Return
113193 ** SQLITE_OK on success.
113194 */
113195 static int fts3CursorSeek(sqlite3_context *pContext, Fts3Cursor *pCsr){
113196 if( pCsr->isRequireSeek ){
 
113197 sqlite3_bind_int64(pCsr->pStmt, 1, pCsr->iPrevId);
113198 pCsr->isRequireSeek = 0;
113199 if( SQLITE_ROW==sqlite3_step(pCsr->pStmt) ){
113200 return SQLITE_OK;
113201 }else{
113202 int rc = sqlite3_reset(pCsr->pStmt);
113203 if( rc==SQLITE_OK ){
@@ -113145,21 +113374,21 @@
113374 if( rc==SQLITE_OK && iHeight>1 ){
113375 char *zBlob = 0; /* Blob read from %_segments table */
113376 int nBlob; /* Size of zBlob in bytes */
113377
113378 if( piLeaf && piLeaf2 && (*piLeaf!=*piLeaf2) ){
113379 rc = sqlite3Fts3ReadBlock(p, *piLeaf, &zBlob, &nBlob, 0);
113380 if( rc==SQLITE_OK ){
113381 rc = fts3SelectLeaf(p, zTerm, nTerm, zBlob, nBlob, piLeaf, 0);
113382 }
113383 sqlite3_free(zBlob);
113384 piLeaf = 0;
113385 zBlob = 0;
113386 }
113387
113388 if( rc==SQLITE_OK ){
113389 rc = sqlite3Fts3ReadBlock(p, piLeaf?*piLeaf:*piLeaf2, &zBlob, &nBlob, 0);
113390 }
113391 if( rc==SQLITE_OK ){
113392 rc = fts3SelectLeaf(p, zTerm, nTerm, zBlob, nBlob, piLeaf, piLeaf2);
113393 }
113394 sqlite3_free(zBlob);
@@ -113531,11 +113760,23 @@
113760 *pp = p;
113761 return 1;
113762 }
113763
113764 /*
113765 ** Merge two position-lists as required by the NEAR operator. The argument
113766 ** position lists correspond to the left and right phrases of an expression
113767 ** like:
113768 **
113769 ** "phrase 1" NEAR "phrase number 2"
113770 **
113771 ** Position list *pp1 corresponds to the left-hand side of the NEAR
113772 ** expression and *pp2 to the right. As usual, the indexes in the position
113773 ** lists are the offsets of the last token in each phrase (tokens "1" and "2"
113774 ** in the example above).
113775 **
113776 ** The output position list - written to *pp - is a copy of *pp2 with those
113777 ** entries that are not sufficiently NEAR entries in *pp1 removed.
113778 */
113779 static int fts3PoslistNearMerge(
113780 char **pp, /* Output buffer */
113781 char *aTmp, /* Temporary buffer space */
113782 int nRight, /* Maximum difference in token positions */
@@ -113544,218 +113785,31 @@
113785 char **pp2 /* IN/OUT: Right input list */
113786 ){
113787 char *p1 = *pp1;
113788 char *p2 = *pp2;
113789
113790 char *pTmp1 = aTmp;
113791 char *pTmp2;
113792 char *aTmp2;
113793 int res = 1;
113794
113795 fts3PoslistPhraseMerge(&pTmp1, nRight, 0, 0, pp1, pp2);
113796 aTmp2 = pTmp2 = pTmp1;
113797 *pp1 = p1;
113798 *pp2 = p2;
113799 fts3PoslistPhraseMerge(&pTmp2, nLeft, 1, 0, pp2, pp1);
113800 if( pTmp1!=aTmp && pTmp2!=aTmp2 ){
113801 fts3PoslistMerge(pp, &aTmp, &aTmp2);
113802 }else if( pTmp1!=aTmp ){
113803 fts3PoslistCopy(pp, &aTmp);
113804 }else if( pTmp2!=aTmp2 ){
113805 fts3PoslistCopy(pp, &aTmp2);
113806 }else{
113807 res = 0;
113808 }
113809
113810 return res;
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
113811 }
113812
113813 /*
113814 ** A pointer to an instance of this structure is used as the context
113815 ** argument to sqlite3Fts3SegReaderIterate()
@@ -113764,10 +113818,152 @@
113818 struct TermSelect {
113819 int isReqPos;
113820 char *aaOutput[16]; /* Malloc'd output buffer */
113821 int anOutput[16]; /* Size of output in bytes */
113822 };
113823
113824
113825 static void fts3GetDeltaVarint3(
113826 char **pp,
113827 char *pEnd,
113828 int bDescIdx,
113829 sqlite3_int64 *pVal
113830 ){
113831 if( *pp>=pEnd ){
113832 *pp = 0;
113833 }else{
113834 sqlite3_int64 iVal;
113835 *pp += sqlite3Fts3GetVarint(*pp, &iVal);
113836 if( bDescIdx ){
113837 *pVal -= iVal;
113838 }else{
113839 *pVal += iVal;
113840 }
113841 }
113842 }
113843
113844 static void fts3PutDeltaVarint3(
113845 char **pp, /* IN/OUT: Output pointer */
113846 int bDescIdx, /* True for descending docids */
113847 sqlite3_int64 *piPrev, /* IN/OUT: Previous value written to list */
113848 int *pbFirst, /* IN/OUT: True after first int written */
113849 sqlite3_int64 iVal /* Write this value to the list */
113850 ){
113851 sqlite3_int64 iWrite;
113852 if( bDescIdx==0 || *pbFirst==0 ){
113853 iWrite = iVal - *piPrev;
113854 }else{
113855 iWrite = *piPrev - iVal;
113856 }
113857 assert( *pbFirst || *piPrev==0 );
113858 assert( *pbFirst==0 || iWrite>0 );
113859 *pp += sqlite3Fts3PutVarint(*pp, iWrite);
113860 *piPrev = iVal;
113861 *pbFirst = 1;
113862 }
113863
113864 #define COMPARE_DOCID(i1, i2) ((bDescIdx?-1:1) * (i1-i2))
113865
113866 static int fts3DoclistOrMerge(
113867 int bDescIdx, /* True if arguments are desc */
113868 char *a1, int n1, /* First doclist */
113869 char *a2, int n2, /* Second doclist */
113870 char **paOut, int *pnOut /* OUT: Malloc'd doclist */
113871 ){
113872 sqlite3_int64 i1 = 0;
113873 sqlite3_int64 i2 = 0;
113874 sqlite3_int64 iPrev = 0;
113875 char *pEnd1 = &a1[n1];
113876 char *pEnd2 = &a2[n2];
113877 char *p1 = a1;
113878 char *p2 = a2;
113879 char *p;
113880 char *aOut;
113881 int bFirstOut = 0;
113882
113883 *paOut = 0;
113884 *pnOut = 0;
113885 aOut = sqlite3_malloc(n1+n2);
113886 if( !aOut ) return SQLITE_NOMEM;
113887
113888 p = aOut;
113889 fts3GetDeltaVarint3(&p1, pEnd1, 0, &i1);
113890 fts3GetDeltaVarint3(&p2, pEnd2, 0, &i2);
113891 while( p1 || p2 ){
113892 sqlite3_int64 iDiff = COMPARE_DOCID(i1, i2);
113893
113894 if( p2 && p1 && iDiff==0 ){
113895 fts3PutDeltaVarint3(&p, bDescIdx, &iPrev, &bFirstOut, i1);
113896 fts3PoslistMerge(&p, &p1, &p2);
113897 fts3GetDeltaVarint3(&p1, pEnd1, bDescIdx, &i1);
113898 fts3GetDeltaVarint3(&p2, pEnd2, bDescIdx, &i2);
113899 }else if( !p2 || (p1 && iDiff<0) ){
113900 fts3PutDeltaVarint3(&p, bDescIdx, &iPrev, &bFirstOut, i1);
113901 fts3PoslistCopy(&p, &p1);
113902 fts3GetDeltaVarint3(&p1, pEnd1, bDescIdx, &i1);
113903 }else{
113904 fts3PutDeltaVarint3(&p, bDescIdx, &iPrev, &bFirstOut, i2);
113905 fts3PoslistCopy(&p, &p2);
113906 fts3GetDeltaVarint3(&p2, pEnd2, bDescIdx, &i2);
113907 }
113908 }
113909
113910 *paOut = aOut;
113911 *pnOut = (p-aOut);
113912 return SQLITE_OK;
113913 }
113914
113915 static void fts3DoclistPhraseMerge(
113916 int bDescIdx, /* True if arguments are desc */
113917 int nDist, /* Distance from left to right (1=adjacent) */
113918 char *aLeft, int nLeft, /* Left doclist */
113919 char *aRight, int *pnRight /* IN/OUT: Right/output doclist */
113920 ){
113921 sqlite3_int64 i1 = 0;
113922 sqlite3_int64 i2 = 0;
113923 sqlite3_int64 iPrev = 0;
113924 char *pEnd1 = &aLeft[nLeft];
113925 char *pEnd2 = &aRight[*pnRight];
113926 char *p1 = aLeft;
113927 char *p2 = aRight;
113928 char *p;
113929 int bFirstOut = 0;
113930 char *aOut = aRight;
113931
113932 assert( nDist>0 );
113933
113934 p = aOut;
113935 fts3GetDeltaVarint3(&p1, pEnd1, 0, &i1);
113936 fts3GetDeltaVarint3(&p2, pEnd2, 0, &i2);
113937
113938 while( p1 && p2 ){
113939 sqlite3_int64 iDiff = COMPARE_DOCID(i1, i2);
113940 if( iDiff==0 ){
113941 char *pSave = p;
113942 sqlite3_int64 iPrevSave = iPrev;
113943 int bFirstOutSave = bFirstOut;
113944
113945 fts3PutDeltaVarint3(&p, bDescIdx, &iPrev, &bFirstOut, i1);
113946 if( 0==fts3PoslistPhraseMerge(&p, nDist, 0, 1, &p1, &p2) ){
113947 p = pSave;
113948 iPrev = iPrevSave;
113949 bFirstOut = bFirstOutSave;
113950 }
113951 fts3GetDeltaVarint3(&p1, pEnd1, bDescIdx, &i1);
113952 fts3GetDeltaVarint3(&p2, pEnd2, bDescIdx, &i2);
113953 }else if( iDiff<0 ){
113954 fts3PoslistCopy(0, &p1);
113955 fts3GetDeltaVarint3(&p1, pEnd1, bDescIdx, &i1);
113956 }else{
113957 fts3PoslistCopy(0, &p2);
113958 fts3GetDeltaVarint3(&p2, pEnd2, bDescIdx, &i2);
113959 }
113960 }
113961
113962 *pnRight = p - aOut;
113963 }
113964
113965
113966 /*
113967 ** Merge all doclists in the TermSelect.aaOutput[] array into a single
113968 ** doclist stored in TermSelect.aaOutput[0]. If successful, delete all
113969 ** other doclists (except the aaOutput[0] one) and return SQLITE_OK.
@@ -113774,12 +113970,11 @@
113970 **
113971 ** If an OOM error occurs, return SQLITE_NOMEM. In this case it is
113972 ** the responsibility of the caller to free any doclists left in the
113973 ** TermSelect.aaOutput[] array.
113974 */
113975 static int fts3TermSelectMerge(Fts3Table *p, TermSelect *pTS){
 
113976 char *aOut = 0;
113977 int nOut = 0;
113978 int i;
113979
113980 /* Loop through the doclists in the aaOutput[] array. Merge them all
@@ -113790,19 +113985,21 @@
113985 if( !aOut ){
113986 aOut = pTS->aaOutput[i];
113987 nOut = pTS->anOutput[i];
113988 pTS->aaOutput[i] = 0;
113989 }else{
113990 int nNew;
113991 char *aNew;
113992
113993 int rc = fts3DoclistOrMerge(p->bDescIdx,
113994 pTS->aaOutput[i], pTS->anOutput[i], aOut, nOut, &aNew, &nNew
113995 );
113996 if( rc!=SQLITE_OK ){
113997 sqlite3_free(aOut);
113998 return rc;
113999 }
114000
 
 
114001 sqlite3_free(pTS->aaOutput[i]);
114002 sqlite3_free(aOut);
114003 pTS->aaOutput[i] = 0;
114004 aOut = aNew;
114005 nOut = nNew;
@@ -113834,217 +114031,241 @@
114031 UNUSED_PARAMETER(zTerm);
114032 UNUSED_PARAMETER(nTerm);
114033
114034 if( pTS->aaOutput[0]==0 ){
114035 /* If this is the first term selected, copy the doclist to the output
114036 ** buffer using memcpy(). */
 
 
114037 pTS->aaOutput[0] = sqlite3_malloc(nDoclist);
114038 pTS->anOutput[0] = nDoclist;
114039 if( pTS->aaOutput[0] ){
114040 memcpy(pTS->aaOutput[0], aDoclist, nDoclist);
114041 }else{
114042 return SQLITE_NOMEM;
114043 }
114044 }else{
 
114045 char *aMerge = aDoclist;
114046 int nMerge = nDoclist;
114047 int iOut;
114048
114049 for(iOut=0; iOut<SizeofArray(pTS->aaOutput); iOut++){
 
 
114050 if( pTS->aaOutput[iOut]==0 ){
114051 assert( iOut>0 );
114052 pTS->aaOutput[iOut] = aMerge;
114053 pTS->anOutput[iOut] = nMerge;
114054 break;
114055 }else{
114056 char *aNew;
114057 int nNew;
114058
114059 int rc = fts3DoclistOrMerge(p->bDescIdx, aMerge, nMerge,
114060 pTS->aaOutput[iOut], pTS->anOutput[iOut], &aNew, &nNew
114061 );
114062 if( rc!=SQLITE_OK ){
114063 if( aMerge!=aDoclist ) sqlite3_free(aMerge);
114064 return rc;
114065 }
114066
114067 if( aMerge!=aDoclist ) sqlite3_free(aMerge);
114068 sqlite3_free(pTS->aaOutput[iOut]);
114069 pTS->aaOutput[iOut] = 0;
114070
114071 aMerge = aNew;
114072 nMerge = nNew;
114073 if( (iOut+1)==SizeofArray(pTS->aaOutput) ){
114074 pTS->aaOutput[iOut] = aMerge;
114075 pTS->anOutput[iOut] = nMerge;
114076 }
114077 }
114078 }
114079 }
114080 return SQLITE_OK;
114081 }
114082
114083 /*
114084 ** Append SegReader object pNew to the end of the pCsr->apSegment[] array.
114085 */
114086 static int fts3SegReaderCursorAppend(
114087 Fts3MultiSegReader *pCsr,
114088 Fts3SegReader *pNew
114089 ){
114090 if( (pCsr->nSegment%16)==0 ){
114091 Fts3SegReader **apNew;
114092 int nByte = (pCsr->nSegment + 16)*sizeof(Fts3SegReader*);
114093 apNew = (Fts3SegReader **)sqlite3_realloc(pCsr->apSegment, nByte);
114094 if( !apNew ){
114095 sqlite3Fts3SegReaderFree(pNew);
114096 return SQLITE_NOMEM;
114097 }
114098 pCsr->apSegment = apNew;
114099 }
114100 pCsr->apSegment[pCsr->nSegment++] = pNew;
114101 return SQLITE_OK;
114102 }
114103
114104 static int fts3SegReaderCursor(
 
 
 
 
 
 
 
 
 
114105 Fts3Table *p, /* FTS3 table handle */
114106 int iIndex, /* Index to search (from 0 to p->nIndex-1) */
114107 int iLevel, /* Level of segments to scan */
114108 const char *zTerm, /* Term to query for */
114109 int nTerm, /* Size of zTerm in bytes */
114110 int isPrefix, /* True for a prefix search */
114111 int isScan, /* True to scan from zTerm to EOF */
114112 Fts3MultiSegReader *pCsr /* Cursor object to populate */
114113 ){
114114 int rc = SQLITE_OK;
114115 int rc2;
 
114116 sqlite3_stmt *pStmt = 0;
114117
114118 /* If iLevel is less than 0 and this is not a scan, include a seg-reader
114119 ** for the pending-terms. If this is a scan, then this call must be being
114120 ** made by an fts4aux module, not an FTS table. In this case calling
114121 ** Fts3SegReaderPending might segfault, as the data structures used by
114122 ** fts4aux are not completely populated. So it's easiest to filter these
114123 ** calls out here. */
114124 if( iLevel<0 && p->aIndex ){
114125 Fts3SegReader *pSeg = 0;
114126 rc = sqlite3Fts3SegReaderPending(p, iIndex, zTerm, nTerm, isPrefix, &pSeg);
114127 if( rc==SQLITE_OK && pSeg ){
114128 rc = fts3SegReaderCursorAppend(pCsr, pSeg);
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
114129 }
114130 }
114131
114132 if( iLevel!=FTS3_SEGCURSOR_PENDING ){
114133 if( rc==SQLITE_OK ){
114134 rc = sqlite3Fts3AllSegdirs(p, iIndex, iLevel, &pStmt);
114135 }
114136
114137 while( rc==SQLITE_OK && SQLITE_ROW==(rc = sqlite3_step(pStmt)) ){
114138 Fts3SegReader *pSeg = 0;
114139
114140 /* Read the values returned by the SELECT into local variables. */
114141 sqlite3_int64 iStartBlock = sqlite3_column_int64(pStmt, 1);
114142 sqlite3_int64 iLeavesEndBlock = sqlite3_column_int64(pStmt, 2);
114143 sqlite3_int64 iEndBlock = sqlite3_column_int64(pStmt, 3);
114144 int nRoot = sqlite3_column_bytes(pStmt, 4);
114145 char const *zRoot = sqlite3_column_blob(pStmt, 4);
114146
 
 
 
 
 
 
 
 
 
 
 
 
114147 /* If zTerm is not NULL, and this segment is not stored entirely on its
114148 ** root node, the range of leaves scanned can be reduced. Do this. */
114149 if( iStartBlock && zTerm ){
114150 sqlite3_int64 *pi = (isPrefix ? &iLeavesEndBlock : 0);
114151 rc = fts3SelectLeaf(p, zTerm, nTerm, zRoot, nRoot, &iStartBlock, pi);
114152 if( rc!=SQLITE_OK ) goto finished;
114153 if( isPrefix==0 && isScan==0 ) iLeavesEndBlock = iStartBlock;
114154 }
114155
114156 rc = sqlite3Fts3SegReaderNew(pCsr->nSegment+1,
114157 iStartBlock, iLeavesEndBlock, iEndBlock, zRoot, nRoot, &pSeg
114158 );
114159 if( rc!=SQLITE_OK ) goto finished;
114160 rc = fts3SegReaderCursorAppend(pCsr, pSeg);
 
114161 }
114162 }
114163
114164 finished:
114165 rc2 = sqlite3_reset(pStmt);
114166 if( rc==SQLITE_DONE ) rc = rc2;
 
114167
114168 return rc;
114169 }
114170
114171 /*
114172 ** Set up a cursor object for iterating through a full-text index or a
114173 ** single level therein.
114174 */
114175 SQLITE_PRIVATE int sqlite3Fts3SegReaderCursor(
114176 Fts3Table *p, /* FTS3 table handle */
114177 int iIndex, /* Index to search (from 0 to p->nIndex-1) */
114178 int iLevel, /* Level of segments to scan */
114179 const char *zTerm, /* Term to query for */
114180 int nTerm, /* Size of zTerm in bytes */
114181 int isPrefix, /* True for a prefix search */
114182 int isScan, /* True to scan from zTerm to EOF */
114183 Fts3MultiSegReader *pCsr /* Cursor object to populate */
114184 ){
114185 assert( iIndex>=0 && iIndex<p->nIndex );
114186 assert( iLevel==FTS3_SEGCURSOR_ALL
114187 || iLevel==FTS3_SEGCURSOR_PENDING
114188 || iLevel>=0
114189 );
114190 assert( iLevel<FTS3_SEGDIR_MAXLEVEL );
114191 assert( FTS3_SEGCURSOR_ALL<0 && FTS3_SEGCURSOR_PENDING<0 );
114192 assert( isPrefix==0 || isScan==0 );
114193
114194 /* "isScan" is only set to true by the ft4aux module, an ordinary
114195 ** full-text tables. */
114196 assert( isScan==0 || p->aIndex==0 );
114197
114198 memset(pCsr, 0, sizeof(Fts3MultiSegReader));
114199
114200 return fts3SegReaderCursor(
114201 p, iIndex, iLevel, zTerm, nTerm, isPrefix, isScan, pCsr
114202 );
114203 }
114204
114205 static int fts3SegReaderCursorAddZero(
114206 Fts3Table *p,
114207 const char *zTerm,
114208 int nTerm,
114209 Fts3MultiSegReader *pCsr
114210 ){
114211 return fts3SegReaderCursor(p, 0, FTS3_SEGCURSOR_ALL, zTerm, nTerm, 0, 0,pCsr);
114212 }
114213
114214
114215 SQLITE_PRIVATE int sqlite3Fts3TermSegReaderCursor(
114216 Fts3Cursor *pCsr, /* Virtual table cursor handle */
114217 const char *zTerm, /* Term to query for */
114218 int nTerm, /* Size of zTerm in bytes */
114219 int isPrefix, /* True for a prefix search */
114220 Fts3MultiSegReader **ppSegcsr /* OUT: Allocated seg-reader cursor */
114221 ){
114222 Fts3MultiSegReader *pSegcsr; /* Object to allocate and return */
114223 int rc = SQLITE_NOMEM; /* Return code */
114224
114225 pSegcsr = sqlite3_malloc(sizeof(Fts3MultiSegReader));
114226 if( pSegcsr ){
114227 int i;
114228 int bFound = 0; /* True once an index has been found */
114229 Fts3Table *p = (Fts3Table *)pCsr->base.pVtab;
114230
114231 if( isPrefix ){
114232 for(i=1; bFound==0 && i<p->nIndex; i++){
114233 if( p->aIndex[i].nPrefix==nTerm ){
114234 bFound = 1;
114235 rc = sqlite3Fts3SegReaderCursor(
114236 p, i, FTS3_SEGCURSOR_ALL, zTerm, nTerm, 0, 0, pSegcsr);
114237 pSegcsr->bLookup = 1;
114238 }
114239 }
114240
114241 for(i=1; bFound==0 && i<p->nIndex; i++){
114242 if( p->aIndex[i].nPrefix==nTerm+1 ){
114243 bFound = 1;
114244 rc = sqlite3Fts3SegReaderCursor(
114245 p, i, FTS3_SEGCURSOR_ALL, zTerm, nTerm, 1, 0, pSegcsr
114246 );
114247 if( rc==SQLITE_OK ){
114248 rc = fts3SegReaderCursorAddZero(p, zTerm, nTerm, pSegcsr);
114249 }
114250 }
114251 }
114252 }
114253
114254 if( bFound==0 ){
114255 rc = sqlite3Fts3SegReaderCursor(
114256 p, 0, FTS3_SEGCURSOR_ALL, zTerm, nTerm, isPrefix, 0, pSegcsr
114257 );
114258 pSegcsr->bLookup = !isPrefix;
114259 }
114260 }
114261
114262 *ppSegcsr = pSegcsr;
114263 return rc;
114264 }
114265
114266 static void fts3SegReaderCursorFree(Fts3MultiSegReader *pSegcsr){
114267 sqlite3Fts3SegReaderFinish(pSegcsr);
114268 sqlite3_free(pSegcsr);
114269 }
114270
114271 /*
@@ -114065,11 +114286,11 @@
114286 int isReqPos, /* True to include position lists in output */
114287 int *pnOut, /* OUT: Size of buffer at *ppOut */
114288 char **ppOut /* OUT: Malloced result buffer */
114289 ){
114290 int rc; /* Return code */
114291 Fts3MultiSegReader *pSegcsr; /* Seg-reader cursor for this term */
114292 TermSelect tsc; /* Context object for fts3TermSelectCb() */
114293 Fts3SegFilter filter; /* Segment term filter configuration */
114294
114295 pSegcsr = pTok->pSegcsr;
114296 memset(&tsc, 0, sizeof(TermSelect));
@@ -114091,11 +114312,11 @@
114312 pSegcsr->zTerm, pSegcsr->nTerm, pSegcsr->aDoclist, pSegcsr->nDoclist
114313 );
114314 }
114315
114316 if( rc==SQLITE_OK ){
114317 rc = fts3TermSelectMerge(p, &tsc);
114318 }
114319 if( rc==SQLITE_OK ){
114320 *ppOut = tsc.aaOutput[0];
114321 *pnOut = tsc.anOutput[0];
114322 }else{
@@ -114141,664 +114362,10 @@
114362 }
114363
114364 return nDoc;
114365 }
114366
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
114367 /*
114368 ** Advance the cursor to the next row in the %_content table that
114369 ** matches the search criteria. For a MATCH search, this will be
114370 ** the next row that matches. For a full-table scan, this will be
114371 ** simply the next row in the %_content table. For a docid lookup,
@@ -114807,43 +114374,24 @@
114374 ** Return SQLITE_OK if nothing goes wrong. SQLITE_OK is returned
114375 ** even if we reach end-of-file. The fts3EofMethod() will be called
114376 ** subsequently to determine whether or not an EOF was hit.
114377 */
114378 static int fts3NextMethod(sqlite3_vtab_cursor *pCursor){
114379 int rc;
 
114380 Fts3Cursor *pCsr = (Fts3Cursor *)pCursor;
114381 if( pCsr->eSearch==FTS3_DOCID_SEARCH || pCsr->eSearch==FTS3_FULLSCAN_SEARCH ){
114382 if( SQLITE_ROW!=sqlite3_step(pCsr->pStmt) ){
114383 pCsr->isEof = 1;
114384 rc = sqlite3_reset(pCsr->pStmt);
114385 }else{
 
 
 
 
114386 pCsr->iPrevId = sqlite3_column_int64(pCsr->pStmt, 0);
114387 rc = SQLITE_OK;
114388 }
114389 }else{
114390 rc = sqlite3Fts3EvalNext((Fts3Cursor *)pCursor);
114391 }
114392 assert( ((Fts3Table *)pCsr->base.pVtab)->pSegments==0 );
 
 
 
 
 
 
 
 
 
 
 
 
 
 
114393 return rc;
114394 }
114395
114396 /*
114397 ** This is the xFilter interface for the virtual table. See
@@ -114866,15 +114414,11 @@
114414 int idxNum, /* Strategy index */
114415 const char *idxStr, /* Unused */
114416 int nVal, /* Number of elements in apVal */
114417 sqlite3_value **apVal /* Arguments for the indexing scheme */
114418 ){
114419 int rc;
 
 
 
 
114420 char *zSql; /* SQL statement used to access %_content */
114421 Fts3Table *p = (Fts3Table *)pCursor->pVtab;
114422 Fts3Cursor *pCsr = (Fts3Cursor *)pCursor;
114423
114424 UNUSED_PARAMETER(idxStr);
@@ -114889,10 +114433,17 @@
114433 sqlite3_finalize(pCsr->pStmt);
114434 sqlite3_free(pCsr->aDoclist);
114435 sqlite3Fts3ExprFree(pCsr->pExpr);
114436 memset(&pCursor[1], 0, sizeof(Fts3Cursor)-sizeof(sqlite3_vtab_cursor));
114437
114438 if( idxStr ){
114439 pCsr->bDesc = (idxStr[0]=='D');
114440 }else{
114441 pCsr->bDesc = p->bDescIdx;
114442 }
114443 pCsr->eSearch = (i16)idxNum;
114444
114445 if( idxNum!=FTS3_DOCID_SEARCH && idxNum!=FTS3_FULLSCAN_SEARCH ){
114446 int iCol = idxNum-FTS3_FULLTEXT_SEARCH;
114447 const char *zQuery = (const char *)sqlite3_value_text(apVal[0]);
114448
114449 if( zQuery==0 && sqlite3_value_type(apVal[0])!=SQLITE_NULL ){
@@ -114902,20 +114453,21 @@
114453 rc = sqlite3Fts3ExprParse(p->pTokenizer, p->azColumn, p->nColumn,
114454 iCol, zQuery, -1, &pCsr->pExpr
114455 );
114456 if( rc!=SQLITE_OK ){
114457 if( rc==SQLITE_ERROR ){
114458 static const char *zErr = "malformed MATCH expression: [%s]";
114459 p->base.zErrMsg = sqlite3_mprintf(zErr, zQuery);
114460 }
114461 return rc;
114462 }
114463
114464 rc = sqlite3Fts3ReadLock(p);
114465 if( rc!=SQLITE_OK ) return rc;
114466
114467 rc = sqlite3Fts3EvalStart(pCsr, pCsr->pExpr, 1);
114468
114469 sqlite3Fts3SegmentsClose(p);
114470 if( rc!=SQLITE_OK ) return rc;
114471 pCsr->pNextId = pCsr->aDoclist;
114472 pCsr->iPrevId = 0;
114473 }
@@ -114923,41 +114475,28 @@
114475 /* Compile a SELECT statement for this cursor. For a full-table-scan, the
114476 ** statement loops through all rows of the %_content table. For a
114477 ** full-text query or docid lookup, the statement retrieves a single
114478 ** row by docid.
114479 */
114480 if( idxNum==FTS3_FULLSCAN_SEARCH ){
114481 const char *zSort = (pCsr->bDesc ? "DESC" : "ASC");
114482 const char *zTmpl = "SELECT %s FROM %Q.'%q_content' AS x ORDER BY docid %s";
114483 zSql = sqlite3_mprintf(zTmpl, p->zReadExprlist, p->zDb, p->zName, zSort);
 
 
114484 }else{
114485 const char *zTmpl = "SELECT %s FROM %Q.'%q_content' AS x WHERE docid = ?";
114486 zSql = sqlite3_mprintf(zTmpl, p->zReadExprlist, p->zDb, p->zName);
114487 }
114488 if( !zSql ) return SQLITE_NOMEM;
114489 rc = sqlite3_prepare_v2(p->db, zSql, -1, &pCsr->pStmt, 0);
114490 sqlite3_free(zSql);
114491 if( rc!=SQLITE_OK ) return rc;
114492
114493 if( idxNum==FTS3_DOCID_SEARCH ){
114494 rc = sqlite3_bind_value(pCsr->pStmt, 1, apVal[0]);
114495 if( rc!=SQLITE_OK ) return rc;
114496 }
114497
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
114498 return fts3NextMethod(pCursor);
114499 }
114500
114501 /*
114502 ** This is the xEof method of the virtual table. SQLite calls this
@@ -114973,20 +114512,11 @@
114512 ** exposes %_content.docid as the rowid for the virtual table. The
114513 ** rowid should be written to *pRowid.
114514 */
114515 static int fts3RowidMethod(sqlite3_vtab_cursor *pCursor, sqlite_int64 *pRowid){
114516 Fts3Cursor *pCsr = (Fts3Cursor *) pCursor;
114517 *pRowid = pCsr->iPrevId;
 
 
 
 
 
 
 
 
 
114518 return SQLITE_OK;
114519 }
114520
114521 /*
114522 ** This is the xColumn method, called by SQLite to request a value from
@@ -114995,11 +114525,11 @@
114525 static int fts3ColumnMethod(
114526 sqlite3_vtab_cursor *pCursor, /* Cursor to retrieve value from */
114527 sqlite3_context *pContext, /* Context for sqlite3_result_xxx() calls */
114528 int iCol /* Index of column to read value from */
114529 ){
114530 int rc = SQLITE_OK; /* Return Code */
114531 Fts3Cursor *pCsr = (Fts3Cursor *) pCursor;
114532 Fts3Table *p = (Fts3Table *)pCursor->pVtab;
114533
114534 /* The column value supplied by SQLite must be in range. */
114535 assert( iCol>=0 && iCol<=p->nColumn+1 );
@@ -115006,25 +114536,24 @@
114536
114537 if( iCol==p->nColumn+1 ){
114538 /* This call is a request for the "docid" column. Since "docid" is an
114539 ** alias for "rowid", use the xRowid() method to obtain the value.
114540 */
114541 sqlite3_result_int64(pContext, pCsr->iPrevId);
 
 
114542 }else if( iCol==p->nColumn ){
114543 /* The extra column whose name is the same as the table.
114544 ** Return a blob which is a pointer to the cursor.
114545 */
114546 sqlite3_result_blob(pContext, &pCsr, sizeof(pCsr), SQLITE_TRANSIENT);
 
114547 }else{
114548 rc = fts3CursorSeek(0, pCsr);
114549 if( rc==SQLITE_OK ){
114550 sqlite3_result_value(pContext, sqlite3_column_value(pCsr->pStmt, iCol+1));
114551 }
114552 }
114553
114554 assert( ((Fts3Table *)pCsr->base.pVtab)->pSegments==0 );
114555 return rc;
114556 }
114557
114558 /*
114559 ** This function is the implementation of the xUpdate callback used by
@@ -115054,10 +114583,11 @@
114583 ** Implementation of xBegin() method. This is a no-op.
114584 */
114585 static int fts3BeginMethod(sqlite3_vtab *pVtab){
114586 UNUSED_PARAMETER(pVtab);
114587 TESTONLY( Fts3Table *p = (Fts3Table*)pVtab );
114588 assert( p->pSegments==0 );
114589 assert( p->nPendingData==0 );
114590 assert( p->inTransaction!=1 );
114591 TESTONLY( p->inTransaction = 1 );
114592 TESTONLY( p->mxSavepoint = -1; );
114593 return SQLITE_OK;
@@ -115071,10 +114601,11 @@
114601 static int fts3CommitMethod(sqlite3_vtab *pVtab){
114602 UNUSED_PARAMETER(pVtab);
114603 TESTONLY( Fts3Table *p = (Fts3Table*)pVtab );
114604 assert( p->nPendingData==0 );
114605 assert( p->inTransaction!=0 );
114606 assert( p->pSegments==0 );
114607 TESTONLY( p->inTransaction = 0 );
114608 TESTONLY( p->mxSavepoint = -1; );
114609 return SQLITE_OK;
114610 }
114611
@@ -115088,132 +114619,30 @@
114619 assert( p->inTransaction!=0 );
114620 TESTONLY( p->inTransaction = 0 );
114621 TESTONLY( p->mxSavepoint = -1; );
114622 return SQLITE_OK;
114623 }
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
114624
114625 /*
114626 ** When called, *ppPoslist must point to the byte immediately following the
114627 ** end of a position-list. i.e. ( (*ppPoslist)[-1]==POS_END ). This function
114628 ** moves *ppPoslist so that it instead points to the first byte of the
114629 ** same position list.
114630 */
114631 static void fts3ReversePoslist(char *pStart, char **ppPoslist){
114632 char *p = &(*ppPoslist)[-2];
114633 char c;
114634
114635 while( p>pStart && (c=*p--)==0 );
114636 while( p>pStart && (*p & 0x80) | c ){
114637 c = *p--;
114638 }
114639 if( p>pStart ){ p = &p[2]; }
114640 while( *p++&0x80 );
114641 *ppPoslist = p;
114642 }
114643
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
114644 /*
114645 ** Helper function used by the implementation of the overloaded snippet(),
114646 ** offsets() and optimize() SQL functions.
114647 **
114648 ** If the value passed as the third argument is a blob of size
@@ -115441,16 +114870,15 @@
114870 );
114871 return rc;
114872 }
114873
114874 static int fts3SavepointMethod(sqlite3_vtab *pVtab, int iSavepoint){
 
114875 UNUSED_PARAMETER(iSavepoint);
114876 assert( ((Fts3Table *)pVtab)->inTransaction );
114877 assert( ((Fts3Table *)pVtab)->mxSavepoint < iSavepoint );
114878 TESTONLY( ((Fts3Table *)pVtab)->mxSavepoint = iSavepoint );
114879 return fts3SyncMethod(pVtab);
114880 }
114881 static int fts3ReleaseMethod(sqlite3_vtab *pVtab, int iSavepoint){
114882 TESTONLY( Fts3Table *p = (Fts3Table*)pVtab );
114883 UNUSED_PARAMETER(iSavepoint);
114884 UNUSED_PARAMETER(pVtab);
@@ -115617,10 +115045,1282 @@
115045 SQLITE_EXTENSION_INIT2(pApi)
115046 return sqlite3Fts3Init(db);
115047 }
115048 #endif
115049
115050
115051 /*
115052 ** Allocate an Fts3MultiSegReader for each token in the expression headed
115053 ** by pExpr.
115054 **
115055 ** An Fts3SegReader object is a cursor that can seek or scan a range of
115056 ** entries within a single segment b-tree. An Fts3MultiSegReader uses multiple
115057 ** Fts3SegReader objects internally to provide an interface to seek or scan
115058 ** within the union of all segments of a b-tree. Hence the name.
115059 **
115060 ** If the allocated Fts3MultiSegReader just seeks to a single entry in a
115061 ** segment b-tree (if the term is not a prefix or it is a prefix for which
115062 ** there exists prefix b-tree of the right length) then it may be traversed
115063 ** and merged incrementally. Otherwise, it has to be merged into an in-memory
115064 ** doclist and then traversed.
115065 */
115066 static void fts3EvalAllocateReaders(
115067 Fts3Cursor *pCsr,
115068 Fts3Expr *pExpr,
115069 int *pnToken, /* OUT: Total number of tokens in phrase. */
115070 int *pnOr, /* OUT: Total number of OR nodes in expr. */
115071 int *pRc
115072 ){
115073 if( pExpr && SQLITE_OK==*pRc ){
115074 if( pExpr->eType==FTSQUERY_PHRASE ){
115075 int i;
115076 int nToken = pExpr->pPhrase->nToken;
115077 *pnToken += nToken;
115078 for(i=0; i<nToken; i++){
115079 Fts3PhraseToken *pToken = &pExpr->pPhrase->aToken[i];
115080 int rc = sqlite3Fts3TermSegReaderCursor(pCsr,
115081 pToken->z, pToken->n, pToken->isPrefix, &pToken->pSegcsr
115082 );
115083 if( rc!=SQLITE_OK ){
115084 *pRc = rc;
115085 return;
115086 }
115087 }
115088 }else{
115089 *pnOr += (pExpr->eType==FTSQUERY_OR);
115090 fts3EvalAllocateReaders(pCsr, pExpr->pLeft, pnToken, pnOr, pRc);
115091 fts3EvalAllocateReaders(pCsr, pExpr->pRight, pnToken, pnOr, pRc);
115092 }
115093 }
115094 }
115095
115096 static int fts3EvalPhraseLoad(
115097 Fts3Cursor *pCsr,
115098 Fts3Phrase *p
115099 ){
115100 Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab;
115101 int iToken;
115102 int rc = SQLITE_OK;
115103
115104 char *aDoclist = 0;
115105 int nDoclist = 0;
115106 int iPrev = -1;
115107
115108 for(iToken=0; rc==SQLITE_OK && iToken<p->nToken; iToken++){
115109 Fts3PhraseToken *pToken = &p->aToken[iToken];
115110 assert( pToken->pSegcsr || pToken->pDeferred );
115111
115112 if( pToken->pDeferred==0 ){
115113 int nThis = 0;
115114 char *pThis = 0;
115115 rc = fts3TermSelect(pTab, pToken, p->iColumn, 1, &nThis, &pThis);
115116 if( rc==SQLITE_OK ){
115117 if( pThis==0 ){
115118 sqlite3_free(aDoclist);
115119 aDoclist = 0;
115120 nDoclist = 0;
115121 break;
115122 }else if( aDoclist==0 ){
115123 aDoclist = pThis;
115124 nDoclist = nThis;
115125 }else{
115126 assert( iPrev>=0 );
115127 fts3DoclistPhraseMerge(pTab->bDescIdx,
115128 iToken-iPrev, aDoclist, nDoclist, pThis, &nThis
115129 );
115130 sqlite3_free(aDoclist);
115131 aDoclist = pThis;
115132 nDoclist = nThis;
115133 }
115134 iPrev = iToken;
115135 }
115136 }
115137 }
115138
115139 if( rc==SQLITE_OK ){
115140 p->doclist.aAll = aDoclist;
115141 p->doclist.nAll = nDoclist;
115142 }else{
115143 sqlite3_free(aDoclist);
115144 }
115145 return rc;
115146 }
115147
115148 static int fts3EvalDeferredPhrase(Fts3Cursor *pCsr, Fts3Phrase *pPhrase){
115149 int iToken;
115150 int rc = SQLITE_OK;
115151
115152 int nMaxUndeferred = -1;
115153 char *aPoslist = 0;
115154 int nPoslist = 0;
115155 int iPrev = -1;
115156
115157 assert( pPhrase->doclist.bFreeList==0 );
115158
115159 for(iToken=0; rc==SQLITE_OK && iToken<pPhrase->nToken; iToken++){
115160 Fts3PhraseToken *pToken = &pPhrase->aToken[iToken];
115161 Fts3DeferredToken *pDeferred = pToken->pDeferred;
115162
115163 if( pDeferred ){
115164 char *pList;
115165 int nList;
115166 rc = sqlite3Fts3DeferredTokenList(pDeferred, &pList, &nList);
115167 if( rc!=SQLITE_OK ) return rc;
115168
115169 if( pList==0 ){
115170 sqlite3_free(aPoslist);
115171 pPhrase->doclist.pList = 0;
115172 pPhrase->doclist.nList = 0;
115173 return SQLITE_OK;
115174
115175 }else if( aPoslist==0 ){
115176 aPoslist = pList;
115177 nPoslist = nList;
115178
115179 }else{
115180 char *aOut = pList;
115181 char *p1 = aPoslist;
115182 char *p2 = aOut;
115183
115184 assert( iPrev>=0 );
115185 fts3PoslistPhraseMerge(&aOut, iToken-iPrev, 0, 1, &p1, &p2);
115186 sqlite3_free(aPoslist);
115187 aPoslist = pList;
115188 nPoslist = aOut - aPoslist;
115189 if( nPoslist==0 ){
115190 sqlite3_free(aPoslist);
115191 pPhrase->doclist.pList = 0;
115192 pPhrase->doclist.nList = 0;
115193 return SQLITE_OK;
115194 }
115195 }
115196 iPrev = iToken;
115197 }else{
115198 nMaxUndeferred = iToken;
115199 }
115200 }
115201
115202 if( iPrev>=0 ){
115203 if( nMaxUndeferred<0 ){
115204 pPhrase->doclist.pList = aPoslist;
115205 pPhrase->doclist.nList = nPoslist;
115206 pPhrase->doclist.iDocid = pCsr->iPrevId;
115207 pPhrase->doclist.bFreeList = 1;
115208 }else{
115209 int nDistance;
115210 char *p1;
115211 char *p2;
115212 char *aOut;
115213
115214 if( nMaxUndeferred>iPrev ){
115215 p1 = aPoslist;
115216 p2 = pPhrase->doclist.pList;
115217 nDistance = nMaxUndeferred - iPrev;
115218 }else{
115219 p1 = pPhrase->doclist.pList;
115220 p2 = aPoslist;
115221 nDistance = iPrev - nMaxUndeferred;
115222 }
115223
115224 aOut = (char *)sqlite3_malloc(nPoslist+8);
115225 if( !aOut ){
115226 sqlite3_free(aPoslist);
115227 return SQLITE_NOMEM;
115228 }
115229
115230 pPhrase->doclist.pList = aOut;
115231 if( fts3PoslistPhraseMerge(&aOut, nDistance, 0, 1, &p1, &p2) ){
115232 pPhrase->doclist.bFreeList = 1;
115233 pPhrase->doclist.nList = (aOut - pPhrase->doclist.pList);
115234 }else{
115235 sqlite3_free(aOut);
115236 pPhrase->doclist.pList = 0;
115237 pPhrase->doclist.nList = 0;
115238 }
115239 sqlite3_free(aPoslist);
115240 }
115241 }
115242
115243 return SQLITE_OK;
115244 }
115245
115246 /*
115247 ** This function is called for each Fts3Phrase in a full-text query
115248 ** expression to initialize the mechanism for returning rows. Once this
115249 ** function has been called successfully on an Fts3Phrase, it may be
115250 ** used with fts3EvalPhraseNext() to iterate through the matching docids.
115251 */
115252 static int fts3EvalPhraseStart(Fts3Cursor *pCsr, int bOptOk, Fts3Phrase *p){
115253 int rc;
115254 Fts3PhraseToken *pFirst = &p->aToken[0];
115255 Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab;
115256
115257 assert( p->doclist.aAll==0 );
115258 if( pCsr->bDesc==pTab->bDescIdx && bOptOk==1 && p->nToken==1
115259 && pFirst->pSegcsr && pFirst->pSegcsr->bLookup
115260 ){
115261 /* Use the incremental approach. */
115262 int iCol = (p->iColumn >= pTab->nColumn ? -1 : p->iColumn);
115263 rc = sqlite3Fts3MsrIncrStart(
115264 pTab, pFirst->pSegcsr, iCol, pFirst->z, pFirst->n);
115265 p->bIncr = 1;
115266
115267 }else{
115268 /* Load the full doclist for the phrase into memory. */
115269 rc = fts3EvalPhraseLoad(pCsr, p);
115270 p->bIncr = 0;
115271 }
115272
115273 assert( rc!=SQLITE_OK || p->nToken<1 || p->aToken[0].pSegcsr==0 || p->bIncr );
115274 return rc;
115275 }
115276
115277 /*
115278 ** This function is used to iterate backwards (from the end to start)
115279 ** through doclists.
115280 */
115281 SQLITE_PRIVATE void sqlite3Fts3DoclistPrev(
115282 int bDescIdx, /* True if the doclist is desc */
115283 char *aDoclist, /* Pointer to entire doclist */
115284 int nDoclist, /* Length of aDoclist in bytes */
115285 char **ppIter, /* IN/OUT: Iterator pointer */
115286 sqlite3_int64 *piDocid, /* IN/OUT: Docid pointer */
115287 int *pnList, /* IN/OUT: List length pointer */
115288 u8 *pbEof /* OUT: End-of-file flag */
115289 ){
115290 char *p = *ppIter;
115291
115292 assert( nDoclist>0 );
115293 assert( *pbEof==0 );
115294 assert( p || *piDocid==0 );
115295 assert( !p || (p>aDoclist && p<&aDoclist[nDoclist]) );
115296
115297 if( p==0 ){
115298 sqlite3_int64 iDocid = 0;
115299 char *pNext = 0;
115300 char *pDocid = aDoclist;
115301 char *pEnd = &aDoclist[nDoclist];
115302 int iMul = 1;
115303
115304 while( pDocid<pEnd ){
115305 sqlite3_int64 iDelta;
115306 pDocid += sqlite3Fts3GetVarint(pDocid, &iDelta);
115307 iDocid += (iMul * iDelta);
115308 pNext = pDocid;
115309 fts3PoslistCopy(0, &pDocid);
115310 while( pDocid<pEnd && *pDocid==0 ) pDocid++;
115311 iMul = (bDescIdx ? -1 : 1);
115312 }
115313
115314 *pnList = pEnd - pNext;
115315 *ppIter = pNext;
115316 *piDocid = iDocid;
115317 }else{
115318 int iMul = (bDescIdx ? -1 : 1);
115319 sqlite3_int64 iDelta;
115320 fts3GetReverseVarint(&p, aDoclist, &iDelta);
115321 *piDocid -= (iMul * iDelta);
115322
115323 if( p==aDoclist ){
115324 *pbEof = 1;
115325 }else{
115326 char *pSave = p;
115327 fts3ReversePoslist(aDoclist, &p);
115328 *pnList = (pSave - p);
115329 }
115330 *ppIter = p;
115331 }
115332 }
115333
115334 /*
115335 ** Attempt to move the phrase iterator to point to the next matching docid.
115336 ** If an error occurs, return an SQLite error code. Otherwise, return
115337 ** SQLITE_OK.
115338 **
115339 ** If there is no "next" entry and no error occurs, then *pbEof is set to
115340 ** 1 before returning. Otherwise, if no error occurs and the iterator is
115341 ** successfully advanced, *pbEof is set to 0.
115342 */
115343 static int fts3EvalPhraseNext(
115344 Fts3Cursor *pCsr,
115345 Fts3Phrase *p,
115346 u8 *pbEof
115347 ){
115348 int rc = SQLITE_OK;
115349 Fts3Doclist *pDL = &p->doclist;
115350 Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab;
115351
115352 if( p->bIncr ){
115353 assert( p->nToken==1 );
115354 assert( pDL->pNextDocid==0 );
115355 rc = sqlite3Fts3MsrIncrNext(pTab, p->aToken[0].pSegcsr,
115356 &pDL->iDocid, &pDL->pList, &pDL->nList
115357 );
115358 if( rc==SQLITE_OK && !pDL->pList ){
115359 *pbEof = 1;
115360 }
115361 }else if( pCsr->bDesc!=pTab->bDescIdx && pDL->nAll ){
115362 sqlite3Fts3DoclistPrev(pTab->bDescIdx, pDL->aAll, pDL->nAll,
115363 &pDL->pNextDocid, &pDL->iDocid, &pDL->nList, pbEof
115364 );
115365 pDL->pList = pDL->pNextDocid;
115366 }else{
115367 char *pIter; /* Used to iterate through aAll */
115368 char *pEnd = &pDL->aAll[pDL->nAll]; /* 1 byte past end of aAll */
115369 if( pDL->pNextDocid ){
115370 pIter = pDL->pNextDocid;
115371 }else{
115372 pIter = pDL->aAll;
115373 }
115374
115375 if( pIter>=pEnd ){
115376 /* We have already reached the end of this doclist. EOF. */
115377 *pbEof = 1;
115378 }else{
115379 sqlite3_int64 iDelta;
115380 pIter += sqlite3Fts3GetVarint(pIter, &iDelta);
115381 if( pTab->bDescIdx==0 || pDL->pNextDocid==0 ){
115382 pDL->iDocid += iDelta;
115383 }else{
115384 pDL->iDocid -= iDelta;
115385 }
115386 pDL->pList = pIter;
115387 fts3PoslistCopy(0, &pIter);
115388 pDL->nList = (pIter - pDL->pList);
115389
115390 /* pIter now points just past the 0x00 that terminates the position-
115391 ** list for document pDL->iDocid. However, if this position-list was
115392 ** edited in place by fts3EvalNearTrim2(), then pIter may not actually
115393 ** point to the start of the next docid value. The following line deals
115394 ** with this case by advancing pIter past the zero-padding added by
115395 ** fts3EvalNearTrim2(). */
115396 while( pIter<pEnd && *pIter==0 ) pIter++;
115397
115398 pDL->pNextDocid = pIter;
115399 assert( *pIter || pIter>=&pDL->aAll[pDL->nAll] );
115400 *pbEof = 0;
115401 }
115402 }
115403
115404 return rc;
115405 }
115406
115407 static void fts3EvalStartReaders(
115408 Fts3Cursor *pCsr,
115409 Fts3Expr *pExpr,
115410 int bOptOk,
115411 int *pRc
115412 ){
115413 if( pExpr && SQLITE_OK==*pRc ){
115414 if( pExpr->eType==FTSQUERY_PHRASE ){
115415 int i;
115416 int nToken = pExpr->pPhrase->nToken;
115417 for(i=0; i<nToken; i++){
115418 if( pExpr->pPhrase->aToken[i].pDeferred==0 ) break;
115419 }
115420 pExpr->bDeferred = (i==nToken);
115421 *pRc = fts3EvalPhraseStart(pCsr, bOptOk, pExpr->pPhrase);
115422 }else{
115423 fts3EvalStartReaders(pCsr, pExpr->pLeft, bOptOk, pRc);
115424 fts3EvalStartReaders(pCsr, pExpr->pRight, bOptOk, pRc);
115425 pExpr->bDeferred = (pExpr->pLeft->bDeferred && pExpr->pRight->bDeferred);
115426 }
115427 }
115428 }
115429
115430
115431 typedef struct Fts3TokenAndCost Fts3TokenAndCost;
115432 struct Fts3TokenAndCost {
115433 Fts3PhraseToken *pToken;
115434 Fts3Expr *pRoot;
115435 int nOvfl;
115436 int iCol;
115437 };
115438
115439 static void fts3EvalTokenCosts(
115440 Fts3Cursor *pCsr,
115441 Fts3Expr *pRoot,
115442 Fts3Expr *pExpr,
115443 Fts3TokenAndCost **ppTC,
115444 Fts3Expr ***ppOr,
115445 int *pRc
115446 ){
115447 if( *pRc==SQLITE_OK && pExpr ){
115448 if( pExpr->eType==FTSQUERY_PHRASE ){
115449 Fts3Phrase *pPhrase = pExpr->pPhrase;
115450 int i;
115451 for(i=0; *pRc==SQLITE_OK && i<pPhrase->nToken; i++){
115452 Fts3TokenAndCost *pTC = (*ppTC)++;
115453 pTC->pRoot = pRoot;
115454 pTC->pToken = &pPhrase->aToken[i];
115455 pTC->iCol = pPhrase->iColumn;
115456 *pRc = sqlite3Fts3MsrOvfl(pCsr, pTC->pToken->pSegcsr, &pTC->nOvfl);
115457 }
115458 }else if( pExpr->eType!=FTSQUERY_NOT ){
115459 if( pExpr->eType==FTSQUERY_OR ){
115460 pRoot = pExpr->pLeft;
115461 **ppOr = pRoot;
115462 (*ppOr)++;
115463 }
115464 fts3EvalTokenCosts(pCsr, pRoot, pExpr->pLeft, ppTC, ppOr, pRc);
115465 if( pExpr->eType==FTSQUERY_OR ){
115466 pRoot = pExpr->pRight;
115467 **ppOr = pRoot;
115468 (*ppOr)++;
115469 }
115470 fts3EvalTokenCosts(pCsr, pRoot, pExpr->pRight, ppTC, ppOr, pRc);
115471 }
115472 }
115473 }
115474
115475 static int fts3EvalAverageDocsize(Fts3Cursor *pCsr, int *pnPage){
115476 if( pCsr->nRowAvg==0 ){
115477 /* The average document size, which is required to calculate the cost
115478 ** of each doclist, has not yet been determined. Read the required
115479 ** data from the %_stat table to calculate it.
115480 **
115481 ** Entry 0 of the %_stat table is a blob containing (nCol+1) FTS3
115482 ** varints, where nCol is the number of columns in the FTS3 table.
115483 ** The first varint is the number of documents currently stored in
115484 ** the table. The following nCol varints contain the total amount of
115485 ** data stored in all rows of each column of the table, from left
115486 ** to right.
115487 */
115488 int rc;
115489 Fts3Table *p = (Fts3Table*)pCsr->base.pVtab;
115490 sqlite3_stmt *pStmt;
115491 sqlite3_int64 nDoc = 0;
115492 sqlite3_int64 nByte = 0;
115493 const char *pEnd;
115494 const char *a;
115495
115496 rc = sqlite3Fts3SelectDoctotal(p, &pStmt);
115497 if( rc!=SQLITE_OK ) return rc;
115498 a = sqlite3_column_blob(pStmt, 0);
115499 assert( a );
115500
115501 pEnd = &a[sqlite3_column_bytes(pStmt, 0)];
115502 a += sqlite3Fts3GetVarint(a, &nDoc);
115503 while( a<pEnd ){
115504 a += sqlite3Fts3GetVarint(a, &nByte);
115505 }
115506 if( nDoc==0 || nByte==0 ){
115507 sqlite3_reset(pStmt);
115508 return SQLITE_CORRUPT_VTAB;
115509 }
115510
115511 pCsr->nDoc = nDoc;
115512 pCsr->nRowAvg = (int)(((nByte / nDoc) + p->nPgsz) / p->nPgsz);
115513 assert( pCsr->nRowAvg>0 );
115514 rc = sqlite3_reset(pStmt);
115515 if( rc!=SQLITE_OK ) return rc;
115516 }
115517
115518 *pnPage = pCsr->nRowAvg;
115519 return SQLITE_OK;
115520 }
115521
115522 static int fts3EvalSelectDeferred(
115523 Fts3Cursor *pCsr,
115524 Fts3Expr *pRoot,
115525 Fts3TokenAndCost *aTC,
115526 int nTC
115527 ){
115528 int nDocSize = 0;
115529 int nDocEst = 0;
115530 int rc = SQLITE_OK;
115531 Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab;
115532 int ii;
115533
115534 int nOvfl = 0;
115535 int nTerm = 0;
115536
115537 for(ii=0; ii<nTC; ii++){
115538 if( aTC[ii].pRoot==pRoot ){
115539 nOvfl += aTC[ii].nOvfl;
115540 nTerm++;
115541 }
115542 }
115543 if( nOvfl==0 || nTerm<2 ) return SQLITE_OK;
115544
115545 rc = fts3EvalAverageDocsize(pCsr, &nDocSize);
115546
115547 for(ii=0; ii<nTerm && rc==SQLITE_OK; ii++){
115548 int jj;
115549 Fts3TokenAndCost *pTC = 0;
115550
115551 for(jj=0; jj<nTC; jj++){
115552 if( aTC[jj].pToken && aTC[jj].pRoot==pRoot
115553 && (!pTC || aTC[jj].nOvfl<pTC->nOvfl)
115554 ){
115555 pTC = &aTC[jj];
115556 }
115557 }
115558 assert( pTC );
115559
115560 /* At this point pTC points to the cheapest remaining token. */
115561 if( ii==0 ){
115562 if( pTC->nOvfl ){
115563 nDocEst = (pTC->nOvfl * pTab->nPgsz + pTab->nPgsz) / 10;
115564 }else{
115565 /* TODO: Fix this so that the doclist need not be read twice. */
115566 Fts3PhraseToken *pToken = pTC->pToken;
115567 int nList = 0;
115568 char *pList = 0;
115569 rc = fts3TermSelect(pTab, pToken, pTC->iCol, 1, &nList, &pList);
115570 if( rc==SQLITE_OK ){
115571 nDocEst = fts3DoclistCountDocids(1, pList, nList);
115572 }
115573 sqlite3_free(pList);
115574 if( rc==SQLITE_OK ){
115575 rc = sqlite3Fts3TermSegReaderCursor(pCsr,
115576 pToken->z, pToken->n, pToken->isPrefix, &pToken->pSegcsr
115577 );
115578 }
115579 }
115580 }else{
115581 if( pTC->nOvfl>=(nDocEst*nDocSize) ){
115582 Fts3PhraseToken *pToken = pTC->pToken;
115583 rc = sqlite3Fts3DeferToken(pCsr, pToken, pTC->iCol);
115584 fts3SegReaderCursorFree(pToken->pSegcsr);
115585 pToken->pSegcsr = 0;
115586 }
115587 nDocEst = 1 + (nDocEst/4);
115588 }
115589 pTC->pToken = 0;
115590 }
115591
115592 return rc;
115593 }
115594
115595 SQLITE_PRIVATE int sqlite3Fts3EvalStart(Fts3Cursor *pCsr, Fts3Expr *pExpr, int bOptOk){
115596 Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab;
115597 int rc = SQLITE_OK;
115598 int nToken = 0;
115599 int nOr = 0;
115600
115601 /* Allocate a MultiSegReader for each token in the expression. */
115602 fts3EvalAllocateReaders(pCsr, pExpr, &nToken, &nOr, &rc);
115603
115604 /* Call fts3EvalPhraseStart() on all phrases in the expression. TODO:
115605 ** This call will eventually also be responsible for determining which
115606 ** tokens are 'deferred' until the document text is loaded into memory.
115607 **
115608 ** Each token in each phrase is dealt with using one of the following
115609 ** three strategies:
115610 **
115611 ** 1. Entire doclist loaded into memory as part of the
115612 ** fts3EvalStartReaders() call.
115613 **
115614 ** 2. Doclist loaded into memory incrementally, as part of each
115615 ** sqlite3Fts3EvalNext() call.
115616 **
115617 ** 3. Token doclist is never loaded. Instead, documents are loaded into
115618 ** memory and scanned for the token as part of the sqlite3Fts3EvalNext()
115619 ** call. This is known as a "deferred" token.
115620 */
115621
115622 /* If bOptOk is true, check if there are any tokens that should be deferred.
115623 */
115624 if( rc==SQLITE_OK && bOptOk && nToken>1 && pTab->bHasStat ){
115625 Fts3TokenAndCost *aTC;
115626 Fts3Expr **apOr;
115627 aTC = (Fts3TokenAndCost *)sqlite3_malloc(
115628 sizeof(Fts3TokenAndCost) * nToken
115629 + sizeof(Fts3Expr *) * nOr * 2
115630 );
115631 apOr = (Fts3Expr **)&aTC[nToken];
115632
115633 if( !aTC ){
115634 rc = SQLITE_NOMEM;
115635 }else{
115636 int ii;
115637 Fts3TokenAndCost *pTC = aTC;
115638 Fts3Expr **ppOr = apOr;
115639
115640 fts3EvalTokenCosts(pCsr, 0, pExpr, &pTC, &ppOr, &rc);
115641 nToken = pTC-aTC;
115642 nOr = ppOr-apOr;
115643
115644 if( rc==SQLITE_OK ){
115645 rc = fts3EvalSelectDeferred(pCsr, 0, aTC, nToken);
115646 for(ii=0; rc==SQLITE_OK && ii<nOr; ii++){
115647 rc = fts3EvalSelectDeferred(pCsr, apOr[ii], aTC, nToken);
115648 }
115649 }
115650
115651 sqlite3_free(aTC);
115652 }
115653 }
115654
115655 fts3EvalStartReaders(pCsr, pExpr, bOptOk, &rc);
115656 return rc;
115657 }
115658
115659 static void fts3EvalZeroPoslist(Fts3Phrase *pPhrase){
115660 if( pPhrase->doclist.bFreeList ){
115661 sqlite3_free(pPhrase->doclist.pList);
115662 }
115663 pPhrase->doclist.pList = 0;
115664 pPhrase->doclist.nList = 0;
115665 pPhrase->doclist.bFreeList = 0;
115666 }
115667
115668 static int fts3EvalNearTrim2(
115669 int nNear,
115670 char *aTmp, /* Temporary space to use */
115671 char **paPoslist, /* IN/OUT: Position list */
115672 int *pnToken, /* IN/OUT: Tokens in phrase of *paPoslist */
115673 Fts3Phrase *pPhrase /* The phrase object to trim the doclist of */
115674 ){
115675 int nParam1 = nNear + pPhrase->nToken;
115676 int nParam2 = nNear + *pnToken;
115677 int nNew;
115678 char *p2;
115679 char *pOut;
115680 int res;
115681
115682 assert( pPhrase->doclist.pList );
115683
115684 p2 = pOut = pPhrase->doclist.pList;
115685 res = fts3PoslistNearMerge(
115686 &pOut, aTmp, nParam1, nParam2, paPoslist, &p2
115687 );
115688 if( res ){
115689 nNew = (pOut - pPhrase->doclist.pList) - 1;
115690 assert( pPhrase->doclist.pList[nNew]=='\0' );
115691 assert( nNew<=pPhrase->doclist.nList && nNew>0 );
115692 memset(&pPhrase->doclist.pList[nNew], 0, pPhrase->doclist.nList - nNew);
115693 pPhrase->doclist.nList = nNew;
115694 *paPoslist = pPhrase->doclist.pList;
115695 *pnToken = pPhrase->nToken;
115696 }
115697
115698 return res;
115699 }
115700
115701 static int fts3EvalNearTest(Fts3Expr *pExpr, int *pRc){
115702 int res = 1;
115703
115704 /* The following block runs if pExpr is the root of a NEAR query.
115705 ** For example, the query:
115706 **
115707 ** "w" NEAR "x" NEAR "y" NEAR "z"
115708 **
115709 ** which is represented in tree form as:
115710 **
115711 ** |
115712 ** +--NEAR--+ <-- root of NEAR query
115713 ** | |
115714 ** +--NEAR--+ "z"
115715 ** | |
115716 ** +--NEAR--+ "y"
115717 ** | |
115718 ** "w" "x"
115719 **
115720 ** The right-hand child of a NEAR node is always a phrase. The
115721 ** left-hand child may be either a phrase or a NEAR node. There are
115722 ** no exceptions to this.
115723 */
115724 if( *pRc==SQLITE_OK
115725 && pExpr->eType==FTSQUERY_NEAR
115726 && pExpr->bEof==0
115727 && (pExpr->pParent==0 || pExpr->pParent->eType!=FTSQUERY_NEAR)
115728 ){
115729 Fts3Expr *p;
115730 int nTmp = 0; /* Bytes of temp space */
115731 char *aTmp; /* Temp space for PoslistNearMerge() */
115732
115733 /* Allocate temporary working space. */
115734 for(p=pExpr; p->pLeft; p=p->pLeft){
115735 nTmp += p->pRight->pPhrase->doclist.nList;
115736 }
115737 nTmp += p->pPhrase->doclist.nList;
115738 aTmp = sqlite3_malloc(nTmp*2);
115739 if( !aTmp ){
115740 *pRc = SQLITE_NOMEM;
115741 res = 0;
115742 }else{
115743 char *aPoslist = p->pPhrase->doclist.pList;
115744 int nToken = p->pPhrase->nToken;
115745
115746 for(p=p->pParent;res && p && p->eType==FTSQUERY_NEAR; p=p->pParent){
115747 Fts3Phrase *pPhrase = p->pRight->pPhrase;
115748 int nNear = p->nNear;
115749 res = fts3EvalNearTrim2(nNear, aTmp, &aPoslist, &nToken, pPhrase);
115750 }
115751
115752 aPoslist = pExpr->pRight->pPhrase->doclist.pList;
115753 nToken = pExpr->pRight->pPhrase->nToken;
115754 for(p=pExpr->pLeft; p && res; p=p->pLeft){
115755 int nNear = p->pParent->nNear;
115756 Fts3Phrase *pPhrase = (
115757 p->eType==FTSQUERY_NEAR ? p->pRight->pPhrase : p->pPhrase
115758 );
115759 res = fts3EvalNearTrim2(nNear, aTmp, &aPoslist, &nToken, pPhrase);
115760 }
115761 }
115762
115763 sqlite3_free(aTmp);
115764 }
115765
115766 return res;
115767 }
115768
115769 /*
115770 ** This macro is used by the fts3EvalNext() function. The two arguments are
115771 ** 64-bit docid values. If the current query is "ORDER BY docid ASC", then
115772 ** the macro returns (i1 - i2). Or if it is "ORDER BY docid DESC", then
115773 ** it returns (i2 - i1). This allows the same code to be used for merging
115774 ** doclists in ascending or descending order.
115775 */
115776 #define DOCID_CMP(i1, i2) ((pCsr->bDesc?-1:1) * (i1-i2))
115777
115778 static void fts3EvalNext(
115779 Fts3Cursor *pCsr,
115780 Fts3Expr *pExpr,
115781 int *pRc
115782 ){
115783 if( *pRc==SQLITE_OK ){
115784 assert( pExpr->bEof==0 );
115785 pExpr->bStart = 1;
115786
115787 switch( pExpr->eType ){
115788 case FTSQUERY_NEAR:
115789 case FTSQUERY_AND: {
115790 Fts3Expr *pLeft = pExpr->pLeft;
115791 Fts3Expr *pRight = pExpr->pRight;
115792 assert( !pLeft->bDeferred || !pRight->bDeferred );
115793 if( pLeft->bDeferred ){
115794 fts3EvalNext(pCsr, pRight, pRc);
115795 pExpr->iDocid = pRight->iDocid;
115796 pExpr->bEof = pRight->bEof;
115797 }else if( pRight->bDeferred ){
115798 fts3EvalNext(pCsr, pLeft, pRc);
115799 pExpr->iDocid = pLeft->iDocid;
115800 pExpr->bEof = pLeft->bEof;
115801 }else{
115802 fts3EvalNext(pCsr, pLeft, pRc);
115803 fts3EvalNext(pCsr, pRight, pRc);
115804
115805 while( !pLeft->bEof && !pRight->bEof && *pRc==SQLITE_OK ){
115806 sqlite3_int64 iDiff = DOCID_CMP(pLeft->iDocid, pRight->iDocid);
115807 if( iDiff==0 ) break;
115808 if( iDiff<0 ){
115809 fts3EvalNext(pCsr, pLeft, pRc);
115810 }else{
115811 fts3EvalNext(pCsr, pRight, pRc);
115812 }
115813 }
115814
115815 pExpr->iDocid = pLeft->iDocid;
115816 pExpr->bEof = (pLeft->bEof || pRight->bEof);
115817 }
115818 break;
115819 }
115820
115821 case FTSQUERY_OR: {
115822 Fts3Expr *pLeft = pExpr->pLeft;
115823 Fts3Expr *pRight = pExpr->pRight;
115824 sqlite3_int64 iCmp = DOCID_CMP(pLeft->iDocid, pRight->iDocid);
115825
115826 assert( pLeft->bStart || pLeft->iDocid==pRight->iDocid );
115827 assert( pRight->bStart || pLeft->iDocid==pRight->iDocid );
115828
115829 if( pRight->bEof || (pLeft->bEof==0 && iCmp<0) ){
115830 fts3EvalNext(pCsr, pLeft, pRc);
115831 }else if( pLeft->bEof || (pRight->bEof==0 && iCmp>0) ){
115832 fts3EvalNext(pCsr, pRight, pRc);
115833 }else{
115834 fts3EvalNext(pCsr, pLeft, pRc);
115835 fts3EvalNext(pCsr, pRight, pRc);
115836 }
115837
115838 pExpr->bEof = (pLeft->bEof && pRight->bEof);
115839 iCmp = DOCID_CMP(pLeft->iDocid, pRight->iDocid);
115840 if( pRight->bEof || (pLeft->bEof==0 && iCmp<0) ){
115841 pExpr->iDocid = pLeft->iDocid;
115842 }else{
115843 pExpr->iDocid = pRight->iDocid;
115844 }
115845
115846 break;
115847 }
115848
115849 case FTSQUERY_NOT: {
115850 Fts3Expr *pLeft = pExpr->pLeft;
115851 Fts3Expr *pRight = pExpr->pRight;
115852
115853 if( pRight->bStart==0 ){
115854 fts3EvalNext(pCsr, pRight, pRc);
115855 assert( *pRc!=SQLITE_OK || pRight->bStart );
115856 }
115857
115858 fts3EvalNext(pCsr, pLeft, pRc);
115859 if( pLeft->bEof==0 ){
115860 while( !*pRc
115861 && !pRight->bEof
115862 && DOCID_CMP(pLeft->iDocid, pRight->iDocid)>0
115863 ){
115864 fts3EvalNext(pCsr, pRight, pRc);
115865 }
115866 }
115867 pExpr->iDocid = pLeft->iDocid;
115868 pExpr->bEof = pLeft->bEof;
115869 break;
115870 }
115871
115872 default: {
115873 Fts3Phrase *pPhrase = pExpr->pPhrase;
115874 fts3EvalZeroPoslist(pPhrase);
115875 *pRc = fts3EvalPhraseNext(pCsr, pPhrase, &pExpr->bEof);
115876 pExpr->iDocid = pPhrase->doclist.iDocid;
115877 break;
115878 }
115879 }
115880 }
115881 }
115882
115883 static int fts3EvalDeferredTest(Fts3Cursor *pCsr, Fts3Expr *pExpr, int *pRc){
115884 int bHit = 1;
115885 if( *pRc==SQLITE_OK ){
115886 switch( pExpr->eType ){
115887 case FTSQUERY_NEAR:
115888 case FTSQUERY_AND:
115889 bHit = (
115890 fts3EvalDeferredTest(pCsr, pExpr->pLeft, pRc)
115891 && fts3EvalDeferredTest(pCsr, pExpr->pRight, pRc)
115892 && fts3EvalNearTest(pExpr, pRc)
115893 );
115894
115895 /* If the NEAR expression does not match any rows, zero the doclist for
115896 ** all phrases involved in the NEAR. This is because the snippet(),
115897 ** offsets() and matchinfo() functions are not supposed to recognize
115898 ** any instances of phrases that are part of unmatched NEAR queries.
115899 ** For example if this expression:
115900 **
115901 ** ... MATCH 'a OR (b NEAR c)'
115902 **
115903 ** is matched against a row containing:
115904 **
115905 ** 'a b d e'
115906 **
115907 ** then any snippet() should ony highlight the "a" term, not the "b"
115908 ** (as "b" is part of a non-matching NEAR clause).
115909 */
115910 if( bHit==0
115911 && pExpr->eType==FTSQUERY_NEAR
115912 && (pExpr->pParent==0 || pExpr->pParent->eType!=FTSQUERY_NEAR)
115913 ){
115914 Fts3Expr *p;
115915 for(p=pExpr; p->pPhrase==0; p=p->pLeft){
115916 if( p->pRight->iDocid==pCsr->iPrevId ){
115917 fts3EvalZeroPoslist(p->pRight->pPhrase);
115918 }
115919 }
115920 if( p->iDocid==pCsr->iPrevId ){
115921 fts3EvalZeroPoslist(p->pPhrase);
115922 }
115923 }
115924
115925 break;
115926
115927 case FTSQUERY_OR: {
115928 int bHit1 = fts3EvalDeferredTest(pCsr, pExpr->pLeft, pRc);
115929 int bHit2 = fts3EvalDeferredTest(pCsr, pExpr->pRight, pRc);
115930 bHit = bHit1 || bHit2;
115931 break;
115932 }
115933
115934 case FTSQUERY_NOT:
115935 bHit = (
115936 fts3EvalDeferredTest(pCsr, pExpr->pLeft, pRc)
115937 && !fts3EvalDeferredTest(pCsr, pExpr->pRight, pRc)
115938 );
115939 break;
115940
115941 default: {
115942 if( pCsr->pDeferred
115943 && (pExpr->iDocid==pCsr->iPrevId || pExpr->bDeferred)
115944 ){
115945 Fts3Phrase *pPhrase = pExpr->pPhrase;
115946 assert( pExpr->bDeferred || pPhrase->doclist.bFreeList==0 );
115947 if( pExpr->bDeferred ){
115948 fts3EvalZeroPoslist(pPhrase);
115949 }
115950 *pRc = fts3EvalDeferredPhrase(pCsr, pPhrase);
115951 bHit = (pPhrase->doclist.pList!=0);
115952 pExpr->iDocid = pCsr->iPrevId;
115953 }else{
115954 bHit = (pExpr->bEof==0 && pExpr->iDocid==pCsr->iPrevId);
115955 }
115956 break;
115957 }
115958 }
115959 }
115960 return bHit;
115961 }
115962
115963 /*
115964 ** Return 1 if both of the following are true:
115965 **
115966 ** 1. *pRc is SQLITE_OK when this function returns, and
115967 **
115968 ** 2. After scanning the current FTS table row for the deferred tokens,
115969 ** it is determined that the row does not match the query.
115970 **
115971 ** Or, if no error occurs and it seems the current row does match the FTS
115972 ** query, return 0.
115973 */
115974 static int fts3EvalLoadDeferred(Fts3Cursor *pCsr, int *pRc){
115975 int rc = *pRc;
115976 int bMiss = 0;
115977 if( rc==SQLITE_OK ){
115978 if( pCsr->pDeferred ){
115979 rc = fts3CursorSeek(0, pCsr);
115980 if( rc==SQLITE_OK ){
115981 rc = sqlite3Fts3CacheDeferredDoclists(pCsr);
115982 }
115983 }
115984 bMiss = (0==fts3EvalDeferredTest(pCsr, pCsr->pExpr, &rc));
115985 sqlite3Fts3FreeDeferredDoclists(pCsr);
115986 *pRc = rc;
115987 }
115988 return (rc==SQLITE_OK && bMiss);
115989 }
115990
115991 /*
115992 ** Advance to the next document that matches the FTS expression in
115993 ** Fts3Cursor.pExpr.
115994 */
115995 SQLITE_PRIVATE int sqlite3Fts3EvalNext(Fts3Cursor *pCsr){
115996 int rc = SQLITE_OK; /* Return Code */
115997 Fts3Expr *pExpr = pCsr->pExpr;
115998 assert( pCsr->isEof==0 );
115999 if( pExpr==0 ){
116000 pCsr->isEof = 1;
116001 }else{
116002 do {
116003 if( pCsr->isRequireSeek==0 ){
116004 sqlite3_reset(pCsr->pStmt);
116005 }
116006 assert( sqlite3_data_count(pCsr->pStmt)==0 );
116007 fts3EvalNext(pCsr, pExpr, &rc);
116008 pCsr->isEof = pExpr->bEof;
116009 pCsr->isRequireSeek = 1;
116010 pCsr->isMatchinfoNeeded = 1;
116011 pCsr->iPrevId = pExpr->iDocid;
116012 }while( pCsr->isEof==0 && fts3EvalLoadDeferred(pCsr, &rc) );
116013 }
116014 return rc;
116015 }
116016
116017 /*
116018 ** Restart interation for expression pExpr so that the next call to
116019 ** sqlite3Fts3EvalNext() visits the first row. Do not allow incremental
116020 ** loading or merging of phrase doclists for this iteration.
116021 **
116022 ** If *pRc is other than SQLITE_OK when this function is called, it is
116023 ** a no-op. If an error occurs within this function, *pRc is set to an
116024 ** SQLite error code before returning.
116025 */
116026 static void fts3EvalRestart(
116027 Fts3Cursor *pCsr,
116028 Fts3Expr *pExpr,
116029 int *pRc
116030 ){
116031 if( pExpr && *pRc==SQLITE_OK ){
116032 Fts3Phrase *pPhrase = pExpr->pPhrase;
116033
116034 if( pPhrase ){
116035 fts3EvalZeroPoslist(pPhrase);
116036 if( pPhrase->bIncr ){
116037 sqlite3Fts3EvalPhraseCleanup(pPhrase);
116038 memset(&pPhrase->doclist, 0, sizeof(Fts3Doclist));
116039 *pRc = sqlite3Fts3EvalStart(pCsr, pExpr, 0);
116040 }else{
116041 pPhrase->doclist.pNextDocid = 0;
116042 pPhrase->doclist.iDocid = 0;
116043 }
116044 }
116045
116046 pExpr->iDocid = 0;
116047 pExpr->bEof = 0;
116048 pExpr->bStart = 0;
116049
116050 fts3EvalRestart(pCsr, pExpr->pLeft, pRc);
116051 fts3EvalRestart(pCsr, pExpr->pRight, pRc);
116052 }
116053 }
116054
116055 /*
116056 ** After allocating the Fts3Expr.aMI[] array for each phrase in the
116057 ** expression rooted at pExpr, the cursor iterates through all rows matched
116058 ** by pExpr, calling this function for each row. This function increments
116059 ** the values in Fts3Expr.aMI[] according to the position-list currently
116060 ** found in Fts3Expr.pPhrase->doclist.pList for each of the phrase
116061 ** expression nodes.
116062 */
116063 static void fts3EvalUpdateCounts(Fts3Expr *pExpr){
116064 if( pExpr ){
116065 Fts3Phrase *pPhrase = pExpr->pPhrase;
116066 if( pPhrase && pPhrase->doclist.pList ){
116067 int iCol = 0;
116068 char *p = pPhrase->doclist.pList;
116069
116070 assert( *p );
116071 while( 1 ){
116072 u8 c = 0;
116073 int iCnt = 0;
116074 while( 0xFE & (*p | c) ){
116075 if( (c&0x80)==0 ) iCnt++;
116076 c = *p++ & 0x80;
116077 }
116078
116079 /* aMI[iCol*3 + 1] = Number of occurrences
116080 ** aMI[iCol*3 + 2] = Number of rows containing at least one instance
116081 */
116082 pExpr->aMI[iCol*3 + 1] += iCnt;
116083 pExpr->aMI[iCol*3 + 2] += (iCnt>0);
116084 if( *p==0x00 ) break;
116085 p++;
116086 p += sqlite3Fts3GetVarint32(p, &iCol);
116087 }
116088 }
116089
116090 fts3EvalUpdateCounts(pExpr->pLeft);
116091 fts3EvalUpdateCounts(pExpr->pRight);
116092 }
116093 }
116094
116095 /*
116096 ** Expression pExpr must be of type FTSQUERY_PHRASE.
116097 **
116098 ** If it is not already allocated and populated, this function allocates and
116099 ** populates the Fts3Expr.aMI[] array for expression pExpr. If pExpr is part
116100 ** of a NEAR expression, then it also allocates and populates the same array
116101 ** for all other phrases that are part of the NEAR expression.
116102 **
116103 ** SQLITE_OK is returned if the aMI[] array is successfully allocated and
116104 ** populated. Otherwise, if an error occurs, an SQLite error code is returned.
116105 */
116106 static int fts3EvalGatherStats(
116107 Fts3Cursor *pCsr, /* Cursor object */
116108 Fts3Expr *pExpr /* FTSQUERY_PHRASE expression */
116109 ){
116110 int rc = SQLITE_OK; /* Return code */
116111
116112 assert( pExpr->eType==FTSQUERY_PHRASE );
116113 if( pExpr->aMI==0 ){
116114 Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab;
116115 Fts3Expr *pRoot; /* Root of NEAR expression */
116116 Fts3Expr *p; /* Iterator used for several purposes */
116117
116118 sqlite3_int64 iPrevId = pCsr->iPrevId;
116119 sqlite3_int64 iDocid;
116120 u8 bEof;
116121
116122 /* Find the root of the NEAR expression */
116123 pRoot = pExpr;
116124 while( pRoot->pParent && pRoot->pParent->eType==FTSQUERY_NEAR ){
116125 pRoot = pRoot->pParent;
116126 }
116127 iDocid = pRoot->iDocid;
116128 bEof = pRoot->bEof;
116129 assert( pRoot->bStart );
116130
116131 /* Allocate space for the aMSI[] array of each FTSQUERY_PHRASE node */
116132 for(p=pRoot; p; p=p->pLeft){
116133 Fts3Expr *pE = (p->eType==FTSQUERY_PHRASE?p:p->pRight);
116134 assert( pE->aMI==0 );
116135 pE->aMI = (u32 *)sqlite3_malloc(pTab->nColumn * 3 * sizeof(u32));
116136 if( !pE->aMI ) return SQLITE_NOMEM;
116137 memset(pE->aMI, 0, pTab->nColumn * 3 * sizeof(u32));
116138 }
116139
116140 fts3EvalRestart(pCsr, pRoot, &rc);
116141
116142 while( pCsr->isEof==0 && rc==SQLITE_OK ){
116143
116144 do {
116145 /* Ensure the %_content statement is reset. */
116146 if( pCsr->isRequireSeek==0 ) sqlite3_reset(pCsr->pStmt);
116147 assert( sqlite3_data_count(pCsr->pStmt)==0 );
116148
116149 /* Advance to the next document */
116150 fts3EvalNext(pCsr, pRoot, &rc);
116151 pCsr->isEof = pRoot->bEof;
116152 pCsr->isRequireSeek = 1;
116153 pCsr->isMatchinfoNeeded = 1;
116154 pCsr->iPrevId = pRoot->iDocid;
116155 }while( pCsr->isEof==0
116156 && pRoot->eType==FTSQUERY_NEAR
116157 && fts3EvalLoadDeferred(pCsr, &rc)
116158 );
116159
116160 if( rc==SQLITE_OK && pCsr->isEof==0 ){
116161 fts3EvalUpdateCounts(pRoot);
116162 }
116163 }
116164
116165 pCsr->isEof = 0;
116166 pCsr->iPrevId = iPrevId;
116167
116168 if( bEof ){
116169 pRoot->bEof = bEof;
116170 }else{
116171 /* Caution: pRoot may iterate through docids in ascending or descending
116172 ** order. For this reason, even though it seems more defensive, the
116173 ** do loop can not be written:
116174 **
116175 ** do {...} while( pRoot->iDocid<iDocid && rc==SQLITE_OK );
116176 */
116177 fts3EvalRestart(pCsr, pRoot, &rc);
116178 do {
116179 fts3EvalNext(pCsr, pRoot, &rc);
116180 assert( pRoot->bEof==0 );
116181 }while( pRoot->iDocid!=iDocid && rc==SQLITE_OK );
116182 fts3EvalLoadDeferred(pCsr, &rc);
116183 }
116184 }
116185 return rc;
116186 }
116187
116188 /*
116189 ** This function is used by the matchinfo() module to query a phrase
116190 ** expression node for the following information:
116191 **
116192 ** 1. The total number of occurrences of the phrase in each column of
116193 ** the FTS table (considering all rows), and
116194 **
116195 ** 2. For each column, the number of rows in the table for which the
116196 ** column contains at least one instance of the phrase.
116197 **
116198 ** If no error occurs, SQLITE_OK is returned and the values for each column
116199 ** written into the array aiOut as follows:
116200 **
116201 ** aiOut[iCol*3 + 1] = Number of occurrences
116202 ** aiOut[iCol*3 + 2] = Number of rows containing at least one instance
116203 **
116204 ** Caveats:
116205 **
116206 ** * If a phrase consists entirely of deferred tokens, then all output
116207 ** values are set to the number of documents in the table. In other
116208 ** words we assume that very common tokens occur exactly once in each
116209 ** column of each row of the table.
116210 **
116211 ** * If a phrase contains some deferred tokens (and some non-deferred
116212 ** tokens), count the potential occurrence identified by considering
116213 ** the non-deferred tokens instead of actual phrase occurrences.
116214 **
116215 ** * If the phrase is part of a NEAR expression, then only phrase instances
116216 ** that meet the NEAR constraint are included in the counts.
116217 */
116218 SQLITE_PRIVATE int sqlite3Fts3EvalPhraseStats(
116219 Fts3Cursor *pCsr, /* FTS cursor handle */
116220 Fts3Expr *pExpr, /* Phrase expression */
116221 u32 *aiOut /* Array to write results into (see above) */
116222 ){
116223 Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab;
116224 int rc = SQLITE_OK;
116225 int iCol;
116226
116227 if( pExpr->bDeferred && pExpr->pParent->eType!=FTSQUERY_NEAR ){
116228 assert( pCsr->nDoc>0 );
116229 for(iCol=0; iCol<pTab->nColumn; iCol++){
116230 aiOut[iCol*3 + 1] = pCsr->nDoc;
116231 aiOut[iCol*3 + 2] = pCsr->nDoc;
116232 }
116233 }else{
116234 rc = fts3EvalGatherStats(pCsr, pExpr);
116235 if( rc==SQLITE_OK ){
116236 assert( pExpr->aMI );
116237 for(iCol=0; iCol<pTab->nColumn; iCol++){
116238 aiOut[iCol*3 + 1] = pExpr->aMI[iCol*3 + 1];
116239 aiOut[iCol*3 + 2] = pExpr->aMI[iCol*3 + 2];
116240 }
116241 }
116242 }
116243
116244 return rc;
116245 }
116246
116247 /*
116248 ** The expression pExpr passed as the second argument to this function
116249 ** must be of type FTSQUERY_PHRASE.
116250 **
116251 ** The returned value is either NULL or a pointer to a buffer containing
116252 ** a position-list indicating the occurrences of the phrase in column iCol
116253 ** of the current row.
116254 **
116255 ** More specifically, the returned buffer contains 1 varint for each
116256 ** occurence of the phrase in the column, stored using the normal (delta+2)
116257 ** compression and is terminated by either an 0x01 or 0x00 byte. For example,
116258 ** if the requested column contains "a b X c d X X" and the position-list
116259 ** for 'X' is requested, the buffer returned may contain:
116260 **
116261 ** 0x04 0x05 0x03 0x01 or 0x04 0x05 0x03 0x00
116262 **
116263 ** This function works regardless of whether or not the phrase is deferred,
116264 ** incremental, or neither.
116265 */
116266 SQLITE_PRIVATE char *sqlite3Fts3EvalPhrasePoslist(
116267 Fts3Cursor *pCsr, /* FTS3 cursor object */
116268 Fts3Expr *pExpr, /* Phrase to return doclist for */
116269 int iCol /* Column to return position list for */
116270 ){
116271 Fts3Phrase *pPhrase = pExpr->pPhrase;
116272 Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab;
116273 char *pIter = pPhrase->doclist.pList;
116274 int iThis;
116275
116276 assert( iCol>=0 && iCol<pTab->nColumn );
116277 if( !pIter
116278 || pExpr->bEof
116279 || pExpr->iDocid!=pCsr->iPrevId
116280 || (pPhrase->iColumn<pTab->nColumn && pPhrase->iColumn!=iCol)
116281 ){
116282 return 0;
116283 }
116284
116285 assert( pPhrase->doclist.nList>0 );
116286 if( *pIter==0x01 ){
116287 pIter++;
116288 pIter += sqlite3Fts3GetVarint32(pIter, &iThis);
116289 }else{
116290 iThis = 0;
116291 }
116292 while( iThis<iCol ){
116293 fts3ColumnlistCopy(0, &pIter);
116294 if( *pIter==0x00 ) return 0;
116295 pIter++;
116296 pIter += sqlite3Fts3GetVarint32(pIter, &iThis);
116297 }
116298
116299 return ((iCol==iThis)?pIter:0);
116300 }
116301
116302 /*
116303 ** Free all components of the Fts3Phrase structure that were allocated by
116304 ** the eval module. Specifically, this means to free:
116305 **
116306 ** * the contents of pPhrase->doclist, and
116307 ** * any Fts3MultiSegReader objects held by phrase tokens.
116308 */
116309 SQLITE_PRIVATE void sqlite3Fts3EvalPhraseCleanup(Fts3Phrase *pPhrase){
116310 if( pPhrase ){
116311 int i;
116312 sqlite3_free(pPhrase->doclist.aAll);
116313 fts3EvalZeroPoslist(pPhrase);
116314 memset(&pPhrase->doclist, 0, sizeof(Fts3Doclist));
116315 for(i=0; i<pPhrase->nToken; i++){
116316 fts3SegReaderCursorFree(pPhrase->aToken[i].pSegcsr);
116317 pPhrase->aToken[i].pSegcsr = 0;
116318 }
116319 }
116320 }
116321
116322 #endif
116323
116324 /************** End of fts3.c ************************************************/
116325 /************** Begin file fts3_aux.c ****************************************/
116326 /*
@@ -115648,11 +116348,11 @@
116348 Fts3Table *pFts3Tab;
116349 };
116350
116351 struct Fts3auxCursor {
116352 sqlite3_vtab_cursor base; /* Base class used by SQLite core */
116353 Fts3MultiSegReader csr; /* Must be right after "base" */
116354 Fts3SegFilter filter;
116355 char *zStop;
116356 int nStop; /* Byte-length of string zStop */
116357 int isEof; /* True if cursor is at EOF */
116358 sqlite3_int64 iRowid; /* Current rowid */
@@ -115716,10 +116416,11 @@
116416
116417 p->pFts3Tab = (Fts3Table *)&p[1];
116418 p->pFts3Tab->zDb = (char *)&p->pFts3Tab[1];
116419 p->pFts3Tab->zName = &p->pFts3Tab->zDb[nDb+1];
116420 p->pFts3Tab->db = db;
116421 p->pFts3Tab->nIndex = 1;
116422
116423 memcpy((char *)p->pFts3Tab->zDb, zDb, nDb);
116424 memcpy((char *)p->pFts3Tab->zName, zFts3, nFts3);
116425 sqlite3Fts3Dequote((char *)p->pFts3Tab->zName);
116426
@@ -115996,11 +116697,11 @@
116697 pCsr->zStop = sqlite3_mprintf("%s", sqlite3_value_text(apVal[iIdx]));
116698 pCsr->nStop = sqlite3_value_bytes(apVal[iIdx]);
116699 if( pCsr->zStop==0 ) return SQLITE_NOMEM;
116700 }
116701
116702 rc = sqlite3Fts3SegReaderCursor(pFts3, 0, FTS3_SEGCURSOR_ALL,
116703 pCsr->filter.zTerm, pCsr->filter.nTerm, 0, isScan, &pCsr->csr
116704 );
116705 if( rc==SQLITE_OK ){
116706 rc = sqlite3Fts3SegReaderStart(pFts3, &pCsr->csr, &pCsr->filter);
116707 }
@@ -116175,16 +116876,25 @@
116876 ** Default span for NEAR operators.
116877 */
116878 #define SQLITE_FTS3_DEFAULT_NEAR_PARAM 10
116879
116880
116881 /*
116882 ** isNot:
116883 ** This variable is used by function getNextNode(). When getNextNode() is
116884 ** called, it sets ParseContext.isNot to true if the 'next node' is a
116885 ** FTSQUERY_PHRASE with a unary "-" attached to it. i.e. "mysql" in the
116886 ** FTS3 query "sqlite -mysql". Otherwise, ParseContext.isNot is set to
116887 ** zero.
116888 */
116889 typedef struct ParseContext ParseContext;
116890 struct ParseContext {
116891 sqlite3_tokenizer *pTokenizer; /* Tokenizer module */
116892 const char **azCol; /* Array of column names for fts3 table */
116893 int nCol; /* Number of entries in azCol[] */
116894 int iDefaultCol; /* Default column to query */
116895 int isNot; /* True if getNextNode() sees a unary - */
116896 sqlite3_context *pCtx; /* Write error message here */
116897 int nNest; /* Number of nested brackets */
116898 };
116899
116900 /*
@@ -116266,11 +116976,11 @@
116976 if( iEnd<n && z[iEnd]=='*' ){
116977 pRet->pPhrase->aToken[0].isPrefix = 1;
116978 iEnd++;
116979 }
116980 if( !sqlite3_fts3_enable_parentheses && iStart>0 && z[iStart-1]=='-' ){
116981 pParse->isNot = 1;
116982 }
116983 }
116984 nConsumed = iEnd;
116985 }
116986
@@ -116318,71 +117028,86 @@
117028 Fts3Expr *p = 0;
117029 sqlite3_tokenizer_cursor *pCursor = 0;
117030 char *zTemp = 0;
117031 int nTemp = 0;
117032
117033 const int nSpace = sizeof(Fts3Expr) + sizeof(Fts3Phrase);
117034 int nToken = 0;
117035
117036 /* The final Fts3Expr data structure, including the Fts3Phrase,
117037 ** Fts3PhraseToken structures token buffers are all stored as a single
117038 ** allocation so that the expression can be freed with a single call to
117039 ** sqlite3_free(). Setting this up requires a two pass approach.
117040 **
117041 ** The first pass, in the block below, uses a tokenizer cursor to iterate
117042 ** through the tokens in the expression. This pass uses fts3ReallocOrFree()
117043 ** to assemble data in two dynamic buffers:
117044 **
117045 ** Buffer p: Points to the Fts3Expr structure, followed by the Fts3Phrase
117046 ** structure, followed by the array of Fts3PhraseToken
117047 ** structures. This pass only populates the Fts3PhraseToken array.
117048 **
117049 ** Buffer zTemp: Contains copies of all tokens.
117050 **
117051 ** The second pass, in the block that begins "if( rc==SQLITE_DONE )" below,
117052 ** appends buffer zTemp to buffer p, and fills in the Fts3Expr and Fts3Phrase
117053 ** structures.
117054 */
117055 rc = pModule->xOpen(pTokenizer, zInput, nInput, &pCursor);
117056 if( rc==SQLITE_OK ){
117057 int ii;
117058 pCursor->pTokenizer = pTokenizer;
117059 for(ii=0; rc==SQLITE_OK; ii++){
117060 const char *zByte;
117061 int nByte, iBegin, iEnd, iPos;
117062 rc = pModule->xNext(pCursor, &zByte, &nByte, &iBegin, &iEnd, &iPos);
117063 if( rc==SQLITE_OK ){
117064 Fts3PhraseToken *pToken;
117065
117066 p = fts3ReallocOrFree(p, nSpace + ii*sizeof(Fts3PhraseToken));
117067 if( !p ) goto no_mem;
117068
117069 zTemp = fts3ReallocOrFree(zTemp, nTemp + nByte);
117070 if( !zTemp ) goto no_mem;
117071
117072 assert( nToken==ii );
117073 pToken = &((Fts3Phrase *)(&p[1]))->aToken[ii];
117074 memset(pToken, 0, sizeof(Fts3PhraseToken));
117075
117076 memcpy(&zTemp[nTemp], zByte, nByte);
117077 nTemp += nByte;
117078
117079 pToken->n = nByte;
117080 pToken->isPrefix = (iEnd<nInput && zInput[iEnd]=='*');
117081 nToken = ii+1;
 
 
 
117082 }
117083 }
117084
117085 pModule->xClose(pCursor);
117086 pCursor = 0;
117087 }
117088
117089 if( rc==SQLITE_DONE ){
117090 int jj;
117091 char *zBuf = 0;
117092
117093 p = fts3ReallocOrFree(p, nSpace + nToken*sizeof(Fts3PhraseToken) + nTemp);
117094 if( !p ) goto no_mem;
117095 memset(p, 0, (char *)&(((Fts3Phrase *)&p[1])->aToken[0])-(char *)p);
117096 p->eType = FTSQUERY_PHRASE;
 
 
 
 
 
 
 
 
117097 p->pPhrase = (Fts3Phrase *)&p[1];
117098 p->pPhrase->iColumn = pParse->iDefaultCol;
117099 p->pPhrase->nToken = nToken;
117100
117101 zBuf = (char *)&p->pPhrase->aToken[nToken];
117102 memcpy(zBuf, zTemp, nTemp);
117103 sqlite3_free(zTemp);
117104
117105 for(jj=0; jj<p->pPhrase->nToken; jj++){
117106 p->pPhrase->aToken[jj].z = zBuf;
117107 zBuf += p->pPhrase->aToken[jj].n;
117108 }
 
 
 
117109 rc = SQLITE_OK;
117110 }
117111
117112 *ppExpr = p;
117113 return rc;
@@ -116434,10 +117159,12 @@
117159 int rc;
117160 Fts3Expr *pRet = 0;
117161
117162 const char *zInput = z;
117163 int nInput = n;
117164
117165 pParse->isNot = 0;
117166
117167 /* Skip over any whitespace before checking for a keyword, an open or
117168 ** close bracket, or a quoted string.
117169 */
117170 while( nInput>0 && fts3isspace(*zInput) ){
@@ -116653,11 +117380,11 @@
117380 rc = getNextNode(pParse, zIn, nIn, &p, &nByte);
117381 if( rc==SQLITE_OK ){
117382 int isPhrase;
117383
117384 if( !sqlite3_fts3_enable_parentheses
117385 && p->eType==FTSQUERY_PHRASE && pParse->isNot
117386 ){
117387 /* Create an implicit NOT operator. */
117388 Fts3Expr *pNot = fts3MallocZero(sizeof(Fts3Expr));
117389 if( !pNot ){
117390 sqlite3Fts3ExprFree(p);
@@ -116671,11 +117398,10 @@
117398 }
117399 pNotBranch = pNot;
117400 p = pPrev;
117401 }else{
117402 int eType = p->eType;
 
117403 isPhrase = (eType==FTSQUERY_PHRASE || p->pLeft);
117404
117405 /* The isRequirePhrase variable is set to true if a phrase or
117406 ** an expression contained in parenthesis is required. If a
117407 ** binary operator (AND, OR, NOT or NEAR) is encounted when
@@ -116834,13 +117560,15 @@
117560 /*
117561 ** Free a parsed fts3 query expression allocated by sqlite3Fts3ExprParse().
117562 */
117563 SQLITE_PRIVATE void sqlite3Fts3ExprFree(Fts3Expr *p){
117564 if( p ){
117565 assert( p->eType==FTSQUERY_PHRASE || p->pPhrase==0 );
117566 sqlite3Fts3ExprFree(p->pLeft);
117567 sqlite3Fts3ExprFree(p->pRight);
117568 sqlite3Fts3EvalPhraseCleanup(p->pPhrase);
117569 sqlite3_free(p->aMI);
117570 sqlite3_free(p);
117571 }
117572 }
117573
117574 /****************************************************************************
@@ -116893,11 +117621,11 @@
117621 switch( pExpr->eType ){
117622 case FTSQUERY_PHRASE: {
117623 Fts3Phrase *pPhrase = pExpr->pPhrase;
117624 int i;
117625 zBuf = sqlite3_mprintf(
117626 "%zPHRASE %d 0", zBuf, pPhrase->iColumn);
117627 for(i=0; zBuf && i<pPhrase->nToken; i++){
117628 zBuf = sqlite3_mprintf("%z %.*s%s", zBuf,
117629 pPhrase->aToken[i].n, pPhrase->aToken[i].z,
117630 (pPhrase->aToken[i].isPrefix?"+":"")
117631 );
@@ -118811,18 +119539,44 @@
119539 ** it is always safe to read up to two varints from it without risking an
119540 ** overread, even if the node data is corrupted.
119541 */
119542 #define FTS3_NODE_PADDING (FTS3_VARINT_MAX*2)
119543
119544 /*
119545 ** Under certain circumstances, b-tree nodes (doclists) can be loaded into
119546 ** memory incrementally instead of all at once. This can be a big performance
119547 ** win (reduced IO and CPU) if SQLite stops calling the virtual table xNext()
119548 ** method before retrieving all query results (as may happen, for example,
119549 ** if a query has a LIMIT clause).
119550 **
119551 ** Incremental loading is used for b-tree nodes FTS3_NODE_CHUNK_THRESHOLD
119552 ** bytes and larger. Nodes are loaded in chunks of FTS3_NODE_CHUNKSIZE bytes.
119553 ** The code is written so that the hard lower-limit for each of these values
119554 ** is 1. Clearly such small values would be inefficient, but can be useful
119555 ** for testing purposes.
119556 **
119557 ** If this module is built with SQLITE_TEST defined, these constants may
119558 ** be overridden at runtime for testing purposes. File fts3_test.c contains
119559 ** a Tcl interface to read and write the values.
119560 */
119561 #ifdef SQLITE_TEST
119562 int test_fts3_node_chunksize = (4*1024);
119563 int test_fts3_node_chunk_threshold = (4*1024)*4;
119564 # define FTS3_NODE_CHUNKSIZE test_fts3_node_chunksize
119565 # define FTS3_NODE_CHUNK_THRESHOLD test_fts3_node_chunk_threshold
119566 #else
119567 # define FTS3_NODE_CHUNKSIZE (4*1024)
119568 # define FTS3_NODE_CHUNK_THRESHOLD (FTS3_NODE_CHUNKSIZE*4)
119569 #endif
119570
119571 typedef struct PendingList PendingList;
119572 typedef struct SegmentNode SegmentNode;
119573 typedef struct SegmentWriter SegmentWriter;
119574
119575 /*
119576 ** An instance of the following data structure is used to build doclists
119577 ** incrementally. See function fts3PendingListAppend() for details.
 
119578 */
119579 struct PendingList {
119580 int nData;
119581 char *aData;
119582 int nSpace;
@@ -118849,11 +119603,10 @@
119603 ** of type Fts3SegReader* are also used by code in fts3.c to iterate through
119604 ** terms when querying the full-text index. See functions:
119605 **
119606 ** sqlite3Fts3SegReaderNew()
119607 ** sqlite3Fts3SegReaderFree()
 
119608 ** sqlite3Fts3SegReaderIterate()
119609 **
119610 ** Methods used to manipulate Fts3SegReader structures:
119611 **
119612 ** fts3SegReaderNext()
@@ -118868,10 +119621,13 @@
119621 sqlite3_int64 iEndBlock; /* Rowid of final block in segment (or 0) */
119622 sqlite3_int64 iCurrentBlock; /* Current leaf block (or 0) */
119623
119624 char *aNode; /* Pointer to node data (or NULL) */
119625 int nNode; /* Size of buffer at aNode (or 0) */
119626 int nPopulate; /* If >0, bytes of buffer aNode[] loaded */
119627 sqlite3_blob *pBlob; /* If not NULL, blob handle to read node */
119628
119629 Fts3HashElem **ppNextElem;
119630
119631 /* Variables set by fts3SegReaderNext(). These may be read directly
119632 ** by the caller. They are valid from the time SegmentReaderNew() returns
119633 ** until SegmentReaderNext() returns something other than SQLITE_OK
@@ -118881,12 +119637,15 @@
119637 char *zTerm; /* Pointer to current term */
119638 int nTermAlloc; /* Allocated size of zTerm buffer */
119639 char *aDoclist; /* Pointer to doclist of current entry */
119640 int nDoclist; /* Size of doclist in current entry */
119641
119642 /* The following variables are used by fts3SegReaderNextDocid() to iterate
119643 ** through the current doclist (aDoclist/nDoclist).
119644 */
119645 char *pOffsetList;
119646 int nOffsetList; /* For descending pending seg-readers only */
119647 sqlite3_int64 iDocid;
119648 };
119649
119650 #define fts3SegReaderIsPending(p) ((p)->ppNextElem!=0)
119651 #define fts3SegReaderIsRootOnly(p) ((p)->aNode==(char *)&(p)[1])
@@ -118920,10 +119679,18 @@
119679 ** within the fts3SegWriterXXX() family of functions described above.
119680 **
119681 ** fts3NodeAddTerm()
119682 ** fts3NodeWrite()
119683 ** fts3NodeFree()
119684 **
119685 ** When a b+tree is written to the database (either as a result of a merge
119686 ** or the pending-terms table being flushed), leaves are written into the
119687 ** database file as soon as they are completely populated. The interior of
119688 ** the tree is assembled in memory and written out only once all leaves have
119689 ** been populated and stored. This is Ok, as the b+-tree fanout is usually
119690 ** very large, meaning that the interior of the tree consumes relatively
119691 ** little memory.
119692 */
119693 struct SegmentNode {
119694 SegmentNode *pParent; /* Parent node (or NULL for root node) */
119695 SegmentNode *pRight; /* Pointer to right-sibling */
119696 SegmentNode *pLeftmost; /* Pointer to left-most node of this depth */
@@ -118950,21 +119717,26 @@
119717 #define SQL_NEXT_SEGMENT_INDEX 8
119718 #define SQL_INSERT_SEGMENTS 9
119719 #define SQL_NEXT_SEGMENTS_ID 10
119720 #define SQL_INSERT_SEGDIR 11
119721 #define SQL_SELECT_LEVEL 12
119722 #define SQL_SELECT_LEVEL_RANGE 13
119723 #define SQL_SELECT_LEVEL_COUNT 14
119724 #define SQL_SELECT_SEGDIR_MAX_LEVEL 15
119725 #define SQL_DELETE_SEGDIR_LEVEL 16
119726 #define SQL_DELETE_SEGMENTS_RANGE 17
119727 #define SQL_CONTENT_INSERT 18
119728 #define SQL_DELETE_DOCSIZE 19
119729 #define SQL_REPLACE_DOCSIZE 20
119730 #define SQL_SELECT_DOCSIZE 21
119731 #define SQL_SELECT_DOCTOTAL 22
119732 #define SQL_REPLACE_DOCTOTAL 23
119733
119734 #define SQL_SELECT_ALL_PREFIX_LEVEL 24
119735 #define SQL_DELETE_ALL_TERMS_SEGDIR 25
119736
119737 #define SQL_DELETE_SEGDIR_RANGE 26
119738
119739 /*
119740 ** This function is used to obtain an SQLite prepared statement handle
119741 ** for the statement identified by the second argument. If successful,
119742 ** *pp is set to the requested statement handle and SQLITE_OK returned.
@@ -118997,23 +119769,29 @@
119769
119770 /* Return segments in order from oldest to newest.*/
119771 /* 12 */ "SELECT idx, start_block, leaves_end_block, end_block, root "
119772 "FROM %Q.'%q_segdir' WHERE level = ? ORDER BY idx ASC",
119773 /* 13 */ "SELECT idx, start_block, leaves_end_block, end_block, root "
119774 "FROM %Q.'%q_segdir' WHERE level BETWEEN ? AND ?"
119775 "ORDER BY level DESC, idx ASC",
119776
119777 /* 14 */ "SELECT count(*) FROM %Q.'%q_segdir' WHERE level = ?",
119778 /* 15 */ "SELECT max(level) FROM %Q.'%q_segdir' WHERE level BETWEEN ? AND ?",
119779
119780 /* 16 */ "DELETE FROM %Q.'%q_segdir' WHERE level = ?",
119781 /* 17 */ "DELETE FROM %Q.'%q_segments' WHERE blockid BETWEEN ? AND ?",
119782 /* 18 */ "INSERT INTO %Q.'%q_content' VALUES(%s)",
119783 /* 19 */ "DELETE FROM %Q.'%q_docsize' WHERE docid = ?",
119784 /* 20 */ "REPLACE INTO %Q.'%q_docsize' VALUES(?,?)",
119785 /* 21 */ "SELECT size FROM %Q.'%q_docsize' WHERE docid=?",
119786 /* 22 */ "SELECT value FROM %Q.'%q_stat' WHERE id=0",
119787 /* 23 */ "REPLACE INTO %Q.'%q_stat' VALUES(0,?)",
119788 /* 24 */ "",
119789 /* 25 */ "",
119790
119791 /* 26 */ "DELETE FROM %Q.'%q_segdir' WHERE level BETWEEN ? AND ?",
119792
119793 };
119794 int rc = SQLITE_OK;
119795 sqlite3_stmt *pStmt;
119796
119797 assert( SizeofArray(azSql)==SizeofArray(p->aStmt) );
@@ -119165,18 +119943,36 @@
119943 ** 1: start_block
119944 ** 2: leaves_end_block
119945 ** 3: end_block
119946 ** 4: root
119947 */
119948 SQLITE_PRIVATE int sqlite3Fts3AllSegdirs(
119949 Fts3Table *p, /* FTS3 table */
119950 int iIndex, /* Index for p->aIndex[] */
119951 int iLevel, /* Level to select */
119952 sqlite3_stmt **ppStmt /* OUT: Compiled statement */
119953 ){
119954 int rc;
119955 sqlite3_stmt *pStmt = 0;
119956
119957 assert( iLevel==FTS3_SEGCURSOR_ALL || iLevel>=0 );
119958 assert( iLevel<FTS3_SEGDIR_MAXLEVEL );
119959 assert( iIndex>=0 && iIndex<p->nIndex );
119960
119961 if( iLevel<0 ){
119962 /* "SELECT * FROM %_segdir WHERE level BETWEEN ? AND ? ORDER BY ..." */
119963 rc = fts3SqlStmt(p, SQL_SELECT_LEVEL_RANGE, &pStmt, 0);
119964 if( rc==SQLITE_OK ){
119965 sqlite3_bind_int(pStmt, 1, iIndex*FTS3_SEGDIR_MAXLEVEL);
119966 sqlite3_bind_int(pStmt, 2, (iIndex+1)*FTS3_SEGDIR_MAXLEVEL-1);
119967 }
119968 }else{
119969 /* "SELECT * FROM %_segdir WHERE level = ? ORDER BY ..." */
119970 rc = fts3SqlStmt(p, SQL_SELECT_LEVEL, &pStmt, 0);
119971 if( rc==SQLITE_OK ){
119972 sqlite3_bind_int(pStmt, 1, iLevel+iIndex*FTS3_SEGDIR_MAXLEVEL);
119973 }
119974 }
119975 *ppStmt = pStmt;
119976 return rc;
119977 }
119978
@@ -119286,10 +120082,51 @@
120082 *pp = p;
120083 return 1;
120084 }
120085 return 0;
120086 }
120087
120088 /*
120089 ** Free a PendingList object allocated by fts3PendingListAppend().
120090 */
120091 static void fts3PendingListDelete(PendingList *pList){
120092 sqlite3_free(pList);
120093 }
120094
120095 /*
120096 ** Add an entry to one of the pending-terms hash tables.
120097 */
120098 static int fts3PendingTermsAddOne(
120099 Fts3Table *p,
120100 int iCol,
120101 int iPos,
120102 Fts3Hash *pHash, /* Pending terms hash table to add entry to */
120103 const char *zToken,
120104 int nToken
120105 ){
120106 PendingList *pList;
120107 int rc = SQLITE_OK;
120108
120109 pList = (PendingList *)fts3HashFind(pHash, zToken, nToken);
120110 if( pList ){
120111 p->nPendingData -= (pList->nData + nToken + sizeof(Fts3HashElem));
120112 }
120113 if( fts3PendingListAppend(&pList, p->iPrevDocid, iCol, iPos, &rc) ){
120114 if( pList==fts3HashInsert(pHash, zToken, nToken, pList) ){
120115 /* Malloc failed while inserting the new entry. This can only
120116 ** happen if there was no previous entry for this token.
120117 */
120118 assert( 0==fts3HashFind(pHash, zToken, nToken) );
120119 sqlite3_free(pList);
120120 rc = SQLITE_NOMEM;
120121 }
120122 }
120123 if( rc==SQLITE_OK ){
120124 p->nPendingData += (pList->nData + nToken + sizeof(Fts3HashElem));
120125 }
120126 return rc;
120127 }
120128
120129 /*
120130 ** Tokenize the nul-terminated string zText and add all tokens to the
120131 ** pending-terms hash-table. The docid used is that currently stored in
120132 ** p->iPrevDocid, and the column is specified by argument iCol.
@@ -119335,12 +120172,11 @@
120172
120173 xNext = pModule->xNext;
120174 while( SQLITE_OK==rc
120175 && SQLITE_OK==(rc = xNext(pCsr, &zToken, &nToken, &iStart, &iEnd, &iPos))
120176 ){
120177 int i;
 
120178 if( iPos>=nWord ) nWord = iPos+1;
120179
120180 /* Positions cannot be negative; we use -1 as a terminator internally.
120181 ** Tokens must have a non-zero length.
120182 */
@@ -119347,26 +120183,23 @@
120183 if( iPos<0 || !zToken || nToken<=0 ){
120184 rc = SQLITE_ERROR;
120185 break;
120186 }
120187
120188 /* Add the term to the terms index */
120189 rc = fts3PendingTermsAddOne(
120190 p, iCol, iPos, &p->aIndex[0].hPending, zToken, nToken
120191 );
120192
120193 /* Add the term to each of the prefix indexes that it is not too
120194 ** short for. */
120195 for(i=1; rc==SQLITE_OK && i<p->nIndex; i++){
120196 struct Fts3Index *pIndex = &p->aIndex[i];
120197 if( nToken<pIndex->nPrefix ) continue;
120198 rc = fts3PendingTermsAddOne(
120199 p, iCol, iPos, &pIndex->hPending, zToken, pIndex->nPrefix
120200 );
 
 
 
120201 }
120202 }
120203
120204 pModule->xClose(pCsr);
120205 *pnWord = nWord;
@@ -119392,18 +120225,23 @@
120225 p->iPrevDocid = iDocid;
120226 return SQLITE_OK;
120227 }
120228
120229 /*
120230 ** Discard the contents of the pending-terms hash tables.
120231 */
120232 SQLITE_PRIVATE void sqlite3Fts3PendingTermsClear(Fts3Table *p){
120233 int i;
120234 for(i=0; i<p->nIndex; i++){
120235 Fts3HashElem *pElem;
120236 Fts3Hash *pHash = &p->aIndex[i].hPending;
120237 for(pElem=fts3HashFirst(pHash); pElem; pElem=fts3HashNext(pElem)){
120238 PendingList *pList = (PendingList *)fts3HashData(pElem);
120239 fts3PendingListDelete(pList);
120240 }
120241 fts3HashClear(pHash);
120242 }
 
120243 p->nPendingData = 0;
120244 }
120245
120246 /*
120247 ** This function is called by the xUpdate() method as part of an INSERT
@@ -119555,11 +120393,11 @@
120393
120394 /*
120395 ** Forward declaration to account for the circular dependency between
120396 ** functions fts3SegmentMerge() and fts3AllocateSegdirIdx().
120397 */
120398 static int fts3SegmentMerge(Fts3Table *, int, int);
120399
120400 /*
120401 ** This function allocates a new level iLevel index in the segdir table.
120402 ** Usually, indexes are allocated within a level sequentially starting
120403 ** with 0, so the allocated index is one greater than the value returned
@@ -119572,19 +120410,24 @@
120410 ** allocated index is 0.
120411 **
120412 ** If successful, *piIdx is set to the allocated index slot and SQLITE_OK
120413 ** returned. Otherwise, an SQLite error code is returned.
120414 */
120415 static int fts3AllocateSegdirIdx(
120416 Fts3Table *p,
120417 int iIndex, /* Index for p->aIndex */
120418 int iLevel,
120419 int *piIdx
120420 ){
120421 int rc; /* Return Code */
120422 sqlite3_stmt *pNextIdx; /* Query for next idx at level iLevel */
120423 int iNext = 0; /* Result of query pNextIdx */
120424
120425 /* Set variable iNext to the next available segdir index at level iLevel. */
120426 rc = fts3SqlStmt(p, SQL_NEXT_SEGMENT_INDEX, &pNextIdx, 0);
120427 if( rc==SQLITE_OK ){
120428 sqlite3_bind_int(pNextIdx, 1, iIndex*FTS3_SEGDIR_MAXLEVEL + iLevel);
120429 if( SQLITE_ROW==sqlite3_step(pNextIdx) ){
120430 iNext = sqlite3_column_int(pNextIdx, 0);
120431 }
120432 rc = sqlite3_reset(pNextIdx);
120433 }
@@ -119594,11 +120437,11 @@
120437 ** full, merge all segments in level iLevel into a single iLevel+1
120438 ** segment and allocate (newly freed) index 0 at level iLevel. Otherwise,
120439 ** if iNext is less than FTS3_MERGE_COUNT, allocate index iNext.
120440 */
120441 if( iNext>=FTS3_MERGE_COUNT ){
120442 rc = fts3SegmentMerge(p, iIndex, iLevel);
120443 *piIdx = 0;
120444 }else{
120445 *piIdx = iNext;
120446 }
120447 }
@@ -119635,11 +120478,12 @@
120478 */
120479 SQLITE_PRIVATE int sqlite3Fts3ReadBlock(
120480 Fts3Table *p, /* FTS3 table handle */
120481 sqlite3_int64 iBlockid, /* Access the row with blockid=$iBlockid */
120482 char **paBlob, /* OUT: Blob data in malloc'd buffer */
120483 int *pnBlob, /* OUT: Size of blob data */
120484 int *pnLoad /* OUT: Bytes actually loaded */
120485 ){
120486 int rc; /* Return code */
120487
120488 /* pnBlob must be non-NULL. paBlob may be NULL or non-NULL. */
120489 assert( pnBlob);
@@ -119656,25 +120500,29 @@
120500 );
120501 }
120502
120503 if( rc==SQLITE_OK ){
120504 int nByte = sqlite3_blob_bytes(p->pSegments);
120505 *pnBlob = nByte;
120506 if( paBlob ){
120507 char *aByte = sqlite3_malloc(nByte + FTS3_NODE_PADDING);
120508 if( !aByte ){
120509 rc = SQLITE_NOMEM;
120510 }else{
120511 if( pnLoad && nByte>(FTS3_NODE_CHUNK_THRESHOLD) ){
120512 nByte = FTS3_NODE_CHUNKSIZE;
120513 *pnLoad = nByte;
120514 }
120515 rc = sqlite3_blob_read(p->pSegments, aByte, nByte, 0);
120516 memset(&aByte[nByte], 0, FTS3_NODE_PADDING);
120517 if( rc!=SQLITE_OK ){
120518 sqlite3_free(aByte);
120519 aByte = 0;
120520 }
120521 }
120522 *paBlob = aByte;
120523 }
 
120524 }
120525
120526 return rc;
120527 }
120528
@@ -119684,17 +120532,59 @@
120532 */
120533 SQLITE_PRIVATE void sqlite3Fts3SegmentsClose(Fts3Table *p){
120534 sqlite3_blob_close(p->pSegments);
120535 p->pSegments = 0;
120536 }
120537
120538 static int fts3SegReaderIncrRead(Fts3SegReader *pReader){
120539 int nRead; /* Number of bytes to read */
120540 int rc; /* Return code */
120541
120542 nRead = MIN(pReader->nNode - pReader->nPopulate, FTS3_NODE_CHUNKSIZE);
120543 rc = sqlite3_blob_read(
120544 pReader->pBlob,
120545 &pReader->aNode[pReader->nPopulate],
120546 nRead,
120547 pReader->nPopulate
120548 );
120549
120550 if( rc==SQLITE_OK ){
120551 pReader->nPopulate += nRead;
120552 memset(&pReader->aNode[pReader->nPopulate], 0, FTS3_NODE_PADDING);
120553 if( pReader->nPopulate==pReader->nNode ){
120554 sqlite3_blob_close(pReader->pBlob);
120555 pReader->pBlob = 0;
120556 pReader->nPopulate = 0;
120557 }
120558 }
120559 return rc;
120560 }
120561
120562 static int fts3SegReaderRequire(Fts3SegReader *pReader, char *pFrom, int nByte){
120563 int rc = SQLITE_OK;
120564 assert( !pReader->pBlob
120565 || (pFrom>=pReader->aNode && pFrom<&pReader->aNode[pReader->nNode])
120566 );
120567 while( pReader->pBlob && rc==SQLITE_OK
120568 && (pFrom - pReader->aNode + nByte)>pReader->nPopulate
120569 ){
120570 rc = fts3SegReaderIncrRead(pReader);
120571 }
120572 return rc;
120573 }
120574
120575 /*
120576 ** Move the iterator passed as the first argument to the next term in the
120577 ** segment. If successful, SQLITE_OK is returned. If there is no next term,
120578 ** SQLITE_DONE. Otherwise, an SQLite error code.
120579 */
120580 static int fts3SegReaderNext(
120581 Fts3Table *p,
120582 Fts3SegReader *pReader,
120583 int bIncr
120584 ){
120585 int rc; /* Return code of various sub-routines */
120586 char *pNext; /* Cursor variable */
120587 int nPrefix; /* Number of bytes in term prefix */
120588 int nSuffix; /* Number of bytes in term suffix */
120589
120590 if( !pReader->aDoclist ){
@@ -119702,11 +120592,10 @@
120592 }else{
120593 pNext = &pReader->aDoclist[pReader->nDoclist];
120594 }
120595
120596 if( !pNext || pNext>=&pReader->aNode[pReader->nNode] ){
 
120597
120598 if( fts3SegReaderIsPending(pReader) ){
120599 Fts3HashElem *pElem = *(pReader->ppNextElem);
120600 if( pElem==0 ){
120601 pReader->aNode = 0;
@@ -119722,10 +120611,12 @@
120611 return SQLITE_OK;
120612 }
120613
120614 if( !fts3SegReaderIsRootOnly(pReader) ){
120615 sqlite3_free(pReader->aNode);
120616 sqlite3_blob_close(pReader->pBlob);
120617 pReader->pBlob = 0;
120618 }
120619 pReader->aNode = 0;
120620
120621 /* If iCurrentBlock>=iLeafEndBlock, this is an EOF condition. All leaf
120622 ** blocks have already been traversed. */
@@ -119733,19 +120624,29 @@
120624 if( pReader->iCurrentBlock>=pReader->iLeafEndBlock ){
120625 return SQLITE_OK;
120626 }
120627
120628 rc = sqlite3Fts3ReadBlock(
120629 p, ++pReader->iCurrentBlock, &pReader->aNode, &pReader->nNode,
120630 (bIncr ? &pReader->nPopulate : 0)
120631 );
120632 if( rc!=SQLITE_OK ) return rc;
120633 assert( pReader->pBlob==0 );
120634 if( bIncr && pReader->nPopulate<pReader->nNode ){
120635 pReader->pBlob = p->pSegments;
120636 p->pSegments = 0;
120637 }
120638 pNext = pReader->aNode;
120639 }
120640
120641 assert( !fts3SegReaderIsPending(pReader) );
120642
120643 rc = fts3SegReaderRequire(pReader, pNext, FTS3_VARINT_MAX*2);
120644 if( rc!=SQLITE_OK ) return rc;
120645
120646 /* Because of the FTS3_NODE_PADDING bytes of padding, the following is
120647 ** safe (no risk of overread) even if the node data is corrupted. */
 
120648 pNext += sqlite3Fts3GetVarint32(pNext, &nPrefix);
120649 pNext += sqlite3Fts3GetVarint32(pNext, &nSuffix);
120650 if( nPrefix<0 || nSuffix<=0
120651 || &pNext[nSuffix]>&pReader->aNode[pReader->nNode]
120652 ){
@@ -119759,10 +120660,14 @@
120660 return SQLITE_NOMEM;
120661 }
120662 pReader->zTerm = zNew;
120663 pReader->nTermAlloc = nNew;
120664 }
120665
120666 rc = fts3SegReaderRequire(pReader, pNext, nSuffix+FTS3_VARINT_MAX);
120667 if( rc!=SQLITE_OK ) return rc;
120668
120669 memcpy(&pReader->zTerm[nPrefix], pNext, nSuffix);
120670 pReader->nTerm = nPrefix+nSuffix;
120671 pNext += nSuffix;
120672 pNext += sqlite3Fts3GetVarint32(pNext, &pReader->nDoclist);
120673 pReader->aDoclist = pNext;
@@ -119771,11 +120676,11 @@
120676 /* Check that the doclist does not appear to extend past the end of the
120677 ** b-tree node. And that the final byte of the doclist is 0x00. If either
120678 ** of these statements is untrue, then the data structure is corrupt.
120679 */
120680 if( &pReader->aDoclist[pReader->nDoclist]>&pReader->aNode[pReader->nNode]
120681 || (pReader->nPopulate==0 && pReader->aDoclist[pReader->nDoclist-1])
120682 ){
120683 return SQLITE_CORRUPT_VTAB;
120684 }
120685 return SQLITE_OK;
120686 }
@@ -119782,16 +120687,30 @@
120687
120688 /*
120689 ** Set the SegReader to point to the first docid in the doclist associated
120690 ** with the current term.
120691 */
120692 static int fts3SegReaderFirstDocid(Fts3Table *pTab, Fts3SegReader *pReader){
120693 int rc = SQLITE_OK;
120694 assert( pReader->aDoclist );
120695 assert( !pReader->pOffsetList );
120696 if( pTab->bDescIdx && fts3SegReaderIsPending(pReader) ){
120697 u8 bEof = 0;
120698 pReader->iDocid = 0;
120699 pReader->nOffsetList = 0;
120700 sqlite3Fts3DoclistPrev(0,
120701 pReader->aDoclist, pReader->nDoclist, &pReader->pOffsetList,
120702 &pReader->iDocid, &pReader->nOffsetList, &bEof
120703 );
120704 }else{
120705 rc = fts3SegReaderRequire(pReader, pReader->aDoclist, FTS3_VARINT_MAX);
120706 if( rc==SQLITE_OK ){
120707 int n = sqlite3Fts3GetVarint(pReader->aDoclist, &pReader->iDocid);
120708 pReader->pOffsetList = &pReader->aDoclist[n];
120709 }
120710 }
120711 return rc;
120712 }
120713
120714 /*
120715 ** Advance the SegReader to point to the next docid in the doclist
120716 ** associated with the current term.
@@ -119800,132 +120719,126 @@
120719 ** *ppOffsetList is set to point to the first column-offset list
120720 ** in the doclist entry (i.e. immediately past the docid varint).
120721 ** *pnOffsetList is set to the length of the set of column-offset
120722 ** lists, not including the nul-terminator byte. For example:
120723 */
120724 static int fts3SegReaderNextDocid(
120725 Fts3Table *pTab,
120726 Fts3SegReader *pReader, /* Reader to advance to next docid */
120727 char **ppOffsetList, /* OUT: Pointer to current position-list */
120728 int *pnOffsetList /* OUT: Length of *ppOffsetList in bytes */
120729 ){
120730 int rc = SQLITE_OK;
120731 char *p = pReader->pOffsetList;
120732 char c = 0;
120733
120734 assert( p );
120735
120736 if( pTab->bDescIdx && fts3SegReaderIsPending(pReader) ){
120737 /* A pending-terms seg-reader for an FTS4 table that uses order=desc.
120738 ** Pending-terms doclists are always built up in ascending order, so
120739 ** we have to iterate through them backwards here. */
120740 u8 bEof = 0;
120741 if( ppOffsetList ){
120742 *ppOffsetList = pReader->pOffsetList;
120743 *pnOffsetList = pReader->nOffsetList - 1;
120744 }
120745 sqlite3Fts3DoclistPrev(0,
120746 pReader->aDoclist, pReader->nDoclist, &p, &pReader->iDocid,
120747 &pReader->nOffsetList, &bEof
120748 );
120749 if( bEof ){
120750 pReader->pOffsetList = 0;
120751 }else{
120752 pReader->pOffsetList = p;
120753 }
 
 
120754 }else{
120755
120756 /* Pointer p currently points at the first byte of an offset list. The
120757 ** following block advances it to point one byte past the end of
120758 ** the same offset list. */
120759 while( 1 ){
120760
120761 /* The following line of code (and the "p++" below the while() loop) is
120762 ** normally all that is required to move pointer p to the desired
120763 ** position. The exception is if this node is being loaded from disk
120764 ** incrementally and pointer "p" now points to the first byte passed
120765 ** the populated part of pReader->aNode[].
120766 */
120767 while( *p | c ) c = *p++ & 0x80;
120768 assert( *p==0 );
120769
120770 if( pReader->pBlob==0 || p<&pReader->aNode[pReader->nPopulate] ) break;
120771 rc = fts3SegReaderIncrRead(pReader);
120772 if( rc!=SQLITE_OK ) return rc;
120773 }
120774 p++;
120775
120776 /* If required, populate the output variables with a pointer to and the
120777 ** size of the previous offset-list.
120778 */
120779 if( ppOffsetList ){
120780 *ppOffsetList = pReader->pOffsetList;
120781 *pnOffsetList = (int)(p - pReader->pOffsetList - 1);
120782 }
120783
120784 /* If there are no more entries in the doclist, set pOffsetList to
120785 ** NULL. Otherwise, set Fts3SegReader.iDocid to the next docid and
120786 ** Fts3SegReader.pOffsetList to point to the next offset list before
120787 ** returning.
120788 */
120789 if( p>=&pReader->aDoclist[pReader->nDoclist] ){
120790 pReader->pOffsetList = 0;
120791 }else{
120792 rc = fts3SegReaderRequire(pReader, p, FTS3_VARINT_MAX);
120793 if( rc==SQLITE_OK ){
120794 sqlite3_int64 iDelta;
120795 pReader->pOffsetList = p + sqlite3Fts3GetVarint(p, &iDelta);
120796 if( pTab->bDescIdx ){
120797 pReader->iDocid -= iDelta;
120798 }else{
120799 pReader->iDocid += iDelta;
120800 }
120801 }
120802 }
120803 }
120804
120805 return SQLITE_OK;
120806 }
120807
120808
120809 SQLITE_PRIVATE int sqlite3Fts3MsrOvfl(
120810 Fts3Cursor *pCsr,
120811 Fts3MultiSegReader *pMsr,
120812 int *pnOvfl
120813 ){
120814 Fts3Table *p = (Fts3Table*)pCsr->base.pVtab;
120815 int nOvfl = 0;
120816 int ii;
120817 int rc = SQLITE_OK;
120818 int pgsz = p->nPgsz;
120819
120820 assert( p->bHasStat );
120821 assert( pgsz>0 );
120822
120823 for(ii=0; rc==SQLITE_OK && ii<pMsr->nSegment; ii++){
120824 Fts3SegReader *pReader = pMsr->apSegment[ii];
120825 if( !fts3SegReaderIsPending(pReader)
120826 && !fts3SegReaderIsRootOnly(pReader)
120827 ){
120828 int jj;
120829 for(jj=pReader->iStartBlock; jj<=pReader->iLeafEndBlock; jj++){
120830 int nBlob;
120831 rc = sqlite3Fts3ReadBlock(p, jj, 0, &nBlob, 0);
120832 if( rc!=SQLITE_OK ) break;
120833 if( (nBlob+35)>pgsz ){
120834 nOvfl += (nBlob + 34)/pgsz;
120835 }
120836 }
120837 }
120838 }
120839 *pnOvfl = nOvfl;
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
120840 return rc;
120841 }
120842
120843 /*
120844 ** Free all allocations associated with the iterator passed as the
@@ -119934,10 +120847,11 @@
120847 SQLITE_PRIVATE void sqlite3Fts3SegReaderFree(Fts3SegReader *pReader){
120848 if( pReader && !fts3SegReaderIsPending(pReader) ){
120849 sqlite3_free(pReader->zTerm);
120850 if( !fts3SegReaderIsRootOnly(pReader) ){
120851 sqlite3_free(pReader->aNode);
120852 sqlite3_blob_close(pReader->pBlob);
120853 }
120854 }
120855 sqlite3_free(pReader);
120856 }
120857
@@ -120010,28 +120924,46 @@
120924 }
120925
120926 /*
120927 ** This function is used to allocate an Fts3SegReader that iterates through
120928 ** a subset of the terms stored in the Fts3Table.pendingTerms array.
120929 **
120930 ** If the isPrefixIter parameter is zero, then the returned SegReader iterates
120931 ** through each term in the pending-terms table. Or, if isPrefixIter is
120932 ** non-zero, it iterates through each term and its prefixes. For example, if
120933 ** the pending terms hash table contains the terms "sqlite", "mysql" and
120934 ** "firebird", then the iterator visits the following 'terms' (in the order
120935 ** shown):
120936 **
120937 ** f fi fir fire fireb firebi firebir firebird
120938 ** m my mys mysq mysql
120939 ** s sq sql sqli sqlit sqlite
120940 **
120941 ** Whereas if isPrefixIter is zero, the terms visited are:
120942 **
120943 ** firebird mysql sqlite
120944 */
120945 SQLITE_PRIVATE int sqlite3Fts3SegReaderPending(
120946 Fts3Table *p, /* Virtual table handle */
120947 int iIndex, /* Index for p->aIndex */
120948 const char *zTerm, /* Term to search for */
120949 int nTerm, /* Size of buffer zTerm */
120950 int bPrefix, /* True for a prefix iterator */
120951 Fts3SegReader **ppReader /* OUT: SegReader for pending-terms */
120952 ){
120953 Fts3SegReader *pReader = 0; /* Fts3SegReader object to return */
120954 Fts3HashElem **aElem = 0; /* Array of term hash entries to scan */
120955 int nElem = 0; /* Size of array at aElem */
120956 int rc = SQLITE_OK; /* Return Code */
120957 Fts3Hash *pHash;
120958
120959 pHash = &p->aIndex[iIndex].hPending;
120960 if( bPrefix ){
120961 int nAlloc = 0; /* Size of allocated array at aElem */
120962 Fts3HashElem *pE = 0; /* Iterator variable */
120963
120964 for(pE=fts3HashFirst(pHash); pE; pE=fts3HashNext(pE)){
120965 char *zKey = (char *)fts3HashKey(pE);
120966 int nKey = fts3HashKeysize(pE);
120967 if( nTerm==0 || (nKey>=nTerm && 0==memcmp(zKey, zTerm, nTerm)) ){
120968 if( nElem==nAlloc ){
120969 Fts3HashElem **aElem2;
@@ -120044,10 +120976,11 @@
120976 nElem = 0;
120977 break;
120978 }
120979 aElem = aElem2;
120980 }
120981
120982 aElem[nElem++] = pE;
120983 }
120984 }
120985
120986 /* If more than one term matches the prefix, sort the Fts3HashElem
@@ -120057,11 +120990,13 @@
120990 if( nElem>1 ){
120991 qsort(aElem, nElem, sizeof(Fts3HashElem *), fts3CompareElemByTerm);
120992 }
120993
120994 }else{
120995 /* The query is a simple term lookup that matches at most one term in
120996 ** the index. All that is required is a straight hash-lookup. */
120997 Fts3HashElem *pE = fts3HashFindElem(pHash, zTerm, nTerm);
120998 if( pE ){
120999 aElem = &pE;
121000 nElem = 1;
121001 }
121002 }
@@ -120077,11 +121012,11 @@
121012 pReader->ppNextElem = (Fts3HashElem **)&pReader[1];
121013 memcpy(pReader->ppNextElem, aElem, nElem*sizeof(Fts3HashElem *));
121014 }
121015 }
121016
121017 if( bPrefix ){
121018 sqlite3_free(aElem);
121019 }
121020 *ppReader = pReader;
121021 return rc;
121022 }
@@ -120137,10 +121072,22 @@
121072 if( pLhs->iDocid==pRhs->iDocid ){
121073 rc = pRhs->iIdx - pLhs->iIdx;
121074 }else{
121075 rc = (pLhs->iDocid > pRhs->iDocid) ? 1 : -1;
121076 }
121077 }
121078 assert( pLhs->aNode && pRhs->aNode );
121079 return rc;
121080 }
121081 static int fts3SegReaderDoclistCmpRev(Fts3SegReader *pLhs, Fts3SegReader *pRhs){
121082 int rc = (pLhs->pOffsetList==0)-(pRhs->pOffsetList==0);
121083 if( rc==0 ){
121084 if( pLhs->iDocid==pRhs->iDocid ){
121085 rc = pRhs->iIdx - pLhs->iIdx;
121086 }else{
121087 rc = (pLhs->iDocid < pRhs->iDocid) ? 1 : -1;
121088 }
121089 }
121090 assert( pLhs->aNode && pRhs->aNode );
121091 return rc;
121092 }
121093
@@ -120689,25 +121636,34 @@
121636 }
121637 return rc;
121638 }
121639
121640 /*
121641 ** Set *pnMax to the largest segment level in the database for the index
121642 ** iIndex.
121643 **
121644 ** Segment levels are stored in the 'level' column of the %_segdir table.
121645 **
121646 ** Return SQLITE_OK if successful, or an SQLite error code if not.
121647 */
121648 static int fts3SegmentMaxLevel(Fts3Table *p, int iIndex, int *pnMax){
121649 sqlite3_stmt *pStmt;
121650 int rc;
121651 assert( iIndex>=0 && iIndex<p->nIndex );
121652
121653 /* Set pStmt to the compiled version of:
121654 **
121655 ** SELECT max(level) FROM %Q.'%q_segdir' WHERE level BETWEEN ? AND ?
121656 **
121657 ** (1024 is actually the value of macro FTS3_SEGDIR_PREFIXLEVEL_STR).
121658 */
121659 rc = fts3SqlStmt(p, SQL_SELECT_SEGDIR_MAX_LEVEL, &pStmt, 0);
121660 if( rc!=SQLITE_OK ) return rc;
121661 sqlite3_bind_int(pStmt, 1, iIndex*FTS3_SEGDIR_MAXLEVEL);
121662 sqlite3_bind_int(pStmt, 2, (iIndex+1)*FTS3_SEGDIR_MAXLEVEL - 1);
121663 if( SQLITE_ROW==sqlite3_step(pStmt) ){
121664 *pnMax = sqlite3_column_int(pStmt, 0);
 
121665 }
121666 return sqlite3_reset(pStmt);
121667 }
121668
121669 /*
@@ -120724,10 +121680,11 @@
121680 **
121681 ** SQLITE_OK is returned if successful, otherwise an SQLite error code.
121682 */
121683 static int fts3DeleteSegdir(
121684 Fts3Table *p, /* Virtual table handle */
121685 int iIndex, /* Index for p->aIndex */
121686 int iLevel, /* Level of %_segdir entries to delete */
121687 Fts3SegReader **apSegment, /* Array of SegReader objects */
121688 int nReader /* Size of array apSegment */
121689 ){
121690 int rc; /* Return Code */
@@ -120746,23 +121703,28 @@
121703 }
121704 if( rc!=SQLITE_OK ){
121705 return rc;
121706 }
121707
121708 assert( iLevel>=0 || iLevel==FTS3_SEGCURSOR_ALL );
121709 if( iLevel==FTS3_SEGCURSOR_ALL ){
121710 rc = fts3SqlStmt(p, SQL_DELETE_SEGDIR_RANGE, &pDelete, 0);
121711 if( rc==SQLITE_OK ){
121712 sqlite3_bind_int(pDelete, 1, iIndex*FTS3_SEGDIR_MAXLEVEL);
121713 sqlite3_bind_int(pDelete, 2, (iIndex+1) * FTS3_SEGDIR_MAXLEVEL - 1);
121714 }
121715 }else{
121716 rc = fts3SqlStmt(p, SQL_DELETE_SEGDIR_LEVEL, &pDelete, 0);
 
121717 if( rc==SQLITE_OK ){
121718 sqlite3_bind_int(pDelete, 1, iIndex*FTS3_SEGDIR_MAXLEVEL + iLevel);
 
 
121719 }
121720 }
121721
121722 if( rc==SQLITE_OK ){
121723 sqlite3_step(pDelete);
121724 rc = sqlite3_reset(pDelete);
121725 }
121726
121727 return rc;
121728 }
121729
121730 /*
@@ -120805,14 +121767,124 @@
121767 }
121768
121769 *ppList = pList;
121770 *pnList = nList;
121771 }
121772
121773 SQLITE_PRIVATE int sqlite3Fts3MsrIncrStart(
121774 Fts3Table *p, /* Virtual table handle */
121775 Fts3MultiSegReader *pCsr, /* Cursor object */
121776 int iCol, /* Column to match on. */
121777 const char *zTerm, /* Term to iterate through a doclist for */
121778 int nTerm /* Number of bytes in zTerm */
121779 ){
121780 int i;
121781 int nSegment = pCsr->nSegment;
121782 int (*xCmp)(Fts3SegReader *, Fts3SegReader *) = (
121783 p->bDescIdx ? fts3SegReaderDoclistCmpRev : fts3SegReaderDoclistCmp
121784 );
121785
121786 assert( pCsr->pFilter==0 );
121787 assert( zTerm && nTerm>0 );
121788
121789 /* Advance each segment iterator until it points to the term zTerm/nTerm. */
121790 for(i=0; i<nSegment; i++){
121791 Fts3SegReader *pSeg = pCsr->apSegment[i];
121792 do {
121793 int rc = fts3SegReaderNext(p, pSeg, 1);
121794 if( rc!=SQLITE_OK ) return rc;
121795 }while( fts3SegReaderTermCmp(pSeg, zTerm, nTerm)<0 );
121796 }
121797 fts3SegReaderSort(pCsr->apSegment, nSegment, nSegment, fts3SegReaderCmp);
121798
121799 /* Determine how many of the segments actually point to zTerm/nTerm. */
121800 for(i=0; i<nSegment; i++){
121801 Fts3SegReader *pSeg = pCsr->apSegment[i];
121802 if( !pSeg->aNode || fts3SegReaderTermCmp(pSeg, zTerm, nTerm) ){
121803 break;
121804 }
121805 }
121806 pCsr->nAdvance = i;
121807
121808 /* Advance each of the segments to point to the first docid. */
121809 for(i=0; i<pCsr->nAdvance; i++){
121810 int rc = fts3SegReaderFirstDocid(p, pCsr->apSegment[i]);
121811 if( rc!=SQLITE_OK ) return rc;
121812 }
121813 fts3SegReaderSort(pCsr->apSegment, i, i, xCmp);
121814
121815 assert( iCol<0 || iCol<p->nColumn );
121816 pCsr->iColFilter = iCol;
121817
121818 return SQLITE_OK;
121819 }
121820
121821 SQLITE_PRIVATE int sqlite3Fts3MsrIncrNext(
121822 Fts3Table *p, /* Virtual table handle */
121823 Fts3MultiSegReader *pMsr, /* Multi-segment-reader handle */
121824 sqlite3_int64 *piDocid, /* OUT: Docid value */
121825 char **paPoslist, /* OUT: Pointer to position list */
121826 int *pnPoslist /* OUT: Size of position list in bytes */
121827 ){
121828 int nMerge = pMsr->nAdvance;
121829 Fts3SegReader **apSegment = pMsr->apSegment;
121830 int (*xCmp)(Fts3SegReader *, Fts3SegReader *) = (
121831 p->bDescIdx ? fts3SegReaderDoclistCmpRev : fts3SegReaderDoclistCmp
121832 );
121833
121834 if( nMerge==0 ){
121835 *paPoslist = 0;
121836 return SQLITE_OK;
121837 }
121838
121839 while( 1 ){
121840 Fts3SegReader *pSeg;
121841 pSeg = pMsr->apSegment[0];
121842
121843 if( pSeg->pOffsetList==0 ){
121844 *paPoslist = 0;
121845 break;
121846 }else{
121847 int rc;
121848 char *pList;
121849 int nList;
121850 int j;
121851 sqlite3_int64 iDocid = apSegment[0]->iDocid;
121852
121853 rc = fts3SegReaderNextDocid(p, apSegment[0], &pList, &nList);
121854 j = 1;
121855 while( rc==SQLITE_OK
121856 && j<nMerge
121857 && apSegment[j]->pOffsetList
121858 && apSegment[j]->iDocid==iDocid
121859 ){
121860 rc = fts3SegReaderNextDocid(p, apSegment[j], 0, 0);
121861 j++;
121862 }
121863 if( rc!=SQLITE_OK ) return rc;
121864 fts3SegReaderSort(pMsr->apSegment, nMerge, j, xCmp);
121865
121866 if( pMsr->iColFilter>=0 ){
121867 fts3ColumnFilter(pMsr->iColFilter, &pList, &nList);
121868 }
121869
121870 if( nList>0 ){
121871 *piDocid = iDocid;
121872 *paPoslist = pList;
121873 *pnPoslist = nList;
121874 break;
121875 }
121876 }
121877
121878 }
121879
121880 return SQLITE_OK;
121881 }
121882
121883 SQLITE_PRIVATE int sqlite3Fts3SegReaderStart(
121884 Fts3Table *p, /* Virtual table handle */
121885 Fts3MultiSegReader *pCsr, /* Cursor object */
121886 Fts3SegFilter *pFilter /* Restrictions on range of iteration */
121887 ){
121888 int i;
121889
121890 /* Initialize the cursor object */
@@ -120827,11 +121899,11 @@
121899 for(i=0; i<pCsr->nSegment; i++){
121900 int nTerm = pFilter->nTerm;
121901 const char *zTerm = pFilter->zTerm;
121902 Fts3SegReader *pSeg = pCsr->apSegment[i];
121903 do {
121904 int rc = fts3SegReaderNext(p, pSeg, 0);
121905 if( rc!=SQLITE_OK ) return rc;
121906 }while( zTerm && fts3SegReaderTermCmp(pSeg, zTerm, nTerm)<0 );
121907 }
121908 fts3SegReaderSort(
121909 pCsr->apSegment, pCsr->nSegment, pCsr->nSegment, fts3SegReaderCmp);
@@ -120839,11 +121911,11 @@
121911 return SQLITE_OK;
121912 }
121913
121914 SQLITE_PRIVATE int sqlite3Fts3SegReaderStep(
121915 Fts3Table *p, /* Virtual table handle */
121916 Fts3MultiSegReader *pCsr /* Cursor object */
121917 ){
121918 int rc = SQLITE_OK;
121919
121920 int isIgnoreEmpty = (pCsr->pFilter->flags & FTS3_SEGMENT_IGNORE_EMPTY);
121921 int isRequirePos = (pCsr->pFilter->flags & FTS3_SEGMENT_REQUIRE_POS);
@@ -120852,10 +121924,13 @@
121924 int isScan = (pCsr->pFilter->flags & FTS3_SEGMENT_SCAN);
121925
121926 Fts3SegReader **apSegment = pCsr->apSegment;
121927 int nSegment = pCsr->nSegment;
121928 Fts3SegFilter *pFilter = pCsr->pFilter;
121929 int (*xCmp)(Fts3SegReader *, Fts3SegReader *) = (
121930 p->bDescIdx ? fts3SegReaderDoclistCmpRev : fts3SegReaderDoclistCmp
121931 );
121932
121933 if( pCsr->nSegment==0 ) return SQLITE_OK;
121934
121935 do {
121936 int nMerge;
@@ -120863,11 +121938,11 @@
121938
121939 /* Advance the first pCsr->nAdvance entries in the apSegment[] array
121940 ** forward. Then sort the list in order of current term again.
121941 */
121942 for(i=0; i<pCsr->nAdvance; i++){
121943 rc = fts3SegReaderNext(p, apSegment[i], 0);
121944 if( rc!=SQLITE_OK ) return rc;
121945 }
121946 fts3SegReaderSort(apSegment, nSegment, pCsr->nAdvance, fts3SegReaderCmp);
121947 pCsr->nAdvance = 0;
121948
@@ -120902,11 +121977,14 @@
121977 ){
121978 nMerge++;
121979 }
121980
121981 assert( isIgnoreEmpty || (isRequirePos && !isColFilter) );
121982 if( nMerge==1
121983 && !isIgnoreEmpty
121984 && (p->bDescIdx==0 || fts3SegReaderIsPending(apSegment[0])==0)
121985 ){
121986 pCsr->aDoclist = apSegment[0]->aDoclist;
121987 pCsr->nDoclist = apSegment[0]->nDoclist;
121988 rc = SQLITE_ROW;
121989 }else{
121990 int nDoclist = 0; /* Size of doclist */
@@ -120915,56 +121993,66 @@
121993 /* The current term of the first nMerge entries in the array
121994 ** of Fts3SegReader objects is the same. The doclists must be merged
121995 ** and a single term returned with the merged doclist.
121996 */
121997 for(i=0; i<nMerge; i++){
121998 fts3SegReaderFirstDocid(p, apSegment[i]);
121999 }
122000 fts3SegReaderSort(apSegment, nMerge, nMerge, xCmp);
122001 while( apSegment[0]->pOffsetList ){
122002 int j; /* Number of segments that share a docid */
122003 char *pList;
122004 int nList;
122005 int nByte;
122006 sqlite3_int64 iDocid = apSegment[0]->iDocid;
122007 fts3SegReaderNextDocid(p, apSegment[0], &pList, &nList);
122008 j = 1;
122009 while( j<nMerge
122010 && apSegment[j]->pOffsetList
122011 && apSegment[j]->iDocid==iDocid
122012 ){
122013 fts3SegReaderNextDocid(p, apSegment[j], 0, 0);
122014 j++;
122015 }
122016
122017 if( isColFilter ){
122018 fts3ColumnFilter(pFilter->iCol, &pList, &nList);
122019 }
122020
122021 if( !isIgnoreEmpty || nList>0 ){
122022
122023 /* Calculate the 'docid' delta value to write into the merged
122024 ** doclist. */
122025 sqlite3_int64 iDelta;
122026 if( p->bDescIdx && nDoclist>0 ){
122027 iDelta = iPrev - iDocid;
122028 }else{
122029 iDelta = iDocid - iPrev;
122030 }
122031 assert( iDelta>0 || (nDoclist==0 && iDelta==iDocid) );
122032 assert( nDoclist>0 || iDelta==iDocid );
122033
122034 nByte = sqlite3Fts3VarintLen(iDelta) + (isRequirePos?nList+1:0);
122035 if( nDoclist+nByte>pCsr->nBuffer ){
122036 char *aNew;
122037 pCsr->nBuffer = (nDoclist+nByte)*2;
122038 aNew = sqlite3_realloc(pCsr->aBuffer, pCsr->nBuffer);
122039 if( !aNew ){
122040 return SQLITE_NOMEM;
122041 }
122042 pCsr->aBuffer = aNew;
122043 }
122044 nDoclist += sqlite3Fts3PutVarint(&pCsr->aBuffer[nDoclist], iDelta);
 
 
122045 iPrev = iDocid;
122046 if( isRequirePos ){
122047 memcpy(&pCsr->aBuffer[nDoclist], pList, nList);
122048 nDoclist += nList;
122049 pCsr->aBuffer[nDoclist++] = '\0';
122050 }
122051 }
122052
122053 fts3SegReaderSort(apSegment, nMerge, j, xCmp);
122054 }
122055 if( nDoclist>0 ){
122056 pCsr->aDoclist = pCsr->aBuffer;
122057 pCsr->nDoclist = nDoclist;
122058 rc = SQLITE_ROW;
@@ -120973,13 +122061,14 @@
122061 pCsr->nAdvance = nMerge;
122062 }while( rc==SQLITE_OK );
122063
122064 return rc;
122065 }
122066
122067
122068 SQLITE_PRIVATE void sqlite3Fts3SegReaderFinish(
122069 Fts3MultiSegReader *pCsr /* Cursor object */
122070 ){
122071 if( pCsr ){
122072 int i;
122073 for(i=0; i<pCsr->nSegment; i++){
122074 sqlite3Fts3SegReaderFree(pCsr->apSegment[i]);
@@ -121002,47 +122091,60 @@
122091 ** If this function is called with iLevel<0, but there is only one
122092 ** segment in the database, SQLITE_DONE is returned immediately.
122093 ** Otherwise, if successful, SQLITE_OK is returned. If an error occurs,
122094 ** an SQLite error code is returned.
122095 */
122096 static int fts3SegmentMerge(Fts3Table *p, int iIndex, int iLevel){
122097 int rc; /* Return code */
122098 int iIdx = 0; /* Index of new segment */
122099 int iNewLevel = 0; /* Level/index to create new segment at */
122100 SegmentWriter *pWriter = 0; /* Used to write the new, merged, segment */
122101 Fts3SegFilter filter; /* Segment term filter condition */
122102 Fts3MultiSegReader csr; /* Cursor to iterate through level(s) */
122103 int bIgnoreEmpty = 0; /* True to ignore empty segments */
122104
122105 assert( iLevel==FTS3_SEGCURSOR_ALL
122106 || iLevel==FTS3_SEGCURSOR_PENDING
122107 || iLevel>=0
122108 );
122109 assert( iLevel<FTS3_SEGDIR_MAXLEVEL );
122110 assert( iIndex>=0 && iIndex<p->nIndex );
122111
122112 rc = sqlite3Fts3SegReaderCursor(p, iIndex, iLevel, 0, 0, 1, 0, &csr);
122113 if( rc!=SQLITE_OK || csr.nSegment==0 ) goto finished;
122114
122115 if( iLevel==FTS3_SEGCURSOR_ALL ){
122116 /* This call is to merge all segments in the database to a single
122117 ** segment. The level of the new segment is equal to the the numerically
122118 ** greatest segment level currently present in the database for this
122119 ** index. The idx of the new segment is always 0. */
 
122120 if( csr.nSegment==1 ){
122121 rc = SQLITE_DONE;
122122 goto finished;
122123 }
122124 rc = fts3SegmentMaxLevel(p, iIndex, &iNewLevel);
122125 bIgnoreEmpty = 1;
122126
122127 }else if( iLevel==FTS3_SEGCURSOR_PENDING ){
122128 iNewLevel = iIndex * FTS3_SEGDIR_MAXLEVEL;
122129 rc = fts3AllocateSegdirIdx(p, iIndex, 0, &iIdx);
122130 }else{
122131 /* This call is to merge all segments at level iLevel. find the next
122132 ** available segment index at level iLevel+1. The call to
122133 ** fts3AllocateSegdirIdx() will merge the segments at level iLevel+1 to
122134 ** a single iLevel+2 segment if necessary. */
122135 rc = fts3AllocateSegdirIdx(p, iIndex, iLevel+1, &iIdx);
122136 iNewLevel = iIndex * FTS3_SEGDIR_MAXLEVEL + iLevel+1;
122137 }
122138 if( rc!=SQLITE_OK ) goto finished;
122139 assert( csr.nSegment>0 );
122140 assert( iNewLevel>=(iIndex*FTS3_SEGDIR_MAXLEVEL) );
122141 assert( iNewLevel<((iIndex+1)*FTS3_SEGDIR_MAXLEVEL) );
122142
122143 memset(&filter, 0, sizeof(Fts3SegFilter));
122144 filter.flags = FTS3_SEGMENT_REQUIRE_POS;
122145 filter.flags |= (bIgnoreEmpty ? FTS3_SEGMENT_IGNORE_EMPTY : 0);
122146
122147 rc = sqlite3Fts3SegReaderStart(p, &csr, &filter);
122148 while( SQLITE_OK==rc ){
122149 rc = sqlite3Fts3SegReaderStep(p, &csr);
122150 if( rc!=SQLITE_ROW ) break;
@@ -121050,12 +122152,14 @@
122152 csr.zTerm, csr.nTerm, csr.aDoclist, csr.nDoclist);
122153 }
122154 if( rc!=SQLITE_OK ) goto finished;
122155 assert( pWriter );
122156
122157 if( iLevel!=FTS3_SEGCURSOR_PENDING ){
122158 rc = fts3DeleteSegdir(p, iIndex, iLevel, csr.apSegment, csr.nSegment);
122159 if( rc!=SQLITE_OK ) goto finished;
122160 }
122161 rc = fts3SegWriterFlush(p, pWriter, iNewLevel, iIdx);
122162
122163 finished:
122164 fts3SegWriterFree(pWriter);
122165 sqlite3Fts3SegReaderFinish(&csr);
@@ -121062,14 +122166,21 @@
122166 return rc;
122167 }
122168
122169
122170 /*
122171 ** Flush the contents of pendingTerms to level 0 segments.
122172 */
122173 SQLITE_PRIVATE int sqlite3Fts3PendingTermsFlush(Fts3Table *p){
122174 int rc = SQLITE_OK;
122175 int i;
122176 for(i=0; rc==SQLITE_OK && i<p->nIndex; i++){
122177 rc = fts3SegmentMerge(p, i, FTS3_SEGCURSOR_PENDING);
122178 if( rc==SQLITE_DONE ) rc = SQLITE_OK;
122179 }
122180 sqlite3Fts3PendingTermsClear(p);
122181 return rc;
122182 }
122183
122184 /*
122185 ** Encode N integers as varints into a blob.
122186 */
@@ -121215,10 +122326,27 @@
122326 sqlite3_bind_blob(pStmt, 1, pBlob, nBlob, SQLITE_STATIC);
122327 sqlite3_step(pStmt);
122328 *pRC = sqlite3_reset(pStmt);
122329 sqlite3_free(a);
122330 }
122331
122332 static int fts3DoOptimize(Fts3Table *p, int bReturnDone){
122333 int i;
122334 int bSeenDone = 0;
122335 int rc = SQLITE_OK;
122336 for(i=0; rc==SQLITE_OK && i<p->nIndex; i++){
122337 rc = fts3SegmentMerge(p, i, FTS3_SEGCURSOR_ALL);
122338 if( rc==SQLITE_DONE ){
122339 bSeenDone = 1;
122340 rc = SQLITE_OK;
122341 }
122342 }
122343 sqlite3Fts3SegmentsClose(p);
122344 sqlite3Fts3PendingTermsClear(p);
122345
122346 return (rc==SQLITE_OK && bReturnDone && bSeenDone) ? SQLITE_DONE : rc;
122347 }
122348
122349 /*
122350 ** Handle a 'special' INSERT of the form:
122351 **
122352 ** "INSERT INTO tbl(tbl) VALUES(<expr>)"
@@ -121232,16 +122360,11 @@
122360 int nVal = sqlite3_value_bytes(pVal);
122361
122362 if( !zVal ){
122363 return SQLITE_NOMEM;
122364 }else if( nVal==8 && 0==sqlite3_strnicmp(zVal, "optimize", 8) ){
122365 rc = fts3DoOptimize(p, 0);
 
 
 
 
 
122366 #ifdef SQLITE_TEST
122367 }else if( nVal>9 && 0==sqlite3_strnicmp(zVal, "nodesize=", 9) ){
122368 p->nNodeSize = atoi(&zVal[9]);
122369 rc = SQLITE_OK;
122370 }else if( nVal>11 && 0==sqlite3_strnicmp(zVal, "maxpending=", 9) ){
@@ -121250,61 +122373,23 @@
122373 #endif
122374 }else{
122375 rc = SQLITE_ERROR;
122376 }
122377
 
122378 return rc;
122379 }
122380
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
122381 /*
122382 ** Delete all cached deferred doclists. Deferred doclists are cached
122383 ** (allocated) by the sqlite3Fts3CacheDeferredDoclists() function.
122384 */
122385 SQLITE_PRIVATE void sqlite3Fts3FreeDeferredDoclists(Fts3Cursor *pCsr){
122386 Fts3DeferredToken *pDef;
122387 for(pDef=pCsr->pDeferred; pDef; pDef=pDef->pNext){
122388 fts3PendingListDelete(pDef->pList);
122389 pDef->pList = 0;
122390 }
 
 
 
122391 }
122392
122393 /*
122394 ** Free all entries in the pCsr->pDeffered list. Entries are added to
122395 ** this list using sqlite3Fts3DeferToken().
@@ -121312,11 +122397,11 @@
122397 SQLITE_PRIVATE void sqlite3Fts3FreeDeferredTokens(Fts3Cursor *pCsr){
122398 Fts3DeferredToken *pDef;
122399 Fts3DeferredToken *pNext;
122400 for(pDef=pCsr->pDeferred; pDef; pDef=pNext){
122401 pNext = pDef->pNext;
122402 fts3PendingListDelete(pDef->pList);
122403 sqlite3_free(pDef);
122404 }
122405 pCsr->pDeferred = 0;
122406 }
122407
@@ -121376,10 +122461,37 @@
122461 }
122462 }
122463
122464 return rc;
122465 }
122466
122467 SQLITE_PRIVATE int sqlite3Fts3DeferredTokenList(
122468 Fts3DeferredToken *p,
122469 char **ppData,
122470 int *pnData
122471 ){
122472 char *pRet;
122473 int nSkip;
122474 sqlite3_int64 dummy;
122475
122476 *ppData = 0;
122477 *pnData = 0;
122478
122479 if( p->pList==0 ){
122480 return SQLITE_OK;
122481 }
122482
122483 pRet = (char *)sqlite3_malloc(p->pList->nData);
122484 if( !pRet ) return SQLITE_NOMEM;
122485
122486 nSkip = sqlite3Fts3GetVarint(p->pList->aData, &dummy);
122487 *pnData = p->pList->nData - nSkip;
122488 *ppData = pRet;
122489
122490 memcpy(pRet, &p->pList->aData[nSkip], *pnData);
122491 return SQLITE_OK;
122492 }
122493
122494 /*
122495 ** Add an entry for token pToken to the pCsr->pDeferred list.
122496 */
122497 SQLITE_PRIVATE int sqlite3Fts3DeferToken(
@@ -121451,11 +122563,11 @@
122563 ){
122564 Fts3Table *p = (Fts3Table *)pVtab;
122565 int rc = SQLITE_OK; /* Return Code */
122566 int isRemove = 0; /* True for an UPDATE or DELETE */
122567 sqlite3_int64 iRemove = 0; /* Rowid removed by UPDATE or DELETE */
122568 u32 *aSzIns = 0; /* Sizes of inserted documents */
122569 u32 *aSzDel; /* Sizes of deleted documents */
122570 int nChng = 0; /* Net change in number of documents */
122571 int bInsertDone = 0;
122572
122573 assert( p->pSegments==0 );
@@ -121466,16 +122578,20 @@
122578 */
122579 if( nArg>1
122580 && sqlite3_value_type(apVal[0])==SQLITE_NULL
122581 && sqlite3_value_type(apVal[p->nColumn+2])!=SQLITE_NULL
122582 ){
122583 rc = fts3SpecialInsert(p, apVal[p->nColumn+2]);
122584 goto update_out;
122585 }
122586
122587 /* Allocate space to hold the change in document sizes */
122588 aSzIns = sqlite3_malloc( sizeof(aSzIns[0])*(p->nColumn+1)*2 );
122589 if( aSzIns==0 ){
122590 rc = SQLITE_NOMEM;
122591 goto update_out;
122592 }
122593 aSzDel = &aSzIns[p->nColumn+1];
122594 memset(aSzIns, 0, sizeof(aSzIns[0])*(p->nColumn+1)*2);
122595
122596 /* If this is an INSERT operation, or an UPDATE that modifies the rowid
122597 ** value, then this operation requires constraint handling.
@@ -121521,12 +122637,11 @@
122637 bInsertDone = 1;
122638 }
122639 }
122640 }
122641 if( rc!=SQLITE_OK ){
122642 goto update_out;
 
122643 }
122644
122645 /* If this is a DELETE or UPDATE operation, remove the old record. */
122646 if( sqlite3_value_type(apVal[0])!=SQLITE_NULL ){
122647 assert( sqlite3_value_type(apVal[0])==SQLITE_INTEGER );
@@ -121555,10 +122670,11 @@
122670
122671 if( p->bHasStat ){
122672 fts3UpdateDocTotals(&rc, p, aSzIns, aSzDel, nChng);
122673 }
122674
122675 update_out:
122676 sqlite3_free(aSzIns);
122677 sqlite3Fts3SegmentsClose(p);
122678 return rc;
122679 }
122680
@@ -121569,16 +122685,14 @@
122685 */
122686 SQLITE_PRIVATE int sqlite3Fts3Optimize(Fts3Table *p){
122687 int rc;
122688 rc = sqlite3_exec(p->db, "SAVEPOINT fts3", 0, 0, 0);
122689 if( rc==SQLITE_OK ){
122690 rc = fts3DoOptimize(p, 1);
122691 if( rc==SQLITE_OK || rc==SQLITE_DONE ){
122692 int rc2 = sqlite3_exec(p->db, "RELEASE fts3", 0, 0, 0);
122693 if( rc2!=SQLITE_OK ) rc = rc2;
 
 
122694 }else{
122695 sqlite3_exec(p->db, "ROLLBACK TO fts3", 0, 0, 0);
122696 sqlite3_exec(p->db, "RELEASE fts3", 0, 0, 0);
122697 }
122698 }
@@ -121763,76 +122877,24 @@
122877 ){
122878 int iPhrase = 0; /* Variable used as the phrase counter */
122879 return fts3ExprIterate2(pExpr, &iPhrase, x, pCtx);
122880 }
122881
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
122882 /*
122883 ** This is an fts3ExprIterate() callback used while loading the doclists
122884 ** for each phrase into Fts3Expr.aDoclist[]/nDoclist. See also
122885 ** fts3ExprLoadDoclists().
122886 */
122887 static int fts3ExprLoadDoclistsCb(Fts3Expr *pExpr, int iPhrase, void *ctx){
122888 int rc = SQLITE_OK;
122889 Fts3Phrase *pPhrase = pExpr->pPhrase;
122890 LoadDoclistCtx *p = (LoadDoclistCtx *)ctx;
122891
122892 UNUSED_PARAMETER(iPhrase);
122893
122894 p->nPhrase++;
122895 p->nToken += pPhrase->nToken;
 
 
 
 
 
 
 
 
122896
122897 return rc;
122898 }
122899
122900 /*
@@ -122002,11 +123064,11 @@
123064 SnippetPhrase *pPhrase = &p->aPhrase[iPhrase];
123065 char *pCsr;
123066
123067 pPhrase->nToken = pExpr->pPhrase->nToken;
123068
123069 pCsr = sqlite3Fts3EvalPhrasePoslist(p->pCsr, pExpr, p->iCol);
123070 if( pCsr ){
123071 int iFirst = 0;
123072 pPhrase->pList = pCsr;
123073 fts3GetDeltaPosition(&pCsr, &iFirst);
123074 pPhrase->pHead = pCsr;
@@ -122359,30 +123421,10 @@
123421
123422 *ppCollist = pEnd;
123423 return nEntry;
123424 }
123425
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
123426 /*
123427 ** fts3ExprIterate() callback used to collect the "global" matchinfo stats
123428 ** for a single query.
123429 **
123430 ** fts3ExprIterate() callback to load the 'global' elements of a
@@ -122412,52 +123454,13 @@
123454 Fts3Expr *pExpr, /* Phrase expression node */
123455 int iPhrase, /* Phrase number (numbered from zero) */
123456 void *pCtx /* Pointer to MatchInfo structure */
123457 ){
123458 MatchInfo *p = (MatchInfo *)pCtx;
123459 return sqlite3Fts3EvalPhraseStats(
123460 p->pCursor, pExpr, &p->aMatchinfo[3*iPhrase*p->nCol]
123461 );
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
123462 }
123463
123464 /*
123465 ** fts3ExprIterate() callback used to collect the "local" part of the
123466 ** FTS3_MATCHINFO_HITS array. The local stats are those elements of the
@@ -122470,18 +123473,17 @@
123473 ){
123474 MatchInfo *p = (MatchInfo *)pCtx;
123475 int iStart = iPhrase * p->nCol * 3;
123476 int i;
123477
123478 for(i=0; i<p->nCol; i++){
 
 
123479 char *pCsr;
123480 pCsr = sqlite3Fts3EvalPhrasePoslist(p->pCursor, pExpr, i);
 
123481 if( pCsr ){
123482 p->aMatchinfo[iStart+i*3] = fts3ColumnlistCount(&pCsr);
123483 }else{
123484 p->aMatchinfo[iStart+i*3] = 0;
123485 }
123486 }
123487
123488 return SQLITE_OK;
123489 }
@@ -122563,13 +123565,12 @@
123565 ** values for a matchinfo() FTS3_MATCHINFO_LCS request.
123566 */
123567 typedef struct LcsIterator LcsIterator;
123568 struct LcsIterator {
123569 Fts3Expr *pExpr; /* Pointer to phrase expression */
 
123570 int iPosOffset; /* Tokens count up to end of this phrase */
123571 char *pRead; /* Cursor used to iterate through aDoclist */
123572 int iPos; /* Current position */
123573 };
123574
123575 /*
123576 ** If LcsIterator.iCol is set to the following value, the iterator has
@@ -122596,21 +123597,14 @@
123597 char *pRead = pIter->pRead;
123598 sqlite3_int64 iRead;
123599 int rc = 0;
123600
123601 pRead += sqlite3Fts3GetVarint(pRead, &iRead);
123602 if( iRead==0 || iRead==1 ){
123603 pRead = 0;
123604 rc = 1;
123605 }else{
 
 
 
 
 
 
 
123606 pIter->iPos += (int)(iRead-2);
123607 }
123608
123609 pIter->pRead = pRead;
123610 return rc;
@@ -122638,46 +123632,38 @@
123632 **/
123633 aIter = sqlite3_malloc(sizeof(LcsIterator) * pCsr->nPhrase);
123634 if( !aIter ) return SQLITE_NOMEM;
123635 memset(aIter, 0, sizeof(LcsIterator) * pCsr->nPhrase);
123636 (void)fts3ExprIterate(pCsr->pExpr, fts3MatchinfoLcsCb, (void*)aIter);
123637
123638 for(i=0; i<pInfo->nPhrase; i++){
123639 LcsIterator *pIter = &aIter[i];
123640 nToken -= pIter->pExpr->pPhrase->nToken;
123641 pIter->iPosOffset = nToken;
 
 
 
 
 
 
 
123642 }
123643
123644 for(iCol=0; iCol<pInfo->nCol; iCol++){
123645 int nLcs = 0; /* LCS value for this column */
123646 int nLive = 0; /* Number of iterators in aIter not at EOF */
123647
 
 
 
123648 for(i=0; i<pInfo->nPhrase; i++){
123649 LcsIterator *pIt = &aIter[i];
123650 pIt->pRead = sqlite3Fts3EvalPhrasePoslist(pCsr, pIt->pExpr, iCol);
123651 if( pIt->pRead ){
123652 pIt->iPos = pIt->iPosOffset;
123653 fts3LcsIteratorAdvance(&aIter[i]);
123654 nLive++;
123655 }
123656 }
123657
 
 
 
 
123658 while( nLive>0 ){
123659 LcsIterator *pAdv = 0; /* The iterator to advance by one position */
123660 int nThisLcs = 0; /* LCS for the current iterator positions */
123661
123662 for(i=0; i<pInfo->nPhrase; i++){
123663 LcsIterator *pIter = &aIter[i];
123664 if( pIter->pRead==0 ){
123665 /* This iterator is already at EOF for this column. */
123666 nThisLcs = 0;
123667 }else{
123668 if( pAdv==0 || pIter->iPos<pAdv->iPos ){
123669 pAdv = pIter;
@@ -123013,11 +123999,11 @@
123999 int iTerm; /* For looping through nTerm phrase terms */
124000 char *pList; /* Pointer to position list for phrase */
124001 int iPos = 0; /* First position in position-list */
124002
124003 UNUSED_PARAMETER(iPhrase);
124004 pList = sqlite3Fts3EvalPhrasePoslist(p->pCsr, pExpr, p->iCol);
124005 nTerm = pExpr->pPhrase->nToken;
124006 if( pList ){
124007 fts3GetDeltaPosition(&pList, &iPos);
124008 assert( iPos>=0 );
124009 }
124010
M src/sqlite3.h
+1 -1
--- src/sqlite3.h
+++ src/sqlite3.h
@@ -107,11 +107,11 @@
107107
 ** [sqlite3_libversion_number()], [sqlite3_sourceid()],
108108
 ** [sqlite_version()] and [sqlite_source_id()].
109109
 */
110110
 #define SQLITE_VERSION "3.7.7"
111111
 #define SQLITE_VERSION_NUMBER 3007007
112
-#define SQLITE_SOURCE_ID "2011-06-03 14:19:10 0206bc6f87bb9393218a380fc5b18039d334a8d8"
112
+#define SQLITE_SOURCE_ID "2011-06-15 13:11:06 f9750870ee04935f338e4d808900fee5a8b2b389"
113113
114114
 /*
115115
 ** CAPI3REF: Run-Time Library Version Numbers
116116
 ** KEYWORDS: sqlite3_version, sqlite3_sourceid
117117
 **
118118
--- src/sqlite3.h
+++ src/sqlite3.h
@@ -107,11 +107,11 @@
107 ** [sqlite3_libversion_number()], [sqlite3_sourceid()],
108 ** [sqlite_version()] and [sqlite_source_id()].
109 */
110 #define SQLITE_VERSION "3.7.7"
111 #define SQLITE_VERSION_NUMBER 3007007
112 #define SQLITE_SOURCE_ID "2011-06-03 14:19:10 0206bc6f87bb9393218a380fc5b18039d334a8d8"
113
114 /*
115 ** CAPI3REF: Run-Time Library Version Numbers
116 ** KEYWORDS: sqlite3_version, sqlite3_sourceid
117 **
118
--- src/sqlite3.h
+++ src/sqlite3.h
@@ -107,11 +107,11 @@
107 ** [sqlite3_libversion_number()], [sqlite3_sourceid()],
108 ** [sqlite_version()] and [sqlite_source_id()].
109 */
110 #define SQLITE_VERSION "3.7.7"
111 #define SQLITE_VERSION_NUMBER 3007007
112 #define SQLITE_SOURCE_ID "2011-06-15 13:11:06 f9750870ee04935f338e4d808900fee5a8b2b389"
113
114 /*
115 ** CAPI3REF: Run-Time Library Version Numbers
116 ** KEYWORDS: sqlite3_version, sqlite3_sourceid
117 **
118

Keyboard Shortcuts

Open search /
Next entry (timeline) j
Previous entry (timeline) k
Open focused entry Enter
Show this help ?
Toggle theme Top nav button