| | @@ -650,11 +650,11 @@ |
| 650 | 650 | ** [sqlite3_libversion_number()], [sqlite3_sourceid()], |
| 651 | 651 | ** [sqlite_version()] and [sqlite_source_id()]. |
| 652 | 652 | */ |
| 653 | 653 | #define SQLITE_VERSION "3.7.7" |
| 654 | 654 | #define SQLITE_VERSION_NUMBER 3007007 |
| 655 | | -#define SQLITE_SOURCE_ID "2011-06-03 14:19:10 0206bc6f87bb9393218a380fc5b18039d334a8d8" |
| 655 | +#define SQLITE_SOURCE_ID "2011-06-15 13:11:06 f9750870ee04935f338e4d808900fee5a8b2b389" |
| 656 | 656 | |
| 657 | 657 | /* |
| 658 | 658 | ** CAPI3REF: Run-Time Library Version Numbers |
| 659 | 659 | ** KEYWORDS: sqlite3_version, sqlite3_sourceid |
| 660 | 660 | ** |
| | @@ -8483,10 +8483,11 @@ |
| 8483 | 8483 | SQLITE_PRIVATE int sqlite3VdbeAddOp2(Vdbe*,int,int,int); |
| 8484 | 8484 | SQLITE_PRIVATE int sqlite3VdbeAddOp3(Vdbe*,int,int,int,int); |
| 8485 | 8485 | SQLITE_PRIVATE int sqlite3VdbeAddOp4(Vdbe*,int,int,int,int,const char *zP4,int); |
| 8486 | 8486 | SQLITE_PRIVATE int sqlite3VdbeAddOp4Int(Vdbe*,int,int,int,int,int); |
| 8487 | 8487 | SQLITE_PRIVATE int sqlite3VdbeAddOpList(Vdbe*, int nOp, VdbeOpList const *aOp); |
| 8488 | +SQLITE_PRIVATE void sqlite3VdbeAddParseSchemaOp(Vdbe*,int,char*); |
| 8488 | 8489 | SQLITE_PRIVATE void sqlite3VdbeChangeP1(Vdbe*, int addr, int P1); |
| 8489 | 8490 | SQLITE_PRIVATE void sqlite3VdbeChangeP2(Vdbe*, int addr, int P2); |
| 8490 | 8491 | SQLITE_PRIVATE void sqlite3VdbeChangeP3(Vdbe*, int addr, int P3); |
| 8491 | 8492 | SQLITE_PRIVATE void sqlite3VdbeChangeP5(Vdbe*, u8 P5); |
| 8492 | 8493 | SQLITE_PRIVATE void sqlite3VdbeJumpHere(Vdbe*, int addr); |
| | @@ -9282,11 +9283,11 @@ |
| 9282 | 9283 | ** sqlite3_close(). |
| 9283 | 9284 | * |
| 9284 | 9285 | ** A thread must be holding a mutex on the corresponding Btree in order |
| 9285 | 9286 | ** to access Schema content. This implies that the thread must also be |
| 9286 | 9287 | ** holding a mutex on the sqlite3 connection pointer that owns the Btree. |
| 9287 | | -** For a TEMP Schema, on the connection mutex is required. |
| 9288 | +** For a TEMP Schema, only the connection mutex is required. |
| 9288 | 9289 | */ |
| 9289 | 9290 | struct Schema { |
| 9290 | 9291 | int schema_cookie; /* Database schema version number for this file */ |
| 9291 | 9292 | int iGeneration; /* Generation counter. Incremented with each change */ |
| 9292 | 9293 | Hash tblHash; /* All tables indexed by name */ |
| | @@ -11479,11 +11480,11 @@ |
| 11479 | 11480 | SQLITE_PRIVATE int sqlite3AtoF(const char *z, double*, int, u8); |
| 11480 | 11481 | SQLITE_PRIVATE int sqlite3GetInt32(const char *, int*); |
| 11481 | 11482 | SQLITE_PRIVATE int sqlite3Atoi(const char*); |
| 11482 | 11483 | SQLITE_PRIVATE int sqlite3Utf16ByteLen(const void *pData, int nChar); |
| 11483 | 11484 | SQLITE_PRIVATE int sqlite3Utf8CharLen(const char *pData, int nByte); |
| 11484 | | -SQLITE_PRIVATE int sqlite3Utf8Read(const u8*, const u8**); |
| 11485 | +SQLITE_PRIVATE u32 sqlite3Utf8Read(const u8*, const u8**); |
| 11485 | 11486 | |
| 11486 | 11487 | /* |
| 11487 | 11488 | ** Routines to read and write variable-length integers. These used to |
| 11488 | 11489 | ** be defined locally, but now we use the varint routines in the util.c |
| 11489 | 11490 | ** file. Code should use the MACRO forms below, as the Varint32 versions |
| | @@ -20086,11 +20087,11 @@ |
| 20086 | 20087 | } \ |
| 20087 | 20088 | if( c<0x80 \ |
| 20088 | 20089 | || (c&0xFFFFF800)==0xD800 \ |
| 20089 | 20090 | || (c&0xFFFFFFFE)==0xFFFE ){ c = 0xFFFD; } \ |
| 20090 | 20091 | } |
| 20091 | | -SQLITE_PRIVATE int sqlite3Utf8Read( |
| 20092 | +SQLITE_PRIVATE u32 sqlite3Utf8Read( |
| 20092 | 20093 | const unsigned char *zIn, /* First byte of UTF-8 character */ |
| 20093 | 20094 | const unsigned char **pzNext /* Write first byte past UTF-8 char here */ |
| 20094 | 20095 | ){ |
| 20095 | 20096 | unsigned int c; |
| 20096 | 20097 | |
| | @@ -35805,11 +35806,11 @@ |
| 35805 | 35806 | memset(pCache, 0, sz); |
| 35806 | 35807 | if( separateCache ){ |
| 35807 | 35808 | pGroup = (PGroup*)&pCache[1]; |
| 35808 | 35809 | pGroup->mxPinned = 10; |
| 35809 | 35810 | }else{ |
| 35810 | | - pGroup = &pcache1_g.grp; |
| 35811 | + pGroup = &pcache1.grp; |
| 35811 | 35812 | } |
| 35812 | 35813 | pCache->pGroup = pGroup; |
| 35813 | 35814 | pCache->szPage = szPage; |
| 35814 | 35815 | pCache->bPurgeable = (bPurgeable ? 1 : 0); |
| 35815 | 35816 | if( bPurgeable ){ |
| | @@ -48324,10 +48325,12 @@ |
| 48324 | 48325 | ** |
| 48325 | 48326 | ** This routine works only for pages that do not contain overflow cells. |
| 48326 | 48327 | */ |
| 48327 | 48328 | #define findCell(P,I) \ |
| 48328 | 48329 | ((P)->aData + ((P)->maskPage & get2byte(&(P)->aData[(P)->cellOffset+2*(I)]))) |
| 48330 | +#define findCellv2(D,M,O,I) (D+(M&get2byte(D+(O+2*(I))))) |
| 48331 | + |
| 48329 | 48332 | |
| 48330 | 48333 | /* |
| 48331 | 48334 | ** This a more complex version of findCell() that works for |
| 48332 | 48335 | ** pages that do contain overflow cells. |
| 48333 | 48336 | */ |
| | @@ -51918,11 +51921,11 @@ |
| 51918 | 51921 | assert( pCur->apPage[pCur->iPage]->nCell==0 ); |
| 51919 | 51922 | return SQLITE_OK; |
| 51920 | 51923 | } |
| 51921 | 51924 | assert( pCur->apPage[0]->intKey || pIdxKey ); |
| 51922 | 51925 | for(;;){ |
| 51923 | | - int lwr, upr; |
| 51926 | + int lwr, upr, idx; |
| 51924 | 51927 | Pgno chldPg; |
| 51925 | 51928 | MemPage *pPage = pCur->apPage[pCur->iPage]; |
| 51926 | 51929 | int c; |
| 51927 | 51930 | |
| 51928 | 51931 | /* pPage->nCell must be greater than zero. If this is the root-page |
| | @@ -51934,18 +51937,18 @@ |
| 51934 | 51937 | assert( pPage->nCell>0 ); |
| 51935 | 51938 | assert( pPage->intKey==(pIdxKey==0) ); |
| 51936 | 51939 | lwr = 0; |
| 51937 | 51940 | upr = pPage->nCell-1; |
| 51938 | 51941 | if( biasRight ){ |
| 51939 | | - pCur->aiIdx[pCur->iPage] = (u16)upr; |
| 51942 | + pCur->aiIdx[pCur->iPage] = (u16)(idx = upr); |
| 51940 | 51943 | }else{ |
| 51941 | | - pCur->aiIdx[pCur->iPage] = (u16)((upr+lwr)/2); |
| 51944 | + pCur->aiIdx[pCur->iPage] = (u16)(idx = (upr+lwr)/2); |
| 51942 | 51945 | } |
| 51943 | 51946 | for(;;){ |
| 51944 | | - int idx = pCur->aiIdx[pCur->iPage]; /* Index of current cell in pPage */ |
| 51945 | 51947 | u8 *pCell; /* Pointer to current cell in pPage */ |
| 51946 | 51948 | |
| 51949 | + assert( idx==pCur->aiIdx[pCur->iPage] ); |
| 51947 | 51950 | pCur->info.nSize = 0; |
| 51948 | 51951 | pCell = findCell(pPage, idx) + pPage->childPtrSize; |
| 51949 | 51952 | if( pPage->intKey ){ |
| 51950 | 51953 | i64 nCellKey; |
| 51951 | 51954 | if( pPage->hasData ){ |
| | @@ -52024,11 +52027,11 @@ |
| 52024 | 52027 | upr = idx-1; |
| 52025 | 52028 | } |
| 52026 | 52029 | if( lwr>upr ){ |
| 52027 | 52030 | break; |
| 52028 | 52031 | } |
| 52029 | | - pCur->aiIdx[pCur->iPage] = (u16)((lwr+upr)/2); |
| 52032 | + pCur->aiIdx[pCur->iPage] = (u16)(idx = (lwr+upr)/2); |
| 52030 | 52033 | } |
| 52031 | 52034 | assert( lwr==upr+1 ); |
| 52032 | 52035 | assert( pPage->isInit ); |
| 52033 | 52036 | if( pPage->leaf ){ |
| 52034 | 52037 | chldPg = 0; |
| | @@ -52886,13 +52889,13 @@ |
| 52886 | 52889 | if( rc ){ |
| 52887 | 52890 | *pRC = rc; |
| 52888 | 52891 | return; |
| 52889 | 52892 | } |
| 52890 | 52893 | endPtr = &data[pPage->cellOffset + 2*pPage->nCell - 2]; |
| 52894 | + assert( (SQLITE_PTR_TO_INT(ptr)&1)==0 ); /* ptr is always 2-byte aligned */ |
| 52891 | 52895 | while( ptr<endPtr ){ |
| 52892 | | - ptr[0] = ptr[2]; |
| 52893 | | - ptr[1] = ptr[3]; |
| 52896 | + *(u16*)ptr = *(u16*)&ptr[2]; |
| 52894 | 52897 | ptr += 2; |
| 52895 | 52898 | } |
| 52896 | 52899 | pPage->nCell--; |
| 52897 | 52900 | put2byte(&data[hdr+3], pPage->nCell); |
| 52898 | 52901 | pPage->nFree += 2; |
| | @@ -52929,10 +52932,11 @@ |
| 52929 | 52932 | int end; /* First byte past the last cell pointer in data[] */ |
| 52930 | 52933 | int ins; /* Index in data[] where new cell pointer is inserted */ |
| 52931 | 52934 | int cellOffset; /* Address of first cell pointer in data[] */ |
| 52932 | 52935 | u8 *data; /* The content of the whole page */ |
| 52933 | 52936 | u8 *ptr; /* Used for moving information around in data[] */ |
| 52937 | + u8 *endPtr; /* End of the loop */ |
| 52934 | 52938 | |
| 52935 | 52939 | int nSkip = (iChild ? 4 : 0); |
| 52936 | 52940 | |
| 52937 | 52941 | if( *pRC ) return; |
| 52938 | 52942 | |
| | @@ -52979,13 +52983,16 @@ |
| 52979 | 52983 | pPage->nFree -= (u16)(2 + sz); |
| 52980 | 52984 | memcpy(&data[idx+nSkip], pCell+nSkip, sz-nSkip); |
| 52981 | 52985 | if( iChild ){ |
| 52982 | 52986 | put4byte(&data[idx], iChild); |
| 52983 | 52987 | } |
| 52984 | | - for(j=end, ptr=&data[j]; j>ins; j-=2, ptr-=2){ |
| 52985 | | - ptr[0] = ptr[-2]; |
| 52986 | | - ptr[1] = ptr[-1]; |
| 52988 | + ptr = &data[end]; |
| 52989 | + endPtr = &data[ins]; |
| 52990 | + assert( (SQLITE_PTR_TO_INT(ptr)&1)==0 ); /* ptr is always 2-byte aligned */ |
| 52991 | + while( ptr>endPtr ){ |
| 52992 | + *(u16*)ptr = *(u16*)&ptr[-2]; |
| 52993 | + ptr -= 2; |
| 52987 | 52994 | } |
| 52988 | 52995 | put2byte(&data[ins], idx); |
| 52989 | 52996 | put2byte(&data[pPage->hdrOffset+3], pPage->nCell); |
| 52990 | 52997 | #ifndef SQLITE_OMIT_AUTOVACUUM |
| 52991 | 52998 | if( pPage->pBt->autoVacuum ){ |
| | @@ -53026,14 +53033,15 @@ |
| 53026 | 53033 | assert( get2byteNotZero(&data[hdr+5])==nUsable ); |
| 53027 | 53034 | |
| 53028 | 53035 | pCellptr = &data[pPage->cellOffset + nCell*2]; |
| 53029 | 53036 | cellbody = nUsable; |
| 53030 | 53037 | for(i=nCell-1; i>=0; i--){ |
| 53038 | + u16 sz = aSize[i]; |
| 53031 | 53039 | pCellptr -= 2; |
| 53032 | | - cellbody -= aSize[i]; |
| 53040 | + cellbody -= sz; |
| 53033 | 53041 | put2byte(pCellptr, cellbody); |
| 53034 | | - memcpy(&data[cellbody], apCell[i], aSize[i]); |
| 53042 | + memcpy(&data[cellbody], apCell[i], sz); |
| 53035 | 53043 | } |
| 53036 | 53044 | put2byte(&data[hdr+3], nCell); |
| 53037 | 53045 | put2byte(&data[hdr+5], cellbody); |
| 53038 | 53046 | pPage->nFree -= (nCell*2 + nUsable - cellbody); |
| 53039 | 53047 | pPage->nCell = (u16)nCell; |
| | @@ -53483,16 +53491,28 @@ |
| 53483 | 53491 | memcpy(pOld, apOld[i], sizeof(MemPage)); |
| 53484 | 53492 | pOld->aData = (void*)&pOld[1]; |
| 53485 | 53493 | memcpy(pOld->aData, apOld[i]->aData, pBt->pageSize); |
| 53486 | 53494 | |
| 53487 | 53495 | limit = pOld->nCell+pOld->nOverflow; |
| 53488 | | - for(j=0; j<limit; j++){ |
| 53489 | | - assert( nCell<nMaxCells ); |
| 53490 | | - apCell[nCell] = findOverflowCell(pOld, j); |
| 53491 | | - szCell[nCell] = cellSizePtr(pOld, apCell[nCell]); |
| 53492 | | - nCell++; |
| 53493 | | - } |
| 53496 | + if( pOld->nOverflow>0 ){ |
| 53497 | + for(j=0; j<limit; j++){ |
| 53498 | + assert( nCell<nMaxCells ); |
| 53499 | + apCell[nCell] = findOverflowCell(pOld, j); |
| 53500 | + szCell[nCell] = cellSizePtr(pOld, apCell[nCell]); |
| 53501 | + nCell++; |
| 53502 | + } |
| 53503 | + }else{ |
| 53504 | + u8 *aData = pOld->aData; |
| 53505 | + u16 maskPage = pOld->maskPage; |
| 53506 | + u16 cellOffset = pOld->cellOffset; |
| 53507 | + for(j=0; j<limit; j++){ |
| 53508 | + assert( nCell<nMaxCells ); |
| 53509 | + apCell[nCell] = findCellv2(aData, maskPage, cellOffset, j); |
| 53510 | + szCell[nCell] = cellSizePtr(pOld, apCell[nCell]); |
| 53511 | + nCell++; |
| 53512 | + } |
| 53513 | + } |
| 53494 | 53514 | if( i<nOld-1 && !leafData){ |
| 53495 | 53515 | u16 sz = (u16)szNew[i]; |
| 53496 | 53516 | u8 *pTemp; |
| 53497 | 53517 | assert( nCell<nMaxCells ); |
| 53498 | 53518 | szCell[nCell] = sz; |
| | @@ -57637,17 +57657,10 @@ |
| 57637 | 57657 | pOp->p1 = p1; |
| 57638 | 57658 | pOp->p2 = p2; |
| 57639 | 57659 | pOp->p3 = p3; |
| 57640 | 57660 | pOp->p4.p = 0; |
| 57641 | 57661 | pOp->p4type = P4_NOTUSED; |
| 57642 | | - p->expired = 0; |
| 57643 | | - if( op==OP_ParseSchema ){ |
| 57644 | | - /* Any program that uses the OP_ParseSchema opcode needs to lock |
| 57645 | | - ** all btrees. */ |
| 57646 | | - int j; |
| 57647 | | - for(j=0; j<p->db->nDb; j++) sqlite3VdbeUsesBtree(p, j); |
| 57648 | | - } |
| 57649 | 57662 | #ifdef SQLITE_DEBUG |
| 57650 | 57663 | pOp->zComment = 0; |
| 57651 | 57664 | if( sqlite3VdbeAddopTrace ) sqlite3VdbePrintOp(0, i, &p->aOp[i]); |
| 57652 | 57665 | #endif |
| 57653 | 57666 | #ifdef VDBE_PROFILE |
| | @@ -57681,10 +57694,24 @@ |
| 57681 | 57694 | ){ |
| 57682 | 57695 | int addr = sqlite3VdbeAddOp3(p, op, p1, p2, p3); |
| 57683 | 57696 | sqlite3VdbeChangeP4(p, addr, zP4, p4type); |
| 57684 | 57697 | return addr; |
| 57685 | 57698 | } |
| 57699 | + |
| 57700 | +/* |
| 57701 | +** Add an OP_ParseSchema opcode. This routine is broken out from |
| 57702 | +** sqlite3VdbeAddOp4() since it needs to also local all btrees. |
| 57703 | +** |
| 57704 | +** The zWhere string must have been obtained from sqlite3_malloc(). |
| 57705 | +** This routine will take ownership of the allocated memory. |
| 57706 | +*/ |
| 57707 | +SQLITE_PRIVATE void sqlite3VdbeAddParseSchemaOp(Vdbe *p, int iDb, char *zWhere){ |
| 57708 | + int j; |
| 57709 | + int addr = sqlite3VdbeAddOp3(p, OP_ParseSchema, iDb, 0, 0); |
| 57710 | + sqlite3VdbeChangeP4(p, addr, zWhere, P4_DYNAMIC); |
| 57711 | + for(j=0; j<p->db->nDb; j++) sqlite3VdbeUsesBtree(p, j); |
| 57712 | +} |
| 57686 | 57713 | |
| 57687 | 57714 | /* |
| 57688 | 57715 | ** Add an opcode that includes the p4 value as an integer. |
| 57689 | 57716 | */ |
| 57690 | 57717 | SQLITE_PRIVATE int sqlite3VdbeAddOp4Int( |
| | @@ -64083,10 +64110,20 @@ |
| 64083 | 64110 | u.ag.ctx.pColl = pOp[-1].p4.pColl; |
| 64084 | 64111 | } |
| 64085 | 64112 | db->lastRowid = lastRowid; |
| 64086 | 64113 | (*u.ag.ctx.pFunc->xFunc)(&u.ag.ctx, u.ag.n, u.ag.apVal); /* IMP: R-24505-23230 */ |
| 64087 | 64114 | lastRowid = db->lastRowid; |
| 64115 | + |
| 64116 | + /* If any auxiliary data functions have been called by this user function, |
| 64117 | + ** immediately call the destructor for any non-static values. |
| 64118 | + */ |
| 64119 | + if( u.ag.ctx.pVdbeFunc ){ |
| 64120 | + sqlite3VdbeDeleteAuxData(u.ag.ctx.pVdbeFunc, pOp->p1); |
| 64121 | + pOp->p4.pVdbeFunc = u.ag.ctx.pVdbeFunc; |
| 64122 | + pOp->p4type = P4_VDBEFUNC; |
| 64123 | + } |
| 64124 | + |
| 64088 | 64125 | if( db->mallocFailed ){ |
| 64089 | 64126 | /* Even though a malloc() has failed, the implementation of the |
| 64090 | 64127 | ** user function may have called an sqlite3_result_XXX() function |
| 64091 | 64128 | ** to return a value. The following call releases any resources |
| 64092 | 64129 | ** associated with such a value. |
| | @@ -64093,19 +64130,10 @@ |
| 64093 | 64130 | */ |
| 64094 | 64131 | sqlite3VdbeMemRelease(&u.ag.ctx.s); |
| 64095 | 64132 | goto no_mem; |
| 64096 | 64133 | } |
| 64097 | 64134 | |
| 64098 | | - /* If any auxiliary data functions have been called by this user function, |
| 64099 | | - ** immediately call the destructor for any non-static values. |
| 64100 | | - */ |
| 64101 | | - if( u.ag.ctx.pVdbeFunc ){ |
| 64102 | | - sqlite3VdbeDeleteAuxData(u.ag.ctx.pVdbeFunc, pOp->p1); |
| 64103 | | - pOp->p4.pVdbeFunc = u.ag.ctx.pVdbeFunc; |
| 64104 | | - pOp->p4type = P4_VDBEFUNC; |
| 64105 | | - } |
| 64106 | | - |
| 64107 | 64135 | /* If the function returned an error, throw an exception */ |
| 64108 | 64136 | if( u.ag.ctx.isError ){ |
| 64109 | 64137 | sqlite3SetString(&p->zErrMsg, db, "%s", sqlite3_value_text(&u.ag.ctx.s)); |
| 64110 | 64138 | rc = u.ag.ctx.isError; |
| 64111 | 64139 | } |
| | @@ -75255,18 +75283,18 @@ |
| 75255 | 75283 | sqlite3VdbeAddOp4(v, OP_DropTable, iDb, 0, 0, pTab->zName, 0); |
| 75256 | 75284 | |
| 75257 | 75285 | /* Reload the table, index and permanent trigger schemas. */ |
| 75258 | 75286 | zWhere = sqlite3MPrintf(pParse->db, "tbl_name=%Q", zName); |
| 75259 | 75287 | if( !zWhere ) return; |
| 75260 | | - sqlite3VdbeAddOp4(v, OP_ParseSchema, iDb, 0, 0, zWhere, P4_DYNAMIC); |
| 75288 | + sqlite3VdbeAddParseSchemaOp(v, iDb, zWhere); |
| 75261 | 75289 | |
| 75262 | 75290 | #ifndef SQLITE_OMIT_TRIGGER |
| 75263 | 75291 | /* Now, if the table is not stored in the temp database, reload any temp |
| 75264 | 75292 | ** triggers. Don't use IN(...) in case SQLITE_OMIT_SUBQUERY is defined. |
| 75265 | 75293 | */ |
| 75266 | 75294 | if( (zWhere=whereTempTriggers(pParse, pTab))!=0 ){ |
| 75267 | | - sqlite3VdbeAddOp4(v, OP_ParseSchema, 1, 0, 0, zWhere, P4_DYNAMIC); |
| 75295 | + sqlite3VdbeAddParseSchemaOp(v, 1, zWhere); |
| 75268 | 75296 | } |
| 75269 | 75297 | #endif |
| 75270 | 75298 | } |
| 75271 | 75299 | |
| 75272 | 75300 | /* |
| | @@ -78875,12 +78903,12 @@ |
| 78875 | 78903 | } |
| 78876 | 78904 | } |
| 78877 | 78905 | #endif |
| 78878 | 78906 | |
| 78879 | 78907 | /* Reparse everything to update our internal data structures */ |
| 78880 | | - sqlite3VdbeAddOp4(v, OP_ParseSchema, iDb, 0, 0, |
| 78881 | | - sqlite3MPrintf(db, "tbl_name='%q'",p->zName), P4_DYNAMIC); |
| 78908 | + sqlite3VdbeAddParseSchemaOp(v, iDb, |
| 78909 | + sqlite3MPrintf(db, "tbl_name='%q'", p->zName)); |
| 78882 | 78910 | } |
| 78883 | 78911 | |
| 78884 | 78912 | |
| 78885 | 78913 | /* Add the table to the in-memory representation of the database. |
| 78886 | 78914 | */ |
| | @@ -80073,13 +80101,12 @@ |
| 80073 | 80101 | ** to invalidate all pre-compiled statements. |
| 80074 | 80102 | */ |
| 80075 | 80103 | if( pTblName ){ |
| 80076 | 80104 | sqlite3RefillIndex(pParse, pIndex, iMem); |
| 80077 | 80105 | sqlite3ChangeCookie(pParse, iDb); |
| 80078 | | - sqlite3VdbeAddOp4(v, OP_ParseSchema, iDb, 0, 0, |
| 80079 | | - sqlite3MPrintf(db, "name='%q' AND type='index'", pIndex->zName), |
| 80080 | | - P4_DYNAMIC); |
| 80106 | + sqlite3VdbeAddParseSchemaOp(v, iDb, |
| 80107 | + sqlite3MPrintf(db, "name='%q' AND type='index'", pIndex->zName)); |
| 80081 | 80108 | sqlite3VdbeAddOp1(v, OP_Expire, 0); |
| 80082 | 80109 | } |
| 80083 | 80110 | } |
| 80084 | 80111 | |
| 80085 | 80112 | /* When adding an index to the list of indices for a table, make |
| | @@ -82621,14 +82648,14 @@ |
| 82621 | 82648 | ** character is exactly one byte in size. Also, all characters are |
| 82622 | 82649 | ** able to participate in upper-case-to-lower-case mappings in EBCDIC |
| 82623 | 82650 | ** whereas only characters less than 0x80 do in ASCII. |
| 82624 | 82651 | */ |
| 82625 | 82652 | #if defined(SQLITE_EBCDIC) |
| 82626 | | -# define sqlite3Utf8Read(A,C) (*(A++)) |
| 82627 | | -# define GlogUpperToLower(A) A = sqlite3UpperToLower[A] |
| 82653 | +# define sqlite3Utf8Read(A,C) (*(A++)) |
| 82654 | +# define GlogUpperToLower(A) A = sqlite3UpperToLower[A] |
| 82628 | 82655 | #else |
| 82629 | | -# define GlogUpperToLower(A) if( A<0x80 ){ A = sqlite3UpperToLower[A]; } |
| 82656 | +# define GlogUpperToLower(A) if( !((A)&~0x7f) ){ A = sqlite3UpperToLower[A]; } |
| 82630 | 82657 | #endif |
| 82631 | 82658 | |
| 82632 | 82659 | static const struct compareInfo globInfo = { '*', '?', '[', 0 }; |
| 82633 | 82660 | /* The correct SQL-92 behavior is for the LIKE operator to ignore |
| 82634 | 82661 | ** case. Thus 'a' LIKE 'A' would be true. */ |
| | @@ -82667,13 +82694,13 @@ |
| 82667 | 82694 | */ |
| 82668 | 82695 | static int patternCompare( |
| 82669 | 82696 | const u8 *zPattern, /* The glob pattern */ |
| 82670 | 82697 | const u8 *zString, /* The string to compare against the glob */ |
| 82671 | 82698 | const struct compareInfo *pInfo, /* Information about how to do the compare */ |
| 82672 | | - const int esc /* The escape character */ |
| 82699 | + u32 esc /* The escape character */ |
| 82673 | 82700 | ){ |
| 82674 | | - int c, c2; |
| 82701 | + u32 c, c2; |
| 82675 | 82702 | int invert; |
| 82676 | 82703 | int seen; |
| 82677 | 82704 | u8 matchOne = pInfo->matchOne; |
| 82678 | 82705 | u8 matchAll = pInfo->matchAll; |
| 82679 | 82706 | u8 matchSet = pInfo->matchSet; |
| | @@ -82723,11 +82750,11 @@ |
| 82723 | 82750 | }else if( !prevEscape && c==matchOne ){ |
| 82724 | 82751 | if( sqlite3Utf8Read(zString, &zString)==0 ){ |
| 82725 | 82752 | return 0; |
| 82726 | 82753 | } |
| 82727 | 82754 | }else if( c==matchSet ){ |
| 82728 | | - int prior_c = 0; |
| 82755 | + u32 prior_c = 0; |
| 82729 | 82756 | assert( esc==0 ); /* This only occurs for GLOB, not LIKE */ |
| 82730 | 82757 | seen = 0; |
| 82731 | 82758 | invert = 0; |
| 82732 | 82759 | c = sqlite3Utf8Read(zString, &zString); |
| 82733 | 82760 | if( c==0 ) return 0; |
| | @@ -82799,11 +82826,11 @@ |
| 82799 | 82826 | sqlite3_context *context, |
| 82800 | 82827 | int argc, |
| 82801 | 82828 | sqlite3_value **argv |
| 82802 | 82829 | ){ |
| 82803 | 82830 | const unsigned char *zA, *zB; |
| 82804 | | - int escape = 0; |
| 82831 | + u32 escape = 0; |
| 82805 | 82832 | int nPat; |
| 82806 | 82833 | sqlite3 *db = sqlite3_context_db_handle(context); |
| 82807 | 82834 | |
| 82808 | 82835 | zB = sqlite3_value_text(argv[0]); |
| 82809 | 82836 | zA = sqlite3_value_text(argv[1]); |
| | @@ -84110,17 +84137,29 @@ |
| 84110 | 84137 | } |
| 84111 | 84138 | |
| 84112 | 84139 | /* If the parent table is the same as the child table, and we are about |
| 84113 | 84140 | ** to increment the constraint-counter (i.e. this is an INSERT operation), |
| 84114 | 84141 | ** then check if the row being inserted matches itself. If so, do not |
| 84115 | | - ** increment the constraint-counter. */ |
| 84142 | + ** increment the constraint-counter. |
| 84143 | + ** |
| 84144 | + ** If any of the parent-key values are NULL, then the row cannot match |
| 84145 | + ** itself. So set JUMPIFNULL to make sure we do the OP_Found if any |
| 84146 | + ** of the parent-key values are NULL (at this point it is known that |
| 84147 | + ** none of the child key values are). |
| 84148 | + */ |
| 84116 | 84149 | if( pTab==pFKey->pFrom && nIncr==1 ){ |
| 84117 | 84150 | int iJump = sqlite3VdbeCurrentAddr(v) + nCol + 1; |
| 84118 | 84151 | for(i=0; i<nCol; i++){ |
| 84119 | 84152 | int iChild = aiCol[i]+1+regData; |
| 84120 | 84153 | int iParent = pIdx->aiColumn[i]+1+regData; |
| 84154 | + assert( aiCol[i]!=pTab->iPKey ); |
| 84155 | + if( pIdx->aiColumn[i]==pTab->iPKey ){ |
| 84156 | + /* The parent key is a composite key that includes the IPK column */ |
| 84157 | + iParent = regData; |
| 84158 | + } |
| 84121 | 84159 | sqlite3VdbeAddOp3(v, OP_Ne, iChild, iJump, iParent); |
| 84160 | + sqlite3VdbeChangeP5(v, SQLITE_JUMPIFNULL); |
| 84122 | 84161 | } |
| 84123 | 84162 | sqlite3VdbeAddOp2(v, OP_Goto, 0, iOk); |
| 84124 | 84163 | } |
| 84125 | 84164 | |
| 84126 | 84165 | sqlite3VdbeAddOp3(v, OP_MakeRecord, regTemp, nCol, regRec); |
| | @@ -95336,13 +95375,12 @@ |
| 95336 | 95375 | "INSERT INTO %Q.%s VALUES('trigger',%Q,%Q,0,'CREATE TRIGGER %q')", |
| 95337 | 95376 | db->aDb[iDb].zName, SCHEMA_TABLE(iDb), zName, |
| 95338 | 95377 | pTrig->table, z); |
| 95339 | 95378 | sqlite3DbFree(db, z); |
| 95340 | 95379 | sqlite3ChangeCookie(pParse, iDb); |
| 95341 | | - sqlite3VdbeAddOp4(v, OP_ParseSchema, iDb, 0, 0, sqlite3MPrintf( |
| 95342 | | - db, "type='trigger' AND name='%q'", zName), P4_DYNAMIC |
| 95343 | | - ); |
| 95380 | + sqlite3VdbeAddParseSchemaOp(v, iDb, |
| 95381 | + sqlite3MPrintf(db, "type='trigger' AND name='%q'", zName)); |
| 95344 | 95382 | } |
| 95345 | 95383 | |
| 95346 | 95384 | if( db->init.busy ){ |
| 95347 | 95385 | Trigger *pLink = pTrig; |
| 95348 | 95386 | Hash *pHash = &db->aDb[iDb].pSchema->trigHash; |
| | @@ -96392,11 +96430,11 @@ |
| 96392 | 96430 | aRegIdx = sqlite3DbMallocRaw(db, sizeof(Index*) * nIdx ); |
| 96393 | 96431 | if( aRegIdx==0 ) goto update_cleanup; |
| 96394 | 96432 | } |
| 96395 | 96433 | for(j=0, pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext, j++){ |
| 96396 | 96434 | int reg; |
| 96397 | | - if( chngRowid ){ |
| 96435 | + if( hasFK || chngRowid ){ |
| 96398 | 96436 | reg = ++pParse->nMem; |
| 96399 | 96437 | }else{ |
| 96400 | 96438 | reg = 0; |
| 96401 | 96439 | for(i=0; i<pIdx->nColumn; i++){ |
| 96402 | 96440 | if( aXRef[pIdx->aiColumn[i]]>=0 ){ |
| | @@ -97547,11 +97585,11 @@ |
| 97547 | 97585 | v = sqlite3GetVdbe(pParse); |
| 97548 | 97586 | sqlite3ChangeCookie(pParse, iDb); |
| 97549 | 97587 | |
| 97550 | 97588 | sqlite3VdbeAddOp2(v, OP_Expire, 0, 0); |
| 97551 | 97589 | zWhere = sqlite3MPrintf(db, "name='%q' AND type='table'", pTab->zName); |
| 97552 | | - sqlite3VdbeAddOp4(v, OP_ParseSchema, iDb, 0, 0, zWhere, P4_DYNAMIC); |
| 97590 | + sqlite3VdbeAddParseSchemaOp(v, iDb, zWhere); |
| 97553 | 97591 | sqlite3VdbeAddOp4(v, OP_VCreate, iDb, 0, 0, |
| 97554 | 97592 | pTab->zName, sqlite3Strlen30(pTab->zName) + 1); |
| 97555 | 97593 | } |
| 97556 | 97594 | |
| 97557 | 97595 | /* If we are rereading the sqlite_master table create the in-memory |
| | @@ -107274,17 +107312,16 @@ |
| 107274 | 107312 | #ifndef SQLITE_OMIT_BLOB_LITERAL |
| 107275 | 107313 | case 'x': case 'X': { |
| 107276 | 107314 | testcase( z[0]=='x' ); testcase( z[0]=='X' ); |
| 107277 | 107315 | if( z[1]=='\'' ){ |
| 107278 | 107316 | *tokenType = TK_BLOB; |
| 107279 | | - for(i=2; (c=z[i])!=0 && c!='\''; i++){ |
| 107280 | | - if( !sqlite3Isxdigit(c) ){ |
| 107281 | | - *tokenType = TK_ILLEGAL; |
| 107282 | | - } |
| 107317 | + for(i=2; sqlite3Isxdigit(z[i]); i++){} |
| 107318 | + if( z[i]!='\'' || i%2 ){ |
| 107319 | + *tokenType = TK_ILLEGAL; |
| 107320 | + while( z[i] && z[i]!='\'' ){ i++; } |
| 107283 | 107321 | } |
| 107284 | | - if( i%2 || !c ) *tokenType = TK_ILLEGAL; |
| 107285 | | - if( c ) i++; |
| 107322 | + if( z[i] ) i++; |
| 107286 | 107323 | return i; |
| 107287 | 107324 | } |
| 107288 | 107325 | /* Otherwise fall through to the next case */ |
| 107289 | 107326 | } |
| 107290 | 107327 | #endif |
| | @@ -111710,16 +111747,39 @@ |
| 111710 | 111747 | ** similar macro called ArraySize(). Use a different name to avoid |
| 111711 | 111748 | ** a collision when building an amalgamation with built-in FTS3. |
| 111712 | 111749 | */ |
| 111713 | 111750 | #define SizeofArray(X) ((int)(sizeof(X)/sizeof(X[0]))) |
| 111714 | 111751 | |
| 111752 | + |
| 111753 | +#ifndef MIN |
| 111754 | +# define MIN(x,y) ((x)<(y)?(x):(y)) |
| 111755 | +#endif |
| 111756 | + |
| 111715 | 111757 | /* |
| 111716 | 111758 | ** Maximum length of a varint encoded integer. The varint format is different |
| 111717 | 111759 | ** from that used by SQLite, so the maximum length is 10, not 9. |
| 111718 | 111760 | */ |
| 111719 | 111761 | #define FTS3_VARINT_MAX 10 |
| 111720 | 111762 | |
| 111763 | +/* |
| 111764 | +** FTS4 virtual tables may maintain multiple indexes - one index of all terms |
| 111765 | +** in the document set and zero or more prefix indexes. All indexes are stored |
| 111766 | +** as one or more b+-trees in the %_segments and %_segdir tables. |
| 111767 | +** |
| 111768 | +** It is possible to determine which index a b+-tree belongs to based on the |
| 111769 | +** value stored in the "%_segdir.level" column. Given this value L, the index |
| 111770 | +** that the b+-tree belongs to is (L<<10). In other words, all b+-trees with |
| 111771 | +** level values between 0 and 1023 (inclusive) belong to index 0, all levels |
| 111772 | +** between 1024 and 2047 to index 1, and so on. |
| 111773 | +** |
| 111774 | +** It is considered impossible for an index to use more than 1024 levels. In |
| 111775 | +** theory though this may happen, but only after at least |
| 111776 | +** (FTS3_MERGE_COUNT^1024) separate flushes of the pending-terms tables. |
| 111777 | +*/ |
| 111778 | +#define FTS3_SEGDIR_MAXLEVEL 1024 |
| 111779 | +#define FTS3_SEGDIR_MAXLEVEL_STR "1024" |
| 111780 | + |
| 111721 | 111781 | /* |
| 111722 | 111782 | ** The testcase() macro is only used by the amalgamation. If undefined, |
| 111723 | 111783 | ** make it a no-op. |
| 111724 | 111784 | */ |
| 111725 | 111785 | #ifndef testcase |
| | @@ -111787,14 +111847,15 @@ |
| 111787 | 111847 | typedef struct Fts3Cursor Fts3Cursor; |
| 111788 | 111848 | typedef struct Fts3Expr Fts3Expr; |
| 111789 | 111849 | typedef struct Fts3Phrase Fts3Phrase; |
| 111790 | 111850 | typedef struct Fts3PhraseToken Fts3PhraseToken; |
| 111791 | 111851 | |
| 111852 | +typedef struct Fts3Doclist Fts3Doclist; |
| 111792 | 111853 | typedef struct Fts3SegFilter Fts3SegFilter; |
| 111793 | 111854 | typedef struct Fts3DeferredToken Fts3DeferredToken; |
| 111794 | 111855 | typedef struct Fts3SegReader Fts3SegReader; |
| 111795 | | -typedef struct Fts3SegReaderCursor Fts3SegReaderCursor; |
| 111856 | +typedef struct Fts3MultiSegReader Fts3MultiSegReader; |
| 111796 | 111857 | |
| 111797 | 111858 | /* |
| 111798 | 111859 | ** A connection to a fulltext index is an instance of the following |
| 111799 | 111860 | ** structure. The xCreate and xConnect methods create an instance |
| 111800 | 111861 | ** of this structure and xDestroy and xDisconnect free that instance. |
| | @@ -111811,33 +111872,45 @@ |
| 111811 | 111872 | sqlite3_tokenizer *pTokenizer; /* tokenizer for inserts and queries */ |
| 111812 | 111873 | |
| 111813 | 111874 | /* Precompiled statements used by the implementation. Each of these |
| 111814 | 111875 | ** statements is run and reset within a single virtual table API call. |
| 111815 | 111876 | */ |
| 111816 | | - sqlite3_stmt *aStmt[24]; |
| 111877 | + sqlite3_stmt *aStmt[27]; |
| 111817 | 111878 | |
| 111818 | 111879 | char *zReadExprlist; |
| 111819 | 111880 | char *zWriteExprlist; |
| 111820 | 111881 | |
| 111821 | 111882 | int nNodeSize; /* Soft limit for node size */ |
| 111822 | 111883 | u8 bHasStat; /* True if %_stat table exists */ |
| 111823 | 111884 | u8 bHasDocsize; /* True if %_docsize table exists */ |
| 111885 | + u8 bDescIdx; /* True if doclists are in reverse order */ |
| 111824 | 111886 | int nPgsz; /* Page size for host database */ |
| 111825 | 111887 | char *zSegmentsTbl; /* Name of %_segments table */ |
| 111826 | 111888 | sqlite3_blob *pSegments; /* Blob handle open on %_segments table */ |
| 111827 | 111889 | |
| 111828 | | - /* The following hash table is used to buffer pending index updates during |
| 111890 | + /* TODO: Fix the first paragraph of this comment. |
| 111891 | + ** |
| 111892 | + ** The following hash table is used to buffer pending index updates during |
| 111829 | 111893 | ** transactions. Variable nPendingData estimates the memory size of the |
| 111830 | 111894 | ** pending data, including hash table overhead, but not malloc overhead. |
| 111831 | 111895 | ** When nPendingData exceeds nMaxPendingData, the buffer is flushed |
| 111832 | 111896 | ** automatically. Variable iPrevDocid is the docid of the most recently |
| 111833 | 111897 | ** inserted record. |
| 111898 | + ** |
| 111899 | + ** A single FTS4 table may have multiple full-text indexes. For each index |
| 111900 | + ** there is an entry in the aIndex[] array. Index 0 is an index of all the |
| 111901 | + ** terms that appear in the document set. Each subsequent index in aIndex[] |
| 111902 | + ** is an index of prefixes of a specific length. |
| 111834 | 111903 | */ |
| 111835 | | - int nMaxPendingData; |
| 111836 | | - int nPendingData; |
| 111837 | | - sqlite_int64 iPrevDocid; |
| 111838 | | - Fts3Hash pendingTerms; |
| 111904 | + int nIndex; /* Size of aIndex[] */ |
| 111905 | + struct Fts3Index { |
| 111906 | + int nPrefix; /* Prefix length (0 for main terms index) */ |
| 111907 | + Fts3Hash hPending; /* Pending terms table for this index */ |
| 111908 | + } *aIndex; |
| 111909 | + int nMaxPendingData; /* Max pending data before flush to disk */ |
| 111910 | + int nPendingData; /* Current bytes of pending data */ |
| 111911 | + sqlite_int64 iPrevDocid; /* Docid of most recently inserted document */ |
| 111839 | 111912 | |
| 111840 | 111913 | #if defined(SQLITE_DEBUG) |
| 111841 | 111914 | /* State variables used for validating that the transaction control |
| 111842 | 111915 | ** methods of the virtual table are called at appropriate times. These |
| 111843 | 111916 | ** values do not contribution to the FTS computation; they are used for |
| | @@ -111864,13 +111937,14 @@ |
| 111864 | 111937 | Fts3DeferredToken *pDeferred; /* Deferred search tokens, if any */ |
| 111865 | 111938 | sqlite3_int64 iPrevId; /* Previous id read from aDoclist */ |
| 111866 | 111939 | char *pNextId; /* Pointer into the body of aDoclist */ |
| 111867 | 111940 | char *aDoclist; /* List of docids for full-text queries */ |
| 111868 | 111941 | int nDoclist; /* Size of buffer at aDoclist */ |
| 111869 | | - int desc; /* True to sort in descending order */ |
| 111942 | + u8 bDesc; /* True to sort in descending order */ |
| 111870 | 111943 | int eEvalmode; /* An FTS3_EVAL_XX constant */ |
| 111871 | 111944 | int nRowAvg; /* Average size of database rows, in pages */ |
| 111945 | + int nDoc; /* Documents in table */ |
| 111872 | 111946 | |
| 111873 | 111947 | int isMatchinfoNeeded; /* True when aMatchinfo[] needs filling in */ |
| 111874 | 111948 | u32 *aMatchinfo; /* Information about most recent match */ |
| 111875 | 111949 | int nMatchinfo; /* Number of elements in aMatchinfo[] */ |
| 111876 | 111950 | char *zMatchinfo; /* Matchinfo specification */ |
| | @@ -111897,66 +111971,90 @@ |
| 111897 | 111971 | */ |
| 111898 | 111972 | #define FTS3_FULLSCAN_SEARCH 0 /* Linear scan of %_content table */ |
| 111899 | 111973 | #define FTS3_DOCID_SEARCH 1 /* Lookup by rowid on %_content table */ |
| 111900 | 111974 | #define FTS3_FULLTEXT_SEARCH 2 /* Full-text index search */ |
| 111901 | 111975 | |
| 111976 | + |
| 111977 | +struct Fts3Doclist { |
| 111978 | + char *aAll; /* Array containing doclist (or NULL) */ |
| 111979 | + int nAll; /* Size of a[] in bytes */ |
| 111980 | + char *pNextDocid; /* Pointer to next docid */ |
| 111981 | + |
| 111982 | + sqlite3_int64 iDocid; /* Current docid (if pList!=0) */ |
| 111983 | + int bFreeList; /* True if pList should be sqlite3_free()d */ |
| 111984 | + char *pList; /* Pointer to position list following iDocid */ |
| 111985 | + int nList; /* Length of position list */ |
| 111986 | +} doclist; |
| 111987 | + |
| 111902 | 111988 | /* |
| 111903 | 111989 | ** A "phrase" is a sequence of one or more tokens that must match in |
| 111904 | 111990 | ** sequence. A single token is the base case and the most common case. |
| 111905 | 111991 | ** For a sequence of tokens contained in double-quotes (i.e. "one two three") |
| 111906 | 111992 | ** nToken will be the number of tokens in the string. |
| 111907 | | -** |
| 111908 | | -** The nDocMatch and nMatch variables contain data that may be used by the |
| 111909 | | -** matchinfo() function. They are populated when the full-text index is |
| 111910 | | -** queried for hits on the phrase. If one or more tokens in the phrase |
| 111911 | | -** are deferred, the nDocMatch and nMatch variables are populated based |
| 111912 | | -** on the assumption that the |
| 111913 | 111993 | */ |
| 111914 | 111994 | struct Fts3PhraseToken { |
| 111915 | 111995 | char *z; /* Text of the token */ |
| 111916 | 111996 | int n; /* Number of bytes in buffer z */ |
| 111917 | 111997 | int isPrefix; /* True if token ends with a "*" character */ |
| 111998 | + |
| 111999 | + /* Variables above this point are populated when the expression is |
| 112000 | + ** parsed (by code in fts3_expr.c). Below this point the variables are |
| 112001 | + ** used when evaluating the expression. */ |
| 111918 | 112002 | int bFulltext; /* True if full-text index was used */ |
| 111919 | | - Fts3SegReaderCursor *pSegcsr; /* Segment-reader for this token */ |
| 111920 | 112003 | Fts3DeferredToken *pDeferred; /* Deferred token object for this token */ |
| 112004 | + Fts3MultiSegReader *pSegcsr; /* Segment-reader for this token */ |
| 111921 | 112005 | }; |
| 111922 | 112006 | |
| 111923 | 112007 | struct Fts3Phrase { |
| 111924 | | - /* Variables populated by fts3_expr.c when parsing a MATCH expression */ |
| 112008 | + /* Cache of doclist for this phrase. */ |
| 112009 | + Fts3Doclist doclist; |
| 112010 | + int bIncr; /* True if doclist is loaded incrementally */ |
| 112011 | + |
| 112012 | + /* Variables below this point are populated by fts3_expr.c when parsing |
| 112013 | + ** a MATCH expression. Everything above is part of the evaluation phase. |
| 112014 | + */ |
| 111925 | 112015 | int nToken; /* Number of tokens in the phrase */ |
| 111926 | 112016 | int iColumn; /* Index of column this phrase must match */ |
| 111927 | | - int isNot; /* Phrase prefixed by unary not (-) operator */ |
| 111928 | 112017 | Fts3PhraseToken aToken[1]; /* One entry for each token in the phrase */ |
| 111929 | 112018 | }; |
| 111930 | 112019 | |
| 111931 | 112020 | /* |
| 111932 | 112021 | ** A tree of these objects forms the RHS of a MATCH operator. |
| 111933 | 112022 | ** |
| 111934 | | -** If Fts3Expr.eType is either FTSQUERY_NEAR or FTSQUERY_PHRASE and isLoaded |
| 111935 | | -** is true, then aDoclist points to a malloced buffer, size nDoclist bytes, |
| 111936 | | -** containing the results of the NEAR or phrase query in FTS3 doclist |
| 111937 | | -** format. As usual, the initial "Length" field found in doclists stored |
| 111938 | | -** on disk is omitted from this buffer. |
| 112023 | +** If Fts3Expr.eType is FTSQUERY_PHRASE and isLoaded is true, then aDoclist |
| 112024 | +** points to a malloced buffer, size nDoclist bytes, containing the results |
| 112025 | +** of this phrase query in FTS3 doclist format. As usual, the initial |
| 112026 | +** "Length" field found in doclists stored on disk is omitted from this |
| 112027 | +** buffer. |
| 111939 | 112028 | ** |
| 111940 | | -** Variable pCurrent always points to the start of a docid field within |
| 111941 | | -** aDoclist. Since the doclist is usually scanned in docid order, this can |
| 111942 | | -** be used to accelerate seeking to the required docid within the doclist. |
| 112029 | +** Variable aMI is used only for FTSQUERY_NEAR nodes to store the global |
| 112030 | +** matchinfo data. If it is not NULL, it points to an array of size nCol*3, |
| 112031 | +** where nCol is the number of columns in the queried FTS table. The array |
| 112032 | +** is populated as follows: |
| 112033 | +** |
| 112034 | +** aMI[iCol*3 + 0] = Undefined |
| 112035 | +** aMI[iCol*3 + 1] = Number of occurrences |
| 112036 | +** aMI[iCol*3 + 2] = Number of rows containing at least one instance |
| 112037 | +** |
| 112038 | +** The aMI array is allocated using sqlite3_malloc(). It should be freed |
| 112039 | +** when the expression node is. |
| 111943 | 112040 | */ |
| 111944 | 112041 | struct Fts3Expr { |
| 111945 | 112042 | int eType; /* One of the FTSQUERY_XXX values defined below */ |
| 111946 | 112043 | int nNear; /* Valid if eType==FTSQUERY_NEAR */ |
| 111947 | 112044 | Fts3Expr *pParent; /* pParent->pLeft==this or pParent->pRight==this */ |
| 111948 | 112045 | Fts3Expr *pLeft; /* Left operand */ |
| 111949 | 112046 | Fts3Expr *pRight; /* Right operand */ |
| 111950 | 112047 | Fts3Phrase *pPhrase; /* Valid if eType==FTSQUERY_PHRASE */ |
| 111951 | 112048 | |
| 111952 | | - int isLoaded; /* True if aDoclist/nDoclist are initialized. */ |
| 111953 | | - char *aDoclist; /* Buffer containing doclist */ |
| 111954 | | - int nDoclist; /* Size of aDoclist in bytes */ |
| 112049 | + /* The following are used by the fts3_eval.c module. */ |
| 112050 | + sqlite3_int64 iDocid; /* Current docid */ |
| 112051 | + u8 bEof; /* True this expression is at EOF already */ |
| 112052 | + u8 bStart; /* True if iDocid is valid */ |
| 112053 | + u8 bDeferred; /* True if this expression is entirely deferred */ |
| 111955 | 112054 | |
| 111956 | | - sqlite3_int64 iCurrent; |
| 111957 | | - char *pCurrent; |
| 112055 | + u32 *aMI; |
| 111958 | 112056 | }; |
| 111959 | 112057 | |
| 111960 | 112058 | /* |
| 111961 | 112059 | ** Candidate values for Fts3Query.eType. Note that the order of the first |
| 111962 | 112060 | ** four values is in order of precedence when parsing expressions. For |
| | @@ -111980,35 +112078,36 @@ |
| 111980 | 112078 | SQLITE_PRIVATE int sqlite3Fts3PendingTermsFlush(Fts3Table *); |
| 111981 | 112079 | SQLITE_PRIVATE void sqlite3Fts3PendingTermsClear(Fts3Table *); |
| 111982 | 112080 | SQLITE_PRIVATE int sqlite3Fts3Optimize(Fts3Table *); |
| 111983 | 112081 | SQLITE_PRIVATE int sqlite3Fts3SegReaderNew(int, sqlite3_int64, |
| 111984 | 112082 | sqlite3_int64, sqlite3_int64, const char *, int, Fts3SegReader**); |
| 111985 | | -SQLITE_PRIVATE int sqlite3Fts3SegReaderPending(Fts3Table*,const char*,int,int,Fts3SegReader**); |
| 112083 | +SQLITE_PRIVATE int sqlite3Fts3SegReaderPending( |
| 112084 | + Fts3Table*,int,const char*,int,int,Fts3SegReader**); |
| 111986 | 112085 | SQLITE_PRIVATE void sqlite3Fts3SegReaderFree(Fts3SegReader *); |
| 111987 | | -SQLITE_PRIVATE int sqlite3Fts3SegReaderCost(Fts3Cursor *, Fts3SegReader *, int *); |
| 111988 | | -SQLITE_PRIVATE int sqlite3Fts3AllSegdirs(Fts3Table*, int, sqlite3_stmt **); |
| 112086 | +SQLITE_PRIVATE int sqlite3Fts3AllSegdirs(Fts3Table*, int, int, sqlite3_stmt **); |
| 111989 | 112087 | SQLITE_PRIVATE int sqlite3Fts3ReadLock(Fts3Table *); |
| 111990 | | -SQLITE_PRIVATE int sqlite3Fts3ReadBlock(Fts3Table*, sqlite3_int64, char **, int*); |
| 112088 | +SQLITE_PRIVATE int sqlite3Fts3ReadBlock(Fts3Table*, sqlite3_int64, char **, int*, int*); |
| 111991 | 112089 | |
| 111992 | 112090 | SQLITE_PRIVATE int sqlite3Fts3SelectDoctotal(Fts3Table *, sqlite3_stmt **); |
| 111993 | 112091 | SQLITE_PRIVATE int sqlite3Fts3SelectDocsize(Fts3Table *, sqlite3_int64, sqlite3_stmt **); |
| 111994 | 112092 | |
| 111995 | 112093 | SQLITE_PRIVATE void sqlite3Fts3FreeDeferredTokens(Fts3Cursor *); |
| 111996 | 112094 | SQLITE_PRIVATE int sqlite3Fts3DeferToken(Fts3Cursor *, Fts3PhraseToken *, int); |
| 111997 | 112095 | SQLITE_PRIVATE int sqlite3Fts3CacheDeferredDoclists(Fts3Cursor *); |
| 111998 | 112096 | SQLITE_PRIVATE void sqlite3Fts3FreeDeferredDoclists(Fts3Cursor *); |
| 111999 | | -SQLITE_PRIVATE char *sqlite3Fts3DeferredDoclist(Fts3DeferredToken *, int *); |
| 112000 | 112097 | SQLITE_PRIVATE void sqlite3Fts3SegmentsClose(Fts3Table *); |
| 112001 | 112098 | |
| 112002 | | -#define FTS3_SEGCURSOR_PENDING -1 |
| 112003 | | -#define FTS3_SEGCURSOR_ALL -2 |
| 112099 | +/* Special values interpreted by sqlite3SegReaderCursor() */ |
| 112100 | +#define FTS3_SEGCURSOR_PENDING -1 |
| 112101 | +#define FTS3_SEGCURSOR_ALL -2 |
| 112004 | 112102 | |
| 112005 | | -SQLITE_PRIVATE int sqlite3Fts3SegReaderStart(Fts3Table*, Fts3SegReaderCursor*, Fts3SegFilter*); |
| 112006 | | -SQLITE_PRIVATE int sqlite3Fts3SegReaderStep(Fts3Table *, Fts3SegReaderCursor *); |
| 112007 | | -SQLITE_PRIVATE void sqlite3Fts3SegReaderFinish(Fts3SegReaderCursor *); |
| 112103 | +SQLITE_PRIVATE int sqlite3Fts3SegReaderStart(Fts3Table*, Fts3MultiSegReader*, Fts3SegFilter*); |
| 112104 | +SQLITE_PRIVATE int sqlite3Fts3SegReaderStep(Fts3Table *, Fts3MultiSegReader *); |
| 112105 | +SQLITE_PRIVATE void sqlite3Fts3SegReaderFinish(Fts3MultiSegReader *); |
| 112106 | + |
| 112008 | 112107 | SQLITE_PRIVATE int sqlite3Fts3SegReaderCursor( |
| 112009 | | - Fts3Table *, int, const char *, int, int, int, Fts3SegReaderCursor *); |
| 112108 | + Fts3Table *, int, int, const char *, int, int, int, Fts3MultiSegReader *); |
| 112010 | 112109 | |
| 112011 | 112110 | /* Flags allowed as part of the 4th argument to SegmentReaderIterate() */ |
| 112012 | 112111 | #define FTS3_SEGMENT_REQUIRE_POS 0x00000001 |
| 112013 | 112112 | #define FTS3_SEGMENT_IGNORE_EMPTY 0x00000002 |
| 112014 | 112113 | #define FTS3_SEGMENT_COLUMN_FILTER 0x00000004 |
| | @@ -112021,21 +112120,24 @@ |
| 112021 | 112120 | int nTerm; |
| 112022 | 112121 | int iCol; |
| 112023 | 112122 | int flags; |
| 112024 | 112123 | }; |
| 112025 | 112124 | |
| 112026 | | -struct Fts3SegReaderCursor { |
| 112125 | +struct Fts3MultiSegReader { |
| 112027 | 112126 | /* Used internally by sqlite3Fts3SegReaderXXX() calls */ |
| 112028 | 112127 | Fts3SegReader **apSegment; /* Array of Fts3SegReader objects */ |
| 112029 | 112128 | int nSegment; /* Size of apSegment array */ |
| 112030 | 112129 | int nAdvance; /* How many seg-readers to advance */ |
| 112031 | 112130 | Fts3SegFilter *pFilter; /* Pointer to filter object */ |
| 112032 | 112131 | char *aBuffer; /* Buffer to merge doclists in */ |
| 112033 | 112132 | int nBuffer; /* Allocated size of aBuffer[] in bytes */ |
| 112034 | 112133 | |
| 112035 | | - /* Cost of running this iterator. Used by fts3.c only. */ |
| 112036 | | - int nCost; |
| 112134 | + int iColFilter; /* If >=0, filter for this column */ |
| 112135 | + |
| 112136 | + /* Used by fts3.c only. */ |
| 112137 | + int nCost; /* Cost of running iterator */ |
| 112138 | + int bLookup; /* True if a lookup of a single entry. */ |
| 112037 | 112139 | |
| 112038 | 112140 | /* Output values. Valid only after Fts3SegReaderStep() returns SQLITE_ROW. */ |
| 112039 | 112141 | char *zTerm; /* Pointer to term buffer */ |
| 112040 | 112142 | int nTerm; /* Size of zTerm in bytes */ |
| 112041 | 112143 | char *aDoclist; /* Pointer to doclist buffer */ |
| | @@ -112046,15 +112148,13 @@ |
| 112046 | 112148 | SQLITE_PRIVATE int sqlite3Fts3PutVarint(char *, sqlite3_int64); |
| 112047 | 112149 | SQLITE_PRIVATE int sqlite3Fts3GetVarint(const char *, sqlite_int64 *); |
| 112048 | 112150 | SQLITE_PRIVATE int sqlite3Fts3GetVarint32(const char *, int *); |
| 112049 | 112151 | SQLITE_PRIVATE int sqlite3Fts3VarintLen(sqlite3_uint64); |
| 112050 | 112152 | SQLITE_PRIVATE void sqlite3Fts3Dequote(char *); |
| 112153 | +SQLITE_PRIVATE void sqlite3Fts3DoclistPrev(int,char*,int,char**,sqlite3_int64*,int*,u8*); |
| 112051 | 112154 | |
| 112052 | | -SQLITE_PRIVATE char *sqlite3Fts3FindPositions(Fts3Cursor *, Fts3Expr *, sqlite3_int64, int); |
| 112053 | | -SQLITE_PRIVATE int sqlite3Fts3ExprLoadDoclist(Fts3Cursor *, Fts3Expr *); |
| 112054 | | -SQLITE_PRIVATE int sqlite3Fts3ExprLoadFtDoclist(Fts3Cursor *, Fts3Expr *, char **, int *); |
| 112055 | | -SQLITE_PRIVATE int sqlite3Fts3ExprNearTrim(Fts3Expr *, Fts3Expr *, int); |
| 112155 | +SQLITE_PRIVATE int sqlite3Fts3EvalPhraseStats(Fts3Cursor *, Fts3Expr *, u32 *); |
| 112056 | 112156 | |
| 112057 | 112157 | /* fts3_tokenizer.c */ |
| 112058 | 112158 | SQLITE_PRIVATE const char *sqlite3Fts3NextToken(const char *, int *); |
| 112059 | 112159 | SQLITE_PRIVATE int sqlite3Fts3InitHashTable(sqlite3 *, Fts3Hash *, const char *); |
| 112060 | 112160 | SQLITE_PRIVATE int sqlite3Fts3InitTokenizer(Fts3Hash *pHash, const char *, |
| | @@ -112079,10 +112179,33 @@ |
| 112079 | 112179 | SQLITE_PRIVATE int sqlite3Fts3InitTerm(sqlite3 *db); |
| 112080 | 112180 | #endif |
| 112081 | 112181 | |
| 112082 | 112182 | /* fts3_aux.c */ |
| 112083 | 112183 | SQLITE_PRIVATE int sqlite3Fts3InitAux(sqlite3 *db); |
| 112184 | + |
| 112185 | +SQLITE_PRIVATE int sqlite3Fts3TermSegReaderCursor( |
| 112186 | + Fts3Cursor *pCsr, /* Virtual table cursor handle */ |
| 112187 | + const char *zTerm, /* Term to query for */ |
| 112188 | + int nTerm, /* Size of zTerm in bytes */ |
| 112189 | + int isPrefix, /* True for a prefix search */ |
| 112190 | + Fts3MultiSegReader **ppSegcsr /* OUT: Allocated seg-reader cursor */ |
| 112191 | +); |
| 112192 | + |
| 112193 | +SQLITE_PRIVATE void sqlite3Fts3EvalPhraseCleanup(Fts3Phrase *); |
| 112194 | + |
| 112195 | +SQLITE_PRIVATE int sqlite3Fts3EvalStart(Fts3Cursor *, Fts3Expr *, int); |
| 112196 | +SQLITE_PRIVATE int sqlite3Fts3EvalNext(Fts3Cursor *pCsr); |
| 112197 | + |
| 112198 | +SQLITE_PRIVATE int sqlite3Fts3MsrIncrStart( |
| 112199 | + Fts3Table*, Fts3MultiSegReader*, int, const char*, int); |
| 112200 | +SQLITE_PRIVATE int sqlite3Fts3MsrIncrNext( |
| 112201 | + Fts3Table *, Fts3MultiSegReader *, sqlite3_int64 *, char **, int *); |
| 112202 | +SQLITE_PRIVATE char *sqlite3Fts3EvalPhrasePoslist(Fts3Cursor *, Fts3Expr *, int iCol); |
| 112203 | +SQLITE_PRIVATE int sqlite3Fts3MsrOvfl(Fts3Cursor *, Fts3MultiSegReader *, int *); |
| 112204 | + |
| 112205 | +SQLITE_PRIVATE int sqlite3Fts3DeferredTokenList(Fts3DeferredToken *, char **, int *); |
| 112206 | + |
| 112084 | 112207 | |
| 112085 | 112208 | #endif /* _FTSINT_H */ |
| 112086 | 112209 | |
| 112087 | 112210 | /************** End of fts3Int.h *********************************************/ |
| 112088 | 112211 | /************** Continuing where we left off in fts3.c ***********************/ |
| | @@ -112200,16 +112323,16 @@ |
| 112200 | 112323 | |
| 112201 | 112324 | /* |
| 112202 | 112325 | ** When this function is called, *pp points to the first byte following a |
| 112203 | 112326 | ** varint that is part of a doclist (or position-list, or any other list |
| 112204 | 112327 | ** of varints). This function moves *pp to point to the start of that varint, |
| 112205 | | -** and decrements the value stored in *pVal by the varint value. |
| 112328 | +** and sets *pVal by the varint value. |
| 112206 | 112329 | ** |
| 112207 | 112330 | ** Argument pStart points to the first byte of the doclist that the |
| 112208 | 112331 | ** varint is part of. |
| 112209 | 112332 | */ |
| 112210 | | -static void fts3GetReverseDeltaVarint( |
| 112333 | +static void fts3GetReverseVarint( |
| 112211 | 112334 | char **pp, |
| 112212 | 112335 | char *pStart, |
| 112213 | 112336 | sqlite3_int64 *pVal |
| 112214 | 112337 | ){ |
| 112215 | 112338 | sqlite3_int64 iVal; |
| | @@ -112221,25 +112344,11 @@ |
| 112221 | 112344 | for(p = (*pp)-2; p>=pStart && *p&0x80; p--); |
| 112222 | 112345 | p++; |
| 112223 | 112346 | *pp = p; |
| 112224 | 112347 | |
| 112225 | 112348 | sqlite3Fts3GetVarint(p, &iVal); |
| 112226 | | - *pVal -= iVal; |
| 112227 | | -} |
| 112228 | | - |
| 112229 | | -/* |
| 112230 | | -** As long as *pp has not reached its end (pEnd), then do the same |
| 112231 | | -** as fts3GetDeltaVarint(): read a single varint and add it to *pVal. |
| 112232 | | -** But if we have reached the end of the varint, just set *pp=0 and |
| 112233 | | -** leave *pVal unchanged. |
| 112234 | | -*/ |
| 112235 | | -static void fts3GetDeltaVarint2(char **pp, char *pEnd, sqlite3_int64 *pVal){ |
| 112236 | | - if( *pp>=pEnd ){ |
| 112237 | | - *pp = 0; |
| 112238 | | - }else{ |
| 112239 | | - fts3GetDeltaVarint(pp, pVal); |
| 112240 | | - } |
| 112349 | + *pVal = iVal; |
| 112241 | 112350 | } |
| 112242 | 112351 | |
| 112243 | 112352 | /* |
| 112244 | 112353 | ** The xDisconnect() virtual table method. |
| 112245 | 112354 | */ |
| | @@ -112608,10 +112717,62 @@ |
| 112608 | 112717 | fts3Appendf(pRc, &zRet, ",%s(?)", zFunction); |
| 112609 | 112718 | } |
| 112610 | 112719 | sqlite3_free(zFree); |
| 112611 | 112720 | return zRet; |
| 112612 | 112721 | } |
| 112722 | + |
| 112723 | +static int fts3GobbleInt(const char **pp, int *pnOut){ |
| 112724 | + const char *p = *pp; |
| 112725 | + int nInt = 0; |
| 112726 | + for(p=*pp; p[0]>='0' && p[0]<='9'; p++){ |
| 112727 | + nInt = nInt * 10 + (p[0] - '0'); |
| 112728 | + } |
| 112729 | + if( p==*pp ) return SQLITE_ERROR; |
| 112730 | + *pnOut = nInt; |
| 112731 | + *pp = p; |
| 112732 | + return SQLITE_OK; |
| 112733 | +} |
| 112734 | + |
| 112735 | + |
| 112736 | +static int fts3PrefixParameter( |
| 112737 | + const char *zParam, /* ABC in prefix=ABC parameter to parse */ |
| 112738 | + int *pnIndex, /* OUT: size of *apIndex[] array */ |
| 112739 | + struct Fts3Index **apIndex, /* OUT: Array of indexes for this table */ |
| 112740 | + struct Fts3Index **apFree /* OUT: Free this with sqlite3_free() */ |
| 112741 | +){ |
| 112742 | + struct Fts3Index *aIndex; |
| 112743 | + int nIndex = 1; |
| 112744 | + |
| 112745 | + if( zParam && zParam[0] ){ |
| 112746 | + const char *p; |
| 112747 | + nIndex++; |
| 112748 | + for(p=zParam; *p; p++){ |
| 112749 | + if( *p==',' ) nIndex++; |
| 112750 | + } |
| 112751 | + } |
| 112752 | + |
| 112753 | + aIndex = sqlite3_malloc(sizeof(struct Fts3Index) * nIndex); |
| 112754 | + *apIndex = *apFree = aIndex; |
| 112755 | + *pnIndex = nIndex; |
| 112756 | + if( !aIndex ){ |
| 112757 | + return SQLITE_NOMEM; |
| 112758 | + } |
| 112759 | + |
| 112760 | + memset(aIndex, 0, sizeof(struct Fts3Index) * nIndex); |
| 112761 | + if( zParam ){ |
| 112762 | + const char *p = zParam; |
| 112763 | + int i; |
| 112764 | + for(i=1; i<nIndex; i++){ |
| 112765 | + int nPrefix; |
| 112766 | + if( fts3GobbleInt(&p, &nPrefix) ) return SQLITE_ERROR; |
| 112767 | + aIndex[i].nPrefix = nPrefix; |
| 112768 | + p++; |
| 112769 | + } |
| 112770 | + } |
| 112771 | + |
| 112772 | + return SQLITE_OK; |
| 112773 | +} |
| 112613 | 112774 | |
| 112614 | 112775 | /* |
| 112615 | 112776 | ** This function is the implementation of both the xConnect and xCreate |
| 112616 | 112777 | ** methods of the FTS3 virtual table. |
| 112617 | 112778 | ** |
| | @@ -112641,16 +112802,23 @@ |
| 112641 | 112802 | int nCol = 0; /* Number of columns in the FTS table */ |
| 112642 | 112803 | char *zCsr; /* Space for holding column names */ |
| 112643 | 112804 | int nDb; /* Bytes required to hold database name */ |
| 112644 | 112805 | int nName; /* Bytes required to hold table name */ |
| 112645 | 112806 | int isFts4 = (argv[0][3]=='4'); /* True for FTS4, false for FTS3 */ |
| 112646 | | - int bNoDocsize = 0; /* True to omit %_docsize table */ |
| 112647 | 112807 | const char **aCol; /* Array of column names */ |
| 112648 | 112808 | sqlite3_tokenizer *pTokenizer = 0; /* Tokenizer for this table */ |
| 112649 | 112809 | |
| 112650 | | - char *zCompress = 0; |
| 112651 | | - char *zUncompress = 0; |
| 112810 | + int nIndex; /* Size of aIndex[] array */ |
| 112811 | + struct Fts3Index *aIndex; /* Array of indexes for this table */ |
| 112812 | + struct Fts3Index *aFree = 0; /* Free this before returning */ |
| 112813 | + |
| 112814 | + /* The results of parsing supported FTS4 key=value options: */ |
| 112815 | + int bNoDocsize = 0; /* True to omit %_docsize table */ |
| 112816 | + int bDescIdx = 0; /* True to store descending indexes */ |
| 112817 | + char *zPrefix = 0; /* Prefix parameter value (or NULL) */ |
| 112818 | + char *zCompress = 0; /* compress=? parameter (or NULL) */ |
| 112819 | + char *zUncompress = 0; /* uncompress=? parameter (or NULL) */ |
| 112652 | 112820 | |
| 112653 | 112821 | assert( strlen(argv[0])==4 ); |
| 112654 | 112822 | assert( (sqlite3_strnicmp(argv[0], "fts4", 4)==0 && isFts4) |
| 112655 | 112823 | || (sqlite3_strnicmp(argv[0], "fts3", 4)==0 && !isFts4) |
| 112656 | 112824 | ); |
| | @@ -112687,32 +112855,76 @@ |
| 112687 | 112855 | rc = sqlite3Fts3InitTokenizer(pHash, &z[9], &pTokenizer, pzErr); |
| 112688 | 112856 | } |
| 112689 | 112857 | |
| 112690 | 112858 | /* Check if it is an FTS4 special argument. */ |
| 112691 | 112859 | else if( isFts4 && fts3IsSpecialColumn(z, &nKey, &zVal) ){ |
| 112860 | + struct Fts4Option { |
| 112861 | + const char *zOpt; |
| 112862 | + int nOpt; |
| 112863 | + char **pzVar; |
| 112864 | + } aFts4Opt[] = { |
| 112865 | + { "matchinfo", 9, 0 }, /* 0 -> MATCHINFO */ |
| 112866 | + { "prefix", 6, 0 }, /* 1 -> PREFIX */ |
| 112867 | + { "compress", 8, 0 }, /* 2 -> COMPRESS */ |
| 112868 | + { "uncompress", 10, 0 }, /* 3 -> UNCOMPRESS */ |
| 112869 | + { "order", 5, 0 } /* 4 -> ORDER */ |
| 112870 | + }; |
| 112871 | + |
| 112872 | + int iOpt; |
| 112692 | 112873 | if( !zVal ){ |
| 112693 | 112874 | rc = SQLITE_NOMEM; |
| 112694 | | - goto fts3_init_out; |
| 112695 | | - } |
| 112696 | | - if( nKey==9 && 0==sqlite3_strnicmp(z, "matchinfo", 9) ){ |
| 112697 | | - if( strlen(zVal)==4 && 0==sqlite3_strnicmp(zVal, "fts3", 4) ){ |
| 112698 | | - bNoDocsize = 1; |
| 112699 | | - }else{ |
| 112700 | | - *pzErr = sqlite3_mprintf("unrecognized matchinfo: %s", zVal); |
| 112701 | | - rc = SQLITE_ERROR; |
| 112702 | | - } |
| 112703 | | - }else if( nKey==8 && 0==sqlite3_strnicmp(z, "compress", 8) ){ |
| 112704 | | - zCompress = zVal; |
| 112705 | | - zVal = 0; |
| 112706 | | - }else if( nKey==10 && 0==sqlite3_strnicmp(z, "uncompress", 10) ){ |
| 112707 | | - zUncompress = zVal; |
| 112708 | | - zVal = 0; |
| 112709 | | - }else{ |
| 112710 | | - *pzErr = sqlite3_mprintf("unrecognized parameter: %s", z); |
| 112711 | | - rc = SQLITE_ERROR; |
| 112712 | | - } |
| 112713 | | - sqlite3_free(zVal); |
| 112875 | + }else{ |
| 112876 | + for(iOpt=0; iOpt<SizeofArray(aFts4Opt); iOpt++){ |
| 112877 | + struct Fts4Option *pOp = &aFts4Opt[iOpt]; |
| 112878 | + if( nKey==pOp->nOpt && !sqlite3_strnicmp(z, pOp->zOpt, pOp->nOpt) ){ |
| 112879 | + break; |
| 112880 | + } |
| 112881 | + } |
| 112882 | + if( iOpt==SizeofArray(aFts4Opt) ){ |
| 112883 | + *pzErr = sqlite3_mprintf("unrecognized parameter: %s", z); |
| 112884 | + rc = SQLITE_ERROR; |
| 112885 | + }else{ |
| 112886 | + switch( iOpt ){ |
| 112887 | + case 0: /* MATCHINFO */ |
| 112888 | + if( strlen(zVal)!=4 || sqlite3_strnicmp(zVal, "fts3", 4) ){ |
| 112889 | + *pzErr = sqlite3_mprintf("unrecognized matchinfo: %s", zVal); |
| 112890 | + rc = SQLITE_ERROR; |
| 112891 | + } |
| 112892 | + bNoDocsize = 1; |
| 112893 | + break; |
| 112894 | + |
| 112895 | + case 1: /* PREFIX */ |
| 112896 | + sqlite3_free(zPrefix); |
| 112897 | + zPrefix = zVal; |
| 112898 | + zVal = 0; |
| 112899 | + break; |
| 112900 | + |
| 112901 | + case 2: /* COMPRESS */ |
| 112902 | + sqlite3_free(zCompress); |
| 112903 | + zCompress = zVal; |
| 112904 | + zVal = 0; |
| 112905 | + break; |
| 112906 | + |
| 112907 | + case 3: /* UNCOMPRESS */ |
| 112908 | + sqlite3_free(zUncompress); |
| 112909 | + zUncompress = zVal; |
| 112910 | + zVal = 0; |
| 112911 | + break; |
| 112912 | + |
| 112913 | + case 4: /* ORDER */ |
| 112914 | + if( (strlen(zVal)!=3 || sqlite3_strnicmp(zVal, "asc", 3)) |
| 112915 | + && (strlen(zVal)!=4 || sqlite3_strnicmp(zVal, "desc", 3)) |
| 112916 | + ){ |
| 112917 | + *pzErr = sqlite3_mprintf("unrecognized order: %s", zVal); |
| 112918 | + rc = SQLITE_ERROR; |
| 112919 | + } |
| 112920 | + bDescIdx = (zVal[0]=='d' || zVal[0]=='D'); |
| 112921 | + break; |
| 112922 | + } |
| 112923 | + } |
| 112924 | + sqlite3_free(zVal); |
| 112925 | + } |
| 112714 | 112926 | } |
| 112715 | 112927 | |
| 112716 | 112928 | /* Otherwise, the argument is a column name. */ |
| 112717 | 112929 | else { |
| 112718 | 112930 | nString += (int)(strlen(z) + 1); |
| | @@ -112732,14 +112944,21 @@ |
| 112732 | 112944 | rc = sqlite3Fts3InitTokenizer(pHash, "simple", &pTokenizer, pzErr); |
| 112733 | 112945 | if( rc!=SQLITE_OK ) goto fts3_init_out; |
| 112734 | 112946 | } |
| 112735 | 112947 | assert( pTokenizer ); |
| 112736 | 112948 | |
| 112949 | + rc = fts3PrefixParameter(zPrefix, &nIndex, &aIndex, &aFree); |
| 112950 | + if( rc==SQLITE_ERROR ){ |
| 112951 | + assert( zPrefix ); |
| 112952 | + *pzErr = sqlite3_mprintf("error parsing prefix parameter: %s", zPrefix); |
| 112953 | + } |
| 112954 | + if( rc!=SQLITE_OK ) goto fts3_init_out; |
| 112737 | 112955 | |
| 112738 | 112956 | /* Allocate and populate the Fts3Table structure. */ |
| 112739 | | - nByte = sizeof(Fts3Table) + /* Fts3Table */ |
| 112957 | + nByte = sizeof(Fts3Table) + /* Fts3Table */ |
| 112740 | 112958 | nCol * sizeof(char *) + /* azColumn */ |
| 112959 | + nIndex * sizeof(struct Fts3Index) + /* aIndex */ |
| 112741 | 112960 | nName + /* zName */ |
| 112742 | 112961 | nDb + /* zDb */ |
| 112743 | 112962 | nString; /* Space for azColumn strings */ |
| 112744 | 112963 | p = (Fts3Table*)sqlite3_malloc(nByte); |
| 112745 | 112964 | if( p==0 ){ |
| | @@ -112750,20 +112969,26 @@ |
| 112750 | 112969 | p->db = db; |
| 112751 | 112970 | p->nColumn = nCol; |
| 112752 | 112971 | p->nPendingData = 0; |
| 112753 | 112972 | p->azColumn = (char **)&p[1]; |
| 112754 | 112973 | p->pTokenizer = pTokenizer; |
| 112755 | | - p->nNodeSize = 1000; |
| 112756 | 112974 | p->nMaxPendingData = FTS3_MAX_PENDING_DATA; |
| 112757 | 112975 | p->bHasDocsize = (isFts4 && bNoDocsize==0); |
| 112758 | 112976 | p->bHasStat = isFts4; |
| 112977 | + p->bDescIdx = bDescIdx; |
| 112759 | 112978 | TESTONLY( p->inTransaction = -1 ); |
| 112760 | 112979 | TESTONLY( p->mxSavepoint = -1 ); |
| 112761 | | - fts3HashInit(&p->pendingTerms, FTS3_HASH_STRING, 1); |
| 112980 | + |
| 112981 | + p->aIndex = (struct Fts3Index *)&p->azColumn[nCol]; |
| 112982 | + memcpy(p->aIndex, aIndex, sizeof(struct Fts3Index) * nIndex); |
| 112983 | + p->nIndex = nIndex; |
| 112984 | + for(i=0; i<nIndex; i++){ |
| 112985 | + fts3HashInit(&p->aIndex[i].hPending, FTS3_HASH_STRING, 1); |
| 112986 | + } |
| 112762 | 112987 | |
| 112763 | 112988 | /* Fill in the zName and zDb fields of the vtab structure. */ |
| 112764 | | - zCsr = (char *)&p->azColumn[nCol]; |
| 112989 | + zCsr = (char *)&p->aIndex[nIndex]; |
| 112765 | 112990 | p->zName = zCsr; |
| 112766 | 112991 | memcpy(zCsr, argv[2], nName); |
| 112767 | 112992 | zCsr += nName; |
| 112768 | 112993 | p->zDb = zCsr; |
| 112769 | 112994 | memcpy(zCsr, argv[1], nDb); |
| | @@ -112797,19 +113022,20 @@ |
| 112797 | 113022 | if( isCreate ){ |
| 112798 | 113023 | rc = fts3CreateTables(p); |
| 112799 | 113024 | } |
| 112800 | 113025 | |
| 112801 | 113026 | /* Figure out the page-size for the database. This is required in order to |
| 112802 | | - ** estimate the cost of loading large doclists from the database (see |
| 112803 | | - ** function sqlite3Fts3SegReaderCost() for details). |
| 112804 | | - */ |
| 113027 | + ** estimate the cost of loading large doclists from the database. */ |
| 112805 | 113028 | fts3DatabasePageSize(&rc, p); |
| 113029 | + p->nNodeSize = p->nPgsz-35; |
| 112806 | 113030 | |
| 112807 | 113031 | /* Declare the table schema to SQLite. */ |
| 112808 | 113032 | fts3DeclareVtab(&rc, p); |
| 112809 | 113033 | |
| 112810 | 113034 | fts3_init_out: |
| 113035 | + sqlite3_free(zPrefix); |
| 113036 | + sqlite3_free(aFree); |
| 112811 | 113037 | sqlite3_free(zCompress); |
| 112812 | 113038 | sqlite3_free(zUncompress); |
| 112813 | 113039 | sqlite3_free((void *)aCol); |
| 112814 | 113040 | if( rc!=SQLITE_OK ){ |
| 112815 | 113041 | if( p ){ |
| | @@ -112816,10 +113042,11 @@ |
| 112816 | 113042 | fts3DisconnectMethod((sqlite3_vtab *)p); |
| 112817 | 113043 | }else if( pTokenizer ){ |
| 112818 | 113044 | pTokenizer->pModule->xDestroy(pTokenizer); |
| 112819 | 113045 | } |
| 112820 | 113046 | }else{ |
| 113047 | + assert( p->pSegments==0 ); |
| 112821 | 113048 | *ppVTab = &p->base; |
| 112822 | 113049 | } |
| 112823 | 113050 | return rc; |
| 112824 | 113051 | } |
| 112825 | 113052 | |
| | @@ -112913,14 +113140,15 @@ |
| 112913 | 113140 | if( pOrder->desc ){ |
| 112914 | 113141 | pInfo->idxStr = "DESC"; |
| 112915 | 113142 | }else{ |
| 112916 | 113143 | pInfo->idxStr = "ASC"; |
| 112917 | 113144 | } |
| 113145 | + pInfo->orderByConsumed = 1; |
| 112918 | 113146 | } |
| 112919 | | - pInfo->orderByConsumed = 1; |
| 112920 | 113147 | } |
| 112921 | 113148 | |
| 113149 | + assert( p->pSegments==0 ); |
| 112922 | 113150 | return SQLITE_OK; |
| 112923 | 113151 | } |
| 112924 | 113152 | |
| 112925 | 113153 | /* |
| 112926 | 113154 | ** Implementation of xOpen method. |
| | @@ -112952,10 +113180,11 @@ |
| 112952 | 113180 | sqlite3_finalize(pCsr->pStmt); |
| 112953 | 113181 | sqlite3Fts3ExprFree(pCsr->pExpr); |
| 112954 | 113182 | sqlite3Fts3FreeDeferredTokens(pCsr); |
| 112955 | 113183 | sqlite3_free(pCsr->aDoclist); |
| 112956 | 113184 | sqlite3_free(pCsr->aMatchinfo); |
| 113185 | + assert( ((Fts3Table *)pCsr->base.pVtab)->pSegments==0 ); |
| 112957 | 113186 | sqlite3_free(pCsr); |
| 112958 | 113187 | return SQLITE_OK; |
| 112959 | 113188 | } |
| 112960 | 113189 | |
| 112961 | 113190 | /* |
| | @@ -112963,12 +113192,12 @@ |
| 112963 | 113192 | ** of the %_content table that contains the last match. Return |
| 112964 | 113193 | ** SQLITE_OK on success. |
| 112965 | 113194 | */ |
| 112966 | 113195 | static int fts3CursorSeek(sqlite3_context *pContext, Fts3Cursor *pCsr){ |
| 112967 | 113196 | if( pCsr->isRequireSeek ){ |
| 112968 | | - pCsr->isRequireSeek = 0; |
| 112969 | 113197 | sqlite3_bind_int64(pCsr->pStmt, 1, pCsr->iPrevId); |
| 113198 | + pCsr->isRequireSeek = 0; |
| 112970 | 113199 | if( SQLITE_ROW==sqlite3_step(pCsr->pStmt) ){ |
| 112971 | 113200 | return SQLITE_OK; |
| 112972 | 113201 | }else{ |
| 112973 | 113202 | int rc = sqlite3_reset(pCsr->pStmt); |
| 112974 | 113203 | if( rc==SQLITE_OK ){ |
| | @@ -113145,21 +113374,21 @@ |
| 113145 | 113374 | if( rc==SQLITE_OK && iHeight>1 ){ |
| 113146 | 113375 | char *zBlob = 0; /* Blob read from %_segments table */ |
| 113147 | 113376 | int nBlob; /* Size of zBlob in bytes */ |
| 113148 | 113377 | |
| 113149 | 113378 | if( piLeaf && piLeaf2 && (*piLeaf!=*piLeaf2) ){ |
| 113150 | | - rc = sqlite3Fts3ReadBlock(p, *piLeaf, &zBlob, &nBlob); |
| 113379 | + rc = sqlite3Fts3ReadBlock(p, *piLeaf, &zBlob, &nBlob, 0); |
| 113151 | 113380 | if( rc==SQLITE_OK ){ |
| 113152 | 113381 | rc = fts3SelectLeaf(p, zTerm, nTerm, zBlob, nBlob, piLeaf, 0); |
| 113153 | 113382 | } |
| 113154 | 113383 | sqlite3_free(zBlob); |
| 113155 | 113384 | piLeaf = 0; |
| 113156 | 113385 | zBlob = 0; |
| 113157 | 113386 | } |
| 113158 | 113387 | |
| 113159 | 113388 | if( rc==SQLITE_OK ){ |
| 113160 | | - rc = sqlite3Fts3ReadBlock(p, piLeaf ? *piLeaf : *piLeaf2, &zBlob, &nBlob); |
| 113389 | + rc = sqlite3Fts3ReadBlock(p, piLeaf?*piLeaf:*piLeaf2, &zBlob, &nBlob, 0); |
| 113161 | 113390 | } |
| 113162 | 113391 | if( rc==SQLITE_OK ){ |
| 113163 | 113392 | rc = fts3SelectLeaf(p, zTerm, nTerm, zBlob, nBlob, piLeaf, piLeaf2); |
| 113164 | 113393 | } |
| 113165 | 113394 | sqlite3_free(zBlob); |
| | @@ -113531,11 +113760,23 @@ |
| 113531 | 113760 | *pp = p; |
| 113532 | 113761 | return 1; |
| 113533 | 113762 | } |
| 113534 | 113763 | |
| 113535 | 113764 | /* |
| 113536 | | -** Merge two position-lists as required by the NEAR operator. |
| 113765 | +** Merge two position-lists as required by the NEAR operator. The argument |
| 113766 | +** position lists correspond to the left and right phrases of an expression |
| 113767 | +** like: |
| 113768 | +** |
| 113769 | +** "phrase 1" NEAR "phrase number 2" |
| 113770 | +** |
| 113771 | +** Position list *pp1 corresponds to the left-hand side of the NEAR |
| 113772 | +** expression and *pp2 to the right. As usual, the indexes in the position |
| 113773 | +** lists are the offsets of the last token in each phrase (tokens "1" and "2" |
| 113774 | +** in the example above). |
| 113775 | +** |
| 113776 | +** The output position list - written to *pp - is a copy of *pp2 with those |
| 113777 | +** entries that are not sufficiently NEAR entries in *pp1 removed. |
| 113537 | 113778 | */ |
| 113538 | 113779 | static int fts3PoslistNearMerge( |
| 113539 | 113780 | char **pp, /* Output buffer */ |
| 113540 | 113781 | char *aTmp, /* Temporary buffer space */ |
| 113541 | 113782 | int nRight, /* Maximum difference in token positions */ |
| | @@ -113544,218 +113785,31 @@ |
| 113544 | 113785 | char **pp2 /* IN/OUT: Right input list */ |
| 113545 | 113786 | ){ |
| 113546 | 113787 | char *p1 = *pp1; |
| 113547 | 113788 | char *p2 = *pp2; |
| 113548 | 113789 | |
| 113549 | | - if( !pp ){ |
| 113550 | | - if( fts3PoslistPhraseMerge(0, nRight, 0, 0, pp1, pp2) ) return 1; |
| 113551 | | - *pp1 = p1; |
| 113552 | | - *pp2 = p2; |
| 113553 | | - return fts3PoslistPhraseMerge(0, nLeft, 0, 0, pp2, pp1); |
| 113554 | | - }else{ |
| 113555 | | - char *pTmp1 = aTmp; |
| 113556 | | - char *pTmp2; |
| 113557 | | - char *aTmp2; |
| 113558 | | - int res = 1; |
| 113559 | | - |
| 113560 | | - fts3PoslistPhraseMerge(&pTmp1, nRight, 0, 0, pp1, pp2); |
| 113561 | | - aTmp2 = pTmp2 = pTmp1; |
| 113562 | | - *pp1 = p1; |
| 113563 | | - *pp2 = p2; |
| 113564 | | - fts3PoslistPhraseMerge(&pTmp2, nLeft, 1, 0, pp2, pp1); |
| 113565 | | - if( pTmp1!=aTmp && pTmp2!=aTmp2 ){ |
| 113566 | | - fts3PoslistMerge(pp, &aTmp, &aTmp2); |
| 113567 | | - }else if( pTmp1!=aTmp ){ |
| 113568 | | - fts3PoslistCopy(pp, &aTmp); |
| 113569 | | - }else if( pTmp2!=aTmp2 ){ |
| 113570 | | - fts3PoslistCopy(pp, &aTmp2); |
| 113571 | | - }else{ |
| 113572 | | - res = 0; |
| 113573 | | - } |
| 113574 | | - |
| 113575 | | - return res; |
| 113576 | | - } |
| 113577 | | -} |
| 113578 | | - |
| 113579 | | -/* |
| 113580 | | -** Values that may be used as the first parameter to fts3DoclistMerge(). |
| 113581 | | -*/ |
| 113582 | | -#define MERGE_NOT 2 /* D + D -> D */ |
| 113583 | | -#define MERGE_AND 3 /* D + D -> D */ |
| 113584 | | -#define MERGE_OR 4 /* D + D -> D */ |
| 113585 | | -#define MERGE_POS_OR 5 /* P + P -> P */ |
| 113586 | | -#define MERGE_PHRASE 6 /* P + P -> D */ |
| 113587 | | -#define MERGE_POS_PHRASE 7 /* P + P -> P */ |
| 113588 | | -#define MERGE_NEAR 8 /* P + P -> D */ |
| 113589 | | -#define MERGE_POS_NEAR 9 /* P + P -> P */ |
| 113590 | | - |
| 113591 | | -/* |
| 113592 | | -** Merge the two doclists passed in buffer a1 (size n1 bytes) and a2 |
| 113593 | | -** (size n2 bytes). The output is written to pre-allocated buffer aBuffer, |
| 113594 | | -** which is guaranteed to be large enough to hold the results. The number |
| 113595 | | -** of bytes written to aBuffer is stored in *pnBuffer before returning. |
| 113596 | | -** |
| 113597 | | -** If successful, SQLITE_OK is returned. Otherwise, if a malloc error |
| 113598 | | -** occurs while allocating a temporary buffer as part of the merge operation, |
| 113599 | | -** SQLITE_NOMEM is returned. |
| 113600 | | -*/ |
| 113601 | | -static int fts3DoclistMerge( |
| 113602 | | - int mergetype, /* One of the MERGE_XXX constants */ |
| 113603 | | - int nParam1, /* Used by MERGE_NEAR and MERGE_POS_NEAR */ |
| 113604 | | - int nParam2, /* Used by MERGE_NEAR and MERGE_POS_NEAR */ |
| 113605 | | - char *aBuffer, /* Pre-allocated output buffer */ |
| 113606 | | - int *pnBuffer, /* OUT: Bytes written to aBuffer */ |
| 113607 | | - char *a1, /* Buffer containing first doclist */ |
| 113608 | | - int n1, /* Size of buffer a1 */ |
| 113609 | | - char *a2, /* Buffer containing second doclist */ |
| 113610 | | - int n2, /* Size of buffer a2 */ |
| 113611 | | - int *pnDoc /* OUT: Number of docids in output */ |
| 113612 | | -){ |
| 113613 | | - sqlite3_int64 i1 = 0; |
| 113614 | | - sqlite3_int64 i2 = 0; |
| 113615 | | - sqlite3_int64 iPrev = 0; |
| 113616 | | - |
| 113617 | | - char *p = aBuffer; |
| 113618 | | - char *p1 = a1; |
| 113619 | | - char *p2 = a2; |
| 113620 | | - char *pEnd1 = &a1[n1]; |
| 113621 | | - char *pEnd2 = &a2[n2]; |
| 113622 | | - int nDoc = 0; |
| 113623 | | - |
| 113624 | | - assert( mergetype==MERGE_OR || mergetype==MERGE_POS_OR |
| 113625 | | - || mergetype==MERGE_AND || mergetype==MERGE_NOT |
| 113626 | | - || mergetype==MERGE_PHRASE || mergetype==MERGE_POS_PHRASE |
| 113627 | | - || mergetype==MERGE_NEAR || mergetype==MERGE_POS_NEAR |
| 113628 | | - ); |
| 113629 | | - |
| 113630 | | - if( !aBuffer ){ |
| 113631 | | - *pnBuffer = 0; |
| 113632 | | - return SQLITE_NOMEM; |
| 113633 | | - } |
| 113634 | | - |
| 113635 | | - /* Read the first docid from each doclist */ |
| 113636 | | - fts3GetDeltaVarint2(&p1, pEnd1, &i1); |
| 113637 | | - fts3GetDeltaVarint2(&p2, pEnd2, &i2); |
| 113638 | | - |
| 113639 | | - switch( mergetype ){ |
| 113640 | | - case MERGE_OR: |
| 113641 | | - case MERGE_POS_OR: |
| 113642 | | - while( p1 || p2 ){ |
| 113643 | | - if( p2 && p1 && i1==i2 ){ |
| 113644 | | - fts3PutDeltaVarint(&p, &iPrev, i1); |
| 113645 | | - if( mergetype==MERGE_POS_OR ) fts3PoslistMerge(&p, &p1, &p2); |
| 113646 | | - fts3GetDeltaVarint2(&p1, pEnd1, &i1); |
| 113647 | | - fts3GetDeltaVarint2(&p2, pEnd2, &i2); |
| 113648 | | - }else if( !p2 || (p1 && i1<i2) ){ |
| 113649 | | - fts3PutDeltaVarint(&p, &iPrev, i1); |
| 113650 | | - if( mergetype==MERGE_POS_OR ) fts3PoslistCopy(&p, &p1); |
| 113651 | | - fts3GetDeltaVarint2(&p1, pEnd1, &i1); |
| 113652 | | - }else{ |
| 113653 | | - fts3PutDeltaVarint(&p, &iPrev, i2); |
| 113654 | | - if( mergetype==MERGE_POS_OR ) fts3PoslistCopy(&p, &p2); |
| 113655 | | - fts3GetDeltaVarint2(&p2, pEnd2, &i2); |
| 113656 | | - } |
| 113657 | | - } |
| 113658 | | - break; |
| 113659 | | - |
| 113660 | | - case MERGE_AND: |
| 113661 | | - while( p1 && p2 ){ |
| 113662 | | - if( i1==i2 ){ |
| 113663 | | - fts3PutDeltaVarint(&p, &iPrev, i1); |
| 113664 | | - fts3GetDeltaVarint2(&p1, pEnd1, &i1); |
| 113665 | | - fts3GetDeltaVarint2(&p2, pEnd2, &i2); |
| 113666 | | - nDoc++; |
| 113667 | | - }else if( i1<i2 ){ |
| 113668 | | - fts3GetDeltaVarint2(&p1, pEnd1, &i1); |
| 113669 | | - }else{ |
| 113670 | | - fts3GetDeltaVarint2(&p2, pEnd2, &i2); |
| 113671 | | - } |
| 113672 | | - } |
| 113673 | | - break; |
| 113674 | | - |
| 113675 | | - case MERGE_NOT: |
| 113676 | | - while( p1 ){ |
| 113677 | | - if( p2 && i1==i2 ){ |
| 113678 | | - fts3GetDeltaVarint2(&p1, pEnd1, &i1); |
| 113679 | | - fts3GetDeltaVarint2(&p2, pEnd2, &i2); |
| 113680 | | - }else if( !p2 || i1<i2 ){ |
| 113681 | | - fts3PutDeltaVarint(&p, &iPrev, i1); |
| 113682 | | - fts3GetDeltaVarint2(&p1, pEnd1, &i1); |
| 113683 | | - }else{ |
| 113684 | | - fts3GetDeltaVarint2(&p2, pEnd2, &i2); |
| 113685 | | - } |
| 113686 | | - } |
| 113687 | | - break; |
| 113688 | | - |
| 113689 | | - case MERGE_POS_PHRASE: |
| 113690 | | - case MERGE_PHRASE: { |
| 113691 | | - char **ppPos = (mergetype==MERGE_PHRASE ? 0 : &p); |
| 113692 | | - while( p1 && p2 ){ |
| 113693 | | - if( i1==i2 ){ |
| 113694 | | - char *pSave = p; |
| 113695 | | - sqlite3_int64 iPrevSave = iPrev; |
| 113696 | | - fts3PutDeltaVarint(&p, &iPrev, i1); |
| 113697 | | - if( 0==fts3PoslistPhraseMerge(ppPos, nParam1, 0, 1, &p1, &p2) ){ |
| 113698 | | - p = pSave; |
| 113699 | | - iPrev = iPrevSave; |
| 113700 | | - }else{ |
| 113701 | | - nDoc++; |
| 113702 | | - } |
| 113703 | | - fts3GetDeltaVarint2(&p1, pEnd1, &i1); |
| 113704 | | - fts3GetDeltaVarint2(&p2, pEnd2, &i2); |
| 113705 | | - }else if( i1<i2 ){ |
| 113706 | | - fts3PoslistCopy(0, &p1); |
| 113707 | | - fts3GetDeltaVarint2(&p1, pEnd1, &i1); |
| 113708 | | - }else{ |
| 113709 | | - fts3PoslistCopy(0, &p2); |
| 113710 | | - fts3GetDeltaVarint2(&p2, pEnd2, &i2); |
| 113711 | | - } |
| 113712 | | - } |
| 113713 | | - break; |
| 113714 | | - } |
| 113715 | | - |
| 113716 | | - default: assert( mergetype==MERGE_POS_NEAR || mergetype==MERGE_NEAR ); { |
| 113717 | | - char *aTmp = 0; |
| 113718 | | - char **ppPos = 0; |
| 113719 | | - |
| 113720 | | - if( mergetype==MERGE_POS_NEAR ){ |
| 113721 | | - ppPos = &p; |
| 113722 | | - aTmp = sqlite3_malloc(2*(n1+n2+1)); |
| 113723 | | - if( !aTmp ){ |
| 113724 | | - return SQLITE_NOMEM; |
| 113725 | | - } |
| 113726 | | - } |
| 113727 | | - |
| 113728 | | - while( p1 && p2 ){ |
| 113729 | | - if( i1==i2 ){ |
| 113730 | | - char *pSave = p; |
| 113731 | | - sqlite3_int64 iPrevSave = iPrev; |
| 113732 | | - fts3PutDeltaVarint(&p, &iPrev, i1); |
| 113733 | | - |
| 113734 | | - if( !fts3PoslistNearMerge(ppPos, aTmp, nParam1, nParam2, &p1, &p2) ){ |
| 113735 | | - iPrev = iPrevSave; |
| 113736 | | - p = pSave; |
| 113737 | | - } |
| 113738 | | - |
| 113739 | | - fts3GetDeltaVarint2(&p1, pEnd1, &i1); |
| 113740 | | - fts3GetDeltaVarint2(&p2, pEnd2, &i2); |
| 113741 | | - }else if( i1<i2 ){ |
| 113742 | | - fts3PoslistCopy(0, &p1); |
| 113743 | | - fts3GetDeltaVarint2(&p1, pEnd1, &i1); |
| 113744 | | - }else{ |
| 113745 | | - fts3PoslistCopy(0, &p2); |
| 113746 | | - fts3GetDeltaVarint2(&p2, pEnd2, &i2); |
| 113747 | | - } |
| 113748 | | - } |
| 113749 | | - sqlite3_free(aTmp); |
| 113750 | | - break; |
| 113751 | | - } |
| 113752 | | - } |
| 113753 | | - |
| 113754 | | - if( pnDoc ) *pnDoc = nDoc; |
| 113755 | | - *pnBuffer = (int)(p-aBuffer); |
| 113756 | | - return SQLITE_OK; |
| 113790 | + char *pTmp1 = aTmp; |
| 113791 | + char *pTmp2; |
| 113792 | + char *aTmp2; |
| 113793 | + int res = 1; |
| 113794 | + |
| 113795 | + fts3PoslistPhraseMerge(&pTmp1, nRight, 0, 0, pp1, pp2); |
| 113796 | + aTmp2 = pTmp2 = pTmp1; |
| 113797 | + *pp1 = p1; |
| 113798 | + *pp2 = p2; |
| 113799 | + fts3PoslistPhraseMerge(&pTmp2, nLeft, 1, 0, pp2, pp1); |
| 113800 | + if( pTmp1!=aTmp && pTmp2!=aTmp2 ){ |
| 113801 | + fts3PoslistMerge(pp, &aTmp, &aTmp2); |
| 113802 | + }else if( pTmp1!=aTmp ){ |
| 113803 | + fts3PoslistCopy(pp, &aTmp); |
| 113804 | + }else if( pTmp2!=aTmp2 ){ |
| 113805 | + fts3PoslistCopy(pp, &aTmp2); |
| 113806 | + }else{ |
| 113807 | + res = 0; |
| 113808 | + } |
| 113809 | + |
| 113810 | + return res; |
| 113757 | 113811 | } |
| 113758 | 113812 | |
| 113759 | 113813 | /* |
| 113760 | 113814 | ** A pointer to an instance of this structure is used as the context |
| 113761 | 113815 | ** argument to sqlite3Fts3SegReaderIterate() |
| | @@ -113764,10 +113818,152 @@ |
| 113764 | 113818 | struct TermSelect { |
| 113765 | 113819 | int isReqPos; |
| 113766 | 113820 | char *aaOutput[16]; /* Malloc'd output buffer */ |
| 113767 | 113821 | int anOutput[16]; /* Size of output in bytes */ |
| 113768 | 113822 | }; |
| 113823 | + |
| 113824 | + |
| 113825 | +static void fts3GetDeltaVarint3( |
| 113826 | + char **pp, |
| 113827 | + char *pEnd, |
| 113828 | + int bDescIdx, |
| 113829 | + sqlite3_int64 *pVal |
| 113830 | +){ |
| 113831 | + if( *pp>=pEnd ){ |
| 113832 | + *pp = 0; |
| 113833 | + }else{ |
| 113834 | + sqlite3_int64 iVal; |
| 113835 | + *pp += sqlite3Fts3GetVarint(*pp, &iVal); |
| 113836 | + if( bDescIdx ){ |
| 113837 | + *pVal -= iVal; |
| 113838 | + }else{ |
| 113839 | + *pVal += iVal; |
| 113840 | + } |
| 113841 | + } |
| 113842 | +} |
| 113843 | + |
| 113844 | +static void fts3PutDeltaVarint3( |
| 113845 | + char **pp, /* IN/OUT: Output pointer */ |
| 113846 | + int bDescIdx, /* True for descending docids */ |
| 113847 | + sqlite3_int64 *piPrev, /* IN/OUT: Previous value written to list */ |
| 113848 | + int *pbFirst, /* IN/OUT: True after first int written */ |
| 113849 | + sqlite3_int64 iVal /* Write this value to the list */ |
| 113850 | +){ |
| 113851 | + sqlite3_int64 iWrite; |
| 113852 | + if( bDescIdx==0 || *pbFirst==0 ){ |
| 113853 | + iWrite = iVal - *piPrev; |
| 113854 | + }else{ |
| 113855 | + iWrite = *piPrev - iVal; |
| 113856 | + } |
| 113857 | + assert( *pbFirst || *piPrev==0 ); |
| 113858 | + assert( *pbFirst==0 || iWrite>0 ); |
| 113859 | + *pp += sqlite3Fts3PutVarint(*pp, iWrite); |
| 113860 | + *piPrev = iVal; |
| 113861 | + *pbFirst = 1; |
| 113862 | +} |
| 113863 | + |
| 113864 | +#define COMPARE_DOCID(i1, i2) ((bDescIdx?-1:1) * (i1-i2)) |
| 113865 | + |
| 113866 | +static int fts3DoclistOrMerge( |
| 113867 | + int bDescIdx, /* True if arguments are desc */ |
| 113868 | + char *a1, int n1, /* First doclist */ |
| 113869 | + char *a2, int n2, /* Second doclist */ |
| 113870 | + char **paOut, int *pnOut /* OUT: Malloc'd doclist */ |
| 113871 | +){ |
| 113872 | + sqlite3_int64 i1 = 0; |
| 113873 | + sqlite3_int64 i2 = 0; |
| 113874 | + sqlite3_int64 iPrev = 0; |
| 113875 | + char *pEnd1 = &a1[n1]; |
| 113876 | + char *pEnd2 = &a2[n2]; |
| 113877 | + char *p1 = a1; |
| 113878 | + char *p2 = a2; |
| 113879 | + char *p; |
| 113880 | + char *aOut; |
| 113881 | + int bFirstOut = 0; |
| 113882 | + |
| 113883 | + *paOut = 0; |
| 113884 | + *pnOut = 0; |
| 113885 | + aOut = sqlite3_malloc(n1+n2); |
| 113886 | + if( !aOut ) return SQLITE_NOMEM; |
| 113887 | + |
| 113888 | + p = aOut; |
| 113889 | + fts3GetDeltaVarint3(&p1, pEnd1, 0, &i1); |
| 113890 | + fts3GetDeltaVarint3(&p2, pEnd2, 0, &i2); |
| 113891 | + while( p1 || p2 ){ |
| 113892 | + sqlite3_int64 iDiff = COMPARE_DOCID(i1, i2); |
| 113893 | + |
| 113894 | + if( p2 && p1 && iDiff==0 ){ |
| 113895 | + fts3PutDeltaVarint3(&p, bDescIdx, &iPrev, &bFirstOut, i1); |
| 113896 | + fts3PoslistMerge(&p, &p1, &p2); |
| 113897 | + fts3GetDeltaVarint3(&p1, pEnd1, bDescIdx, &i1); |
| 113898 | + fts3GetDeltaVarint3(&p2, pEnd2, bDescIdx, &i2); |
| 113899 | + }else if( !p2 || (p1 && iDiff<0) ){ |
| 113900 | + fts3PutDeltaVarint3(&p, bDescIdx, &iPrev, &bFirstOut, i1); |
| 113901 | + fts3PoslistCopy(&p, &p1); |
| 113902 | + fts3GetDeltaVarint3(&p1, pEnd1, bDescIdx, &i1); |
| 113903 | + }else{ |
| 113904 | + fts3PutDeltaVarint3(&p, bDescIdx, &iPrev, &bFirstOut, i2); |
| 113905 | + fts3PoslistCopy(&p, &p2); |
| 113906 | + fts3GetDeltaVarint3(&p2, pEnd2, bDescIdx, &i2); |
| 113907 | + } |
| 113908 | + } |
| 113909 | + |
| 113910 | + *paOut = aOut; |
| 113911 | + *pnOut = (p-aOut); |
| 113912 | + return SQLITE_OK; |
| 113913 | +} |
| 113914 | + |
| 113915 | +static void fts3DoclistPhraseMerge( |
| 113916 | + int bDescIdx, /* True if arguments are desc */ |
| 113917 | + int nDist, /* Distance from left to right (1=adjacent) */ |
| 113918 | + char *aLeft, int nLeft, /* Left doclist */ |
| 113919 | + char *aRight, int *pnRight /* IN/OUT: Right/output doclist */ |
| 113920 | +){ |
| 113921 | + sqlite3_int64 i1 = 0; |
| 113922 | + sqlite3_int64 i2 = 0; |
| 113923 | + sqlite3_int64 iPrev = 0; |
| 113924 | + char *pEnd1 = &aLeft[nLeft]; |
| 113925 | + char *pEnd2 = &aRight[*pnRight]; |
| 113926 | + char *p1 = aLeft; |
| 113927 | + char *p2 = aRight; |
| 113928 | + char *p; |
| 113929 | + int bFirstOut = 0; |
| 113930 | + char *aOut = aRight; |
| 113931 | + |
| 113932 | + assert( nDist>0 ); |
| 113933 | + |
| 113934 | + p = aOut; |
| 113935 | + fts3GetDeltaVarint3(&p1, pEnd1, 0, &i1); |
| 113936 | + fts3GetDeltaVarint3(&p2, pEnd2, 0, &i2); |
| 113937 | + |
| 113938 | + while( p1 && p2 ){ |
| 113939 | + sqlite3_int64 iDiff = COMPARE_DOCID(i1, i2); |
| 113940 | + if( iDiff==0 ){ |
| 113941 | + char *pSave = p; |
| 113942 | + sqlite3_int64 iPrevSave = iPrev; |
| 113943 | + int bFirstOutSave = bFirstOut; |
| 113944 | + |
| 113945 | + fts3PutDeltaVarint3(&p, bDescIdx, &iPrev, &bFirstOut, i1); |
| 113946 | + if( 0==fts3PoslistPhraseMerge(&p, nDist, 0, 1, &p1, &p2) ){ |
| 113947 | + p = pSave; |
| 113948 | + iPrev = iPrevSave; |
| 113949 | + bFirstOut = bFirstOutSave; |
| 113950 | + } |
| 113951 | + fts3GetDeltaVarint3(&p1, pEnd1, bDescIdx, &i1); |
| 113952 | + fts3GetDeltaVarint3(&p2, pEnd2, bDescIdx, &i2); |
| 113953 | + }else if( iDiff<0 ){ |
| 113954 | + fts3PoslistCopy(0, &p1); |
| 113955 | + fts3GetDeltaVarint3(&p1, pEnd1, bDescIdx, &i1); |
| 113956 | + }else{ |
| 113957 | + fts3PoslistCopy(0, &p2); |
| 113958 | + fts3GetDeltaVarint3(&p2, pEnd2, bDescIdx, &i2); |
| 113959 | + } |
| 113960 | + } |
| 113961 | + |
| 113962 | + *pnRight = p - aOut; |
| 113963 | +} |
| 113964 | + |
| 113769 | 113965 | |
| 113770 | 113966 | /* |
| 113771 | 113967 | ** Merge all doclists in the TermSelect.aaOutput[] array into a single |
| 113772 | 113968 | ** doclist stored in TermSelect.aaOutput[0]. If successful, delete all |
| 113773 | 113969 | ** other doclists (except the aaOutput[0] one) and return SQLITE_OK. |
| | @@ -113774,12 +113970,11 @@ |
| 113774 | 113970 | ** |
| 113775 | 113971 | ** If an OOM error occurs, return SQLITE_NOMEM. In this case it is |
| 113776 | 113972 | ** the responsibility of the caller to free any doclists left in the |
| 113777 | 113973 | ** TermSelect.aaOutput[] array. |
| 113778 | 113974 | */ |
| 113779 | | -static int fts3TermSelectMerge(TermSelect *pTS){ |
| 113780 | | - int mergetype = (pTS->isReqPos ? MERGE_POS_OR : MERGE_OR); |
| 113975 | +static int fts3TermSelectMerge(Fts3Table *p, TermSelect *pTS){ |
| 113781 | 113976 | char *aOut = 0; |
| 113782 | 113977 | int nOut = 0; |
| 113783 | 113978 | int i; |
| 113784 | 113979 | |
| 113785 | 113980 | /* Loop through the doclists in the aaOutput[] array. Merge them all |
| | @@ -113790,19 +113985,21 @@ |
| 113790 | 113985 | if( !aOut ){ |
| 113791 | 113986 | aOut = pTS->aaOutput[i]; |
| 113792 | 113987 | nOut = pTS->anOutput[i]; |
| 113793 | 113988 | pTS->aaOutput[i] = 0; |
| 113794 | 113989 | }else{ |
| 113795 | | - int nNew = nOut + pTS->anOutput[i]; |
| 113796 | | - char *aNew = sqlite3_malloc(nNew); |
| 113797 | | - if( !aNew ){ |
| 113990 | + int nNew; |
| 113991 | + char *aNew; |
| 113992 | + |
| 113993 | + int rc = fts3DoclistOrMerge(p->bDescIdx, |
| 113994 | + pTS->aaOutput[i], pTS->anOutput[i], aOut, nOut, &aNew, &nNew |
| 113995 | + ); |
| 113996 | + if( rc!=SQLITE_OK ){ |
| 113798 | 113997 | sqlite3_free(aOut); |
| 113799 | | - return SQLITE_NOMEM; |
| 113998 | + return rc; |
| 113800 | 113999 | } |
| 113801 | | - fts3DoclistMerge(mergetype, 0, 0, |
| 113802 | | - aNew, &nNew, pTS->aaOutput[i], pTS->anOutput[i], aOut, nOut, 0 |
| 113803 | | - ); |
| 114000 | + |
| 113804 | 114001 | sqlite3_free(pTS->aaOutput[i]); |
| 113805 | 114002 | sqlite3_free(aOut); |
| 113806 | 114003 | pTS->aaOutput[i] = 0; |
| 113807 | 114004 | aOut = aNew; |
| 113808 | 114005 | nOut = nNew; |
| | @@ -113834,217 +114031,241 @@ |
| 113834 | 114031 | UNUSED_PARAMETER(zTerm); |
| 113835 | 114032 | UNUSED_PARAMETER(nTerm); |
| 113836 | 114033 | |
| 113837 | 114034 | if( pTS->aaOutput[0]==0 ){ |
| 113838 | 114035 | /* If this is the first term selected, copy the doclist to the output |
| 113839 | | - ** buffer using memcpy(). TODO: Add a way to transfer control of the |
| 113840 | | - ** aDoclist buffer from the caller so as to avoid the memcpy(). |
| 113841 | | - */ |
| 114036 | + ** buffer using memcpy(). */ |
| 113842 | 114037 | pTS->aaOutput[0] = sqlite3_malloc(nDoclist); |
| 113843 | 114038 | pTS->anOutput[0] = nDoclist; |
| 113844 | 114039 | if( pTS->aaOutput[0] ){ |
| 113845 | 114040 | memcpy(pTS->aaOutput[0], aDoclist, nDoclist); |
| 113846 | 114041 | }else{ |
| 113847 | 114042 | return SQLITE_NOMEM; |
| 113848 | 114043 | } |
| 113849 | 114044 | }else{ |
| 113850 | | - int mergetype = (pTS->isReqPos ? MERGE_POS_OR : MERGE_OR); |
| 113851 | 114045 | char *aMerge = aDoclist; |
| 113852 | 114046 | int nMerge = nDoclist; |
| 113853 | 114047 | int iOut; |
| 113854 | 114048 | |
| 113855 | 114049 | for(iOut=0; iOut<SizeofArray(pTS->aaOutput); iOut++){ |
| 113856 | | - char *aNew; |
| 113857 | | - int nNew; |
| 113858 | 114050 | if( pTS->aaOutput[iOut]==0 ){ |
| 113859 | 114051 | assert( iOut>0 ); |
| 113860 | 114052 | pTS->aaOutput[iOut] = aMerge; |
| 113861 | 114053 | pTS->anOutput[iOut] = nMerge; |
| 113862 | 114054 | break; |
| 113863 | | - } |
| 113864 | | - |
| 113865 | | - nNew = nMerge + pTS->anOutput[iOut]; |
| 113866 | | - aNew = sqlite3_malloc(nNew); |
| 113867 | | - if( !aNew ){ |
| 113868 | | - if( aMerge!=aDoclist ){ |
| 113869 | | - sqlite3_free(aMerge); |
| 113870 | | - } |
| 113871 | | - return SQLITE_NOMEM; |
| 113872 | | - } |
| 113873 | | - fts3DoclistMerge(mergetype, 0, 0, aNew, &nNew, |
| 113874 | | - pTS->aaOutput[iOut], pTS->anOutput[iOut], aMerge, nMerge, 0 |
| 113875 | | - ); |
| 113876 | | - |
| 113877 | | - if( iOut>0 ) sqlite3_free(aMerge); |
| 113878 | | - sqlite3_free(pTS->aaOutput[iOut]); |
| 113879 | | - pTS->aaOutput[iOut] = 0; |
| 113880 | | - |
| 113881 | | - aMerge = aNew; |
| 113882 | | - nMerge = nNew; |
| 113883 | | - if( (iOut+1)==SizeofArray(pTS->aaOutput) ){ |
| 113884 | | - pTS->aaOutput[iOut] = aMerge; |
| 113885 | | - pTS->anOutput[iOut] = nMerge; |
| 113886 | | - } |
| 113887 | | - } |
| 113888 | | - } |
| 113889 | | - return SQLITE_OK; |
| 113890 | | -} |
| 113891 | | - |
| 113892 | | -static int fts3DeferredTermSelect( |
| 113893 | | - Fts3DeferredToken *pToken, /* Phrase token */ |
| 113894 | | - int isTermPos, /* True to include positions */ |
| 113895 | | - int *pnOut, /* OUT: Size of list */ |
| 113896 | | - char **ppOut /* OUT: Body of list */ |
| 113897 | | -){ |
| 113898 | | - char *aSource; |
| 113899 | | - int nSource; |
| 113900 | | - |
| 113901 | | - aSource = sqlite3Fts3DeferredDoclist(pToken, &nSource); |
| 113902 | | - if( !aSource ){ |
| 113903 | | - *pnOut = 0; |
| 113904 | | - *ppOut = 0; |
| 113905 | | - }else if( isTermPos ){ |
| 113906 | | - *ppOut = sqlite3_malloc(nSource); |
| 113907 | | - if( !*ppOut ) return SQLITE_NOMEM; |
| 113908 | | - memcpy(*ppOut, aSource, nSource); |
| 113909 | | - *pnOut = nSource; |
| 113910 | | - }else{ |
| 113911 | | - sqlite3_int64 docid; |
| 113912 | | - *pnOut = sqlite3Fts3GetVarint(aSource, &docid); |
| 113913 | | - *ppOut = sqlite3_malloc(*pnOut); |
| 113914 | | - if( !*ppOut ) return SQLITE_NOMEM; |
| 113915 | | - sqlite3Fts3PutVarint(*ppOut, docid); |
| 113916 | | - } |
| 113917 | | - |
| 113918 | | - return SQLITE_OK; |
| 113919 | | -} |
| 113920 | | - |
| 113921 | | -SQLITE_PRIVATE int sqlite3Fts3SegReaderCursor( |
| 114055 | + }else{ |
| 114056 | + char *aNew; |
| 114057 | + int nNew; |
| 114058 | + |
| 114059 | + int rc = fts3DoclistOrMerge(p->bDescIdx, aMerge, nMerge, |
| 114060 | + pTS->aaOutput[iOut], pTS->anOutput[iOut], &aNew, &nNew |
| 114061 | + ); |
| 114062 | + if( rc!=SQLITE_OK ){ |
| 114063 | + if( aMerge!=aDoclist ) sqlite3_free(aMerge); |
| 114064 | + return rc; |
| 114065 | + } |
| 114066 | + |
| 114067 | + if( aMerge!=aDoclist ) sqlite3_free(aMerge); |
| 114068 | + sqlite3_free(pTS->aaOutput[iOut]); |
| 114069 | + pTS->aaOutput[iOut] = 0; |
| 114070 | + |
| 114071 | + aMerge = aNew; |
| 114072 | + nMerge = nNew; |
| 114073 | + if( (iOut+1)==SizeofArray(pTS->aaOutput) ){ |
| 114074 | + pTS->aaOutput[iOut] = aMerge; |
| 114075 | + pTS->anOutput[iOut] = nMerge; |
| 114076 | + } |
| 114077 | + } |
| 114078 | + } |
| 114079 | + } |
| 114080 | + return SQLITE_OK; |
| 114081 | +} |
| 114082 | + |
| 114083 | +/* |
| 114084 | +** Append SegReader object pNew to the end of the pCsr->apSegment[] array. |
| 114085 | +*/ |
| 114086 | +static int fts3SegReaderCursorAppend( |
| 114087 | + Fts3MultiSegReader *pCsr, |
| 114088 | + Fts3SegReader *pNew |
| 114089 | +){ |
| 114090 | + if( (pCsr->nSegment%16)==0 ){ |
| 114091 | + Fts3SegReader **apNew; |
| 114092 | + int nByte = (pCsr->nSegment + 16)*sizeof(Fts3SegReader*); |
| 114093 | + apNew = (Fts3SegReader **)sqlite3_realloc(pCsr->apSegment, nByte); |
| 114094 | + if( !apNew ){ |
| 114095 | + sqlite3Fts3SegReaderFree(pNew); |
| 114096 | + return SQLITE_NOMEM; |
| 114097 | + } |
| 114098 | + pCsr->apSegment = apNew; |
| 114099 | + } |
| 114100 | + pCsr->apSegment[pCsr->nSegment++] = pNew; |
| 114101 | + return SQLITE_OK; |
| 114102 | +} |
| 114103 | + |
| 114104 | +static int fts3SegReaderCursor( |
| 113922 | 114105 | Fts3Table *p, /* FTS3 table handle */ |
| 114106 | + int iIndex, /* Index to search (from 0 to p->nIndex-1) */ |
| 113923 | 114107 | int iLevel, /* Level of segments to scan */ |
| 113924 | 114108 | const char *zTerm, /* Term to query for */ |
| 113925 | 114109 | int nTerm, /* Size of zTerm in bytes */ |
| 113926 | 114110 | int isPrefix, /* True for a prefix search */ |
| 113927 | 114111 | int isScan, /* True to scan from zTerm to EOF */ |
| 113928 | | - Fts3SegReaderCursor *pCsr /* Cursor object to populate */ |
| 114112 | + Fts3MultiSegReader *pCsr /* Cursor object to populate */ |
| 113929 | 114113 | ){ |
| 113930 | 114114 | int rc = SQLITE_OK; |
| 113931 | 114115 | int rc2; |
| 113932 | | - int iAge = 0; |
| 113933 | 114116 | sqlite3_stmt *pStmt = 0; |
| 113934 | | - Fts3SegReader *pPending = 0; |
| 113935 | | - |
| 113936 | | - assert( iLevel==FTS3_SEGCURSOR_ALL |
| 113937 | | - || iLevel==FTS3_SEGCURSOR_PENDING |
| 113938 | | - || iLevel>=0 |
| 113939 | | - ); |
| 113940 | | - assert( FTS3_SEGCURSOR_PENDING<0 ); |
| 113941 | | - assert( FTS3_SEGCURSOR_ALL<0 ); |
| 113942 | | - assert( iLevel==FTS3_SEGCURSOR_ALL || (zTerm==0 && isPrefix==1) ); |
| 113943 | | - assert( isPrefix==0 || isScan==0 ); |
| 113944 | | - |
| 113945 | | - |
| 113946 | | - memset(pCsr, 0, sizeof(Fts3SegReaderCursor)); |
| 113947 | | - |
| 113948 | | - /* If iLevel is less than 0, include a seg-reader for the pending-terms. */ |
| 113949 | | - assert( isScan==0 || fts3HashCount(&p->pendingTerms)==0 ); |
| 113950 | | - if( iLevel<0 && isScan==0 ){ |
| 113951 | | - rc = sqlite3Fts3SegReaderPending(p, zTerm, nTerm, isPrefix, &pPending); |
| 113952 | | - if( rc==SQLITE_OK && pPending ){ |
| 113953 | | - int nByte = (sizeof(Fts3SegReader *) * 16); |
| 113954 | | - pCsr->apSegment = (Fts3SegReader **)sqlite3_malloc(nByte); |
| 113955 | | - if( pCsr->apSegment==0 ){ |
| 113956 | | - rc = SQLITE_NOMEM; |
| 113957 | | - }else{ |
| 113958 | | - pCsr->apSegment[0] = pPending; |
| 113959 | | - pCsr->nSegment = 1; |
| 113960 | | - pPending = 0; |
| 113961 | | - } |
| 114117 | + |
| 114118 | + /* If iLevel is less than 0 and this is not a scan, include a seg-reader |
| 114119 | + ** for the pending-terms. If this is a scan, then this call must be being |
| 114120 | + ** made by an fts4aux module, not an FTS table. In this case calling |
| 114121 | + ** Fts3SegReaderPending might segfault, as the data structures used by |
| 114122 | + ** fts4aux are not completely populated. So it's easiest to filter these |
| 114123 | + ** calls out here. */ |
| 114124 | + if( iLevel<0 && p->aIndex ){ |
| 114125 | + Fts3SegReader *pSeg = 0; |
| 114126 | + rc = sqlite3Fts3SegReaderPending(p, iIndex, zTerm, nTerm, isPrefix, &pSeg); |
| 114127 | + if( rc==SQLITE_OK && pSeg ){ |
| 114128 | + rc = fts3SegReaderCursorAppend(pCsr, pSeg); |
| 113962 | 114129 | } |
| 113963 | 114130 | } |
| 113964 | 114131 | |
| 113965 | 114132 | if( iLevel!=FTS3_SEGCURSOR_PENDING ){ |
| 113966 | 114133 | if( rc==SQLITE_OK ){ |
| 113967 | | - rc = sqlite3Fts3AllSegdirs(p, iLevel, &pStmt); |
| 114134 | + rc = sqlite3Fts3AllSegdirs(p, iIndex, iLevel, &pStmt); |
| 113968 | 114135 | } |
| 114136 | + |
| 113969 | 114137 | while( rc==SQLITE_OK && SQLITE_ROW==(rc = sqlite3_step(pStmt)) ){ |
| 114138 | + Fts3SegReader *pSeg = 0; |
| 113970 | 114139 | |
| 113971 | 114140 | /* Read the values returned by the SELECT into local variables. */ |
| 113972 | 114141 | sqlite3_int64 iStartBlock = sqlite3_column_int64(pStmt, 1); |
| 113973 | 114142 | sqlite3_int64 iLeavesEndBlock = sqlite3_column_int64(pStmt, 2); |
| 113974 | 114143 | sqlite3_int64 iEndBlock = sqlite3_column_int64(pStmt, 3); |
| 113975 | 114144 | int nRoot = sqlite3_column_bytes(pStmt, 4); |
| 113976 | 114145 | char const *zRoot = sqlite3_column_blob(pStmt, 4); |
| 113977 | 114146 | |
| 113978 | | - /* If nSegment is a multiple of 16 the array needs to be extended. */ |
| 113979 | | - if( (pCsr->nSegment%16)==0 ){ |
| 113980 | | - Fts3SegReader **apNew; |
| 113981 | | - int nByte = (pCsr->nSegment + 16)*sizeof(Fts3SegReader*); |
| 113982 | | - apNew = (Fts3SegReader **)sqlite3_realloc(pCsr->apSegment, nByte); |
| 113983 | | - if( !apNew ){ |
| 113984 | | - rc = SQLITE_NOMEM; |
| 113985 | | - goto finished; |
| 113986 | | - } |
| 113987 | | - pCsr->apSegment = apNew; |
| 113988 | | - } |
| 113989 | | - |
| 113990 | 114147 | /* If zTerm is not NULL, and this segment is not stored entirely on its |
| 113991 | 114148 | ** root node, the range of leaves scanned can be reduced. Do this. */ |
| 113992 | 114149 | if( iStartBlock && zTerm ){ |
| 113993 | 114150 | sqlite3_int64 *pi = (isPrefix ? &iLeavesEndBlock : 0); |
| 113994 | 114151 | rc = fts3SelectLeaf(p, zTerm, nTerm, zRoot, nRoot, &iStartBlock, pi); |
| 113995 | 114152 | if( rc!=SQLITE_OK ) goto finished; |
| 113996 | 114153 | if( isPrefix==0 && isScan==0 ) iLeavesEndBlock = iStartBlock; |
| 113997 | 114154 | } |
| 113998 | 114155 | |
| 113999 | | - rc = sqlite3Fts3SegReaderNew(iAge, iStartBlock, iLeavesEndBlock, |
| 114000 | | - iEndBlock, zRoot, nRoot, &pCsr->apSegment[pCsr->nSegment] |
| 114156 | + rc = sqlite3Fts3SegReaderNew(pCsr->nSegment+1, |
| 114157 | + iStartBlock, iLeavesEndBlock, iEndBlock, zRoot, nRoot, &pSeg |
| 114001 | 114158 | ); |
| 114002 | 114159 | if( rc!=SQLITE_OK ) goto finished; |
| 114003 | | - pCsr->nSegment++; |
| 114004 | | - iAge++; |
| 114160 | + rc = fts3SegReaderCursorAppend(pCsr, pSeg); |
| 114005 | 114161 | } |
| 114006 | 114162 | } |
| 114007 | 114163 | |
| 114008 | 114164 | finished: |
| 114009 | 114165 | rc2 = sqlite3_reset(pStmt); |
| 114010 | 114166 | if( rc==SQLITE_DONE ) rc = rc2; |
| 114011 | | - sqlite3Fts3SegReaderFree(pPending); |
| 114012 | 114167 | |
| 114013 | 114168 | return rc; |
| 114014 | 114169 | } |
| 114015 | 114170 | |
| 114171 | +/* |
| 114172 | +** Set up a cursor object for iterating through a full-text index or a |
| 114173 | +** single level therein. |
| 114174 | +*/ |
| 114175 | +SQLITE_PRIVATE int sqlite3Fts3SegReaderCursor( |
| 114176 | + Fts3Table *p, /* FTS3 table handle */ |
| 114177 | + int iIndex, /* Index to search (from 0 to p->nIndex-1) */ |
| 114178 | + int iLevel, /* Level of segments to scan */ |
| 114179 | + const char *zTerm, /* Term to query for */ |
| 114180 | + int nTerm, /* Size of zTerm in bytes */ |
| 114181 | + int isPrefix, /* True for a prefix search */ |
| 114182 | + int isScan, /* True to scan from zTerm to EOF */ |
| 114183 | + Fts3MultiSegReader *pCsr /* Cursor object to populate */ |
| 114184 | +){ |
| 114185 | + assert( iIndex>=0 && iIndex<p->nIndex ); |
| 114186 | + assert( iLevel==FTS3_SEGCURSOR_ALL |
| 114187 | + || iLevel==FTS3_SEGCURSOR_PENDING |
| 114188 | + || iLevel>=0 |
| 114189 | + ); |
| 114190 | + assert( iLevel<FTS3_SEGDIR_MAXLEVEL ); |
| 114191 | + assert( FTS3_SEGCURSOR_ALL<0 && FTS3_SEGCURSOR_PENDING<0 ); |
| 114192 | + assert( isPrefix==0 || isScan==0 ); |
| 114016 | 114193 | |
| 114017 | | -static int fts3TermSegReaderCursor( |
| 114194 | + /* "isScan" is only set to true by the ft4aux module, an ordinary |
| 114195 | + ** full-text tables. */ |
| 114196 | + assert( isScan==0 || p->aIndex==0 ); |
| 114197 | + |
| 114198 | + memset(pCsr, 0, sizeof(Fts3MultiSegReader)); |
| 114199 | + |
| 114200 | + return fts3SegReaderCursor( |
| 114201 | + p, iIndex, iLevel, zTerm, nTerm, isPrefix, isScan, pCsr |
| 114202 | + ); |
| 114203 | +} |
| 114204 | + |
| 114205 | +static int fts3SegReaderCursorAddZero( |
| 114206 | + Fts3Table *p, |
| 114207 | + const char *zTerm, |
| 114208 | + int nTerm, |
| 114209 | + Fts3MultiSegReader *pCsr |
| 114210 | +){ |
| 114211 | + return fts3SegReaderCursor(p, 0, FTS3_SEGCURSOR_ALL, zTerm, nTerm, 0, 0,pCsr); |
| 114212 | +} |
| 114213 | + |
| 114214 | + |
| 114215 | +SQLITE_PRIVATE int sqlite3Fts3TermSegReaderCursor( |
| 114018 | 114216 | Fts3Cursor *pCsr, /* Virtual table cursor handle */ |
| 114019 | 114217 | const char *zTerm, /* Term to query for */ |
| 114020 | 114218 | int nTerm, /* Size of zTerm in bytes */ |
| 114021 | 114219 | int isPrefix, /* True for a prefix search */ |
| 114022 | | - Fts3SegReaderCursor **ppSegcsr /* OUT: Allocated seg-reader cursor */ |
| 114220 | + Fts3MultiSegReader **ppSegcsr /* OUT: Allocated seg-reader cursor */ |
| 114023 | 114221 | ){ |
| 114024 | | - Fts3SegReaderCursor *pSegcsr; /* Object to allocate and return */ |
| 114222 | + Fts3MultiSegReader *pSegcsr; /* Object to allocate and return */ |
| 114025 | 114223 | int rc = SQLITE_NOMEM; /* Return code */ |
| 114026 | 114224 | |
| 114027 | | - pSegcsr = sqlite3_malloc(sizeof(Fts3SegReaderCursor)); |
| 114225 | + pSegcsr = sqlite3_malloc(sizeof(Fts3MultiSegReader)); |
| 114028 | 114226 | if( pSegcsr ){ |
| 114227 | + int i; |
| 114228 | + int bFound = 0; /* True once an index has been found */ |
| 114029 | 114229 | Fts3Table *p = (Fts3Table *)pCsr->base.pVtab; |
| 114030 | | - int i; |
| 114031 | | - int nCost = 0; |
| 114032 | | - rc = sqlite3Fts3SegReaderCursor( |
| 114033 | | - p, FTS3_SEGCURSOR_ALL, zTerm, nTerm, isPrefix, 0, pSegcsr); |
| 114034 | | - |
| 114035 | | - for(i=0; rc==SQLITE_OK && i<pSegcsr->nSegment; i++){ |
| 114036 | | - rc = sqlite3Fts3SegReaderCost(pCsr, pSegcsr->apSegment[i], &nCost); |
| 114037 | | - } |
| 114038 | | - pSegcsr->nCost = nCost; |
| 114230 | + |
| 114231 | + if( isPrefix ){ |
| 114232 | + for(i=1; bFound==0 && i<p->nIndex; i++){ |
| 114233 | + if( p->aIndex[i].nPrefix==nTerm ){ |
| 114234 | + bFound = 1; |
| 114235 | + rc = sqlite3Fts3SegReaderCursor( |
| 114236 | + p, i, FTS3_SEGCURSOR_ALL, zTerm, nTerm, 0, 0, pSegcsr); |
| 114237 | + pSegcsr->bLookup = 1; |
| 114238 | + } |
| 114239 | + } |
| 114240 | + |
| 114241 | + for(i=1; bFound==0 && i<p->nIndex; i++){ |
| 114242 | + if( p->aIndex[i].nPrefix==nTerm+1 ){ |
| 114243 | + bFound = 1; |
| 114244 | + rc = sqlite3Fts3SegReaderCursor( |
| 114245 | + p, i, FTS3_SEGCURSOR_ALL, zTerm, nTerm, 1, 0, pSegcsr |
| 114246 | + ); |
| 114247 | + if( rc==SQLITE_OK ){ |
| 114248 | + rc = fts3SegReaderCursorAddZero(p, zTerm, nTerm, pSegcsr); |
| 114249 | + } |
| 114250 | + } |
| 114251 | + } |
| 114252 | + } |
| 114253 | + |
| 114254 | + if( bFound==0 ){ |
| 114255 | + rc = sqlite3Fts3SegReaderCursor( |
| 114256 | + p, 0, FTS3_SEGCURSOR_ALL, zTerm, nTerm, isPrefix, 0, pSegcsr |
| 114257 | + ); |
| 114258 | + pSegcsr->bLookup = !isPrefix; |
| 114259 | + } |
| 114039 | 114260 | } |
| 114040 | 114261 | |
| 114041 | 114262 | *ppSegcsr = pSegcsr; |
| 114042 | 114263 | return rc; |
| 114043 | 114264 | } |
| 114044 | 114265 | |
| 114045 | | -static void fts3SegReaderCursorFree(Fts3SegReaderCursor *pSegcsr){ |
| 114266 | +static void fts3SegReaderCursorFree(Fts3MultiSegReader *pSegcsr){ |
| 114046 | 114267 | sqlite3Fts3SegReaderFinish(pSegcsr); |
| 114047 | 114268 | sqlite3_free(pSegcsr); |
| 114048 | 114269 | } |
| 114049 | 114270 | |
| 114050 | 114271 | /* |
| | @@ -114065,11 +114286,11 @@ |
| 114065 | 114286 | int isReqPos, /* True to include position lists in output */ |
| 114066 | 114287 | int *pnOut, /* OUT: Size of buffer at *ppOut */ |
| 114067 | 114288 | char **ppOut /* OUT: Malloced result buffer */ |
| 114068 | 114289 | ){ |
| 114069 | 114290 | int rc; /* Return code */ |
| 114070 | | - Fts3SegReaderCursor *pSegcsr; /* Seg-reader cursor for this term */ |
| 114291 | + Fts3MultiSegReader *pSegcsr; /* Seg-reader cursor for this term */ |
| 114071 | 114292 | TermSelect tsc; /* Context object for fts3TermSelectCb() */ |
| 114072 | 114293 | Fts3SegFilter filter; /* Segment term filter configuration */ |
| 114073 | 114294 | |
| 114074 | 114295 | pSegcsr = pTok->pSegcsr; |
| 114075 | 114296 | memset(&tsc, 0, sizeof(TermSelect)); |
| | @@ -114091,11 +114312,11 @@ |
| 114091 | 114312 | pSegcsr->zTerm, pSegcsr->nTerm, pSegcsr->aDoclist, pSegcsr->nDoclist |
| 114092 | 114313 | ); |
| 114093 | 114314 | } |
| 114094 | 114315 | |
| 114095 | 114316 | if( rc==SQLITE_OK ){ |
| 114096 | | - rc = fts3TermSelectMerge(&tsc); |
| 114317 | + rc = fts3TermSelectMerge(p, &tsc); |
| 114097 | 114318 | } |
| 114098 | 114319 | if( rc==SQLITE_OK ){ |
| 114099 | 114320 | *ppOut = tsc.aaOutput[0]; |
| 114100 | 114321 | *pnOut = tsc.anOutput[0]; |
| 114101 | 114322 | }else{ |
| | @@ -114141,664 +114362,10 @@ |
| 114141 | 114362 | } |
| 114142 | 114363 | |
| 114143 | 114364 | return nDoc; |
| 114144 | 114365 | } |
| 114145 | 114366 | |
| 114146 | | -/* |
| 114147 | | -** Call sqlite3Fts3DeferToken() for each token in the expression pExpr. |
| 114148 | | -*/ |
| 114149 | | -static int fts3DeferExpression(Fts3Cursor *pCsr, Fts3Expr *pExpr){ |
| 114150 | | - int rc = SQLITE_OK; |
| 114151 | | - if( pExpr ){ |
| 114152 | | - rc = fts3DeferExpression(pCsr, pExpr->pLeft); |
| 114153 | | - if( rc==SQLITE_OK ){ |
| 114154 | | - rc = fts3DeferExpression(pCsr, pExpr->pRight); |
| 114155 | | - } |
| 114156 | | - if( pExpr->eType==FTSQUERY_PHRASE ){ |
| 114157 | | - int iCol = pExpr->pPhrase->iColumn; |
| 114158 | | - int i; |
| 114159 | | - for(i=0; rc==SQLITE_OK && i<pExpr->pPhrase->nToken; i++){ |
| 114160 | | - Fts3PhraseToken *pToken = &pExpr->pPhrase->aToken[i]; |
| 114161 | | - if( pToken->pDeferred==0 ){ |
| 114162 | | - rc = sqlite3Fts3DeferToken(pCsr, pToken, iCol); |
| 114163 | | - } |
| 114164 | | - } |
| 114165 | | - } |
| 114166 | | - } |
| 114167 | | - return rc; |
| 114168 | | -} |
| 114169 | | - |
| 114170 | | -/* |
| 114171 | | -** This function removes the position information from a doclist. When |
| 114172 | | -** called, buffer aList (size *pnList bytes) contains a doclist that includes |
| 114173 | | -** position information. This function removes the position information so |
| 114174 | | -** that aList contains only docids, and adjusts *pnList to reflect the new |
| 114175 | | -** (possibly reduced) size of the doclist. |
| 114176 | | -*/ |
| 114177 | | -static void fts3DoclistStripPositions( |
| 114178 | | - char *aList, /* IN/OUT: Buffer containing doclist */ |
| 114179 | | - int *pnList /* IN/OUT: Size of doclist in bytes */ |
| 114180 | | -){ |
| 114181 | | - if( aList ){ |
| 114182 | | - char *aEnd = &aList[*pnList]; /* Pointer to one byte after EOF */ |
| 114183 | | - char *p = aList; /* Input cursor */ |
| 114184 | | - char *pOut = aList; /* Output cursor */ |
| 114185 | | - |
| 114186 | | - while( p<aEnd ){ |
| 114187 | | - sqlite3_int64 delta; |
| 114188 | | - p += sqlite3Fts3GetVarint(p, &delta); |
| 114189 | | - fts3PoslistCopy(0, &p); |
| 114190 | | - pOut += sqlite3Fts3PutVarint(pOut, delta); |
| 114191 | | - } |
| 114192 | | - |
| 114193 | | - *pnList = (int)(pOut - aList); |
| 114194 | | - } |
| 114195 | | -} |
| 114196 | | - |
| 114197 | | -/* |
| 114198 | | -** Return a DocList corresponding to the phrase *pPhrase. |
| 114199 | | -** |
| 114200 | | -** If this function returns SQLITE_OK, but *pnOut is set to a negative value, |
| 114201 | | -** then no tokens in the phrase were looked up in the full-text index. This |
| 114202 | | -** is only possible when this function is called from within xFilter(). The |
| 114203 | | -** caller should assume that all documents match the phrase. The actual |
| 114204 | | -** filtering will take place in xNext(). |
| 114205 | | -*/ |
| 114206 | | -static int fts3PhraseSelect( |
| 114207 | | - Fts3Cursor *pCsr, /* Virtual table cursor handle */ |
| 114208 | | - Fts3Phrase *pPhrase, /* Phrase to return a doclist for */ |
| 114209 | | - int isReqPos, /* True if output should contain positions */ |
| 114210 | | - char **paOut, /* OUT: Pointer to malloc'd result buffer */ |
| 114211 | | - int *pnOut /* OUT: Size of buffer at *paOut */ |
| 114212 | | -){ |
| 114213 | | - char *pOut = 0; |
| 114214 | | - int nOut = 0; |
| 114215 | | - int rc = SQLITE_OK; |
| 114216 | | - int ii; |
| 114217 | | - int iCol = pPhrase->iColumn; |
| 114218 | | - int isTermPos = (pPhrase->nToken>1 || isReqPos); |
| 114219 | | - Fts3Table *p = (Fts3Table *)pCsr->base.pVtab; |
| 114220 | | - int isFirst = 1; |
| 114221 | | - |
| 114222 | | - int iPrevTok = 0; |
| 114223 | | - int nDoc = 0; |
| 114224 | | - |
| 114225 | | - /* If this is an xFilter() evaluation, create a segment-reader for each |
| 114226 | | - ** phrase token. Or, if this is an xNext() or snippet/offsets/matchinfo |
| 114227 | | - ** evaluation, only create segment-readers if there are no Fts3DeferredToken |
| 114228 | | - ** objects attached to the phrase-tokens. |
| 114229 | | - */ |
| 114230 | | - for(ii=0; ii<pPhrase->nToken; ii++){ |
| 114231 | | - Fts3PhraseToken *pTok = &pPhrase->aToken[ii]; |
| 114232 | | - if( pTok->pSegcsr==0 ){ |
| 114233 | | - if( (pCsr->eEvalmode==FTS3_EVAL_FILTER) |
| 114234 | | - || (pCsr->eEvalmode==FTS3_EVAL_NEXT && pCsr->pDeferred==0) |
| 114235 | | - || (pCsr->eEvalmode==FTS3_EVAL_MATCHINFO && pTok->bFulltext) |
| 114236 | | - ){ |
| 114237 | | - rc = fts3TermSegReaderCursor( |
| 114238 | | - pCsr, pTok->z, pTok->n, pTok->isPrefix, &pTok->pSegcsr |
| 114239 | | - ); |
| 114240 | | - if( rc!=SQLITE_OK ) return rc; |
| 114241 | | - } |
| 114242 | | - } |
| 114243 | | - } |
| 114244 | | - |
| 114245 | | - for(ii=0; ii<pPhrase->nToken; ii++){ |
| 114246 | | - Fts3PhraseToken *pTok; /* Token to find doclist for */ |
| 114247 | | - int iTok = 0; /* The token being queried this iteration */ |
| 114248 | | - char *pList = 0; /* Pointer to token doclist */ |
| 114249 | | - int nList = 0; /* Size of buffer at pList */ |
| 114250 | | - |
| 114251 | | - /* Select a token to process. If this is an xFilter() call, then tokens |
| 114252 | | - ** are processed in order from least to most costly. Otherwise, tokens |
| 114253 | | - ** are processed in the order in which they occur in the phrase. |
| 114254 | | - */ |
| 114255 | | - if( pCsr->eEvalmode==FTS3_EVAL_MATCHINFO ){ |
| 114256 | | - assert( isReqPos ); |
| 114257 | | - iTok = ii; |
| 114258 | | - pTok = &pPhrase->aToken[iTok]; |
| 114259 | | - if( pTok->bFulltext==0 ) continue; |
| 114260 | | - }else if( pCsr->eEvalmode==FTS3_EVAL_NEXT || isReqPos ){ |
| 114261 | | - iTok = ii; |
| 114262 | | - pTok = &pPhrase->aToken[iTok]; |
| 114263 | | - }else{ |
| 114264 | | - int nMinCost = 0x7FFFFFFF; |
| 114265 | | - int jj; |
| 114266 | | - |
| 114267 | | - /* Find the remaining token with the lowest cost. */ |
| 114268 | | - for(jj=0; jj<pPhrase->nToken; jj++){ |
| 114269 | | - Fts3SegReaderCursor *pSegcsr = pPhrase->aToken[jj].pSegcsr; |
| 114270 | | - if( pSegcsr && pSegcsr->nCost<nMinCost ){ |
| 114271 | | - iTok = jj; |
| 114272 | | - nMinCost = pSegcsr->nCost; |
| 114273 | | - } |
| 114274 | | - } |
| 114275 | | - pTok = &pPhrase->aToken[iTok]; |
| 114276 | | - |
| 114277 | | - /* This branch is taken if it is determined that loading the doclist |
| 114278 | | - ** for the next token would require more IO than loading all documents |
| 114279 | | - ** currently identified by doclist pOut/nOut. No further doclists will |
| 114280 | | - ** be loaded from the full-text index for this phrase. |
| 114281 | | - */ |
| 114282 | | - if( nMinCost>nDoc && ii>0 ){ |
| 114283 | | - rc = fts3DeferExpression(pCsr, pCsr->pExpr); |
| 114284 | | - break; |
| 114285 | | - } |
| 114286 | | - } |
| 114287 | | - |
| 114288 | | - if( pCsr->eEvalmode==FTS3_EVAL_NEXT && pTok->pDeferred ){ |
| 114289 | | - rc = fts3DeferredTermSelect(pTok->pDeferred, isTermPos, &nList, &pList); |
| 114290 | | - }else{ |
| 114291 | | - if( pTok->pSegcsr ){ |
| 114292 | | - rc = fts3TermSelect(p, pTok, iCol, isTermPos, &nList, &pList); |
| 114293 | | - } |
| 114294 | | - pTok->bFulltext = 1; |
| 114295 | | - } |
| 114296 | | - assert( rc!=SQLITE_OK || pCsr->eEvalmode || pTok->pSegcsr==0 ); |
| 114297 | | - if( rc!=SQLITE_OK ) break; |
| 114298 | | - |
| 114299 | | - if( isFirst ){ |
| 114300 | | - pOut = pList; |
| 114301 | | - nOut = nList; |
| 114302 | | - if( pCsr->eEvalmode==FTS3_EVAL_FILTER && pPhrase->nToken>1 ){ |
| 114303 | | - nDoc = fts3DoclistCountDocids(1, pOut, nOut); |
| 114304 | | - } |
| 114305 | | - isFirst = 0; |
| 114306 | | - iPrevTok = iTok; |
| 114307 | | - }else{ |
| 114308 | | - /* Merge the new term list and the current output. */ |
| 114309 | | - char *aLeft, *aRight; |
| 114310 | | - int nLeft, nRight; |
| 114311 | | - int nDist; |
| 114312 | | - int mt; |
| 114313 | | - |
| 114314 | | - /* If this is the final token of the phrase, and positions were not |
| 114315 | | - ** requested by the caller, use MERGE_PHRASE instead of POS_PHRASE. |
| 114316 | | - ** This drops the position information from the output list. |
| 114317 | | - */ |
| 114318 | | - mt = MERGE_POS_PHRASE; |
| 114319 | | - if( ii==pPhrase->nToken-1 && !isReqPos ) mt = MERGE_PHRASE; |
| 114320 | | - |
| 114321 | | - assert( iPrevTok!=iTok ); |
| 114322 | | - if( iPrevTok<iTok ){ |
| 114323 | | - aLeft = pOut; |
| 114324 | | - nLeft = nOut; |
| 114325 | | - aRight = pList; |
| 114326 | | - nRight = nList; |
| 114327 | | - nDist = iTok-iPrevTok; |
| 114328 | | - iPrevTok = iTok; |
| 114329 | | - }else{ |
| 114330 | | - aRight = pOut; |
| 114331 | | - nRight = nOut; |
| 114332 | | - aLeft = pList; |
| 114333 | | - nLeft = nList; |
| 114334 | | - nDist = iPrevTok-iTok; |
| 114335 | | - } |
| 114336 | | - pOut = aRight; |
| 114337 | | - fts3DoclistMerge( |
| 114338 | | - mt, nDist, 0, pOut, &nOut, aLeft, nLeft, aRight, nRight, &nDoc |
| 114339 | | - ); |
| 114340 | | - sqlite3_free(aLeft); |
| 114341 | | - } |
| 114342 | | - assert( nOut==0 || pOut!=0 ); |
| 114343 | | - } |
| 114344 | | - |
| 114345 | | - if( rc==SQLITE_OK ){ |
| 114346 | | - if( ii!=pPhrase->nToken ){ |
| 114347 | | - assert( pCsr->eEvalmode==FTS3_EVAL_FILTER && isReqPos==0 ); |
| 114348 | | - fts3DoclistStripPositions(pOut, &nOut); |
| 114349 | | - } |
| 114350 | | - *paOut = pOut; |
| 114351 | | - *pnOut = nOut; |
| 114352 | | - }else{ |
| 114353 | | - sqlite3_free(pOut); |
| 114354 | | - } |
| 114355 | | - return rc; |
| 114356 | | -} |
| 114357 | | - |
| 114358 | | -/* |
| 114359 | | -** This function merges two doclists according to the requirements of a |
| 114360 | | -** NEAR operator. |
| 114361 | | -** |
| 114362 | | -** Both input doclists must include position information. The output doclist |
| 114363 | | -** includes position information if the first argument to this function |
| 114364 | | -** is MERGE_POS_NEAR, or does not if it is MERGE_NEAR. |
| 114365 | | -*/ |
| 114366 | | -static int fts3NearMerge( |
| 114367 | | - int mergetype, /* MERGE_POS_NEAR or MERGE_NEAR */ |
| 114368 | | - int nNear, /* Parameter to NEAR operator */ |
| 114369 | | - int nTokenLeft, /* Number of tokens in LHS phrase arg */ |
| 114370 | | - char *aLeft, /* Doclist for LHS (incl. positions) */ |
| 114371 | | - int nLeft, /* Size of LHS doclist in bytes */ |
| 114372 | | - int nTokenRight, /* As nTokenLeft */ |
| 114373 | | - char *aRight, /* As aLeft */ |
| 114374 | | - int nRight, /* As nRight */ |
| 114375 | | - char **paOut, /* OUT: Results of merge (malloced) */ |
| 114376 | | - int *pnOut /* OUT: Sized of output buffer */ |
| 114377 | | -){ |
| 114378 | | - char *aOut; /* Buffer to write output doclist to */ |
| 114379 | | - int rc; /* Return code */ |
| 114380 | | - |
| 114381 | | - assert( mergetype==MERGE_POS_NEAR || MERGE_NEAR ); |
| 114382 | | - |
| 114383 | | - aOut = sqlite3_malloc(nLeft+nRight+1); |
| 114384 | | - if( aOut==0 ){ |
| 114385 | | - rc = SQLITE_NOMEM; |
| 114386 | | - }else{ |
| 114387 | | - rc = fts3DoclistMerge(mergetype, nNear+nTokenRight, nNear+nTokenLeft, |
| 114388 | | - aOut, pnOut, aLeft, nLeft, aRight, nRight, 0 |
| 114389 | | - ); |
| 114390 | | - if( rc!=SQLITE_OK ){ |
| 114391 | | - sqlite3_free(aOut); |
| 114392 | | - aOut = 0; |
| 114393 | | - } |
| 114394 | | - } |
| 114395 | | - |
| 114396 | | - *paOut = aOut; |
| 114397 | | - return rc; |
| 114398 | | -} |
| 114399 | | - |
| 114400 | | -/* |
| 114401 | | -** This function is used as part of the processing for the snippet() and |
| 114402 | | -** offsets() functions. |
| 114403 | | -** |
| 114404 | | -** Both pLeft and pRight are expression nodes of type FTSQUERY_PHRASE. Both |
| 114405 | | -** have their respective doclists (including position information) loaded |
| 114406 | | -** in Fts3Expr.aDoclist/nDoclist. This function removes all entries from |
| 114407 | | -** each doclist that are not within nNear tokens of a corresponding entry |
| 114408 | | -** in the other doclist. |
| 114409 | | -*/ |
| 114410 | | -SQLITE_PRIVATE int sqlite3Fts3ExprNearTrim(Fts3Expr *pLeft, Fts3Expr *pRight, int nNear){ |
| 114411 | | - int rc; /* Return code */ |
| 114412 | | - |
| 114413 | | - assert( pLeft->eType==FTSQUERY_PHRASE ); |
| 114414 | | - assert( pRight->eType==FTSQUERY_PHRASE ); |
| 114415 | | - assert( pLeft->isLoaded && pRight->isLoaded ); |
| 114416 | | - |
| 114417 | | - if( pLeft->aDoclist==0 || pRight->aDoclist==0 ){ |
| 114418 | | - sqlite3_free(pLeft->aDoclist); |
| 114419 | | - sqlite3_free(pRight->aDoclist); |
| 114420 | | - pRight->aDoclist = 0; |
| 114421 | | - pLeft->aDoclist = 0; |
| 114422 | | - rc = SQLITE_OK; |
| 114423 | | - }else{ |
| 114424 | | - char *aOut; /* Buffer in which to assemble new doclist */ |
| 114425 | | - int nOut; /* Size of buffer aOut in bytes */ |
| 114426 | | - |
| 114427 | | - rc = fts3NearMerge(MERGE_POS_NEAR, nNear, |
| 114428 | | - pLeft->pPhrase->nToken, pLeft->aDoclist, pLeft->nDoclist, |
| 114429 | | - pRight->pPhrase->nToken, pRight->aDoclist, pRight->nDoclist, |
| 114430 | | - &aOut, &nOut |
| 114431 | | - ); |
| 114432 | | - if( rc!=SQLITE_OK ) return rc; |
| 114433 | | - sqlite3_free(pRight->aDoclist); |
| 114434 | | - pRight->aDoclist = aOut; |
| 114435 | | - pRight->nDoclist = nOut; |
| 114436 | | - |
| 114437 | | - rc = fts3NearMerge(MERGE_POS_NEAR, nNear, |
| 114438 | | - pRight->pPhrase->nToken, pRight->aDoclist, pRight->nDoclist, |
| 114439 | | - pLeft->pPhrase->nToken, pLeft->aDoclist, pLeft->nDoclist, |
| 114440 | | - &aOut, &nOut |
| 114441 | | - ); |
| 114442 | | - sqlite3_free(pLeft->aDoclist); |
| 114443 | | - pLeft->aDoclist = aOut; |
| 114444 | | - pLeft->nDoclist = nOut; |
| 114445 | | - } |
| 114446 | | - return rc; |
| 114447 | | -} |
| 114448 | | - |
| 114449 | | - |
| 114450 | | -/* |
| 114451 | | -** Allocate an Fts3SegReaderArray for each token in the expression pExpr. |
| 114452 | | -** The allocated objects are stored in the Fts3PhraseToken.pArray member |
| 114453 | | -** variables of each token structure. |
| 114454 | | -*/ |
| 114455 | | -static int fts3ExprAllocateSegReaders( |
| 114456 | | - Fts3Cursor *pCsr, /* FTS3 table */ |
| 114457 | | - Fts3Expr *pExpr, /* Expression to create seg-readers for */ |
| 114458 | | - int *pnExpr /* OUT: Number of AND'd expressions */ |
| 114459 | | -){ |
| 114460 | | - int rc = SQLITE_OK; /* Return code */ |
| 114461 | | - |
| 114462 | | - assert( pCsr->eEvalmode==FTS3_EVAL_FILTER ); |
| 114463 | | - if( pnExpr && pExpr->eType!=FTSQUERY_AND ){ |
| 114464 | | - (*pnExpr)++; |
| 114465 | | - pnExpr = 0; |
| 114466 | | - } |
| 114467 | | - |
| 114468 | | - if( pExpr->eType==FTSQUERY_PHRASE ){ |
| 114469 | | - Fts3Phrase *pPhrase = pExpr->pPhrase; |
| 114470 | | - int ii; |
| 114471 | | - |
| 114472 | | - for(ii=0; rc==SQLITE_OK && ii<pPhrase->nToken; ii++){ |
| 114473 | | - Fts3PhraseToken *pTok = &pPhrase->aToken[ii]; |
| 114474 | | - if( pTok->pSegcsr==0 ){ |
| 114475 | | - rc = fts3TermSegReaderCursor( |
| 114476 | | - pCsr, pTok->z, pTok->n, pTok->isPrefix, &pTok->pSegcsr |
| 114477 | | - ); |
| 114478 | | - } |
| 114479 | | - } |
| 114480 | | - }else{ |
| 114481 | | - rc = fts3ExprAllocateSegReaders(pCsr, pExpr->pLeft, pnExpr); |
| 114482 | | - if( rc==SQLITE_OK ){ |
| 114483 | | - rc = fts3ExprAllocateSegReaders(pCsr, pExpr->pRight, pnExpr); |
| 114484 | | - } |
| 114485 | | - } |
| 114486 | | - return rc; |
| 114487 | | -} |
| 114488 | | - |
| 114489 | | -/* |
| 114490 | | -** Free the Fts3SegReaderArray objects associated with each token in the |
| 114491 | | -** expression pExpr. In other words, this function frees the resources |
| 114492 | | -** allocated by fts3ExprAllocateSegReaders(). |
| 114493 | | -*/ |
| 114494 | | -static void fts3ExprFreeSegReaders(Fts3Expr *pExpr){ |
| 114495 | | - if( pExpr ){ |
| 114496 | | - Fts3Phrase *pPhrase = pExpr->pPhrase; |
| 114497 | | - if( pPhrase ){ |
| 114498 | | - int kk; |
| 114499 | | - for(kk=0; kk<pPhrase->nToken; kk++){ |
| 114500 | | - fts3SegReaderCursorFree(pPhrase->aToken[kk].pSegcsr); |
| 114501 | | - pPhrase->aToken[kk].pSegcsr = 0; |
| 114502 | | - } |
| 114503 | | - } |
| 114504 | | - fts3ExprFreeSegReaders(pExpr->pLeft); |
| 114505 | | - fts3ExprFreeSegReaders(pExpr->pRight); |
| 114506 | | - } |
| 114507 | | -} |
| 114508 | | - |
| 114509 | | -/* |
| 114510 | | -** Return the sum of the costs of all tokens in the expression pExpr. This |
| 114511 | | -** function must be called after Fts3SegReaderArrays have been allocated |
| 114512 | | -** for all tokens using fts3ExprAllocateSegReaders(). |
| 114513 | | -*/ |
| 114514 | | -static int fts3ExprCost(Fts3Expr *pExpr){ |
| 114515 | | - int nCost; /* Return value */ |
| 114516 | | - if( pExpr->eType==FTSQUERY_PHRASE ){ |
| 114517 | | - Fts3Phrase *pPhrase = pExpr->pPhrase; |
| 114518 | | - int ii; |
| 114519 | | - nCost = 0; |
| 114520 | | - for(ii=0; ii<pPhrase->nToken; ii++){ |
| 114521 | | - Fts3SegReaderCursor *pSegcsr = pPhrase->aToken[ii].pSegcsr; |
| 114522 | | - if( pSegcsr ) nCost += pSegcsr->nCost; |
| 114523 | | - } |
| 114524 | | - }else{ |
| 114525 | | - nCost = fts3ExprCost(pExpr->pLeft) + fts3ExprCost(pExpr->pRight); |
| 114526 | | - } |
| 114527 | | - return nCost; |
| 114528 | | -} |
| 114529 | | - |
| 114530 | | -/* |
| 114531 | | -** The following is a helper function (and type) for fts3EvalExpr(). It |
| 114532 | | -** must be called after Fts3SegReaders have been allocated for every token |
| 114533 | | -** in the expression. See the context it is called from in fts3EvalExpr() |
| 114534 | | -** for further explanation. |
| 114535 | | -*/ |
| 114536 | | -typedef struct ExprAndCost ExprAndCost; |
| 114537 | | -struct ExprAndCost { |
| 114538 | | - Fts3Expr *pExpr; |
| 114539 | | - int nCost; |
| 114540 | | -}; |
| 114541 | | -static void fts3ExprAssignCosts( |
| 114542 | | - Fts3Expr *pExpr, /* Expression to create seg-readers for */ |
| 114543 | | - ExprAndCost **ppExprCost /* OUT: Write to *ppExprCost */ |
| 114544 | | -){ |
| 114545 | | - if( pExpr->eType==FTSQUERY_AND ){ |
| 114546 | | - fts3ExprAssignCosts(pExpr->pLeft, ppExprCost); |
| 114547 | | - fts3ExprAssignCosts(pExpr->pRight, ppExprCost); |
| 114548 | | - }else{ |
| 114549 | | - (*ppExprCost)->pExpr = pExpr; |
| 114550 | | - (*ppExprCost)->nCost = fts3ExprCost(pExpr); |
| 114551 | | - (*ppExprCost)++; |
| 114552 | | - } |
| 114553 | | -} |
| 114554 | | - |
| 114555 | | -/* |
| 114556 | | -** Evaluate the full-text expression pExpr against FTS3 table pTab. Store |
| 114557 | | -** the resulting doclist in *paOut and *pnOut. This routine mallocs for |
| 114558 | | -** the space needed to store the output. The caller is responsible for |
| 114559 | | -** freeing the space when it has finished. |
| 114560 | | -** |
| 114561 | | -** This function is called in two distinct contexts: |
| 114562 | | -** |
| 114563 | | -** * From within the virtual table xFilter() method. In this case, the |
| 114564 | | -** output doclist contains entries for all rows in the table, based on |
| 114565 | | -** data read from the full-text index. |
| 114566 | | -** |
| 114567 | | -** In this case, if the query expression contains one or more tokens that |
| 114568 | | -** are very common, then the returned doclist may contain a superset of |
| 114569 | | -** the documents that actually match the expression. |
| 114570 | | -** |
| 114571 | | -** * From within the virtual table xNext() method. This call is only made |
| 114572 | | -** if the call from within xFilter() found that there were very common |
| 114573 | | -** tokens in the query expression and did return a superset of the |
| 114574 | | -** matching documents. In this case the returned doclist contains only |
| 114575 | | -** entries that correspond to the current row of the table. Instead of |
| 114576 | | -** reading the data for each token from the full-text index, the data is |
| 114577 | | -** already available in-memory in the Fts3PhraseToken.pDeferred structures. |
| 114578 | | -** See fts3EvalDeferred() for how it gets there. |
| 114579 | | -** |
| 114580 | | -** In the first case above, Fts3Cursor.doDeferred==0. In the second (if it is |
| 114581 | | -** required) Fts3Cursor.doDeferred==1. |
| 114582 | | -** |
| 114583 | | -** If the SQLite invokes the snippet(), offsets() or matchinfo() function |
| 114584 | | -** as part of a SELECT on an FTS3 table, this function is called on each |
| 114585 | | -** individual phrase expression in the query. If there were very common tokens |
| 114586 | | -** found in the xFilter() call, then this function is called once for phrase |
| 114587 | | -** for each row visited, and the returned doclist contains entries for the |
| 114588 | | -** current row only. Otherwise, if there were no very common tokens, then this |
| 114589 | | -** function is called once only for each phrase in the query and the returned |
| 114590 | | -** doclist contains entries for all rows of the table. |
| 114591 | | -** |
| 114592 | | -** Fts3Cursor.doDeferred==1 when this function is called on phrases as a |
| 114593 | | -** result of a snippet(), offsets() or matchinfo() invocation. |
| 114594 | | -*/ |
| 114595 | | -static int fts3EvalExpr( |
| 114596 | | - Fts3Cursor *p, /* Virtual table cursor handle */ |
| 114597 | | - Fts3Expr *pExpr, /* Parsed fts3 expression */ |
| 114598 | | - char **paOut, /* OUT: Pointer to malloc'd result buffer */ |
| 114599 | | - int *pnOut, /* OUT: Size of buffer at *paOut */ |
| 114600 | | - int isReqPos /* Require positions in output buffer */ |
| 114601 | | -){ |
| 114602 | | - int rc = SQLITE_OK; /* Return code */ |
| 114603 | | - |
| 114604 | | - /* Zero the output parameters. */ |
| 114605 | | - *paOut = 0; |
| 114606 | | - *pnOut = 0; |
| 114607 | | - |
| 114608 | | - if( pExpr ){ |
| 114609 | | - assert( pExpr->eType==FTSQUERY_NEAR || pExpr->eType==FTSQUERY_OR |
| 114610 | | - || pExpr->eType==FTSQUERY_AND || pExpr->eType==FTSQUERY_NOT |
| 114611 | | - || pExpr->eType==FTSQUERY_PHRASE |
| 114612 | | - ); |
| 114613 | | - assert( pExpr->eType==FTSQUERY_PHRASE || isReqPos==0 ); |
| 114614 | | - |
| 114615 | | - if( pExpr->eType==FTSQUERY_PHRASE ){ |
| 114616 | | - rc = fts3PhraseSelect(p, pExpr->pPhrase, |
| 114617 | | - isReqPos || (pExpr->pParent && pExpr->pParent->eType==FTSQUERY_NEAR), |
| 114618 | | - paOut, pnOut |
| 114619 | | - ); |
| 114620 | | - fts3ExprFreeSegReaders(pExpr); |
| 114621 | | - }else if( p->eEvalmode==FTS3_EVAL_FILTER && pExpr->eType==FTSQUERY_AND ){ |
| 114622 | | - ExprAndCost *aExpr = 0; /* Array of AND'd expressions and costs */ |
| 114623 | | - int nExpr = 0; /* Size of aExpr[] */ |
| 114624 | | - char *aRet = 0; /* Doclist to return to caller */ |
| 114625 | | - int nRet = 0; /* Length of aRet[] in bytes */ |
| 114626 | | - int nDoc = 0x7FFFFFFF; |
| 114627 | | - |
| 114628 | | - assert( !isReqPos ); |
| 114629 | | - |
| 114630 | | - rc = fts3ExprAllocateSegReaders(p, pExpr, &nExpr); |
| 114631 | | - if( rc==SQLITE_OK ){ |
| 114632 | | - assert( nExpr>1 ); |
| 114633 | | - aExpr = sqlite3_malloc(sizeof(ExprAndCost) * nExpr); |
| 114634 | | - if( !aExpr ) rc = SQLITE_NOMEM; |
| 114635 | | - } |
| 114636 | | - if( rc==SQLITE_OK ){ |
| 114637 | | - int ii; /* Used to iterate through expressions */ |
| 114638 | | - |
| 114639 | | - fts3ExprAssignCosts(pExpr, &aExpr); |
| 114640 | | - aExpr -= nExpr; |
| 114641 | | - for(ii=0; ii<nExpr; ii++){ |
| 114642 | | - char *aNew; |
| 114643 | | - int nNew; |
| 114644 | | - int jj; |
| 114645 | | - ExprAndCost *pBest = 0; |
| 114646 | | - |
| 114647 | | - for(jj=0; jj<nExpr; jj++){ |
| 114648 | | - ExprAndCost *pCand = &aExpr[jj]; |
| 114649 | | - if( pCand->pExpr && (pBest==0 || pCand->nCost<pBest->nCost) ){ |
| 114650 | | - pBest = pCand; |
| 114651 | | - } |
| 114652 | | - } |
| 114653 | | - |
| 114654 | | - if( pBest->nCost>nDoc ){ |
| 114655 | | - rc = fts3DeferExpression(p, p->pExpr); |
| 114656 | | - break; |
| 114657 | | - }else{ |
| 114658 | | - rc = fts3EvalExpr(p, pBest->pExpr, &aNew, &nNew, 0); |
| 114659 | | - if( rc!=SQLITE_OK ) break; |
| 114660 | | - pBest->pExpr = 0; |
| 114661 | | - if( ii==0 ){ |
| 114662 | | - aRet = aNew; |
| 114663 | | - nRet = nNew; |
| 114664 | | - nDoc = fts3DoclistCountDocids(0, aRet, nRet); |
| 114665 | | - }else{ |
| 114666 | | - fts3DoclistMerge( |
| 114667 | | - MERGE_AND, 0, 0, aRet, &nRet, aRet, nRet, aNew, nNew, &nDoc |
| 114668 | | - ); |
| 114669 | | - sqlite3_free(aNew); |
| 114670 | | - } |
| 114671 | | - } |
| 114672 | | - } |
| 114673 | | - } |
| 114674 | | - |
| 114675 | | - if( rc==SQLITE_OK ){ |
| 114676 | | - *paOut = aRet; |
| 114677 | | - *pnOut = nRet; |
| 114678 | | - }else{ |
| 114679 | | - assert( *paOut==0 ); |
| 114680 | | - sqlite3_free(aRet); |
| 114681 | | - } |
| 114682 | | - sqlite3_free(aExpr); |
| 114683 | | - fts3ExprFreeSegReaders(pExpr); |
| 114684 | | - |
| 114685 | | - }else{ |
| 114686 | | - char *aLeft; |
| 114687 | | - char *aRight; |
| 114688 | | - int nLeft; |
| 114689 | | - int nRight; |
| 114690 | | - |
| 114691 | | - assert( pExpr->eType==FTSQUERY_NEAR |
| 114692 | | - || pExpr->eType==FTSQUERY_OR |
| 114693 | | - || pExpr->eType==FTSQUERY_NOT |
| 114694 | | - || (pExpr->eType==FTSQUERY_AND && p->eEvalmode==FTS3_EVAL_NEXT) |
| 114695 | | - ); |
| 114696 | | - |
| 114697 | | - if( 0==(rc = fts3EvalExpr(p, pExpr->pRight, &aRight, &nRight, isReqPos)) |
| 114698 | | - && 0==(rc = fts3EvalExpr(p, pExpr->pLeft, &aLeft, &nLeft, isReqPos)) |
| 114699 | | - ){ |
| 114700 | | - switch( pExpr->eType ){ |
| 114701 | | - case FTSQUERY_NEAR: { |
| 114702 | | - Fts3Expr *pLeft; |
| 114703 | | - Fts3Expr *pRight; |
| 114704 | | - int mergetype = MERGE_NEAR; |
| 114705 | | - if( pExpr->pParent && pExpr->pParent->eType==FTSQUERY_NEAR ){ |
| 114706 | | - mergetype = MERGE_POS_NEAR; |
| 114707 | | - } |
| 114708 | | - pLeft = pExpr->pLeft; |
| 114709 | | - while( pLeft->eType==FTSQUERY_NEAR ){ |
| 114710 | | - pLeft=pLeft->pRight; |
| 114711 | | - } |
| 114712 | | - pRight = pExpr->pRight; |
| 114713 | | - assert( pRight->eType==FTSQUERY_PHRASE ); |
| 114714 | | - assert( pLeft->eType==FTSQUERY_PHRASE ); |
| 114715 | | - |
| 114716 | | - rc = fts3NearMerge(mergetype, pExpr->nNear, |
| 114717 | | - pLeft->pPhrase->nToken, aLeft, nLeft, |
| 114718 | | - pRight->pPhrase->nToken, aRight, nRight, |
| 114719 | | - paOut, pnOut |
| 114720 | | - ); |
| 114721 | | - sqlite3_free(aLeft); |
| 114722 | | - break; |
| 114723 | | - } |
| 114724 | | - |
| 114725 | | - case FTSQUERY_OR: { |
| 114726 | | - /* Allocate a buffer for the output. The maximum size is the |
| 114727 | | - ** sum of the sizes of the two input buffers. The +1 term is |
| 114728 | | - ** so that a buffer of zero bytes is never allocated - this can |
| 114729 | | - ** cause fts3DoclistMerge() to incorrectly return SQLITE_NOMEM. |
| 114730 | | - */ |
| 114731 | | - char *aBuffer = sqlite3_malloc(nRight+nLeft+1); |
| 114732 | | - rc = fts3DoclistMerge(MERGE_OR, 0, 0, aBuffer, pnOut, |
| 114733 | | - aLeft, nLeft, aRight, nRight, 0 |
| 114734 | | - ); |
| 114735 | | - *paOut = aBuffer; |
| 114736 | | - sqlite3_free(aLeft); |
| 114737 | | - break; |
| 114738 | | - } |
| 114739 | | - |
| 114740 | | - default: { |
| 114741 | | - assert( FTSQUERY_NOT==MERGE_NOT && FTSQUERY_AND==MERGE_AND ); |
| 114742 | | - fts3DoclistMerge(pExpr->eType, 0, 0, aLeft, pnOut, |
| 114743 | | - aLeft, nLeft, aRight, nRight, 0 |
| 114744 | | - ); |
| 114745 | | - *paOut = aLeft; |
| 114746 | | - break; |
| 114747 | | - } |
| 114748 | | - } |
| 114749 | | - } |
| 114750 | | - sqlite3_free(aRight); |
| 114751 | | - } |
| 114752 | | - } |
| 114753 | | - |
| 114754 | | - assert( rc==SQLITE_OK || *paOut==0 ); |
| 114755 | | - return rc; |
| 114756 | | -} |
| 114757 | | - |
| 114758 | | -/* |
| 114759 | | -** This function is called from within xNext() for each row visited by |
| 114760 | | -** an FTS3 query. If evaluating the FTS3 query expression within xFilter() |
| 114761 | | -** was able to determine the exact set of matching rows, this function sets |
| 114762 | | -** *pbRes to true and returns SQLITE_IO immediately. |
| 114763 | | -** |
| 114764 | | -** Otherwise, if evaluating the query expression within xFilter() returned a |
| 114765 | | -** superset of the matching documents instead of an exact set (this happens |
| 114766 | | -** when the query includes very common tokens and it is deemed too expensive to |
| 114767 | | -** load their doclists from disk), this function tests if the current row |
| 114768 | | -** really does match the FTS3 query. |
| 114769 | | -** |
| 114770 | | -** If an error occurs, an SQLite error code is returned. Otherwise, SQLITE_OK |
| 114771 | | -** is returned and *pbRes is set to true if the current row matches the |
| 114772 | | -** FTS3 query (and should be included in the results returned to SQLite), or |
| 114773 | | -** false otherwise. |
| 114774 | | -*/ |
| 114775 | | -static int fts3EvalDeferred( |
| 114776 | | - Fts3Cursor *pCsr, /* FTS3 cursor pointing at row to test */ |
| 114777 | | - int *pbRes /* OUT: Set to true if row is a match */ |
| 114778 | | -){ |
| 114779 | | - int rc = SQLITE_OK; |
| 114780 | | - if( pCsr->pDeferred==0 ){ |
| 114781 | | - *pbRes = 1; |
| 114782 | | - }else{ |
| 114783 | | - rc = fts3CursorSeek(0, pCsr); |
| 114784 | | - if( rc==SQLITE_OK ){ |
| 114785 | | - sqlite3Fts3FreeDeferredDoclists(pCsr); |
| 114786 | | - rc = sqlite3Fts3CacheDeferredDoclists(pCsr); |
| 114787 | | - } |
| 114788 | | - if( rc==SQLITE_OK ){ |
| 114789 | | - char *a = 0; |
| 114790 | | - int n = 0; |
| 114791 | | - rc = fts3EvalExpr(pCsr, pCsr->pExpr, &a, &n, 0); |
| 114792 | | - assert( n>=0 ); |
| 114793 | | - *pbRes = (n>0); |
| 114794 | | - sqlite3_free(a); |
| 114795 | | - } |
| 114796 | | - } |
| 114797 | | - return rc; |
| 114798 | | -} |
| 114799 | | - |
| 114800 | 114367 | /* |
| 114801 | 114368 | ** Advance the cursor to the next row in the %_content table that |
| 114802 | 114369 | ** matches the search criteria. For a MATCH search, this will be |
| 114803 | 114370 | ** the next row that matches. For a full-table scan, this will be |
| 114804 | 114371 | ** simply the next row in the %_content table. For a docid lookup, |
| | @@ -114807,43 +114374,24 @@ |
| 114807 | 114374 | ** Return SQLITE_OK if nothing goes wrong. SQLITE_OK is returned |
| 114808 | 114375 | ** even if we reach end-of-file. The fts3EofMethod() will be called |
| 114809 | 114376 | ** subsequently to determine whether or not an EOF was hit. |
| 114810 | 114377 | */ |
| 114811 | 114378 | static int fts3NextMethod(sqlite3_vtab_cursor *pCursor){ |
| 114812 | | - int res; |
| 114813 | | - int rc = SQLITE_OK; /* Return code */ |
| 114379 | + int rc; |
| 114814 | 114380 | Fts3Cursor *pCsr = (Fts3Cursor *)pCursor; |
| 114815 | | - |
| 114816 | | - pCsr->eEvalmode = FTS3_EVAL_NEXT; |
| 114817 | | - do { |
| 114818 | | - if( pCsr->aDoclist==0 ){ |
| 114819 | | - if( SQLITE_ROW!=sqlite3_step(pCsr->pStmt) ){ |
| 114820 | | - pCsr->isEof = 1; |
| 114821 | | - rc = sqlite3_reset(pCsr->pStmt); |
| 114822 | | - break; |
| 114823 | | - } |
| 114381 | + if( pCsr->eSearch==FTS3_DOCID_SEARCH || pCsr->eSearch==FTS3_FULLSCAN_SEARCH ){ |
| 114382 | + if( SQLITE_ROW!=sqlite3_step(pCsr->pStmt) ){ |
| 114383 | + pCsr->isEof = 1; |
| 114384 | + rc = sqlite3_reset(pCsr->pStmt); |
| 114385 | + }else{ |
| 114824 | 114386 | pCsr->iPrevId = sqlite3_column_int64(pCsr->pStmt, 0); |
| 114825 | | - }else{ |
| 114826 | | - if( pCsr->desc==0 ){ |
| 114827 | | - if( pCsr->pNextId>=&pCsr->aDoclist[pCsr->nDoclist] ){ |
| 114828 | | - pCsr->isEof = 1; |
| 114829 | | - break; |
| 114830 | | - } |
| 114831 | | - fts3GetDeltaVarint(&pCsr->pNextId, &pCsr->iPrevId); |
| 114832 | | - }else{ |
| 114833 | | - fts3GetReverseDeltaVarint(&pCsr->pNextId,pCsr->aDoclist,&pCsr->iPrevId); |
| 114834 | | - if( pCsr->pNextId<=pCsr->aDoclist ){ |
| 114835 | | - pCsr->isEof = 1; |
| 114836 | | - break; |
| 114837 | | - } |
| 114838 | | - } |
| 114839 | | - sqlite3_reset(pCsr->pStmt); |
| 114840 | | - pCsr->isRequireSeek = 1; |
| 114841 | | - pCsr->isMatchinfoNeeded = 1; |
| 114842 | | - } |
| 114843 | | - }while( SQLITE_OK==(rc = fts3EvalDeferred(pCsr, &res)) && res==0 ); |
| 114844 | | - |
| 114387 | + rc = SQLITE_OK; |
| 114388 | + } |
| 114389 | + }else{ |
| 114390 | + rc = sqlite3Fts3EvalNext((Fts3Cursor *)pCursor); |
| 114391 | + } |
| 114392 | + assert( ((Fts3Table *)pCsr->base.pVtab)->pSegments==0 ); |
| 114845 | 114393 | return rc; |
| 114846 | 114394 | } |
| 114847 | 114395 | |
| 114848 | 114396 | /* |
| 114849 | 114397 | ** This is the xFilter interface for the virtual table. See |
| | @@ -114866,15 +114414,11 @@ |
| 114866 | 114414 | int idxNum, /* Strategy index */ |
| 114867 | 114415 | const char *idxStr, /* Unused */ |
| 114868 | 114416 | int nVal, /* Number of elements in apVal */ |
| 114869 | 114417 | sqlite3_value **apVal /* Arguments for the indexing scheme */ |
| 114870 | 114418 | ){ |
| 114871 | | - const char *azSql[] = { |
| 114872 | | - "SELECT %s FROM %Q.'%q_content' AS x WHERE docid = ?", /* non-full-scan */ |
| 114873 | | - "SELECT %s FROM %Q.'%q_content' AS x ORDER BY docid %s", /* full-scan */ |
| 114874 | | - }; |
| 114875 | | - int rc; /* Return code */ |
| 114419 | + int rc; |
| 114876 | 114420 | char *zSql; /* SQL statement used to access %_content */ |
| 114877 | 114421 | Fts3Table *p = (Fts3Table *)pCursor->pVtab; |
| 114878 | 114422 | Fts3Cursor *pCsr = (Fts3Cursor *)pCursor; |
| 114879 | 114423 | |
| 114880 | 114424 | UNUSED_PARAMETER(idxStr); |
| | @@ -114889,10 +114433,17 @@ |
| 114889 | 114433 | sqlite3_finalize(pCsr->pStmt); |
| 114890 | 114434 | sqlite3_free(pCsr->aDoclist); |
| 114891 | 114435 | sqlite3Fts3ExprFree(pCsr->pExpr); |
| 114892 | 114436 | memset(&pCursor[1], 0, sizeof(Fts3Cursor)-sizeof(sqlite3_vtab_cursor)); |
| 114893 | 114437 | |
| 114438 | + if( idxStr ){ |
| 114439 | + pCsr->bDesc = (idxStr[0]=='D'); |
| 114440 | + }else{ |
| 114441 | + pCsr->bDesc = p->bDescIdx; |
| 114442 | + } |
| 114443 | + pCsr->eSearch = (i16)idxNum; |
| 114444 | + |
| 114894 | 114445 | if( idxNum!=FTS3_DOCID_SEARCH && idxNum!=FTS3_FULLSCAN_SEARCH ){ |
| 114895 | 114446 | int iCol = idxNum-FTS3_FULLTEXT_SEARCH; |
| 114896 | 114447 | const char *zQuery = (const char *)sqlite3_value_text(apVal[0]); |
| 114897 | 114448 | |
| 114898 | 114449 | if( zQuery==0 && sqlite3_value_type(apVal[0])!=SQLITE_NULL ){ |
| | @@ -114902,20 +114453,21 @@ |
| 114902 | 114453 | rc = sqlite3Fts3ExprParse(p->pTokenizer, p->azColumn, p->nColumn, |
| 114903 | 114454 | iCol, zQuery, -1, &pCsr->pExpr |
| 114904 | 114455 | ); |
| 114905 | 114456 | if( rc!=SQLITE_OK ){ |
| 114906 | 114457 | if( rc==SQLITE_ERROR ){ |
| 114907 | | - p->base.zErrMsg = sqlite3_mprintf("malformed MATCH expression: [%s]", |
| 114908 | | - zQuery); |
| 114458 | + static const char *zErr = "malformed MATCH expression: [%s]"; |
| 114459 | + p->base.zErrMsg = sqlite3_mprintf(zErr, zQuery); |
| 114909 | 114460 | } |
| 114910 | 114461 | return rc; |
| 114911 | 114462 | } |
| 114912 | 114463 | |
| 114913 | 114464 | rc = sqlite3Fts3ReadLock(p); |
| 114914 | 114465 | if( rc!=SQLITE_OK ) return rc; |
| 114915 | 114466 | |
| 114916 | | - rc = fts3EvalExpr(pCsr, pCsr->pExpr, &pCsr->aDoclist, &pCsr->nDoclist, 0); |
| 114467 | + rc = sqlite3Fts3EvalStart(pCsr, pCsr->pExpr, 1); |
| 114468 | + |
| 114917 | 114469 | sqlite3Fts3SegmentsClose(p); |
| 114918 | 114470 | if( rc!=SQLITE_OK ) return rc; |
| 114919 | 114471 | pCsr->pNextId = pCsr->aDoclist; |
| 114920 | 114472 | pCsr->iPrevId = 0; |
| 114921 | 114473 | } |
| | @@ -114923,41 +114475,28 @@ |
| 114923 | 114475 | /* Compile a SELECT statement for this cursor. For a full-table-scan, the |
| 114924 | 114476 | ** statement loops through all rows of the %_content table. For a |
| 114925 | 114477 | ** full-text query or docid lookup, the statement retrieves a single |
| 114926 | 114478 | ** row by docid. |
| 114927 | 114479 | */ |
| 114928 | | - zSql = (char *)azSql[idxNum==FTS3_FULLSCAN_SEARCH]; |
| 114929 | | - zSql = sqlite3_mprintf( |
| 114930 | | - zSql, p->zReadExprlist, p->zDb, p->zName, (idxStr ? idxStr : "ASC") |
| 114931 | | - ); |
| 114932 | | - if( !zSql ){ |
| 114933 | | - rc = SQLITE_NOMEM; |
| 114480 | + if( idxNum==FTS3_FULLSCAN_SEARCH ){ |
| 114481 | + const char *zSort = (pCsr->bDesc ? "DESC" : "ASC"); |
| 114482 | + const char *zTmpl = "SELECT %s FROM %Q.'%q_content' AS x ORDER BY docid %s"; |
| 114483 | + zSql = sqlite3_mprintf(zTmpl, p->zReadExprlist, p->zDb, p->zName, zSort); |
| 114934 | 114484 | }else{ |
| 114935 | | - rc = sqlite3_prepare_v2(p->db, zSql, -1, &pCsr->pStmt, 0); |
| 114936 | | - sqlite3_free(zSql); |
| 114485 | + const char *zTmpl = "SELECT %s FROM %Q.'%q_content' AS x WHERE docid = ?"; |
| 114486 | + zSql = sqlite3_mprintf(zTmpl, p->zReadExprlist, p->zDb, p->zName); |
| 114937 | 114487 | } |
| 114938 | | - if( rc==SQLITE_OK && idxNum==FTS3_DOCID_SEARCH ){ |
| 114488 | + if( !zSql ) return SQLITE_NOMEM; |
| 114489 | + rc = sqlite3_prepare_v2(p->db, zSql, -1, &pCsr->pStmt, 0); |
| 114490 | + sqlite3_free(zSql); |
| 114491 | + if( rc!=SQLITE_OK ) return rc; |
| 114492 | + |
| 114493 | + if( idxNum==FTS3_DOCID_SEARCH ){ |
| 114939 | 114494 | rc = sqlite3_bind_value(pCsr->pStmt, 1, apVal[0]); |
| 114940 | | - } |
| 114941 | | - pCsr->eSearch = (i16)idxNum; |
| 114942 | | - |
| 114943 | | - assert( pCsr->desc==0 ); |
| 114944 | | - if( rc!=SQLITE_OK ) return rc; |
| 114945 | | - if( rc==SQLITE_OK && pCsr->nDoclist>0 && idxStr && idxStr[0]=='D' ){ |
| 114946 | | - sqlite3_int64 iDocid = 0; |
| 114947 | | - char *csr = pCsr->aDoclist; |
| 114948 | | - while( csr<&pCsr->aDoclist[pCsr->nDoclist] ){ |
| 114949 | | - fts3GetDeltaVarint(&csr, &iDocid); |
| 114950 | | - } |
| 114951 | | - pCsr->pNextId = csr; |
| 114952 | | - pCsr->iPrevId = iDocid; |
| 114953 | | - pCsr->desc = 1; |
| 114954 | | - pCsr->isRequireSeek = 1; |
| 114955 | | - pCsr->isMatchinfoNeeded = 1; |
| 114956 | | - pCsr->eEvalmode = FTS3_EVAL_NEXT; |
| 114957 | | - return SQLITE_OK; |
| 114958 | | - } |
| 114495 | + if( rc!=SQLITE_OK ) return rc; |
| 114496 | + } |
| 114497 | + |
| 114959 | 114498 | return fts3NextMethod(pCursor); |
| 114960 | 114499 | } |
| 114961 | 114500 | |
| 114962 | 114501 | /* |
| 114963 | 114502 | ** This is the xEof method of the virtual table. SQLite calls this |
| | @@ -114973,20 +114512,11 @@ |
| 114973 | 114512 | ** exposes %_content.docid as the rowid for the virtual table. The |
| 114974 | 114513 | ** rowid should be written to *pRowid. |
| 114975 | 114514 | */ |
| 114976 | 114515 | static int fts3RowidMethod(sqlite3_vtab_cursor *pCursor, sqlite_int64 *pRowid){ |
| 114977 | 114516 | Fts3Cursor *pCsr = (Fts3Cursor *) pCursor; |
| 114978 | | - if( pCsr->aDoclist ){ |
| 114979 | | - *pRowid = pCsr->iPrevId; |
| 114980 | | - }else{ |
| 114981 | | - /* This branch runs if the query is implemented using a full-table scan |
| 114982 | | - ** (not using the full-text index). In this case grab the rowid from the |
| 114983 | | - ** SELECT statement. |
| 114984 | | - */ |
| 114985 | | - assert( pCsr->isRequireSeek==0 ); |
| 114986 | | - *pRowid = sqlite3_column_int64(pCsr->pStmt, 0); |
| 114987 | | - } |
| 114517 | + *pRowid = pCsr->iPrevId; |
| 114988 | 114518 | return SQLITE_OK; |
| 114989 | 114519 | } |
| 114990 | 114520 | |
| 114991 | 114521 | /* |
| 114992 | 114522 | ** This is the xColumn method, called by SQLite to request a value from |
| | @@ -114995,11 +114525,11 @@ |
| 114995 | 114525 | static int fts3ColumnMethod( |
| 114996 | 114526 | sqlite3_vtab_cursor *pCursor, /* Cursor to retrieve value from */ |
| 114997 | 114527 | sqlite3_context *pContext, /* Context for sqlite3_result_xxx() calls */ |
| 114998 | 114528 | int iCol /* Index of column to read value from */ |
| 114999 | 114529 | ){ |
| 115000 | | - int rc; /* Return Code */ |
| 114530 | + int rc = SQLITE_OK; /* Return Code */ |
| 115001 | 114531 | Fts3Cursor *pCsr = (Fts3Cursor *) pCursor; |
| 115002 | 114532 | Fts3Table *p = (Fts3Table *)pCursor->pVtab; |
| 115003 | 114533 | |
| 115004 | 114534 | /* The column value supplied by SQLite must be in range. */ |
| 115005 | 114535 | assert( iCol>=0 && iCol<=p->nColumn+1 ); |
| | @@ -115006,25 +114536,24 @@ |
| 115006 | 114536 | |
| 115007 | 114537 | if( iCol==p->nColumn+1 ){ |
| 115008 | 114538 | /* This call is a request for the "docid" column. Since "docid" is an |
| 115009 | 114539 | ** alias for "rowid", use the xRowid() method to obtain the value. |
| 115010 | 114540 | */ |
| 115011 | | - sqlite3_int64 iRowid; |
| 115012 | | - rc = fts3RowidMethod(pCursor, &iRowid); |
| 115013 | | - sqlite3_result_int64(pContext, iRowid); |
| 114541 | + sqlite3_result_int64(pContext, pCsr->iPrevId); |
| 115014 | 114542 | }else if( iCol==p->nColumn ){ |
| 115015 | 114543 | /* The extra column whose name is the same as the table. |
| 115016 | 114544 | ** Return a blob which is a pointer to the cursor. |
| 115017 | 114545 | */ |
| 115018 | 114546 | sqlite3_result_blob(pContext, &pCsr, sizeof(pCsr), SQLITE_TRANSIENT); |
| 115019 | | - rc = SQLITE_OK; |
| 115020 | 114547 | }else{ |
| 115021 | 114548 | rc = fts3CursorSeek(0, pCsr); |
| 115022 | 114549 | if( rc==SQLITE_OK ){ |
| 115023 | 114550 | sqlite3_result_value(pContext, sqlite3_column_value(pCsr->pStmt, iCol+1)); |
| 115024 | 114551 | } |
| 115025 | 114552 | } |
| 114553 | + |
| 114554 | + assert( ((Fts3Table *)pCsr->base.pVtab)->pSegments==0 ); |
| 115026 | 114555 | return rc; |
| 115027 | 114556 | } |
| 115028 | 114557 | |
| 115029 | 114558 | /* |
| 115030 | 114559 | ** This function is the implementation of the xUpdate callback used by |
| | @@ -115054,10 +114583,11 @@ |
| 115054 | 114583 | ** Implementation of xBegin() method. This is a no-op. |
| 115055 | 114584 | */ |
| 115056 | 114585 | static int fts3BeginMethod(sqlite3_vtab *pVtab){ |
| 115057 | 114586 | UNUSED_PARAMETER(pVtab); |
| 115058 | 114587 | TESTONLY( Fts3Table *p = (Fts3Table*)pVtab ); |
| 114588 | + assert( p->pSegments==0 ); |
| 115059 | 114589 | assert( p->nPendingData==0 ); |
| 115060 | 114590 | assert( p->inTransaction!=1 ); |
| 115061 | 114591 | TESTONLY( p->inTransaction = 1 ); |
| 115062 | 114592 | TESTONLY( p->mxSavepoint = -1; ); |
| 115063 | 114593 | return SQLITE_OK; |
| | @@ -115071,10 +114601,11 @@ |
| 115071 | 114601 | static int fts3CommitMethod(sqlite3_vtab *pVtab){ |
| 115072 | 114602 | UNUSED_PARAMETER(pVtab); |
| 115073 | 114603 | TESTONLY( Fts3Table *p = (Fts3Table*)pVtab ); |
| 115074 | 114604 | assert( p->nPendingData==0 ); |
| 115075 | 114605 | assert( p->inTransaction!=0 ); |
| 114606 | + assert( p->pSegments==0 ); |
| 115076 | 114607 | TESTONLY( p->inTransaction = 0 ); |
| 115077 | 114608 | TESTONLY( p->mxSavepoint = -1; ); |
| 115078 | 114609 | return SQLITE_OK; |
| 115079 | 114610 | } |
| 115080 | 114611 | |
| | @@ -115088,132 +114619,30 @@ |
| 115088 | 114619 | assert( p->inTransaction!=0 ); |
| 115089 | 114620 | TESTONLY( p->inTransaction = 0 ); |
| 115090 | 114621 | TESTONLY( p->mxSavepoint = -1; ); |
| 115091 | 114622 | return SQLITE_OK; |
| 115092 | 114623 | } |
| 115093 | | - |
| 115094 | | -/* |
| 115095 | | -** Load the doclist associated with expression pExpr to pExpr->aDoclist. |
| 115096 | | -** The loaded doclist contains positions as well as the document ids. |
| 115097 | | -** This is used by the matchinfo(), snippet() and offsets() auxillary |
| 115098 | | -** functions. |
| 115099 | | -*/ |
| 115100 | | -SQLITE_PRIVATE int sqlite3Fts3ExprLoadDoclist(Fts3Cursor *pCsr, Fts3Expr *pExpr){ |
| 115101 | | - int rc; |
| 115102 | | - assert( pExpr->eType==FTSQUERY_PHRASE && pExpr->pPhrase ); |
| 115103 | | - assert( pCsr->eEvalmode==FTS3_EVAL_NEXT ); |
| 115104 | | - rc = fts3EvalExpr(pCsr, pExpr, &pExpr->aDoclist, &pExpr->nDoclist, 1); |
| 115105 | | - return rc; |
| 115106 | | -} |
| 115107 | | - |
| 115108 | | -SQLITE_PRIVATE int sqlite3Fts3ExprLoadFtDoclist( |
| 115109 | | - Fts3Cursor *pCsr, |
| 115110 | | - Fts3Expr *pExpr, |
| 115111 | | - char **paDoclist, |
| 115112 | | - int *pnDoclist |
| 115113 | | -){ |
| 115114 | | - int rc; |
| 115115 | | - assert( pCsr->eEvalmode==FTS3_EVAL_NEXT ); |
| 115116 | | - assert( pExpr->eType==FTSQUERY_PHRASE && pExpr->pPhrase ); |
| 115117 | | - pCsr->eEvalmode = FTS3_EVAL_MATCHINFO; |
| 115118 | | - rc = fts3EvalExpr(pCsr, pExpr, paDoclist, pnDoclist, 1); |
| 115119 | | - pCsr->eEvalmode = FTS3_EVAL_NEXT; |
| 115120 | | - return rc; |
| 115121 | | -} |
| 115122 | | - |
| 115123 | 114624 | |
| 115124 | 114625 | /* |
| 115125 | 114626 | ** When called, *ppPoslist must point to the byte immediately following the |
| 115126 | 114627 | ** end of a position-list. i.e. ( (*ppPoslist)[-1]==POS_END ). This function |
| 115127 | 114628 | ** moves *ppPoslist so that it instead points to the first byte of the |
| 115128 | 114629 | ** same position list. |
| 115129 | 114630 | */ |
| 115130 | 114631 | static void fts3ReversePoslist(char *pStart, char **ppPoslist){ |
| 115131 | | - char *p = &(*ppPoslist)[-3]; |
| 115132 | | - char c = p[1]; |
| 114632 | + char *p = &(*ppPoslist)[-2]; |
| 114633 | + char c; |
| 114634 | + |
| 114635 | + while( p>pStart && (c=*p--)==0 ); |
| 115133 | 114636 | while( p>pStart && (*p & 0x80) | c ){ |
| 115134 | 114637 | c = *p--; |
| 115135 | 114638 | } |
| 115136 | 114639 | if( p>pStart ){ p = &p[2]; } |
| 115137 | 114640 | while( *p++&0x80 ); |
| 115138 | 114641 | *ppPoslist = p; |
| 115139 | 114642 | } |
| 115140 | 114643 | |
| 115141 | | - |
| 115142 | | -/* |
| 115143 | | -** After ExprLoadDoclist() (see above) has been called, this function is |
| 115144 | | -** used to iterate/search through the position lists that make up the doclist |
| 115145 | | -** stored in pExpr->aDoclist. |
| 115146 | | -*/ |
| 115147 | | -SQLITE_PRIVATE char *sqlite3Fts3FindPositions( |
| 115148 | | - Fts3Cursor *pCursor, /* Associate FTS3 cursor */ |
| 115149 | | - Fts3Expr *pExpr, /* Access this expressions doclist */ |
| 115150 | | - sqlite3_int64 iDocid, /* Docid associated with requested pos-list */ |
| 115151 | | - int iCol /* Column of requested pos-list */ |
| 115152 | | -){ |
| 115153 | | - assert( pExpr->isLoaded ); |
| 115154 | | - if( pExpr->aDoclist ){ |
| 115155 | | - char *pEnd = &pExpr->aDoclist[pExpr->nDoclist]; |
| 115156 | | - char *pCsr; |
| 115157 | | - |
| 115158 | | - if( pExpr->pCurrent==0 ){ |
| 115159 | | - if( pCursor->desc==0 ){ |
| 115160 | | - pExpr->pCurrent = pExpr->aDoclist; |
| 115161 | | - pExpr->iCurrent = 0; |
| 115162 | | - fts3GetDeltaVarint(&pExpr->pCurrent, &pExpr->iCurrent); |
| 115163 | | - }else{ |
| 115164 | | - pCsr = pExpr->aDoclist; |
| 115165 | | - while( pCsr<pEnd ){ |
| 115166 | | - fts3GetDeltaVarint(&pCsr, &pExpr->iCurrent); |
| 115167 | | - fts3PoslistCopy(0, &pCsr); |
| 115168 | | - } |
| 115169 | | - fts3ReversePoslist(pExpr->aDoclist, &pCsr); |
| 115170 | | - pExpr->pCurrent = pCsr; |
| 115171 | | - } |
| 115172 | | - } |
| 115173 | | - pCsr = pExpr->pCurrent; |
| 115174 | | - assert( pCsr ); |
| 115175 | | - |
| 115176 | | - while( (pCursor->desc==0 && pCsr<pEnd) |
| 115177 | | - || (pCursor->desc && pCsr>pExpr->aDoclist) |
| 115178 | | - ){ |
| 115179 | | - if( pCursor->desc==0 && pExpr->iCurrent<iDocid ){ |
| 115180 | | - fts3PoslistCopy(0, &pCsr); |
| 115181 | | - if( pCsr<pEnd ){ |
| 115182 | | - fts3GetDeltaVarint(&pCsr, &pExpr->iCurrent); |
| 115183 | | - } |
| 115184 | | - pExpr->pCurrent = pCsr; |
| 115185 | | - }else if( pCursor->desc && pExpr->iCurrent>iDocid ){ |
| 115186 | | - fts3GetReverseDeltaVarint(&pCsr, pExpr->aDoclist, &pExpr->iCurrent); |
| 115187 | | - fts3ReversePoslist(pExpr->aDoclist, &pCsr); |
| 115188 | | - pExpr->pCurrent = pCsr; |
| 115189 | | - }else{ |
| 115190 | | - if( pExpr->iCurrent==iDocid ){ |
| 115191 | | - int iThis = 0; |
| 115192 | | - if( iCol<0 ){ |
| 115193 | | - /* If iCol is negative, return a pointer to the start of the |
| 115194 | | - ** position-list (instead of a pointer to the start of a list |
| 115195 | | - ** of offsets associated with a specific column). |
| 115196 | | - */ |
| 115197 | | - return pCsr; |
| 115198 | | - } |
| 115199 | | - while( iThis<iCol ){ |
| 115200 | | - fts3ColumnlistCopy(0, &pCsr); |
| 115201 | | - if( *pCsr==0x00 ) return 0; |
| 115202 | | - pCsr++; |
| 115203 | | - pCsr += sqlite3Fts3GetVarint32(pCsr, &iThis); |
| 115204 | | - } |
| 115205 | | - if( iCol==iThis && (*pCsr&0xFE) ) return pCsr; |
| 115206 | | - } |
| 115207 | | - return 0; |
| 115208 | | - } |
| 115209 | | - } |
| 115210 | | - } |
| 115211 | | - |
| 115212 | | - return 0; |
| 115213 | | -} |
| 115214 | | - |
| 115215 | 114644 | /* |
| 115216 | 114645 | ** Helper function used by the implementation of the overloaded snippet(), |
| 115217 | 114646 | ** offsets() and optimize() SQL functions. |
| 115218 | 114647 | ** |
| 115219 | 114648 | ** If the value passed as the third argument is a blob of size |
| | @@ -115441,16 +114870,15 @@ |
| 115441 | 114870 | ); |
| 115442 | 114871 | return rc; |
| 115443 | 114872 | } |
| 115444 | 114873 | |
| 115445 | 114874 | static int fts3SavepointMethod(sqlite3_vtab *pVtab, int iSavepoint){ |
| 115446 | | - Fts3Table *p = (Fts3Table*)pVtab; |
| 115447 | 114875 | UNUSED_PARAMETER(iSavepoint); |
| 115448 | | - assert( p->inTransaction ); |
| 115449 | | - assert( p->mxSavepoint < iSavepoint ); |
| 115450 | | - TESTONLY( p->mxSavepoint = iSavepoint ); |
| 115451 | | - return sqlite3Fts3PendingTermsFlush(p); |
| 114876 | + assert( ((Fts3Table *)pVtab)->inTransaction ); |
| 114877 | + assert( ((Fts3Table *)pVtab)->mxSavepoint < iSavepoint ); |
| 114878 | + TESTONLY( ((Fts3Table *)pVtab)->mxSavepoint = iSavepoint ); |
| 114879 | + return fts3SyncMethod(pVtab); |
| 115452 | 114880 | } |
| 115453 | 114881 | static int fts3ReleaseMethod(sqlite3_vtab *pVtab, int iSavepoint){ |
| 115454 | 114882 | TESTONLY( Fts3Table *p = (Fts3Table*)pVtab ); |
| 115455 | 114883 | UNUSED_PARAMETER(iSavepoint); |
| 115456 | 114884 | UNUSED_PARAMETER(pVtab); |
| | @@ -115617,10 +115045,1282 @@ |
| 115617 | 115045 | SQLITE_EXTENSION_INIT2(pApi) |
| 115618 | 115046 | return sqlite3Fts3Init(db); |
| 115619 | 115047 | } |
| 115620 | 115048 | #endif |
| 115621 | 115049 | |
| 115050 | + |
| 115051 | +/* |
| 115052 | +** Allocate an Fts3MultiSegReader for each token in the expression headed |
| 115053 | +** by pExpr. |
| 115054 | +** |
| 115055 | +** An Fts3SegReader object is a cursor that can seek or scan a range of |
| 115056 | +** entries within a single segment b-tree. An Fts3MultiSegReader uses multiple |
| 115057 | +** Fts3SegReader objects internally to provide an interface to seek or scan |
| 115058 | +** within the union of all segments of a b-tree. Hence the name. |
| 115059 | +** |
| 115060 | +** If the allocated Fts3MultiSegReader just seeks to a single entry in a |
| 115061 | +** segment b-tree (if the term is not a prefix or it is a prefix for which |
| 115062 | +** there exists prefix b-tree of the right length) then it may be traversed |
| 115063 | +** and merged incrementally. Otherwise, it has to be merged into an in-memory |
| 115064 | +** doclist and then traversed. |
| 115065 | +*/ |
| 115066 | +static void fts3EvalAllocateReaders( |
| 115067 | + Fts3Cursor *pCsr, |
| 115068 | + Fts3Expr *pExpr, |
| 115069 | + int *pnToken, /* OUT: Total number of tokens in phrase. */ |
| 115070 | + int *pnOr, /* OUT: Total number of OR nodes in expr. */ |
| 115071 | + int *pRc |
| 115072 | +){ |
| 115073 | + if( pExpr && SQLITE_OK==*pRc ){ |
| 115074 | + if( pExpr->eType==FTSQUERY_PHRASE ){ |
| 115075 | + int i; |
| 115076 | + int nToken = pExpr->pPhrase->nToken; |
| 115077 | + *pnToken += nToken; |
| 115078 | + for(i=0; i<nToken; i++){ |
| 115079 | + Fts3PhraseToken *pToken = &pExpr->pPhrase->aToken[i]; |
| 115080 | + int rc = sqlite3Fts3TermSegReaderCursor(pCsr, |
| 115081 | + pToken->z, pToken->n, pToken->isPrefix, &pToken->pSegcsr |
| 115082 | + ); |
| 115083 | + if( rc!=SQLITE_OK ){ |
| 115084 | + *pRc = rc; |
| 115085 | + return; |
| 115086 | + } |
| 115087 | + } |
| 115088 | + }else{ |
| 115089 | + *pnOr += (pExpr->eType==FTSQUERY_OR); |
| 115090 | + fts3EvalAllocateReaders(pCsr, pExpr->pLeft, pnToken, pnOr, pRc); |
| 115091 | + fts3EvalAllocateReaders(pCsr, pExpr->pRight, pnToken, pnOr, pRc); |
| 115092 | + } |
| 115093 | + } |
| 115094 | +} |
| 115095 | + |
| 115096 | +static int fts3EvalPhraseLoad( |
| 115097 | + Fts3Cursor *pCsr, |
| 115098 | + Fts3Phrase *p |
| 115099 | +){ |
| 115100 | + Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab; |
| 115101 | + int iToken; |
| 115102 | + int rc = SQLITE_OK; |
| 115103 | + |
| 115104 | + char *aDoclist = 0; |
| 115105 | + int nDoclist = 0; |
| 115106 | + int iPrev = -1; |
| 115107 | + |
| 115108 | + for(iToken=0; rc==SQLITE_OK && iToken<p->nToken; iToken++){ |
| 115109 | + Fts3PhraseToken *pToken = &p->aToken[iToken]; |
| 115110 | + assert( pToken->pSegcsr || pToken->pDeferred ); |
| 115111 | + |
| 115112 | + if( pToken->pDeferred==0 ){ |
| 115113 | + int nThis = 0; |
| 115114 | + char *pThis = 0; |
| 115115 | + rc = fts3TermSelect(pTab, pToken, p->iColumn, 1, &nThis, &pThis); |
| 115116 | + if( rc==SQLITE_OK ){ |
| 115117 | + if( pThis==0 ){ |
| 115118 | + sqlite3_free(aDoclist); |
| 115119 | + aDoclist = 0; |
| 115120 | + nDoclist = 0; |
| 115121 | + break; |
| 115122 | + }else if( aDoclist==0 ){ |
| 115123 | + aDoclist = pThis; |
| 115124 | + nDoclist = nThis; |
| 115125 | + }else{ |
| 115126 | + assert( iPrev>=0 ); |
| 115127 | + fts3DoclistPhraseMerge(pTab->bDescIdx, |
| 115128 | + iToken-iPrev, aDoclist, nDoclist, pThis, &nThis |
| 115129 | + ); |
| 115130 | + sqlite3_free(aDoclist); |
| 115131 | + aDoclist = pThis; |
| 115132 | + nDoclist = nThis; |
| 115133 | + } |
| 115134 | + iPrev = iToken; |
| 115135 | + } |
| 115136 | + } |
| 115137 | + } |
| 115138 | + |
| 115139 | + if( rc==SQLITE_OK ){ |
| 115140 | + p->doclist.aAll = aDoclist; |
| 115141 | + p->doclist.nAll = nDoclist; |
| 115142 | + }else{ |
| 115143 | + sqlite3_free(aDoclist); |
| 115144 | + } |
| 115145 | + return rc; |
| 115146 | +} |
| 115147 | + |
| 115148 | +static int fts3EvalDeferredPhrase(Fts3Cursor *pCsr, Fts3Phrase *pPhrase){ |
| 115149 | + int iToken; |
| 115150 | + int rc = SQLITE_OK; |
| 115151 | + |
| 115152 | + int nMaxUndeferred = -1; |
| 115153 | + char *aPoslist = 0; |
| 115154 | + int nPoslist = 0; |
| 115155 | + int iPrev = -1; |
| 115156 | + |
| 115157 | + assert( pPhrase->doclist.bFreeList==0 ); |
| 115158 | + |
| 115159 | + for(iToken=0; rc==SQLITE_OK && iToken<pPhrase->nToken; iToken++){ |
| 115160 | + Fts3PhraseToken *pToken = &pPhrase->aToken[iToken]; |
| 115161 | + Fts3DeferredToken *pDeferred = pToken->pDeferred; |
| 115162 | + |
| 115163 | + if( pDeferred ){ |
| 115164 | + char *pList; |
| 115165 | + int nList; |
| 115166 | + rc = sqlite3Fts3DeferredTokenList(pDeferred, &pList, &nList); |
| 115167 | + if( rc!=SQLITE_OK ) return rc; |
| 115168 | + |
| 115169 | + if( pList==0 ){ |
| 115170 | + sqlite3_free(aPoslist); |
| 115171 | + pPhrase->doclist.pList = 0; |
| 115172 | + pPhrase->doclist.nList = 0; |
| 115173 | + return SQLITE_OK; |
| 115174 | + |
| 115175 | + }else if( aPoslist==0 ){ |
| 115176 | + aPoslist = pList; |
| 115177 | + nPoslist = nList; |
| 115178 | + |
| 115179 | + }else{ |
| 115180 | + char *aOut = pList; |
| 115181 | + char *p1 = aPoslist; |
| 115182 | + char *p2 = aOut; |
| 115183 | + |
| 115184 | + assert( iPrev>=0 ); |
| 115185 | + fts3PoslistPhraseMerge(&aOut, iToken-iPrev, 0, 1, &p1, &p2); |
| 115186 | + sqlite3_free(aPoslist); |
| 115187 | + aPoslist = pList; |
| 115188 | + nPoslist = aOut - aPoslist; |
| 115189 | + if( nPoslist==0 ){ |
| 115190 | + sqlite3_free(aPoslist); |
| 115191 | + pPhrase->doclist.pList = 0; |
| 115192 | + pPhrase->doclist.nList = 0; |
| 115193 | + return SQLITE_OK; |
| 115194 | + } |
| 115195 | + } |
| 115196 | + iPrev = iToken; |
| 115197 | + }else{ |
| 115198 | + nMaxUndeferred = iToken; |
| 115199 | + } |
| 115200 | + } |
| 115201 | + |
| 115202 | + if( iPrev>=0 ){ |
| 115203 | + if( nMaxUndeferred<0 ){ |
| 115204 | + pPhrase->doclist.pList = aPoslist; |
| 115205 | + pPhrase->doclist.nList = nPoslist; |
| 115206 | + pPhrase->doclist.iDocid = pCsr->iPrevId; |
| 115207 | + pPhrase->doclist.bFreeList = 1; |
| 115208 | + }else{ |
| 115209 | + int nDistance; |
| 115210 | + char *p1; |
| 115211 | + char *p2; |
| 115212 | + char *aOut; |
| 115213 | + |
| 115214 | + if( nMaxUndeferred>iPrev ){ |
| 115215 | + p1 = aPoslist; |
| 115216 | + p2 = pPhrase->doclist.pList; |
| 115217 | + nDistance = nMaxUndeferred - iPrev; |
| 115218 | + }else{ |
| 115219 | + p1 = pPhrase->doclist.pList; |
| 115220 | + p2 = aPoslist; |
| 115221 | + nDistance = iPrev - nMaxUndeferred; |
| 115222 | + } |
| 115223 | + |
| 115224 | + aOut = (char *)sqlite3_malloc(nPoslist+8); |
| 115225 | + if( !aOut ){ |
| 115226 | + sqlite3_free(aPoslist); |
| 115227 | + return SQLITE_NOMEM; |
| 115228 | + } |
| 115229 | + |
| 115230 | + pPhrase->doclist.pList = aOut; |
| 115231 | + if( fts3PoslistPhraseMerge(&aOut, nDistance, 0, 1, &p1, &p2) ){ |
| 115232 | + pPhrase->doclist.bFreeList = 1; |
| 115233 | + pPhrase->doclist.nList = (aOut - pPhrase->doclist.pList); |
| 115234 | + }else{ |
| 115235 | + sqlite3_free(aOut); |
| 115236 | + pPhrase->doclist.pList = 0; |
| 115237 | + pPhrase->doclist.nList = 0; |
| 115238 | + } |
| 115239 | + sqlite3_free(aPoslist); |
| 115240 | + } |
| 115241 | + } |
| 115242 | + |
| 115243 | + return SQLITE_OK; |
| 115244 | +} |
| 115245 | + |
| 115246 | +/* |
| 115247 | +** This function is called for each Fts3Phrase in a full-text query |
| 115248 | +** expression to initialize the mechanism for returning rows. Once this |
| 115249 | +** function has been called successfully on an Fts3Phrase, it may be |
| 115250 | +** used with fts3EvalPhraseNext() to iterate through the matching docids. |
| 115251 | +*/ |
| 115252 | +static int fts3EvalPhraseStart(Fts3Cursor *pCsr, int bOptOk, Fts3Phrase *p){ |
| 115253 | + int rc; |
| 115254 | + Fts3PhraseToken *pFirst = &p->aToken[0]; |
| 115255 | + Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab; |
| 115256 | + |
| 115257 | + assert( p->doclist.aAll==0 ); |
| 115258 | + if( pCsr->bDesc==pTab->bDescIdx && bOptOk==1 && p->nToken==1 |
| 115259 | + && pFirst->pSegcsr && pFirst->pSegcsr->bLookup |
| 115260 | + ){ |
| 115261 | + /* Use the incremental approach. */ |
| 115262 | + int iCol = (p->iColumn >= pTab->nColumn ? -1 : p->iColumn); |
| 115263 | + rc = sqlite3Fts3MsrIncrStart( |
| 115264 | + pTab, pFirst->pSegcsr, iCol, pFirst->z, pFirst->n); |
| 115265 | + p->bIncr = 1; |
| 115266 | + |
| 115267 | + }else{ |
| 115268 | + /* Load the full doclist for the phrase into memory. */ |
| 115269 | + rc = fts3EvalPhraseLoad(pCsr, p); |
| 115270 | + p->bIncr = 0; |
| 115271 | + } |
| 115272 | + |
| 115273 | + assert( rc!=SQLITE_OK || p->nToken<1 || p->aToken[0].pSegcsr==0 || p->bIncr ); |
| 115274 | + return rc; |
| 115275 | +} |
| 115276 | + |
| 115277 | +/* |
| 115278 | +** This function is used to iterate backwards (from the end to start) |
| 115279 | +** through doclists. |
| 115280 | +*/ |
| 115281 | +SQLITE_PRIVATE void sqlite3Fts3DoclistPrev( |
| 115282 | + int bDescIdx, /* True if the doclist is desc */ |
| 115283 | + char *aDoclist, /* Pointer to entire doclist */ |
| 115284 | + int nDoclist, /* Length of aDoclist in bytes */ |
| 115285 | + char **ppIter, /* IN/OUT: Iterator pointer */ |
| 115286 | + sqlite3_int64 *piDocid, /* IN/OUT: Docid pointer */ |
| 115287 | + int *pnList, /* IN/OUT: List length pointer */ |
| 115288 | + u8 *pbEof /* OUT: End-of-file flag */ |
| 115289 | +){ |
| 115290 | + char *p = *ppIter; |
| 115291 | + |
| 115292 | + assert( nDoclist>0 ); |
| 115293 | + assert( *pbEof==0 ); |
| 115294 | + assert( p || *piDocid==0 ); |
| 115295 | + assert( !p || (p>aDoclist && p<&aDoclist[nDoclist]) ); |
| 115296 | + |
| 115297 | + if( p==0 ){ |
| 115298 | + sqlite3_int64 iDocid = 0; |
| 115299 | + char *pNext = 0; |
| 115300 | + char *pDocid = aDoclist; |
| 115301 | + char *pEnd = &aDoclist[nDoclist]; |
| 115302 | + int iMul = 1; |
| 115303 | + |
| 115304 | + while( pDocid<pEnd ){ |
| 115305 | + sqlite3_int64 iDelta; |
| 115306 | + pDocid += sqlite3Fts3GetVarint(pDocid, &iDelta); |
| 115307 | + iDocid += (iMul * iDelta); |
| 115308 | + pNext = pDocid; |
| 115309 | + fts3PoslistCopy(0, &pDocid); |
| 115310 | + while( pDocid<pEnd && *pDocid==0 ) pDocid++; |
| 115311 | + iMul = (bDescIdx ? -1 : 1); |
| 115312 | + } |
| 115313 | + |
| 115314 | + *pnList = pEnd - pNext; |
| 115315 | + *ppIter = pNext; |
| 115316 | + *piDocid = iDocid; |
| 115317 | + }else{ |
| 115318 | + int iMul = (bDescIdx ? -1 : 1); |
| 115319 | + sqlite3_int64 iDelta; |
| 115320 | + fts3GetReverseVarint(&p, aDoclist, &iDelta); |
| 115321 | + *piDocid -= (iMul * iDelta); |
| 115322 | + |
| 115323 | + if( p==aDoclist ){ |
| 115324 | + *pbEof = 1; |
| 115325 | + }else{ |
| 115326 | + char *pSave = p; |
| 115327 | + fts3ReversePoslist(aDoclist, &p); |
| 115328 | + *pnList = (pSave - p); |
| 115329 | + } |
| 115330 | + *ppIter = p; |
| 115331 | + } |
| 115332 | +} |
| 115333 | + |
| 115334 | +/* |
| 115335 | +** Attempt to move the phrase iterator to point to the next matching docid. |
| 115336 | +** If an error occurs, return an SQLite error code. Otherwise, return |
| 115337 | +** SQLITE_OK. |
| 115338 | +** |
| 115339 | +** If there is no "next" entry and no error occurs, then *pbEof is set to |
| 115340 | +** 1 before returning. Otherwise, if no error occurs and the iterator is |
| 115341 | +** successfully advanced, *pbEof is set to 0. |
| 115342 | +*/ |
| 115343 | +static int fts3EvalPhraseNext( |
| 115344 | + Fts3Cursor *pCsr, |
| 115345 | + Fts3Phrase *p, |
| 115346 | + u8 *pbEof |
| 115347 | +){ |
| 115348 | + int rc = SQLITE_OK; |
| 115349 | + Fts3Doclist *pDL = &p->doclist; |
| 115350 | + Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab; |
| 115351 | + |
| 115352 | + if( p->bIncr ){ |
| 115353 | + assert( p->nToken==1 ); |
| 115354 | + assert( pDL->pNextDocid==0 ); |
| 115355 | + rc = sqlite3Fts3MsrIncrNext(pTab, p->aToken[0].pSegcsr, |
| 115356 | + &pDL->iDocid, &pDL->pList, &pDL->nList |
| 115357 | + ); |
| 115358 | + if( rc==SQLITE_OK && !pDL->pList ){ |
| 115359 | + *pbEof = 1; |
| 115360 | + } |
| 115361 | + }else if( pCsr->bDesc!=pTab->bDescIdx && pDL->nAll ){ |
| 115362 | + sqlite3Fts3DoclistPrev(pTab->bDescIdx, pDL->aAll, pDL->nAll, |
| 115363 | + &pDL->pNextDocid, &pDL->iDocid, &pDL->nList, pbEof |
| 115364 | + ); |
| 115365 | + pDL->pList = pDL->pNextDocid; |
| 115366 | + }else{ |
| 115367 | + char *pIter; /* Used to iterate through aAll */ |
| 115368 | + char *pEnd = &pDL->aAll[pDL->nAll]; /* 1 byte past end of aAll */ |
| 115369 | + if( pDL->pNextDocid ){ |
| 115370 | + pIter = pDL->pNextDocid; |
| 115371 | + }else{ |
| 115372 | + pIter = pDL->aAll; |
| 115373 | + } |
| 115374 | + |
| 115375 | + if( pIter>=pEnd ){ |
| 115376 | + /* We have already reached the end of this doclist. EOF. */ |
| 115377 | + *pbEof = 1; |
| 115378 | + }else{ |
| 115379 | + sqlite3_int64 iDelta; |
| 115380 | + pIter += sqlite3Fts3GetVarint(pIter, &iDelta); |
| 115381 | + if( pTab->bDescIdx==0 || pDL->pNextDocid==0 ){ |
| 115382 | + pDL->iDocid += iDelta; |
| 115383 | + }else{ |
| 115384 | + pDL->iDocid -= iDelta; |
| 115385 | + } |
| 115386 | + pDL->pList = pIter; |
| 115387 | + fts3PoslistCopy(0, &pIter); |
| 115388 | + pDL->nList = (pIter - pDL->pList); |
| 115389 | + |
| 115390 | + /* pIter now points just past the 0x00 that terminates the position- |
| 115391 | + ** list for document pDL->iDocid. However, if this position-list was |
| 115392 | + ** edited in place by fts3EvalNearTrim2(), then pIter may not actually |
| 115393 | + ** point to the start of the next docid value. The following line deals |
| 115394 | + ** with this case by advancing pIter past the zero-padding added by |
| 115395 | + ** fts3EvalNearTrim2(). */ |
| 115396 | + while( pIter<pEnd && *pIter==0 ) pIter++; |
| 115397 | + |
| 115398 | + pDL->pNextDocid = pIter; |
| 115399 | + assert( *pIter || pIter>=&pDL->aAll[pDL->nAll] ); |
| 115400 | + *pbEof = 0; |
| 115401 | + } |
| 115402 | + } |
| 115403 | + |
| 115404 | + return rc; |
| 115405 | +} |
| 115406 | + |
| 115407 | +static void fts3EvalStartReaders( |
| 115408 | + Fts3Cursor *pCsr, |
| 115409 | + Fts3Expr *pExpr, |
| 115410 | + int bOptOk, |
| 115411 | + int *pRc |
| 115412 | +){ |
| 115413 | + if( pExpr && SQLITE_OK==*pRc ){ |
| 115414 | + if( pExpr->eType==FTSQUERY_PHRASE ){ |
| 115415 | + int i; |
| 115416 | + int nToken = pExpr->pPhrase->nToken; |
| 115417 | + for(i=0; i<nToken; i++){ |
| 115418 | + if( pExpr->pPhrase->aToken[i].pDeferred==0 ) break; |
| 115419 | + } |
| 115420 | + pExpr->bDeferred = (i==nToken); |
| 115421 | + *pRc = fts3EvalPhraseStart(pCsr, bOptOk, pExpr->pPhrase); |
| 115422 | + }else{ |
| 115423 | + fts3EvalStartReaders(pCsr, pExpr->pLeft, bOptOk, pRc); |
| 115424 | + fts3EvalStartReaders(pCsr, pExpr->pRight, bOptOk, pRc); |
| 115425 | + pExpr->bDeferred = (pExpr->pLeft->bDeferred && pExpr->pRight->bDeferred); |
| 115426 | + } |
| 115427 | + } |
| 115428 | +} |
| 115429 | + |
| 115430 | + |
| 115431 | +typedef struct Fts3TokenAndCost Fts3TokenAndCost; |
| 115432 | +struct Fts3TokenAndCost { |
| 115433 | + Fts3PhraseToken *pToken; |
| 115434 | + Fts3Expr *pRoot; |
| 115435 | + int nOvfl; |
| 115436 | + int iCol; |
| 115437 | +}; |
| 115438 | + |
| 115439 | +static void fts3EvalTokenCosts( |
| 115440 | + Fts3Cursor *pCsr, |
| 115441 | + Fts3Expr *pRoot, |
| 115442 | + Fts3Expr *pExpr, |
| 115443 | + Fts3TokenAndCost **ppTC, |
| 115444 | + Fts3Expr ***ppOr, |
| 115445 | + int *pRc |
| 115446 | +){ |
| 115447 | + if( *pRc==SQLITE_OK && pExpr ){ |
| 115448 | + if( pExpr->eType==FTSQUERY_PHRASE ){ |
| 115449 | + Fts3Phrase *pPhrase = pExpr->pPhrase; |
| 115450 | + int i; |
| 115451 | + for(i=0; *pRc==SQLITE_OK && i<pPhrase->nToken; i++){ |
| 115452 | + Fts3TokenAndCost *pTC = (*ppTC)++; |
| 115453 | + pTC->pRoot = pRoot; |
| 115454 | + pTC->pToken = &pPhrase->aToken[i]; |
| 115455 | + pTC->iCol = pPhrase->iColumn; |
| 115456 | + *pRc = sqlite3Fts3MsrOvfl(pCsr, pTC->pToken->pSegcsr, &pTC->nOvfl); |
| 115457 | + } |
| 115458 | + }else if( pExpr->eType!=FTSQUERY_NOT ){ |
| 115459 | + if( pExpr->eType==FTSQUERY_OR ){ |
| 115460 | + pRoot = pExpr->pLeft; |
| 115461 | + **ppOr = pRoot; |
| 115462 | + (*ppOr)++; |
| 115463 | + } |
| 115464 | + fts3EvalTokenCosts(pCsr, pRoot, pExpr->pLeft, ppTC, ppOr, pRc); |
| 115465 | + if( pExpr->eType==FTSQUERY_OR ){ |
| 115466 | + pRoot = pExpr->pRight; |
| 115467 | + **ppOr = pRoot; |
| 115468 | + (*ppOr)++; |
| 115469 | + } |
| 115470 | + fts3EvalTokenCosts(pCsr, pRoot, pExpr->pRight, ppTC, ppOr, pRc); |
| 115471 | + } |
| 115472 | + } |
| 115473 | +} |
| 115474 | + |
| 115475 | +static int fts3EvalAverageDocsize(Fts3Cursor *pCsr, int *pnPage){ |
| 115476 | + if( pCsr->nRowAvg==0 ){ |
| 115477 | + /* The average document size, which is required to calculate the cost |
| 115478 | + ** of each doclist, has not yet been determined. Read the required |
| 115479 | + ** data from the %_stat table to calculate it. |
| 115480 | + ** |
| 115481 | + ** Entry 0 of the %_stat table is a blob containing (nCol+1) FTS3 |
| 115482 | + ** varints, where nCol is the number of columns in the FTS3 table. |
| 115483 | + ** The first varint is the number of documents currently stored in |
| 115484 | + ** the table. The following nCol varints contain the total amount of |
| 115485 | + ** data stored in all rows of each column of the table, from left |
| 115486 | + ** to right. |
| 115487 | + */ |
| 115488 | + int rc; |
| 115489 | + Fts3Table *p = (Fts3Table*)pCsr->base.pVtab; |
| 115490 | + sqlite3_stmt *pStmt; |
| 115491 | + sqlite3_int64 nDoc = 0; |
| 115492 | + sqlite3_int64 nByte = 0; |
| 115493 | + const char *pEnd; |
| 115494 | + const char *a; |
| 115495 | + |
| 115496 | + rc = sqlite3Fts3SelectDoctotal(p, &pStmt); |
| 115497 | + if( rc!=SQLITE_OK ) return rc; |
| 115498 | + a = sqlite3_column_blob(pStmt, 0); |
| 115499 | + assert( a ); |
| 115500 | + |
| 115501 | + pEnd = &a[sqlite3_column_bytes(pStmt, 0)]; |
| 115502 | + a += sqlite3Fts3GetVarint(a, &nDoc); |
| 115503 | + while( a<pEnd ){ |
| 115504 | + a += sqlite3Fts3GetVarint(a, &nByte); |
| 115505 | + } |
| 115506 | + if( nDoc==0 || nByte==0 ){ |
| 115507 | + sqlite3_reset(pStmt); |
| 115508 | + return SQLITE_CORRUPT_VTAB; |
| 115509 | + } |
| 115510 | + |
| 115511 | + pCsr->nDoc = nDoc; |
| 115512 | + pCsr->nRowAvg = (int)(((nByte / nDoc) + p->nPgsz) / p->nPgsz); |
| 115513 | + assert( pCsr->nRowAvg>0 ); |
| 115514 | + rc = sqlite3_reset(pStmt); |
| 115515 | + if( rc!=SQLITE_OK ) return rc; |
| 115516 | + } |
| 115517 | + |
| 115518 | + *pnPage = pCsr->nRowAvg; |
| 115519 | + return SQLITE_OK; |
| 115520 | +} |
| 115521 | + |
| 115522 | +static int fts3EvalSelectDeferred( |
| 115523 | + Fts3Cursor *pCsr, |
| 115524 | + Fts3Expr *pRoot, |
| 115525 | + Fts3TokenAndCost *aTC, |
| 115526 | + int nTC |
| 115527 | +){ |
| 115528 | + int nDocSize = 0; |
| 115529 | + int nDocEst = 0; |
| 115530 | + int rc = SQLITE_OK; |
| 115531 | + Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab; |
| 115532 | + int ii; |
| 115533 | + |
| 115534 | + int nOvfl = 0; |
| 115535 | + int nTerm = 0; |
| 115536 | + |
| 115537 | + for(ii=0; ii<nTC; ii++){ |
| 115538 | + if( aTC[ii].pRoot==pRoot ){ |
| 115539 | + nOvfl += aTC[ii].nOvfl; |
| 115540 | + nTerm++; |
| 115541 | + } |
| 115542 | + } |
| 115543 | + if( nOvfl==0 || nTerm<2 ) return SQLITE_OK; |
| 115544 | + |
| 115545 | + rc = fts3EvalAverageDocsize(pCsr, &nDocSize); |
| 115546 | + |
| 115547 | + for(ii=0; ii<nTerm && rc==SQLITE_OK; ii++){ |
| 115548 | + int jj; |
| 115549 | + Fts3TokenAndCost *pTC = 0; |
| 115550 | + |
| 115551 | + for(jj=0; jj<nTC; jj++){ |
| 115552 | + if( aTC[jj].pToken && aTC[jj].pRoot==pRoot |
| 115553 | + && (!pTC || aTC[jj].nOvfl<pTC->nOvfl) |
| 115554 | + ){ |
| 115555 | + pTC = &aTC[jj]; |
| 115556 | + } |
| 115557 | + } |
| 115558 | + assert( pTC ); |
| 115559 | + |
| 115560 | + /* At this point pTC points to the cheapest remaining token. */ |
| 115561 | + if( ii==0 ){ |
| 115562 | + if( pTC->nOvfl ){ |
| 115563 | + nDocEst = (pTC->nOvfl * pTab->nPgsz + pTab->nPgsz) / 10; |
| 115564 | + }else{ |
| 115565 | + /* TODO: Fix this so that the doclist need not be read twice. */ |
| 115566 | + Fts3PhraseToken *pToken = pTC->pToken; |
| 115567 | + int nList = 0; |
| 115568 | + char *pList = 0; |
| 115569 | + rc = fts3TermSelect(pTab, pToken, pTC->iCol, 1, &nList, &pList); |
| 115570 | + if( rc==SQLITE_OK ){ |
| 115571 | + nDocEst = fts3DoclistCountDocids(1, pList, nList); |
| 115572 | + } |
| 115573 | + sqlite3_free(pList); |
| 115574 | + if( rc==SQLITE_OK ){ |
| 115575 | + rc = sqlite3Fts3TermSegReaderCursor(pCsr, |
| 115576 | + pToken->z, pToken->n, pToken->isPrefix, &pToken->pSegcsr |
| 115577 | + ); |
| 115578 | + } |
| 115579 | + } |
| 115580 | + }else{ |
| 115581 | + if( pTC->nOvfl>=(nDocEst*nDocSize) ){ |
| 115582 | + Fts3PhraseToken *pToken = pTC->pToken; |
| 115583 | + rc = sqlite3Fts3DeferToken(pCsr, pToken, pTC->iCol); |
| 115584 | + fts3SegReaderCursorFree(pToken->pSegcsr); |
| 115585 | + pToken->pSegcsr = 0; |
| 115586 | + } |
| 115587 | + nDocEst = 1 + (nDocEst/4); |
| 115588 | + } |
| 115589 | + pTC->pToken = 0; |
| 115590 | + } |
| 115591 | + |
| 115592 | + return rc; |
| 115593 | +} |
| 115594 | + |
| 115595 | +SQLITE_PRIVATE int sqlite3Fts3EvalStart(Fts3Cursor *pCsr, Fts3Expr *pExpr, int bOptOk){ |
| 115596 | + Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab; |
| 115597 | + int rc = SQLITE_OK; |
| 115598 | + int nToken = 0; |
| 115599 | + int nOr = 0; |
| 115600 | + |
| 115601 | + /* Allocate a MultiSegReader for each token in the expression. */ |
| 115602 | + fts3EvalAllocateReaders(pCsr, pExpr, &nToken, &nOr, &rc); |
| 115603 | + |
| 115604 | + /* Call fts3EvalPhraseStart() on all phrases in the expression. TODO: |
| 115605 | + ** This call will eventually also be responsible for determining which |
| 115606 | + ** tokens are 'deferred' until the document text is loaded into memory. |
| 115607 | + ** |
| 115608 | + ** Each token in each phrase is dealt with using one of the following |
| 115609 | + ** three strategies: |
| 115610 | + ** |
| 115611 | + ** 1. Entire doclist loaded into memory as part of the |
| 115612 | + ** fts3EvalStartReaders() call. |
| 115613 | + ** |
| 115614 | + ** 2. Doclist loaded into memory incrementally, as part of each |
| 115615 | + ** sqlite3Fts3EvalNext() call. |
| 115616 | + ** |
| 115617 | + ** 3. Token doclist is never loaded. Instead, documents are loaded into |
| 115618 | + ** memory and scanned for the token as part of the sqlite3Fts3EvalNext() |
| 115619 | + ** call. This is known as a "deferred" token. |
| 115620 | + */ |
| 115621 | + |
| 115622 | + /* If bOptOk is true, check if there are any tokens that should be deferred. |
| 115623 | + */ |
| 115624 | + if( rc==SQLITE_OK && bOptOk && nToken>1 && pTab->bHasStat ){ |
| 115625 | + Fts3TokenAndCost *aTC; |
| 115626 | + Fts3Expr **apOr; |
| 115627 | + aTC = (Fts3TokenAndCost *)sqlite3_malloc( |
| 115628 | + sizeof(Fts3TokenAndCost) * nToken |
| 115629 | + + sizeof(Fts3Expr *) * nOr * 2 |
| 115630 | + ); |
| 115631 | + apOr = (Fts3Expr **)&aTC[nToken]; |
| 115632 | + |
| 115633 | + if( !aTC ){ |
| 115634 | + rc = SQLITE_NOMEM; |
| 115635 | + }else{ |
| 115636 | + int ii; |
| 115637 | + Fts3TokenAndCost *pTC = aTC; |
| 115638 | + Fts3Expr **ppOr = apOr; |
| 115639 | + |
| 115640 | + fts3EvalTokenCosts(pCsr, 0, pExpr, &pTC, &ppOr, &rc); |
| 115641 | + nToken = pTC-aTC; |
| 115642 | + nOr = ppOr-apOr; |
| 115643 | + |
| 115644 | + if( rc==SQLITE_OK ){ |
| 115645 | + rc = fts3EvalSelectDeferred(pCsr, 0, aTC, nToken); |
| 115646 | + for(ii=0; rc==SQLITE_OK && ii<nOr; ii++){ |
| 115647 | + rc = fts3EvalSelectDeferred(pCsr, apOr[ii], aTC, nToken); |
| 115648 | + } |
| 115649 | + } |
| 115650 | + |
| 115651 | + sqlite3_free(aTC); |
| 115652 | + } |
| 115653 | + } |
| 115654 | + |
| 115655 | + fts3EvalStartReaders(pCsr, pExpr, bOptOk, &rc); |
| 115656 | + return rc; |
| 115657 | +} |
| 115658 | + |
| 115659 | +static void fts3EvalZeroPoslist(Fts3Phrase *pPhrase){ |
| 115660 | + if( pPhrase->doclist.bFreeList ){ |
| 115661 | + sqlite3_free(pPhrase->doclist.pList); |
| 115662 | + } |
| 115663 | + pPhrase->doclist.pList = 0; |
| 115664 | + pPhrase->doclist.nList = 0; |
| 115665 | + pPhrase->doclist.bFreeList = 0; |
| 115666 | +} |
| 115667 | + |
| 115668 | +static int fts3EvalNearTrim2( |
| 115669 | + int nNear, |
| 115670 | + char *aTmp, /* Temporary space to use */ |
| 115671 | + char **paPoslist, /* IN/OUT: Position list */ |
| 115672 | + int *pnToken, /* IN/OUT: Tokens in phrase of *paPoslist */ |
| 115673 | + Fts3Phrase *pPhrase /* The phrase object to trim the doclist of */ |
| 115674 | +){ |
| 115675 | + int nParam1 = nNear + pPhrase->nToken; |
| 115676 | + int nParam2 = nNear + *pnToken; |
| 115677 | + int nNew; |
| 115678 | + char *p2; |
| 115679 | + char *pOut; |
| 115680 | + int res; |
| 115681 | + |
| 115682 | + assert( pPhrase->doclist.pList ); |
| 115683 | + |
| 115684 | + p2 = pOut = pPhrase->doclist.pList; |
| 115685 | + res = fts3PoslistNearMerge( |
| 115686 | + &pOut, aTmp, nParam1, nParam2, paPoslist, &p2 |
| 115687 | + ); |
| 115688 | + if( res ){ |
| 115689 | + nNew = (pOut - pPhrase->doclist.pList) - 1; |
| 115690 | + assert( pPhrase->doclist.pList[nNew]=='\0' ); |
| 115691 | + assert( nNew<=pPhrase->doclist.nList && nNew>0 ); |
| 115692 | + memset(&pPhrase->doclist.pList[nNew], 0, pPhrase->doclist.nList - nNew); |
| 115693 | + pPhrase->doclist.nList = nNew; |
| 115694 | + *paPoslist = pPhrase->doclist.pList; |
| 115695 | + *pnToken = pPhrase->nToken; |
| 115696 | + } |
| 115697 | + |
| 115698 | + return res; |
| 115699 | +} |
| 115700 | + |
| 115701 | +static int fts3EvalNearTest(Fts3Expr *pExpr, int *pRc){ |
| 115702 | + int res = 1; |
| 115703 | + |
| 115704 | + /* The following block runs if pExpr is the root of a NEAR query. |
| 115705 | + ** For example, the query: |
| 115706 | + ** |
| 115707 | + ** "w" NEAR "x" NEAR "y" NEAR "z" |
| 115708 | + ** |
| 115709 | + ** which is represented in tree form as: |
| 115710 | + ** |
| 115711 | + ** | |
| 115712 | + ** +--NEAR--+ <-- root of NEAR query |
| 115713 | + ** | | |
| 115714 | + ** +--NEAR--+ "z" |
| 115715 | + ** | | |
| 115716 | + ** +--NEAR--+ "y" |
| 115717 | + ** | | |
| 115718 | + ** "w" "x" |
| 115719 | + ** |
| 115720 | + ** The right-hand child of a NEAR node is always a phrase. The |
| 115721 | + ** left-hand child may be either a phrase or a NEAR node. There are |
| 115722 | + ** no exceptions to this. |
| 115723 | + */ |
| 115724 | + if( *pRc==SQLITE_OK |
| 115725 | + && pExpr->eType==FTSQUERY_NEAR |
| 115726 | + && pExpr->bEof==0 |
| 115727 | + && (pExpr->pParent==0 || pExpr->pParent->eType!=FTSQUERY_NEAR) |
| 115728 | + ){ |
| 115729 | + Fts3Expr *p; |
| 115730 | + int nTmp = 0; /* Bytes of temp space */ |
| 115731 | + char *aTmp; /* Temp space for PoslistNearMerge() */ |
| 115732 | + |
| 115733 | + /* Allocate temporary working space. */ |
| 115734 | + for(p=pExpr; p->pLeft; p=p->pLeft){ |
| 115735 | + nTmp += p->pRight->pPhrase->doclist.nList; |
| 115736 | + } |
| 115737 | + nTmp += p->pPhrase->doclist.nList; |
| 115738 | + aTmp = sqlite3_malloc(nTmp*2); |
| 115739 | + if( !aTmp ){ |
| 115740 | + *pRc = SQLITE_NOMEM; |
| 115741 | + res = 0; |
| 115742 | + }else{ |
| 115743 | + char *aPoslist = p->pPhrase->doclist.pList; |
| 115744 | + int nToken = p->pPhrase->nToken; |
| 115745 | + |
| 115746 | + for(p=p->pParent;res && p && p->eType==FTSQUERY_NEAR; p=p->pParent){ |
| 115747 | + Fts3Phrase *pPhrase = p->pRight->pPhrase; |
| 115748 | + int nNear = p->nNear; |
| 115749 | + res = fts3EvalNearTrim2(nNear, aTmp, &aPoslist, &nToken, pPhrase); |
| 115750 | + } |
| 115751 | + |
| 115752 | + aPoslist = pExpr->pRight->pPhrase->doclist.pList; |
| 115753 | + nToken = pExpr->pRight->pPhrase->nToken; |
| 115754 | + for(p=pExpr->pLeft; p && res; p=p->pLeft){ |
| 115755 | + int nNear = p->pParent->nNear; |
| 115756 | + Fts3Phrase *pPhrase = ( |
| 115757 | + p->eType==FTSQUERY_NEAR ? p->pRight->pPhrase : p->pPhrase |
| 115758 | + ); |
| 115759 | + res = fts3EvalNearTrim2(nNear, aTmp, &aPoslist, &nToken, pPhrase); |
| 115760 | + } |
| 115761 | + } |
| 115762 | + |
| 115763 | + sqlite3_free(aTmp); |
| 115764 | + } |
| 115765 | + |
| 115766 | + return res; |
| 115767 | +} |
| 115768 | + |
| 115769 | +/* |
| 115770 | +** This macro is used by the fts3EvalNext() function. The two arguments are |
| 115771 | +** 64-bit docid values. If the current query is "ORDER BY docid ASC", then |
| 115772 | +** the macro returns (i1 - i2). Or if it is "ORDER BY docid DESC", then |
| 115773 | +** it returns (i2 - i1). This allows the same code to be used for merging |
| 115774 | +** doclists in ascending or descending order. |
| 115775 | +*/ |
| 115776 | +#define DOCID_CMP(i1, i2) ((pCsr->bDesc?-1:1) * (i1-i2)) |
| 115777 | + |
| 115778 | +static void fts3EvalNext( |
| 115779 | + Fts3Cursor *pCsr, |
| 115780 | + Fts3Expr *pExpr, |
| 115781 | + int *pRc |
| 115782 | +){ |
| 115783 | + if( *pRc==SQLITE_OK ){ |
| 115784 | + assert( pExpr->bEof==0 ); |
| 115785 | + pExpr->bStart = 1; |
| 115786 | + |
| 115787 | + switch( pExpr->eType ){ |
| 115788 | + case FTSQUERY_NEAR: |
| 115789 | + case FTSQUERY_AND: { |
| 115790 | + Fts3Expr *pLeft = pExpr->pLeft; |
| 115791 | + Fts3Expr *pRight = pExpr->pRight; |
| 115792 | + assert( !pLeft->bDeferred || !pRight->bDeferred ); |
| 115793 | + if( pLeft->bDeferred ){ |
| 115794 | + fts3EvalNext(pCsr, pRight, pRc); |
| 115795 | + pExpr->iDocid = pRight->iDocid; |
| 115796 | + pExpr->bEof = pRight->bEof; |
| 115797 | + }else if( pRight->bDeferred ){ |
| 115798 | + fts3EvalNext(pCsr, pLeft, pRc); |
| 115799 | + pExpr->iDocid = pLeft->iDocid; |
| 115800 | + pExpr->bEof = pLeft->bEof; |
| 115801 | + }else{ |
| 115802 | + fts3EvalNext(pCsr, pLeft, pRc); |
| 115803 | + fts3EvalNext(pCsr, pRight, pRc); |
| 115804 | + |
| 115805 | + while( !pLeft->bEof && !pRight->bEof && *pRc==SQLITE_OK ){ |
| 115806 | + sqlite3_int64 iDiff = DOCID_CMP(pLeft->iDocid, pRight->iDocid); |
| 115807 | + if( iDiff==0 ) break; |
| 115808 | + if( iDiff<0 ){ |
| 115809 | + fts3EvalNext(pCsr, pLeft, pRc); |
| 115810 | + }else{ |
| 115811 | + fts3EvalNext(pCsr, pRight, pRc); |
| 115812 | + } |
| 115813 | + } |
| 115814 | + |
| 115815 | + pExpr->iDocid = pLeft->iDocid; |
| 115816 | + pExpr->bEof = (pLeft->bEof || pRight->bEof); |
| 115817 | + } |
| 115818 | + break; |
| 115819 | + } |
| 115820 | + |
| 115821 | + case FTSQUERY_OR: { |
| 115822 | + Fts3Expr *pLeft = pExpr->pLeft; |
| 115823 | + Fts3Expr *pRight = pExpr->pRight; |
| 115824 | + sqlite3_int64 iCmp = DOCID_CMP(pLeft->iDocid, pRight->iDocid); |
| 115825 | + |
| 115826 | + assert( pLeft->bStart || pLeft->iDocid==pRight->iDocid ); |
| 115827 | + assert( pRight->bStart || pLeft->iDocid==pRight->iDocid ); |
| 115828 | + |
| 115829 | + if( pRight->bEof || (pLeft->bEof==0 && iCmp<0) ){ |
| 115830 | + fts3EvalNext(pCsr, pLeft, pRc); |
| 115831 | + }else if( pLeft->bEof || (pRight->bEof==0 && iCmp>0) ){ |
| 115832 | + fts3EvalNext(pCsr, pRight, pRc); |
| 115833 | + }else{ |
| 115834 | + fts3EvalNext(pCsr, pLeft, pRc); |
| 115835 | + fts3EvalNext(pCsr, pRight, pRc); |
| 115836 | + } |
| 115837 | + |
| 115838 | + pExpr->bEof = (pLeft->bEof && pRight->bEof); |
| 115839 | + iCmp = DOCID_CMP(pLeft->iDocid, pRight->iDocid); |
| 115840 | + if( pRight->bEof || (pLeft->bEof==0 && iCmp<0) ){ |
| 115841 | + pExpr->iDocid = pLeft->iDocid; |
| 115842 | + }else{ |
| 115843 | + pExpr->iDocid = pRight->iDocid; |
| 115844 | + } |
| 115845 | + |
| 115846 | + break; |
| 115847 | + } |
| 115848 | + |
| 115849 | + case FTSQUERY_NOT: { |
| 115850 | + Fts3Expr *pLeft = pExpr->pLeft; |
| 115851 | + Fts3Expr *pRight = pExpr->pRight; |
| 115852 | + |
| 115853 | + if( pRight->bStart==0 ){ |
| 115854 | + fts3EvalNext(pCsr, pRight, pRc); |
| 115855 | + assert( *pRc!=SQLITE_OK || pRight->bStart ); |
| 115856 | + } |
| 115857 | + |
| 115858 | + fts3EvalNext(pCsr, pLeft, pRc); |
| 115859 | + if( pLeft->bEof==0 ){ |
| 115860 | + while( !*pRc |
| 115861 | + && !pRight->bEof |
| 115862 | + && DOCID_CMP(pLeft->iDocid, pRight->iDocid)>0 |
| 115863 | + ){ |
| 115864 | + fts3EvalNext(pCsr, pRight, pRc); |
| 115865 | + } |
| 115866 | + } |
| 115867 | + pExpr->iDocid = pLeft->iDocid; |
| 115868 | + pExpr->bEof = pLeft->bEof; |
| 115869 | + break; |
| 115870 | + } |
| 115871 | + |
| 115872 | + default: { |
| 115873 | + Fts3Phrase *pPhrase = pExpr->pPhrase; |
| 115874 | + fts3EvalZeroPoslist(pPhrase); |
| 115875 | + *pRc = fts3EvalPhraseNext(pCsr, pPhrase, &pExpr->bEof); |
| 115876 | + pExpr->iDocid = pPhrase->doclist.iDocid; |
| 115877 | + break; |
| 115878 | + } |
| 115879 | + } |
| 115880 | + } |
| 115881 | +} |
| 115882 | + |
| 115883 | +static int fts3EvalDeferredTest(Fts3Cursor *pCsr, Fts3Expr *pExpr, int *pRc){ |
| 115884 | + int bHit = 1; |
| 115885 | + if( *pRc==SQLITE_OK ){ |
| 115886 | + switch( pExpr->eType ){ |
| 115887 | + case FTSQUERY_NEAR: |
| 115888 | + case FTSQUERY_AND: |
| 115889 | + bHit = ( |
| 115890 | + fts3EvalDeferredTest(pCsr, pExpr->pLeft, pRc) |
| 115891 | + && fts3EvalDeferredTest(pCsr, pExpr->pRight, pRc) |
| 115892 | + && fts3EvalNearTest(pExpr, pRc) |
| 115893 | + ); |
| 115894 | + |
| 115895 | + /* If the NEAR expression does not match any rows, zero the doclist for |
| 115896 | + ** all phrases involved in the NEAR. This is because the snippet(), |
| 115897 | + ** offsets() and matchinfo() functions are not supposed to recognize |
| 115898 | + ** any instances of phrases that are part of unmatched NEAR queries. |
| 115899 | + ** For example if this expression: |
| 115900 | + ** |
| 115901 | + ** ... MATCH 'a OR (b NEAR c)' |
| 115902 | + ** |
| 115903 | + ** is matched against a row containing: |
| 115904 | + ** |
| 115905 | + ** 'a b d e' |
| 115906 | + ** |
| 115907 | + ** then any snippet() should ony highlight the "a" term, not the "b" |
| 115908 | + ** (as "b" is part of a non-matching NEAR clause). |
| 115909 | + */ |
| 115910 | + if( bHit==0 |
| 115911 | + && pExpr->eType==FTSQUERY_NEAR |
| 115912 | + && (pExpr->pParent==0 || pExpr->pParent->eType!=FTSQUERY_NEAR) |
| 115913 | + ){ |
| 115914 | + Fts3Expr *p; |
| 115915 | + for(p=pExpr; p->pPhrase==0; p=p->pLeft){ |
| 115916 | + if( p->pRight->iDocid==pCsr->iPrevId ){ |
| 115917 | + fts3EvalZeroPoslist(p->pRight->pPhrase); |
| 115918 | + } |
| 115919 | + } |
| 115920 | + if( p->iDocid==pCsr->iPrevId ){ |
| 115921 | + fts3EvalZeroPoslist(p->pPhrase); |
| 115922 | + } |
| 115923 | + } |
| 115924 | + |
| 115925 | + break; |
| 115926 | + |
| 115927 | + case FTSQUERY_OR: { |
| 115928 | + int bHit1 = fts3EvalDeferredTest(pCsr, pExpr->pLeft, pRc); |
| 115929 | + int bHit2 = fts3EvalDeferredTest(pCsr, pExpr->pRight, pRc); |
| 115930 | + bHit = bHit1 || bHit2; |
| 115931 | + break; |
| 115932 | + } |
| 115933 | + |
| 115934 | + case FTSQUERY_NOT: |
| 115935 | + bHit = ( |
| 115936 | + fts3EvalDeferredTest(pCsr, pExpr->pLeft, pRc) |
| 115937 | + && !fts3EvalDeferredTest(pCsr, pExpr->pRight, pRc) |
| 115938 | + ); |
| 115939 | + break; |
| 115940 | + |
| 115941 | + default: { |
| 115942 | + if( pCsr->pDeferred |
| 115943 | + && (pExpr->iDocid==pCsr->iPrevId || pExpr->bDeferred) |
| 115944 | + ){ |
| 115945 | + Fts3Phrase *pPhrase = pExpr->pPhrase; |
| 115946 | + assert( pExpr->bDeferred || pPhrase->doclist.bFreeList==0 ); |
| 115947 | + if( pExpr->bDeferred ){ |
| 115948 | + fts3EvalZeroPoslist(pPhrase); |
| 115949 | + } |
| 115950 | + *pRc = fts3EvalDeferredPhrase(pCsr, pPhrase); |
| 115951 | + bHit = (pPhrase->doclist.pList!=0); |
| 115952 | + pExpr->iDocid = pCsr->iPrevId; |
| 115953 | + }else{ |
| 115954 | + bHit = (pExpr->bEof==0 && pExpr->iDocid==pCsr->iPrevId); |
| 115955 | + } |
| 115956 | + break; |
| 115957 | + } |
| 115958 | + } |
| 115959 | + } |
| 115960 | + return bHit; |
| 115961 | +} |
| 115962 | + |
| 115963 | +/* |
| 115964 | +** Return 1 if both of the following are true: |
| 115965 | +** |
| 115966 | +** 1. *pRc is SQLITE_OK when this function returns, and |
| 115967 | +** |
| 115968 | +** 2. After scanning the current FTS table row for the deferred tokens, |
| 115969 | +** it is determined that the row does not match the query. |
| 115970 | +** |
| 115971 | +** Or, if no error occurs and it seems the current row does match the FTS |
| 115972 | +** query, return 0. |
| 115973 | +*/ |
| 115974 | +static int fts3EvalLoadDeferred(Fts3Cursor *pCsr, int *pRc){ |
| 115975 | + int rc = *pRc; |
| 115976 | + int bMiss = 0; |
| 115977 | + if( rc==SQLITE_OK ){ |
| 115978 | + if( pCsr->pDeferred ){ |
| 115979 | + rc = fts3CursorSeek(0, pCsr); |
| 115980 | + if( rc==SQLITE_OK ){ |
| 115981 | + rc = sqlite3Fts3CacheDeferredDoclists(pCsr); |
| 115982 | + } |
| 115983 | + } |
| 115984 | + bMiss = (0==fts3EvalDeferredTest(pCsr, pCsr->pExpr, &rc)); |
| 115985 | + sqlite3Fts3FreeDeferredDoclists(pCsr); |
| 115986 | + *pRc = rc; |
| 115987 | + } |
| 115988 | + return (rc==SQLITE_OK && bMiss); |
| 115989 | +} |
| 115990 | + |
| 115991 | +/* |
| 115992 | +** Advance to the next document that matches the FTS expression in |
| 115993 | +** Fts3Cursor.pExpr. |
| 115994 | +*/ |
| 115995 | +SQLITE_PRIVATE int sqlite3Fts3EvalNext(Fts3Cursor *pCsr){ |
| 115996 | + int rc = SQLITE_OK; /* Return Code */ |
| 115997 | + Fts3Expr *pExpr = pCsr->pExpr; |
| 115998 | + assert( pCsr->isEof==0 ); |
| 115999 | + if( pExpr==0 ){ |
| 116000 | + pCsr->isEof = 1; |
| 116001 | + }else{ |
| 116002 | + do { |
| 116003 | + if( pCsr->isRequireSeek==0 ){ |
| 116004 | + sqlite3_reset(pCsr->pStmt); |
| 116005 | + } |
| 116006 | + assert( sqlite3_data_count(pCsr->pStmt)==0 ); |
| 116007 | + fts3EvalNext(pCsr, pExpr, &rc); |
| 116008 | + pCsr->isEof = pExpr->bEof; |
| 116009 | + pCsr->isRequireSeek = 1; |
| 116010 | + pCsr->isMatchinfoNeeded = 1; |
| 116011 | + pCsr->iPrevId = pExpr->iDocid; |
| 116012 | + }while( pCsr->isEof==0 && fts3EvalLoadDeferred(pCsr, &rc) ); |
| 116013 | + } |
| 116014 | + return rc; |
| 116015 | +} |
| 116016 | + |
| 116017 | +/* |
| 116018 | +** Restart interation for expression pExpr so that the next call to |
| 116019 | +** sqlite3Fts3EvalNext() visits the first row. Do not allow incremental |
| 116020 | +** loading or merging of phrase doclists for this iteration. |
| 116021 | +** |
| 116022 | +** If *pRc is other than SQLITE_OK when this function is called, it is |
| 116023 | +** a no-op. If an error occurs within this function, *pRc is set to an |
| 116024 | +** SQLite error code before returning. |
| 116025 | +*/ |
| 116026 | +static void fts3EvalRestart( |
| 116027 | + Fts3Cursor *pCsr, |
| 116028 | + Fts3Expr *pExpr, |
| 116029 | + int *pRc |
| 116030 | +){ |
| 116031 | + if( pExpr && *pRc==SQLITE_OK ){ |
| 116032 | + Fts3Phrase *pPhrase = pExpr->pPhrase; |
| 116033 | + |
| 116034 | + if( pPhrase ){ |
| 116035 | + fts3EvalZeroPoslist(pPhrase); |
| 116036 | + if( pPhrase->bIncr ){ |
| 116037 | + sqlite3Fts3EvalPhraseCleanup(pPhrase); |
| 116038 | + memset(&pPhrase->doclist, 0, sizeof(Fts3Doclist)); |
| 116039 | + *pRc = sqlite3Fts3EvalStart(pCsr, pExpr, 0); |
| 116040 | + }else{ |
| 116041 | + pPhrase->doclist.pNextDocid = 0; |
| 116042 | + pPhrase->doclist.iDocid = 0; |
| 116043 | + } |
| 116044 | + } |
| 116045 | + |
| 116046 | + pExpr->iDocid = 0; |
| 116047 | + pExpr->bEof = 0; |
| 116048 | + pExpr->bStart = 0; |
| 116049 | + |
| 116050 | + fts3EvalRestart(pCsr, pExpr->pLeft, pRc); |
| 116051 | + fts3EvalRestart(pCsr, pExpr->pRight, pRc); |
| 116052 | + } |
| 116053 | +} |
| 116054 | + |
| 116055 | +/* |
| 116056 | +** After allocating the Fts3Expr.aMI[] array for each phrase in the |
| 116057 | +** expression rooted at pExpr, the cursor iterates through all rows matched |
| 116058 | +** by pExpr, calling this function for each row. This function increments |
| 116059 | +** the values in Fts3Expr.aMI[] according to the position-list currently |
| 116060 | +** found in Fts3Expr.pPhrase->doclist.pList for each of the phrase |
| 116061 | +** expression nodes. |
| 116062 | +*/ |
| 116063 | +static void fts3EvalUpdateCounts(Fts3Expr *pExpr){ |
| 116064 | + if( pExpr ){ |
| 116065 | + Fts3Phrase *pPhrase = pExpr->pPhrase; |
| 116066 | + if( pPhrase && pPhrase->doclist.pList ){ |
| 116067 | + int iCol = 0; |
| 116068 | + char *p = pPhrase->doclist.pList; |
| 116069 | + |
| 116070 | + assert( *p ); |
| 116071 | + while( 1 ){ |
| 116072 | + u8 c = 0; |
| 116073 | + int iCnt = 0; |
| 116074 | + while( 0xFE & (*p | c) ){ |
| 116075 | + if( (c&0x80)==0 ) iCnt++; |
| 116076 | + c = *p++ & 0x80; |
| 116077 | + } |
| 116078 | + |
| 116079 | + /* aMI[iCol*3 + 1] = Number of occurrences |
| 116080 | + ** aMI[iCol*3 + 2] = Number of rows containing at least one instance |
| 116081 | + */ |
| 116082 | + pExpr->aMI[iCol*3 + 1] += iCnt; |
| 116083 | + pExpr->aMI[iCol*3 + 2] += (iCnt>0); |
| 116084 | + if( *p==0x00 ) break; |
| 116085 | + p++; |
| 116086 | + p += sqlite3Fts3GetVarint32(p, &iCol); |
| 116087 | + } |
| 116088 | + } |
| 116089 | + |
| 116090 | + fts3EvalUpdateCounts(pExpr->pLeft); |
| 116091 | + fts3EvalUpdateCounts(pExpr->pRight); |
| 116092 | + } |
| 116093 | +} |
| 116094 | + |
| 116095 | +/* |
| 116096 | +** Expression pExpr must be of type FTSQUERY_PHRASE. |
| 116097 | +** |
| 116098 | +** If it is not already allocated and populated, this function allocates and |
| 116099 | +** populates the Fts3Expr.aMI[] array for expression pExpr. If pExpr is part |
| 116100 | +** of a NEAR expression, then it also allocates and populates the same array |
| 116101 | +** for all other phrases that are part of the NEAR expression. |
| 116102 | +** |
| 116103 | +** SQLITE_OK is returned if the aMI[] array is successfully allocated and |
| 116104 | +** populated. Otherwise, if an error occurs, an SQLite error code is returned. |
| 116105 | +*/ |
| 116106 | +static int fts3EvalGatherStats( |
| 116107 | + Fts3Cursor *pCsr, /* Cursor object */ |
| 116108 | + Fts3Expr *pExpr /* FTSQUERY_PHRASE expression */ |
| 116109 | +){ |
| 116110 | + int rc = SQLITE_OK; /* Return code */ |
| 116111 | + |
| 116112 | + assert( pExpr->eType==FTSQUERY_PHRASE ); |
| 116113 | + if( pExpr->aMI==0 ){ |
| 116114 | + Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab; |
| 116115 | + Fts3Expr *pRoot; /* Root of NEAR expression */ |
| 116116 | + Fts3Expr *p; /* Iterator used for several purposes */ |
| 116117 | + |
| 116118 | + sqlite3_int64 iPrevId = pCsr->iPrevId; |
| 116119 | + sqlite3_int64 iDocid; |
| 116120 | + u8 bEof; |
| 116121 | + |
| 116122 | + /* Find the root of the NEAR expression */ |
| 116123 | + pRoot = pExpr; |
| 116124 | + while( pRoot->pParent && pRoot->pParent->eType==FTSQUERY_NEAR ){ |
| 116125 | + pRoot = pRoot->pParent; |
| 116126 | + } |
| 116127 | + iDocid = pRoot->iDocid; |
| 116128 | + bEof = pRoot->bEof; |
| 116129 | + assert( pRoot->bStart ); |
| 116130 | + |
| 116131 | + /* Allocate space for the aMSI[] array of each FTSQUERY_PHRASE node */ |
| 116132 | + for(p=pRoot; p; p=p->pLeft){ |
| 116133 | + Fts3Expr *pE = (p->eType==FTSQUERY_PHRASE?p:p->pRight); |
| 116134 | + assert( pE->aMI==0 ); |
| 116135 | + pE->aMI = (u32 *)sqlite3_malloc(pTab->nColumn * 3 * sizeof(u32)); |
| 116136 | + if( !pE->aMI ) return SQLITE_NOMEM; |
| 116137 | + memset(pE->aMI, 0, pTab->nColumn * 3 * sizeof(u32)); |
| 116138 | + } |
| 116139 | + |
| 116140 | + fts3EvalRestart(pCsr, pRoot, &rc); |
| 116141 | + |
| 116142 | + while( pCsr->isEof==0 && rc==SQLITE_OK ){ |
| 116143 | + |
| 116144 | + do { |
| 116145 | + /* Ensure the %_content statement is reset. */ |
| 116146 | + if( pCsr->isRequireSeek==0 ) sqlite3_reset(pCsr->pStmt); |
| 116147 | + assert( sqlite3_data_count(pCsr->pStmt)==0 ); |
| 116148 | + |
| 116149 | + /* Advance to the next document */ |
| 116150 | + fts3EvalNext(pCsr, pRoot, &rc); |
| 116151 | + pCsr->isEof = pRoot->bEof; |
| 116152 | + pCsr->isRequireSeek = 1; |
| 116153 | + pCsr->isMatchinfoNeeded = 1; |
| 116154 | + pCsr->iPrevId = pRoot->iDocid; |
| 116155 | + }while( pCsr->isEof==0 |
| 116156 | + && pRoot->eType==FTSQUERY_NEAR |
| 116157 | + && fts3EvalLoadDeferred(pCsr, &rc) |
| 116158 | + ); |
| 116159 | + |
| 116160 | + if( rc==SQLITE_OK && pCsr->isEof==0 ){ |
| 116161 | + fts3EvalUpdateCounts(pRoot); |
| 116162 | + } |
| 116163 | + } |
| 116164 | + |
| 116165 | + pCsr->isEof = 0; |
| 116166 | + pCsr->iPrevId = iPrevId; |
| 116167 | + |
| 116168 | + if( bEof ){ |
| 116169 | + pRoot->bEof = bEof; |
| 116170 | + }else{ |
| 116171 | + /* Caution: pRoot may iterate through docids in ascending or descending |
| 116172 | + ** order. For this reason, even though it seems more defensive, the |
| 116173 | + ** do loop can not be written: |
| 116174 | + ** |
| 116175 | + ** do {...} while( pRoot->iDocid<iDocid && rc==SQLITE_OK ); |
| 116176 | + */ |
| 116177 | + fts3EvalRestart(pCsr, pRoot, &rc); |
| 116178 | + do { |
| 116179 | + fts3EvalNext(pCsr, pRoot, &rc); |
| 116180 | + assert( pRoot->bEof==0 ); |
| 116181 | + }while( pRoot->iDocid!=iDocid && rc==SQLITE_OK ); |
| 116182 | + fts3EvalLoadDeferred(pCsr, &rc); |
| 116183 | + } |
| 116184 | + } |
| 116185 | + return rc; |
| 116186 | +} |
| 116187 | + |
| 116188 | +/* |
| 116189 | +** This function is used by the matchinfo() module to query a phrase |
| 116190 | +** expression node for the following information: |
| 116191 | +** |
| 116192 | +** 1. The total number of occurrences of the phrase in each column of |
| 116193 | +** the FTS table (considering all rows), and |
| 116194 | +** |
| 116195 | +** 2. For each column, the number of rows in the table for which the |
| 116196 | +** column contains at least one instance of the phrase. |
| 116197 | +** |
| 116198 | +** If no error occurs, SQLITE_OK is returned and the values for each column |
| 116199 | +** written into the array aiOut as follows: |
| 116200 | +** |
| 116201 | +** aiOut[iCol*3 + 1] = Number of occurrences |
| 116202 | +** aiOut[iCol*3 + 2] = Number of rows containing at least one instance |
| 116203 | +** |
| 116204 | +** Caveats: |
| 116205 | +** |
| 116206 | +** * If a phrase consists entirely of deferred tokens, then all output |
| 116207 | +** values are set to the number of documents in the table. In other |
| 116208 | +** words we assume that very common tokens occur exactly once in each |
| 116209 | +** column of each row of the table. |
| 116210 | +** |
| 116211 | +** * If a phrase contains some deferred tokens (and some non-deferred |
| 116212 | +** tokens), count the potential occurrence identified by considering |
| 116213 | +** the non-deferred tokens instead of actual phrase occurrences. |
| 116214 | +** |
| 116215 | +** * If the phrase is part of a NEAR expression, then only phrase instances |
| 116216 | +** that meet the NEAR constraint are included in the counts. |
| 116217 | +*/ |
| 116218 | +SQLITE_PRIVATE int sqlite3Fts3EvalPhraseStats( |
| 116219 | + Fts3Cursor *pCsr, /* FTS cursor handle */ |
| 116220 | + Fts3Expr *pExpr, /* Phrase expression */ |
| 116221 | + u32 *aiOut /* Array to write results into (see above) */ |
| 116222 | +){ |
| 116223 | + Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab; |
| 116224 | + int rc = SQLITE_OK; |
| 116225 | + int iCol; |
| 116226 | + |
| 116227 | + if( pExpr->bDeferred && pExpr->pParent->eType!=FTSQUERY_NEAR ){ |
| 116228 | + assert( pCsr->nDoc>0 ); |
| 116229 | + for(iCol=0; iCol<pTab->nColumn; iCol++){ |
| 116230 | + aiOut[iCol*3 + 1] = pCsr->nDoc; |
| 116231 | + aiOut[iCol*3 + 2] = pCsr->nDoc; |
| 116232 | + } |
| 116233 | + }else{ |
| 116234 | + rc = fts3EvalGatherStats(pCsr, pExpr); |
| 116235 | + if( rc==SQLITE_OK ){ |
| 116236 | + assert( pExpr->aMI ); |
| 116237 | + for(iCol=0; iCol<pTab->nColumn; iCol++){ |
| 116238 | + aiOut[iCol*3 + 1] = pExpr->aMI[iCol*3 + 1]; |
| 116239 | + aiOut[iCol*3 + 2] = pExpr->aMI[iCol*3 + 2]; |
| 116240 | + } |
| 116241 | + } |
| 116242 | + } |
| 116243 | + |
| 116244 | + return rc; |
| 116245 | +} |
| 116246 | + |
| 116247 | +/* |
| 116248 | +** The expression pExpr passed as the second argument to this function |
| 116249 | +** must be of type FTSQUERY_PHRASE. |
| 116250 | +** |
| 116251 | +** The returned value is either NULL or a pointer to a buffer containing |
| 116252 | +** a position-list indicating the occurrences of the phrase in column iCol |
| 116253 | +** of the current row. |
| 116254 | +** |
| 116255 | +** More specifically, the returned buffer contains 1 varint for each |
| 116256 | +** occurence of the phrase in the column, stored using the normal (delta+2) |
| 116257 | +** compression and is terminated by either an 0x01 or 0x00 byte. For example, |
| 116258 | +** if the requested column contains "a b X c d X X" and the position-list |
| 116259 | +** for 'X' is requested, the buffer returned may contain: |
| 116260 | +** |
| 116261 | +** 0x04 0x05 0x03 0x01 or 0x04 0x05 0x03 0x00 |
| 116262 | +** |
| 116263 | +** This function works regardless of whether or not the phrase is deferred, |
| 116264 | +** incremental, or neither. |
| 116265 | +*/ |
| 116266 | +SQLITE_PRIVATE char *sqlite3Fts3EvalPhrasePoslist( |
| 116267 | + Fts3Cursor *pCsr, /* FTS3 cursor object */ |
| 116268 | + Fts3Expr *pExpr, /* Phrase to return doclist for */ |
| 116269 | + int iCol /* Column to return position list for */ |
| 116270 | +){ |
| 116271 | + Fts3Phrase *pPhrase = pExpr->pPhrase; |
| 116272 | + Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab; |
| 116273 | + char *pIter = pPhrase->doclist.pList; |
| 116274 | + int iThis; |
| 116275 | + |
| 116276 | + assert( iCol>=0 && iCol<pTab->nColumn ); |
| 116277 | + if( !pIter |
| 116278 | + || pExpr->bEof |
| 116279 | + || pExpr->iDocid!=pCsr->iPrevId |
| 116280 | + || (pPhrase->iColumn<pTab->nColumn && pPhrase->iColumn!=iCol) |
| 116281 | + ){ |
| 116282 | + return 0; |
| 116283 | + } |
| 116284 | + |
| 116285 | + assert( pPhrase->doclist.nList>0 ); |
| 116286 | + if( *pIter==0x01 ){ |
| 116287 | + pIter++; |
| 116288 | + pIter += sqlite3Fts3GetVarint32(pIter, &iThis); |
| 116289 | + }else{ |
| 116290 | + iThis = 0; |
| 116291 | + } |
| 116292 | + while( iThis<iCol ){ |
| 116293 | + fts3ColumnlistCopy(0, &pIter); |
| 116294 | + if( *pIter==0x00 ) return 0; |
| 116295 | + pIter++; |
| 116296 | + pIter += sqlite3Fts3GetVarint32(pIter, &iThis); |
| 116297 | + } |
| 116298 | + |
| 116299 | + return ((iCol==iThis)?pIter:0); |
| 116300 | +} |
| 116301 | + |
| 116302 | +/* |
| 116303 | +** Free all components of the Fts3Phrase structure that were allocated by |
| 116304 | +** the eval module. Specifically, this means to free: |
| 116305 | +** |
| 116306 | +** * the contents of pPhrase->doclist, and |
| 116307 | +** * any Fts3MultiSegReader objects held by phrase tokens. |
| 116308 | +*/ |
| 116309 | +SQLITE_PRIVATE void sqlite3Fts3EvalPhraseCleanup(Fts3Phrase *pPhrase){ |
| 116310 | + if( pPhrase ){ |
| 116311 | + int i; |
| 116312 | + sqlite3_free(pPhrase->doclist.aAll); |
| 116313 | + fts3EvalZeroPoslist(pPhrase); |
| 116314 | + memset(&pPhrase->doclist, 0, sizeof(Fts3Doclist)); |
| 116315 | + for(i=0; i<pPhrase->nToken; i++){ |
| 116316 | + fts3SegReaderCursorFree(pPhrase->aToken[i].pSegcsr); |
| 116317 | + pPhrase->aToken[i].pSegcsr = 0; |
| 116318 | + } |
| 116319 | + } |
| 116320 | +} |
| 116321 | + |
| 115622 | 116322 | #endif |
| 115623 | 116323 | |
| 115624 | 116324 | /************** End of fts3.c ************************************************/ |
| 115625 | 116325 | /************** Begin file fts3_aux.c ****************************************/ |
| 115626 | 116326 | /* |
| | @@ -115648,11 +116348,11 @@ |
| 115648 | 116348 | Fts3Table *pFts3Tab; |
| 115649 | 116349 | }; |
| 115650 | 116350 | |
| 115651 | 116351 | struct Fts3auxCursor { |
| 115652 | 116352 | sqlite3_vtab_cursor base; /* Base class used by SQLite core */ |
| 115653 | | - Fts3SegReaderCursor csr; /* Must be right after "base" */ |
| 116353 | + Fts3MultiSegReader csr; /* Must be right after "base" */ |
| 115654 | 116354 | Fts3SegFilter filter; |
| 115655 | 116355 | char *zStop; |
| 115656 | 116356 | int nStop; /* Byte-length of string zStop */ |
| 115657 | 116357 | int isEof; /* True if cursor is at EOF */ |
| 115658 | 116358 | sqlite3_int64 iRowid; /* Current rowid */ |
| | @@ -115716,10 +116416,11 @@ |
| 115716 | 116416 | |
| 115717 | 116417 | p->pFts3Tab = (Fts3Table *)&p[1]; |
| 115718 | 116418 | p->pFts3Tab->zDb = (char *)&p->pFts3Tab[1]; |
| 115719 | 116419 | p->pFts3Tab->zName = &p->pFts3Tab->zDb[nDb+1]; |
| 115720 | 116420 | p->pFts3Tab->db = db; |
| 116421 | + p->pFts3Tab->nIndex = 1; |
| 115721 | 116422 | |
| 115722 | 116423 | memcpy((char *)p->pFts3Tab->zDb, zDb, nDb); |
| 115723 | 116424 | memcpy((char *)p->pFts3Tab->zName, zFts3, nFts3); |
| 115724 | 116425 | sqlite3Fts3Dequote((char *)p->pFts3Tab->zName); |
| 115725 | 116426 | |
| | @@ -115996,11 +116697,11 @@ |
| 115996 | 116697 | pCsr->zStop = sqlite3_mprintf("%s", sqlite3_value_text(apVal[iIdx])); |
| 115997 | 116698 | pCsr->nStop = sqlite3_value_bytes(apVal[iIdx]); |
| 115998 | 116699 | if( pCsr->zStop==0 ) return SQLITE_NOMEM; |
| 115999 | 116700 | } |
| 116000 | 116701 | |
| 116001 | | - rc = sqlite3Fts3SegReaderCursor(pFts3, FTS3_SEGCURSOR_ALL, |
| 116702 | + rc = sqlite3Fts3SegReaderCursor(pFts3, 0, FTS3_SEGCURSOR_ALL, |
| 116002 | 116703 | pCsr->filter.zTerm, pCsr->filter.nTerm, 0, isScan, &pCsr->csr |
| 116003 | 116704 | ); |
| 116004 | 116705 | if( rc==SQLITE_OK ){ |
| 116005 | 116706 | rc = sqlite3Fts3SegReaderStart(pFts3, &pCsr->csr, &pCsr->filter); |
| 116006 | 116707 | } |
| | @@ -116175,16 +116876,25 @@ |
| 116175 | 116876 | ** Default span for NEAR operators. |
| 116176 | 116877 | */ |
| 116177 | 116878 | #define SQLITE_FTS3_DEFAULT_NEAR_PARAM 10 |
| 116178 | 116879 | |
| 116179 | 116880 | |
| 116881 | +/* |
| 116882 | +** isNot: |
| 116883 | +** This variable is used by function getNextNode(). When getNextNode() is |
| 116884 | +** called, it sets ParseContext.isNot to true if the 'next node' is a |
| 116885 | +** FTSQUERY_PHRASE with a unary "-" attached to it. i.e. "mysql" in the |
| 116886 | +** FTS3 query "sqlite -mysql". Otherwise, ParseContext.isNot is set to |
| 116887 | +** zero. |
| 116888 | +*/ |
| 116180 | 116889 | typedef struct ParseContext ParseContext; |
| 116181 | 116890 | struct ParseContext { |
| 116182 | 116891 | sqlite3_tokenizer *pTokenizer; /* Tokenizer module */ |
| 116183 | 116892 | const char **azCol; /* Array of column names for fts3 table */ |
| 116184 | 116893 | int nCol; /* Number of entries in azCol[] */ |
| 116185 | 116894 | int iDefaultCol; /* Default column to query */ |
| 116895 | + int isNot; /* True if getNextNode() sees a unary - */ |
| 116186 | 116896 | sqlite3_context *pCtx; /* Write error message here */ |
| 116187 | 116897 | int nNest; /* Number of nested brackets */ |
| 116188 | 116898 | }; |
| 116189 | 116899 | |
| 116190 | 116900 | /* |
| | @@ -116266,11 +116976,11 @@ |
| 116266 | 116976 | if( iEnd<n && z[iEnd]=='*' ){ |
| 116267 | 116977 | pRet->pPhrase->aToken[0].isPrefix = 1; |
| 116268 | 116978 | iEnd++; |
| 116269 | 116979 | } |
| 116270 | 116980 | if( !sqlite3_fts3_enable_parentheses && iStart>0 && z[iStart-1]=='-' ){ |
| 116271 | | - pRet->pPhrase->isNot = 1; |
| 116981 | + pParse->isNot = 1; |
| 116272 | 116982 | } |
| 116273 | 116983 | } |
| 116274 | 116984 | nConsumed = iEnd; |
| 116275 | 116985 | } |
| 116276 | 116986 | |
| | @@ -116318,71 +117028,86 @@ |
| 116318 | 117028 | Fts3Expr *p = 0; |
| 116319 | 117029 | sqlite3_tokenizer_cursor *pCursor = 0; |
| 116320 | 117030 | char *zTemp = 0; |
| 116321 | 117031 | int nTemp = 0; |
| 116322 | 117032 | |
| 117033 | + const int nSpace = sizeof(Fts3Expr) + sizeof(Fts3Phrase); |
| 117034 | + int nToken = 0; |
| 117035 | + |
| 117036 | + /* The final Fts3Expr data structure, including the Fts3Phrase, |
| 117037 | + ** Fts3PhraseToken structures token buffers are all stored as a single |
| 117038 | + ** allocation so that the expression can be freed with a single call to |
| 117039 | + ** sqlite3_free(). Setting this up requires a two pass approach. |
| 117040 | + ** |
| 117041 | + ** The first pass, in the block below, uses a tokenizer cursor to iterate |
| 117042 | + ** through the tokens in the expression. This pass uses fts3ReallocOrFree() |
| 117043 | + ** to assemble data in two dynamic buffers: |
| 117044 | + ** |
| 117045 | + ** Buffer p: Points to the Fts3Expr structure, followed by the Fts3Phrase |
| 117046 | + ** structure, followed by the array of Fts3PhraseToken |
| 117047 | + ** structures. This pass only populates the Fts3PhraseToken array. |
| 117048 | + ** |
| 117049 | + ** Buffer zTemp: Contains copies of all tokens. |
| 117050 | + ** |
| 117051 | + ** The second pass, in the block that begins "if( rc==SQLITE_DONE )" below, |
| 117052 | + ** appends buffer zTemp to buffer p, and fills in the Fts3Expr and Fts3Phrase |
| 117053 | + ** structures. |
| 117054 | + */ |
| 116323 | 117055 | rc = pModule->xOpen(pTokenizer, zInput, nInput, &pCursor); |
| 116324 | 117056 | if( rc==SQLITE_OK ){ |
| 116325 | 117057 | int ii; |
| 116326 | 117058 | pCursor->pTokenizer = pTokenizer; |
| 116327 | 117059 | for(ii=0; rc==SQLITE_OK; ii++){ |
| 116328 | | - const char *zToken; |
| 116329 | | - int nToken, iBegin, iEnd, iPos; |
| 116330 | | - rc = pModule->xNext(pCursor, &zToken, &nToken, &iBegin, &iEnd, &iPos); |
| 117060 | + const char *zByte; |
| 117061 | + int nByte, iBegin, iEnd, iPos; |
| 117062 | + rc = pModule->xNext(pCursor, &zByte, &nByte, &iBegin, &iEnd, &iPos); |
| 116331 | 117063 | if( rc==SQLITE_OK ){ |
| 116332 | | - int nByte = sizeof(Fts3Expr) + sizeof(Fts3Phrase); |
| 116333 | | - p = fts3ReallocOrFree(p, nByte+ii*sizeof(Fts3PhraseToken)); |
| 116334 | | - zTemp = fts3ReallocOrFree(zTemp, nTemp + nToken); |
| 116335 | | - if( !p || !zTemp ){ |
| 116336 | | - goto no_mem; |
| 116337 | | - } |
| 116338 | | - if( ii==0 ){ |
| 116339 | | - memset(p, 0, nByte); |
| 116340 | | - p->pPhrase = (Fts3Phrase *)&p[1]; |
| 116341 | | - } |
| 116342 | | - p->pPhrase = (Fts3Phrase *)&p[1]; |
| 116343 | | - memset(&p->pPhrase->aToken[ii], 0, sizeof(Fts3PhraseToken)); |
| 116344 | | - p->pPhrase->nToken = ii+1; |
| 116345 | | - p->pPhrase->aToken[ii].n = nToken; |
| 116346 | | - memcpy(&zTemp[nTemp], zToken, nToken); |
| 116347 | | - nTemp += nToken; |
| 116348 | | - if( iEnd<nInput && zInput[iEnd]=='*' ){ |
| 116349 | | - p->pPhrase->aToken[ii].isPrefix = 1; |
| 116350 | | - }else{ |
| 116351 | | - p->pPhrase->aToken[ii].isPrefix = 0; |
| 116352 | | - } |
| 117064 | + Fts3PhraseToken *pToken; |
| 117065 | + |
| 117066 | + p = fts3ReallocOrFree(p, nSpace + ii*sizeof(Fts3PhraseToken)); |
| 117067 | + if( !p ) goto no_mem; |
| 117068 | + |
| 117069 | + zTemp = fts3ReallocOrFree(zTemp, nTemp + nByte); |
| 117070 | + if( !zTemp ) goto no_mem; |
| 117071 | + |
| 117072 | + assert( nToken==ii ); |
| 117073 | + pToken = &((Fts3Phrase *)(&p[1]))->aToken[ii]; |
| 117074 | + memset(pToken, 0, sizeof(Fts3PhraseToken)); |
| 117075 | + |
| 117076 | + memcpy(&zTemp[nTemp], zByte, nByte); |
| 117077 | + nTemp += nByte; |
| 117078 | + |
| 117079 | + pToken->n = nByte; |
| 117080 | + pToken->isPrefix = (iEnd<nInput && zInput[iEnd]=='*'); |
| 117081 | + nToken = ii+1; |
| 116353 | 117082 | } |
| 116354 | 117083 | } |
| 116355 | 117084 | |
| 116356 | 117085 | pModule->xClose(pCursor); |
| 116357 | 117086 | pCursor = 0; |
| 116358 | 117087 | } |
| 116359 | 117088 | |
| 116360 | 117089 | if( rc==SQLITE_DONE ){ |
| 116361 | 117090 | int jj; |
| 116362 | | - char *zNew = NULL; |
| 116363 | | - int nNew = 0; |
| 116364 | | - int nByte = sizeof(Fts3Expr) + sizeof(Fts3Phrase); |
| 116365 | | - nByte += (p?(p->pPhrase->nToken-1):0) * sizeof(Fts3PhraseToken); |
| 116366 | | - p = fts3ReallocOrFree(p, nByte + nTemp); |
| 116367 | | - if( !p ){ |
| 116368 | | - goto no_mem; |
| 116369 | | - } |
| 116370 | | - if( zTemp ){ |
| 116371 | | - zNew = &(((char *)p)[nByte]); |
| 116372 | | - memcpy(zNew, zTemp, nTemp); |
| 116373 | | - }else{ |
| 116374 | | - memset(p, 0, nByte+nTemp); |
| 116375 | | - } |
| 117091 | + char *zBuf = 0; |
| 117092 | + |
| 117093 | + p = fts3ReallocOrFree(p, nSpace + nToken*sizeof(Fts3PhraseToken) + nTemp); |
| 117094 | + if( !p ) goto no_mem; |
| 117095 | + memset(p, 0, (char *)&(((Fts3Phrase *)&p[1])->aToken[0])-(char *)p); |
| 117096 | + p->eType = FTSQUERY_PHRASE; |
| 116376 | 117097 | p->pPhrase = (Fts3Phrase *)&p[1]; |
| 117098 | + p->pPhrase->iColumn = pParse->iDefaultCol; |
| 117099 | + p->pPhrase->nToken = nToken; |
| 117100 | + |
| 117101 | + zBuf = (char *)&p->pPhrase->aToken[nToken]; |
| 117102 | + memcpy(zBuf, zTemp, nTemp); |
| 117103 | + sqlite3_free(zTemp); |
| 117104 | + |
| 116377 | 117105 | for(jj=0; jj<p->pPhrase->nToken; jj++){ |
| 116378 | | - p->pPhrase->aToken[jj].z = &zNew[nNew]; |
| 116379 | | - nNew += p->pPhrase->aToken[jj].n; |
| 117106 | + p->pPhrase->aToken[jj].z = zBuf; |
| 117107 | + zBuf += p->pPhrase->aToken[jj].n; |
| 116380 | 117108 | } |
| 116381 | | - sqlite3_free(zTemp); |
| 116382 | | - p->eType = FTSQUERY_PHRASE; |
| 116383 | | - p->pPhrase->iColumn = pParse->iDefaultCol; |
| 116384 | 117109 | rc = SQLITE_OK; |
| 116385 | 117110 | } |
| 116386 | 117111 | |
| 116387 | 117112 | *ppExpr = p; |
| 116388 | 117113 | return rc; |
| | @@ -116434,10 +117159,12 @@ |
| 116434 | 117159 | int rc; |
| 116435 | 117160 | Fts3Expr *pRet = 0; |
| 116436 | 117161 | |
| 116437 | 117162 | const char *zInput = z; |
| 116438 | 117163 | int nInput = n; |
| 117164 | + |
| 117165 | + pParse->isNot = 0; |
| 116439 | 117166 | |
| 116440 | 117167 | /* Skip over any whitespace before checking for a keyword, an open or |
| 116441 | 117168 | ** close bracket, or a quoted string. |
| 116442 | 117169 | */ |
| 116443 | 117170 | while( nInput>0 && fts3isspace(*zInput) ){ |
| | @@ -116653,11 +117380,11 @@ |
| 116653 | 117380 | rc = getNextNode(pParse, zIn, nIn, &p, &nByte); |
| 116654 | 117381 | if( rc==SQLITE_OK ){ |
| 116655 | 117382 | int isPhrase; |
| 116656 | 117383 | |
| 116657 | 117384 | if( !sqlite3_fts3_enable_parentheses |
| 116658 | | - && p->eType==FTSQUERY_PHRASE && p->pPhrase->isNot |
| 117385 | + && p->eType==FTSQUERY_PHRASE && pParse->isNot |
| 116659 | 117386 | ){ |
| 116660 | 117387 | /* Create an implicit NOT operator. */ |
| 116661 | 117388 | Fts3Expr *pNot = fts3MallocZero(sizeof(Fts3Expr)); |
| 116662 | 117389 | if( !pNot ){ |
| 116663 | 117390 | sqlite3Fts3ExprFree(p); |
| | @@ -116671,11 +117398,10 @@ |
| 116671 | 117398 | } |
| 116672 | 117399 | pNotBranch = pNot; |
| 116673 | 117400 | p = pPrev; |
| 116674 | 117401 | }else{ |
| 116675 | 117402 | int eType = p->eType; |
| 116676 | | - assert( eType!=FTSQUERY_PHRASE || !p->pPhrase->isNot ); |
| 116677 | 117403 | isPhrase = (eType==FTSQUERY_PHRASE || p->pLeft); |
| 116678 | 117404 | |
| 116679 | 117405 | /* The isRequirePhrase variable is set to true if a phrase or |
| 116680 | 117406 | ** an expression contained in parenthesis is required. If a |
| 116681 | 117407 | ** binary operator (AND, OR, NOT or NEAR) is encounted when |
| | @@ -116834,13 +117560,15 @@ |
| 116834 | 117560 | /* |
| 116835 | 117561 | ** Free a parsed fts3 query expression allocated by sqlite3Fts3ExprParse(). |
| 116836 | 117562 | */ |
| 116837 | 117563 | SQLITE_PRIVATE void sqlite3Fts3ExprFree(Fts3Expr *p){ |
| 116838 | 117564 | if( p ){ |
| 117565 | + assert( p->eType==FTSQUERY_PHRASE || p->pPhrase==0 ); |
| 116839 | 117566 | sqlite3Fts3ExprFree(p->pLeft); |
| 116840 | 117567 | sqlite3Fts3ExprFree(p->pRight); |
| 116841 | | - sqlite3_free(p->aDoclist); |
| 117568 | + sqlite3Fts3EvalPhraseCleanup(p->pPhrase); |
| 117569 | + sqlite3_free(p->aMI); |
| 116842 | 117570 | sqlite3_free(p); |
| 116843 | 117571 | } |
| 116844 | 117572 | } |
| 116845 | 117573 | |
| 116846 | 117574 | /**************************************************************************** |
| | @@ -116893,11 +117621,11 @@ |
| 116893 | 117621 | switch( pExpr->eType ){ |
| 116894 | 117622 | case FTSQUERY_PHRASE: { |
| 116895 | 117623 | Fts3Phrase *pPhrase = pExpr->pPhrase; |
| 116896 | 117624 | int i; |
| 116897 | 117625 | zBuf = sqlite3_mprintf( |
| 116898 | | - "%zPHRASE %d %d", zBuf, pPhrase->iColumn, pPhrase->isNot); |
| 117626 | + "%zPHRASE %d 0", zBuf, pPhrase->iColumn); |
| 116899 | 117627 | for(i=0; zBuf && i<pPhrase->nToken; i++){ |
| 116900 | 117628 | zBuf = sqlite3_mprintf("%z %.*s%s", zBuf, |
| 116901 | 117629 | pPhrase->aToken[i].n, pPhrase->aToken[i].z, |
| 116902 | 117630 | (pPhrase->aToken[i].isPrefix?"+":"") |
| 116903 | 117631 | ); |
| | @@ -118811,18 +119539,44 @@ |
| 118811 | 119539 | ** it is always safe to read up to two varints from it without risking an |
| 118812 | 119540 | ** overread, even if the node data is corrupted. |
| 118813 | 119541 | */ |
| 118814 | 119542 | #define FTS3_NODE_PADDING (FTS3_VARINT_MAX*2) |
| 118815 | 119543 | |
| 119544 | +/* |
| 119545 | +** Under certain circumstances, b-tree nodes (doclists) can be loaded into |
| 119546 | +** memory incrementally instead of all at once. This can be a big performance |
| 119547 | +** win (reduced IO and CPU) if SQLite stops calling the virtual table xNext() |
| 119548 | +** method before retrieving all query results (as may happen, for example, |
| 119549 | +** if a query has a LIMIT clause). |
| 119550 | +** |
| 119551 | +** Incremental loading is used for b-tree nodes FTS3_NODE_CHUNK_THRESHOLD |
| 119552 | +** bytes and larger. Nodes are loaded in chunks of FTS3_NODE_CHUNKSIZE bytes. |
| 119553 | +** The code is written so that the hard lower-limit for each of these values |
| 119554 | +** is 1. Clearly such small values would be inefficient, but can be useful |
| 119555 | +** for testing purposes. |
| 119556 | +** |
| 119557 | +** If this module is built with SQLITE_TEST defined, these constants may |
| 119558 | +** be overridden at runtime for testing purposes. File fts3_test.c contains |
| 119559 | +** a Tcl interface to read and write the values. |
| 119560 | +*/ |
| 119561 | +#ifdef SQLITE_TEST |
| 119562 | +int test_fts3_node_chunksize = (4*1024); |
| 119563 | +int test_fts3_node_chunk_threshold = (4*1024)*4; |
| 119564 | +# define FTS3_NODE_CHUNKSIZE test_fts3_node_chunksize |
| 119565 | +# define FTS3_NODE_CHUNK_THRESHOLD test_fts3_node_chunk_threshold |
| 119566 | +#else |
| 119567 | +# define FTS3_NODE_CHUNKSIZE (4*1024) |
| 119568 | +# define FTS3_NODE_CHUNK_THRESHOLD (FTS3_NODE_CHUNKSIZE*4) |
| 119569 | +#endif |
| 119570 | + |
| 118816 | 119571 | typedef struct PendingList PendingList; |
| 118817 | 119572 | typedef struct SegmentNode SegmentNode; |
| 118818 | 119573 | typedef struct SegmentWriter SegmentWriter; |
| 118819 | 119574 | |
| 118820 | 119575 | /* |
| 118821 | | -** Data structure used while accumulating terms in the pending-terms hash |
| 118822 | | -** table. The hash table entry maps from term (a string) to a malloc'd |
| 118823 | | -** instance of this structure. |
| 119576 | +** An instance of the following data structure is used to build doclists |
| 119577 | +** incrementally. See function fts3PendingListAppend() for details. |
| 118824 | 119578 | */ |
| 118825 | 119579 | struct PendingList { |
| 118826 | 119580 | int nData; |
| 118827 | 119581 | char *aData; |
| 118828 | 119582 | int nSpace; |
| | @@ -118849,11 +119603,10 @@ |
| 118849 | 119603 | ** of type Fts3SegReader* are also used by code in fts3.c to iterate through |
| 118850 | 119604 | ** terms when querying the full-text index. See functions: |
| 118851 | 119605 | ** |
| 118852 | 119606 | ** sqlite3Fts3SegReaderNew() |
| 118853 | 119607 | ** sqlite3Fts3SegReaderFree() |
| 118854 | | -** sqlite3Fts3SegReaderCost() |
| 118855 | 119608 | ** sqlite3Fts3SegReaderIterate() |
| 118856 | 119609 | ** |
| 118857 | 119610 | ** Methods used to manipulate Fts3SegReader structures: |
| 118858 | 119611 | ** |
| 118859 | 119612 | ** fts3SegReaderNext() |
| | @@ -118868,10 +119621,13 @@ |
| 118868 | 119621 | sqlite3_int64 iEndBlock; /* Rowid of final block in segment (or 0) */ |
| 118869 | 119622 | sqlite3_int64 iCurrentBlock; /* Current leaf block (or 0) */ |
| 118870 | 119623 | |
| 118871 | 119624 | char *aNode; /* Pointer to node data (or NULL) */ |
| 118872 | 119625 | int nNode; /* Size of buffer at aNode (or 0) */ |
| 119626 | + int nPopulate; /* If >0, bytes of buffer aNode[] loaded */ |
| 119627 | + sqlite3_blob *pBlob; /* If not NULL, blob handle to read node */ |
| 119628 | + |
| 118873 | 119629 | Fts3HashElem **ppNextElem; |
| 118874 | 119630 | |
| 118875 | 119631 | /* Variables set by fts3SegReaderNext(). These may be read directly |
| 118876 | 119632 | ** by the caller. They are valid from the time SegmentReaderNew() returns |
| 118877 | 119633 | ** until SegmentReaderNext() returns something other than SQLITE_OK |
| | @@ -118881,12 +119637,15 @@ |
| 118881 | 119637 | char *zTerm; /* Pointer to current term */ |
| 118882 | 119638 | int nTermAlloc; /* Allocated size of zTerm buffer */ |
| 118883 | 119639 | char *aDoclist; /* Pointer to doclist of current entry */ |
| 118884 | 119640 | int nDoclist; /* Size of doclist in current entry */ |
| 118885 | 119641 | |
| 118886 | | - /* The following variables are used to iterate through the current doclist */ |
| 119642 | + /* The following variables are used by fts3SegReaderNextDocid() to iterate |
| 119643 | + ** through the current doclist (aDoclist/nDoclist). |
| 119644 | + */ |
| 118887 | 119645 | char *pOffsetList; |
| 119646 | + int nOffsetList; /* For descending pending seg-readers only */ |
| 118888 | 119647 | sqlite3_int64 iDocid; |
| 118889 | 119648 | }; |
| 118890 | 119649 | |
| 118891 | 119650 | #define fts3SegReaderIsPending(p) ((p)->ppNextElem!=0) |
| 118892 | 119651 | #define fts3SegReaderIsRootOnly(p) ((p)->aNode==(char *)&(p)[1]) |
| | @@ -118920,10 +119679,18 @@ |
| 118920 | 119679 | ** within the fts3SegWriterXXX() family of functions described above. |
| 118921 | 119680 | ** |
| 118922 | 119681 | ** fts3NodeAddTerm() |
| 118923 | 119682 | ** fts3NodeWrite() |
| 118924 | 119683 | ** fts3NodeFree() |
| 119684 | +** |
| 119685 | +** When a b+tree is written to the database (either as a result of a merge |
| 119686 | +** or the pending-terms table being flushed), leaves are written into the |
| 119687 | +** database file as soon as they are completely populated. The interior of |
| 119688 | +** the tree is assembled in memory and written out only once all leaves have |
| 119689 | +** been populated and stored. This is Ok, as the b+-tree fanout is usually |
| 119690 | +** very large, meaning that the interior of the tree consumes relatively |
| 119691 | +** little memory. |
| 118925 | 119692 | */ |
| 118926 | 119693 | struct SegmentNode { |
| 118927 | 119694 | SegmentNode *pParent; /* Parent node (or NULL for root node) */ |
| 118928 | 119695 | SegmentNode *pRight; /* Pointer to right-sibling */ |
| 118929 | 119696 | SegmentNode *pLeftmost; /* Pointer to left-most node of this depth */ |
| | @@ -118950,21 +119717,26 @@ |
| 118950 | 119717 | #define SQL_NEXT_SEGMENT_INDEX 8 |
| 118951 | 119718 | #define SQL_INSERT_SEGMENTS 9 |
| 118952 | 119719 | #define SQL_NEXT_SEGMENTS_ID 10 |
| 118953 | 119720 | #define SQL_INSERT_SEGDIR 11 |
| 118954 | 119721 | #define SQL_SELECT_LEVEL 12 |
| 118955 | | -#define SQL_SELECT_ALL_LEVEL 13 |
| 119722 | +#define SQL_SELECT_LEVEL_RANGE 13 |
| 118956 | 119723 | #define SQL_SELECT_LEVEL_COUNT 14 |
| 118957 | | -#define SQL_SELECT_SEGDIR_COUNT_MAX 15 |
| 118958 | | -#define SQL_DELETE_SEGDIR_BY_LEVEL 16 |
| 119724 | +#define SQL_SELECT_SEGDIR_MAX_LEVEL 15 |
| 119725 | +#define SQL_DELETE_SEGDIR_LEVEL 16 |
| 118959 | 119726 | #define SQL_DELETE_SEGMENTS_RANGE 17 |
| 118960 | 119727 | #define SQL_CONTENT_INSERT 18 |
| 118961 | 119728 | #define SQL_DELETE_DOCSIZE 19 |
| 118962 | 119729 | #define SQL_REPLACE_DOCSIZE 20 |
| 118963 | 119730 | #define SQL_SELECT_DOCSIZE 21 |
| 118964 | 119731 | #define SQL_SELECT_DOCTOTAL 22 |
| 118965 | 119732 | #define SQL_REPLACE_DOCTOTAL 23 |
| 119733 | + |
| 119734 | +#define SQL_SELECT_ALL_PREFIX_LEVEL 24 |
| 119735 | +#define SQL_DELETE_ALL_TERMS_SEGDIR 25 |
| 119736 | + |
| 119737 | +#define SQL_DELETE_SEGDIR_RANGE 26 |
| 118966 | 119738 | |
| 118967 | 119739 | /* |
| 118968 | 119740 | ** This function is used to obtain an SQLite prepared statement handle |
| 118969 | 119741 | ** for the statement identified by the second argument. If successful, |
| 118970 | 119742 | ** *pp is set to the requested statement handle and SQLITE_OK returned. |
| | @@ -118997,23 +119769,29 @@ |
| 118997 | 119769 | |
| 118998 | 119770 | /* Return segments in order from oldest to newest.*/ |
| 118999 | 119771 | /* 12 */ "SELECT idx, start_block, leaves_end_block, end_block, root " |
| 119000 | 119772 | "FROM %Q.'%q_segdir' WHERE level = ? ORDER BY idx ASC", |
| 119001 | 119773 | /* 13 */ "SELECT idx, start_block, leaves_end_block, end_block, root " |
| 119002 | | - "FROM %Q.'%q_segdir' ORDER BY level DESC, idx ASC", |
| 119774 | + "FROM %Q.'%q_segdir' WHERE level BETWEEN ? AND ?" |
| 119775 | + "ORDER BY level DESC, idx ASC", |
| 119003 | 119776 | |
| 119004 | 119777 | /* 14 */ "SELECT count(*) FROM %Q.'%q_segdir' WHERE level = ?", |
| 119005 | | -/* 15 */ "SELECT count(*), max(level) FROM %Q.'%q_segdir'", |
| 119778 | +/* 15 */ "SELECT max(level) FROM %Q.'%q_segdir' WHERE level BETWEEN ? AND ?", |
| 119006 | 119779 | |
| 119007 | 119780 | /* 16 */ "DELETE FROM %Q.'%q_segdir' WHERE level = ?", |
| 119008 | 119781 | /* 17 */ "DELETE FROM %Q.'%q_segments' WHERE blockid BETWEEN ? AND ?", |
| 119009 | 119782 | /* 18 */ "INSERT INTO %Q.'%q_content' VALUES(%s)", |
| 119010 | 119783 | /* 19 */ "DELETE FROM %Q.'%q_docsize' WHERE docid = ?", |
| 119011 | 119784 | /* 20 */ "REPLACE INTO %Q.'%q_docsize' VALUES(?,?)", |
| 119012 | 119785 | /* 21 */ "SELECT size FROM %Q.'%q_docsize' WHERE docid=?", |
| 119013 | 119786 | /* 22 */ "SELECT value FROM %Q.'%q_stat' WHERE id=0", |
| 119014 | 119787 | /* 23 */ "REPLACE INTO %Q.'%q_stat' VALUES(0,?)", |
| 119788 | +/* 24 */ "", |
| 119789 | +/* 25 */ "", |
| 119790 | + |
| 119791 | +/* 26 */ "DELETE FROM %Q.'%q_segdir' WHERE level BETWEEN ? AND ?", |
| 119792 | + |
| 119015 | 119793 | }; |
| 119016 | 119794 | int rc = SQLITE_OK; |
| 119017 | 119795 | sqlite3_stmt *pStmt; |
| 119018 | 119796 | |
| 119019 | 119797 | assert( SizeofArray(azSql)==SizeofArray(p->aStmt) ); |
| | @@ -119165,18 +119943,36 @@ |
| 119165 | 119943 | ** 1: start_block |
| 119166 | 119944 | ** 2: leaves_end_block |
| 119167 | 119945 | ** 3: end_block |
| 119168 | 119946 | ** 4: root |
| 119169 | 119947 | */ |
| 119170 | | -SQLITE_PRIVATE int sqlite3Fts3AllSegdirs(Fts3Table *p, int iLevel, sqlite3_stmt **ppStmt){ |
| 119948 | +SQLITE_PRIVATE int sqlite3Fts3AllSegdirs( |
| 119949 | + Fts3Table *p, /* FTS3 table */ |
| 119950 | + int iIndex, /* Index for p->aIndex[] */ |
| 119951 | + int iLevel, /* Level to select */ |
| 119952 | + sqlite3_stmt **ppStmt /* OUT: Compiled statement */ |
| 119953 | +){ |
| 119171 | 119954 | int rc; |
| 119172 | 119955 | sqlite3_stmt *pStmt = 0; |
| 119956 | + |
| 119957 | + assert( iLevel==FTS3_SEGCURSOR_ALL || iLevel>=0 ); |
| 119958 | + assert( iLevel<FTS3_SEGDIR_MAXLEVEL ); |
| 119959 | + assert( iIndex>=0 && iIndex<p->nIndex ); |
| 119960 | + |
| 119173 | 119961 | if( iLevel<0 ){ |
| 119174 | | - rc = fts3SqlStmt(p, SQL_SELECT_ALL_LEVEL, &pStmt, 0); |
| 119962 | + /* "SELECT * FROM %_segdir WHERE level BETWEEN ? AND ? ORDER BY ..." */ |
| 119963 | + rc = fts3SqlStmt(p, SQL_SELECT_LEVEL_RANGE, &pStmt, 0); |
| 119964 | + if( rc==SQLITE_OK ){ |
| 119965 | + sqlite3_bind_int(pStmt, 1, iIndex*FTS3_SEGDIR_MAXLEVEL); |
| 119966 | + sqlite3_bind_int(pStmt, 2, (iIndex+1)*FTS3_SEGDIR_MAXLEVEL-1); |
| 119967 | + } |
| 119175 | 119968 | }else{ |
| 119969 | + /* "SELECT * FROM %_segdir WHERE level = ? ORDER BY ..." */ |
| 119176 | 119970 | rc = fts3SqlStmt(p, SQL_SELECT_LEVEL, &pStmt, 0); |
| 119177 | | - if( rc==SQLITE_OK ) sqlite3_bind_int(pStmt, 1, iLevel); |
| 119971 | + if( rc==SQLITE_OK ){ |
| 119972 | + sqlite3_bind_int(pStmt, 1, iLevel+iIndex*FTS3_SEGDIR_MAXLEVEL); |
| 119973 | + } |
| 119178 | 119974 | } |
| 119179 | 119975 | *ppStmt = pStmt; |
| 119180 | 119976 | return rc; |
| 119181 | 119977 | } |
| 119182 | 119978 | |
| | @@ -119286,10 +120082,51 @@ |
| 119286 | 120082 | *pp = p; |
| 119287 | 120083 | return 1; |
| 119288 | 120084 | } |
| 119289 | 120085 | return 0; |
| 119290 | 120086 | } |
| 120087 | + |
| 120088 | +/* |
| 120089 | +** Free a PendingList object allocated by fts3PendingListAppend(). |
| 120090 | +*/ |
| 120091 | +static void fts3PendingListDelete(PendingList *pList){ |
| 120092 | + sqlite3_free(pList); |
| 120093 | +} |
| 120094 | + |
| 120095 | +/* |
| 120096 | +** Add an entry to one of the pending-terms hash tables. |
| 120097 | +*/ |
| 120098 | +static int fts3PendingTermsAddOne( |
| 120099 | + Fts3Table *p, |
| 120100 | + int iCol, |
| 120101 | + int iPos, |
| 120102 | + Fts3Hash *pHash, /* Pending terms hash table to add entry to */ |
| 120103 | + const char *zToken, |
| 120104 | + int nToken |
| 120105 | +){ |
| 120106 | + PendingList *pList; |
| 120107 | + int rc = SQLITE_OK; |
| 120108 | + |
| 120109 | + pList = (PendingList *)fts3HashFind(pHash, zToken, nToken); |
| 120110 | + if( pList ){ |
| 120111 | + p->nPendingData -= (pList->nData + nToken + sizeof(Fts3HashElem)); |
| 120112 | + } |
| 120113 | + if( fts3PendingListAppend(&pList, p->iPrevDocid, iCol, iPos, &rc) ){ |
| 120114 | + if( pList==fts3HashInsert(pHash, zToken, nToken, pList) ){ |
| 120115 | + /* Malloc failed while inserting the new entry. This can only |
| 120116 | + ** happen if there was no previous entry for this token. |
| 120117 | + */ |
| 120118 | + assert( 0==fts3HashFind(pHash, zToken, nToken) ); |
| 120119 | + sqlite3_free(pList); |
| 120120 | + rc = SQLITE_NOMEM; |
| 120121 | + } |
| 120122 | + } |
| 120123 | + if( rc==SQLITE_OK ){ |
| 120124 | + p->nPendingData += (pList->nData + nToken + sizeof(Fts3HashElem)); |
| 120125 | + } |
| 120126 | + return rc; |
| 120127 | +} |
| 119291 | 120128 | |
| 119292 | 120129 | /* |
| 119293 | 120130 | ** Tokenize the nul-terminated string zText and add all tokens to the |
| 119294 | 120131 | ** pending-terms hash-table. The docid used is that currently stored in |
| 119295 | 120132 | ** p->iPrevDocid, and the column is specified by argument iCol. |
| | @@ -119335,12 +120172,11 @@ |
| 119335 | 120172 | |
| 119336 | 120173 | xNext = pModule->xNext; |
| 119337 | 120174 | while( SQLITE_OK==rc |
| 119338 | 120175 | && SQLITE_OK==(rc = xNext(pCsr, &zToken, &nToken, &iStart, &iEnd, &iPos)) |
| 119339 | 120176 | ){ |
| 119340 | | - PendingList *pList; |
| 119341 | | - |
| 120177 | + int i; |
| 119342 | 120178 | if( iPos>=nWord ) nWord = iPos+1; |
| 119343 | 120179 | |
| 119344 | 120180 | /* Positions cannot be negative; we use -1 as a terminator internally. |
| 119345 | 120181 | ** Tokens must have a non-zero length. |
| 119346 | 120182 | */ |
| | @@ -119347,26 +120183,23 @@ |
| 119347 | 120183 | if( iPos<0 || !zToken || nToken<=0 ){ |
| 119348 | 120184 | rc = SQLITE_ERROR; |
| 119349 | 120185 | break; |
| 119350 | 120186 | } |
| 119351 | 120187 | |
| 119352 | | - pList = (PendingList *)fts3HashFind(&p->pendingTerms, zToken, nToken); |
| 119353 | | - if( pList ){ |
| 119354 | | - p->nPendingData -= (pList->nData + nToken + sizeof(Fts3HashElem)); |
| 119355 | | - } |
| 119356 | | - if( fts3PendingListAppend(&pList, p->iPrevDocid, iCol, iPos, &rc) ){ |
| 119357 | | - if( pList==fts3HashInsert(&p->pendingTerms, zToken, nToken, pList) ){ |
| 119358 | | - /* Malloc failed while inserting the new entry. This can only |
| 119359 | | - ** happen if there was no previous entry for this token. |
| 119360 | | - */ |
| 119361 | | - assert( 0==fts3HashFind(&p->pendingTerms, zToken, nToken) ); |
| 119362 | | - sqlite3_free(pList); |
| 119363 | | - rc = SQLITE_NOMEM; |
| 119364 | | - } |
| 119365 | | - } |
| 119366 | | - if( rc==SQLITE_OK ){ |
| 119367 | | - p->nPendingData += (pList->nData + nToken + sizeof(Fts3HashElem)); |
| 120188 | + /* Add the term to the terms index */ |
| 120189 | + rc = fts3PendingTermsAddOne( |
| 120190 | + p, iCol, iPos, &p->aIndex[0].hPending, zToken, nToken |
| 120191 | + ); |
| 120192 | + |
| 120193 | + /* Add the term to each of the prefix indexes that it is not too |
| 120194 | + ** short for. */ |
| 120195 | + for(i=1; rc==SQLITE_OK && i<p->nIndex; i++){ |
| 120196 | + struct Fts3Index *pIndex = &p->aIndex[i]; |
| 120197 | + if( nToken<pIndex->nPrefix ) continue; |
| 120198 | + rc = fts3PendingTermsAddOne( |
| 120199 | + p, iCol, iPos, &pIndex->hPending, zToken, pIndex->nPrefix |
| 120200 | + ); |
| 119368 | 120201 | } |
| 119369 | 120202 | } |
| 119370 | 120203 | |
| 119371 | 120204 | pModule->xClose(pCsr); |
| 119372 | 120205 | *pnWord = nWord; |
| | @@ -119392,18 +120225,23 @@ |
| 119392 | 120225 | p->iPrevDocid = iDocid; |
| 119393 | 120226 | return SQLITE_OK; |
| 119394 | 120227 | } |
| 119395 | 120228 | |
| 119396 | 120229 | /* |
| 119397 | | -** Discard the contents of the pending-terms hash table. |
| 120230 | +** Discard the contents of the pending-terms hash tables. |
| 119398 | 120231 | */ |
| 119399 | 120232 | SQLITE_PRIVATE void sqlite3Fts3PendingTermsClear(Fts3Table *p){ |
| 119400 | | - Fts3HashElem *pElem; |
| 119401 | | - for(pElem=fts3HashFirst(&p->pendingTerms); pElem; pElem=fts3HashNext(pElem)){ |
| 119402 | | - sqlite3_free(fts3HashData(pElem)); |
| 120233 | + int i; |
| 120234 | + for(i=0; i<p->nIndex; i++){ |
| 120235 | + Fts3HashElem *pElem; |
| 120236 | + Fts3Hash *pHash = &p->aIndex[i].hPending; |
| 120237 | + for(pElem=fts3HashFirst(pHash); pElem; pElem=fts3HashNext(pElem)){ |
| 120238 | + PendingList *pList = (PendingList *)fts3HashData(pElem); |
| 120239 | + fts3PendingListDelete(pList); |
| 120240 | + } |
| 120241 | + fts3HashClear(pHash); |
| 119403 | 120242 | } |
| 119404 | | - fts3HashClear(&p->pendingTerms); |
| 119405 | 120243 | p->nPendingData = 0; |
| 119406 | 120244 | } |
| 119407 | 120245 | |
| 119408 | 120246 | /* |
| 119409 | 120247 | ** This function is called by the xUpdate() method as part of an INSERT |
| | @@ -119555,11 +120393,11 @@ |
| 119555 | 120393 | |
| 119556 | 120394 | /* |
| 119557 | 120395 | ** Forward declaration to account for the circular dependency between |
| 119558 | 120396 | ** functions fts3SegmentMerge() and fts3AllocateSegdirIdx(). |
| 119559 | 120397 | */ |
| 119560 | | -static int fts3SegmentMerge(Fts3Table *, int); |
| 120398 | +static int fts3SegmentMerge(Fts3Table *, int, int); |
| 119561 | 120399 | |
| 119562 | 120400 | /* |
| 119563 | 120401 | ** This function allocates a new level iLevel index in the segdir table. |
| 119564 | 120402 | ** Usually, indexes are allocated within a level sequentially starting |
| 119565 | 120403 | ** with 0, so the allocated index is one greater than the value returned |
| | @@ -119572,19 +120410,24 @@ |
| 119572 | 120410 | ** allocated index is 0. |
| 119573 | 120411 | ** |
| 119574 | 120412 | ** If successful, *piIdx is set to the allocated index slot and SQLITE_OK |
| 119575 | 120413 | ** returned. Otherwise, an SQLite error code is returned. |
| 119576 | 120414 | */ |
| 119577 | | -static int fts3AllocateSegdirIdx(Fts3Table *p, int iLevel, int *piIdx){ |
| 120415 | +static int fts3AllocateSegdirIdx( |
| 120416 | + Fts3Table *p, |
| 120417 | + int iIndex, /* Index for p->aIndex */ |
| 120418 | + int iLevel, |
| 120419 | + int *piIdx |
| 120420 | +){ |
| 119578 | 120421 | int rc; /* Return Code */ |
| 119579 | 120422 | sqlite3_stmt *pNextIdx; /* Query for next idx at level iLevel */ |
| 119580 | 120423 | int iNext = 0; /* Result of query pNextIdx */ |
| 119581 | 120424 | |
| 119582 | 120425 | /* Set variable iNext to the next available segdir index at level iLevel. */ |
| 119583 | 120426 | rc = fts3SqlStmt(p, SQL_NEXT_SEGMENT_INDEX, &pNextIdx, 0); |
| 119584 | 120427 | if( rc==SQLITE_OK ){ |
| 119585 | | - sqlite3_bind_int(pNextIdx, 1, iLevel); |
| 120428 | + sqlite3_bind_int(pNextIdx, 1, iIndex*FTS3_SEGDIR_MAXLEVEL + iLevel); |
| 119586 | 120429 | if( SQLITE_ROW==sqlite3_step(pNextIdx) ){ |
| 119587 | 120430 | iNext = sqlite3_column_int(pNextIdx, 0); |
| 119588 | 120431 | } |
| 119589 | 120432 | rc = sqlite3_reset(pNextIdx); |
| 119590 | 120433 | } |
| | @@ -119594,11 +120437,11 @@ |
| 119594 | 120437 | ** full, merge all segments in level iLevel into a single iLevel+1 |
| 119595 | 120438 | ** segment and allocate (newly freed) index 0 at level iLevel. Otherwise, |
| 119596 | 120439 | ** if iNext is less than FTS3_MERGE_COUNT, allocate index iNext. |
| 119597 | 120440 | */ |
| 119598 | 120441 | if( iNext>=FTS3_MERGE_COUNT ){ |
| 119599 | | - rc = fts3SegmentMerge(p, iLevel); |
| 120442 | + rc = fts3SegmentMerge(p, iIndex, iLevel); |
| 119600 | 120443 | *piIdx = 0; |
| 119601 | 120444 | }else{ |
| 119602 | 120445 | *piIdx = iNext; |
| 119603 | 120446 | } |
| 119604 | 120447 | } |
| | @@ -119635,11 +120478,12 @@ |
| 119635 | 120478 | */ |
| 119636 | 120479 | SQLITE_PRIVATE int sqlite3Fts3ReadBlock( |
| 119637 | 120480 | Fts3Table *p, /* FTS3 table handle */ |
| 119638 | 120481 | sqlite3_int64 iBlockid, /* Access the row with blockid=$iBlockid */ |
| 119639 | 120482 | char **paBlob, /* OUT: Blob data in malloc'd buffer */ |
| 119640 | | - int *pnBlob /* OUT: Size of blob data */ |
| 120483 | + int *pnBlob, /* OUT: Size of blob data */ |
| 120484 | + int *pnLoad /* OUT: Bytes actually loaded */ |
| 119641 | 120485 | ){ |
| 119642 | 120486 | int rc; /* Return code */ |
| 119643 | 120487 | |
| 119644 | 120488 | /* pnBlob must be non-NULL. paBlob may be NULL or non-NULL. */ |
| 119645 | 120489 | assert( pnBlob); |
| | @@ -119656,25 +120500,29 @@ |
| 119656 | 120500 | ); |
| 119657 | 120501 | } |
| 119658 | 120502 | |
| 119659 | 120503 | if( rc==SQLITE_OK ){ |
| 119660 | 120504 | int nByte = sqlite3_blob_bytes(p->pSegments); |
| 120505 | + *pnBlob = nByte; |
| 119661 | 120506 | if( paBlob ){ |
| 119662 | 120507 | char *aByte = sqlite3_malloc(nByte + FTS3_NODE_PADDING); |
| 119663 | 120508 | if( !aByte ){ |
| 119664 | 120509 | rc = SQLITE_NOMEM; |
| 119665 | 120510 | }else{ |
| 120511 | + if( pnLoad && nByte>(FTS3_NODE_CHUNK_THRESHOLD) ){ |
| 120512 | + nByte = FTS3_NODE_CHUNKSIZE; |
| 120513 | + *pnLoad = nByte; |
| 120514 | + } |
| 119666 | 120515 | rc = sqlite3_blob_read(p->pSegments, aByte, nByte, 0); |
| 119667 | 120516 | memset(&aByte[nByte], 0, FTS3_NODE_PADDING); |
| 119668 | 120517 | if( rc!=SQLITE_OK ){ |
| 119669 | 120518 | sqlite3_free(aByte); |
| 119670 | 120519 | aByte = 0; |
| 119671 | 120520 | } |
| 119672 | 120521 | } |
| 119673 | 120522 | *paBlob = aByte; |
| 119674 | 120523 | } |
| 119675 | | - *pnBlob = nByte; |
| 119676 | 120524 | } |
| 119677 | 120525 | |
| 119678 | 120526 | return rc; |
| 119679 | 120527 | } |
| 119680 | 120528 | |
| | @@ -119684,17 +120532,59 @@ |
| 119684 | 120532 | */ |
| 119685 | 120533 | SQLITE_PRIVATE void sqlite3Fts3SegmentsClose(Fts3Table *p){ |
| 119686 | 120534 | sqlite3_blob_close(p->pSegments); |
| 119687 | 120535 | p->pSegments = 0; |
| 119688 | 120536 | } |
| 120537 | + |
| 120538 | +static int fts3SegReaderIncrRead(Fts3SegReader *pReader){ |
| 120539 | + int nRead; /* Number of bytes to read */ |
| 120540 | + int rc; /* Return code */ |
| 120541 | + |
| 120542 | + nRead = MIN(pReader->nNode - pReader->nPopulate, FTS3_NODE_CHUNKSIZE); |
| 120543 | + rc = sqlite3_blob_read( |
| 120544 | + pReader->pBlob, |
| 120545 | + &pReader->aNode[pReader->nPopulate], |
| 120546 | + nRead, |
| 120547 | + pReader->nPopulate |
| 120548 | + ); |
| 120549 | + |
| 120550 | + if( rc==SQLITE_OK ){ |
| 120551 | + pReader->nPopulate += nRead; |
| 120552 | + memset(&pReader->aNode[pReader->nPopulate], 0, FTS3_NODE_PADDING); |
| 120553 | + if( pReader->nPopulate==pReader->nNode ){ |
| 120554 | + sqlite3_blob_close(pReader->pBlob); |
| 120555 | + pReader->pBlob = 0; |
| 120556 | + pReader->nPopulate = 0; |
| 120557 | + } |
| 120558 | + } |
| 120559 | + return rc; |
| 120560 | +} |
| 120561 | + |
| 120562 | +static int fts3SegReaderRequire(Fts3SegReader *pReader, char *pFrom, int nByte){ |
| 120563 | + int rc = SQLITE_OK; |
| 120564 | + assert( !pReader->pBlob |
| 120565 | + || (pFrom>=pReader->aNode && pFrom<&pReader->aNode[pReader->nNode]) |
| 120566 | + ); |
| 120567 | + while( pReader->pBlob && rc==SQLITE_OK |
| 120568 | + && (pFrom - pReader->aNode + nByte)>pReader->nPopulate |
| 120569 | + ){ |
| 120570 | + rc = fts3SegReaderIncrRead(pReader); |
| 120571 | + } |
| 120572 | + return rc; |
| 120573 | +} |
| 119689 | 120574 | |
| 119690 | 120575 | /* |
| 119691 | 120576 | ** Move the iterator passed as the first argument to the next term in the |
| 119692 | 120577 | ** segment. If successful, SQLITE_OK is returned. If there is no next term, |
| 119693 | 120578 | ** SQLITE_DONE. Otherwise, an SQLite error code. |
| 119694 | 120579 | */ |
| 119695 | | -static int fts3SegReaderNext(Fts3Table *p, Fts3SegReader *pReader){ |
| 120580 | +static int fts3SegReaderNext( |
| 120581 | + Fts3Table *p, |
| 120582 | + Fts3SegReader *pReader, |
| 120583 | + int bIncr |
| 120584 | +){ |
| 120585 | + int rc; /* Return code of various sub-routines */ |
| 119696 | 120586 | char *pNext; /* Cursor variable */ |
| 119697 | 120587 | int nPrefix; /* Number of bytes in term prefix */ |
| 119698 | 120588 | int nSuffix; /* Number of bytes in term suffix */ |
| 119699 | 120589 | |
| 119700 | 120590 | if( !pReader->aDoclist ){ |
| | @@ -119702,11 +120592,10 @@ |
| 119702 | 120592 | }else{ |
| 119703 | 120593 | pNext = &pReader->aDoclist[pReader->nDoclist]; |
| 119704 | 120594 | } |
| 119705 | 120595 | |
| 119706 | 120596 | if( !pNext || pNext>=&pReader->aNode[pReader->nNode] ){ |
| 119707 | | - int rc; /* Return code from Fts3ReadBlock() */ |
| 119708 | 120597 | |
| 119709 | 120598 | if( fts3SegReaderIsPending(pReader) ){ |
| 119710 | 120599 | Fts3HashElem *pElem = *(pReader->ppNextElem); |
| 119711 | 120600 | if( pElem==0 ){ |
| 119712 | 120601 | pReader->aNode = 0; |
| | @@ -119722,10 +120611,12 @@ |
| 119722 | 120611 | return SQLITE_OK; |
| 119723 | 120612 | } |
| 119724 | 120613 | |
| 119725 | 120614 | if( !fts3SegReaderIsRootOnly(pReader) ){ |
| 119726 | 120615 | sqlite3_free(pReader->aNode); |
| 120616 | + sqlite3_blob_close(pReader->pBlob); |
| 120617 | + pReader->pBlob = 0; |
| 119727 | 120618 | } |
| 119728 | 120619 | pReader->aNode = 0; |
| 119729 | 120620 | |
| 119730 | 120621 | /* If iCurrentBlock>=iLeafEndBlock, this is an EOF condition. All leaf |
| 119731 | 120622 | ** blocks have already been traversed. */ |
| | @@ -119733,19 +120624,29 @@ |
| 119733 | 120624 | if( pReader->iCurrentBlock>=pReader->iLeafEndBlock ){ |
| 119734 | 120625 | return SQLITE_OK; |
| 119735 | 120626 | } |
| 119736 | 120627 | |
| 119737 | 120628 | rc = sqlite3Fts3ReadBlock( |
| 119738 | | - p, ++pReader->iCurrentBlock, &pReader->aNode, &pReader->nNode |
| 120629 | + p, ++pReader->iCurrentBlock, &pReader->aNode, &pReader->nNode, |
| 120630 | + (bIncr ? &pReader->nPopulate : 0) |
| 119739 | 120631 | ); |
| 119740 | 120632 | if( rc!=SQLITE_OK ) return rc; |
| 120633 | + assert( pReader->pBlob==0 ); |
| 120634 | + if( bIncr && pReader->nPopulate<pReader->nNode ){ |
| 120635 | + pReader->pBlob = p->pSegments; |
| 120636 | + p->pSegments = 0; |
| 120637 | + } |
| 119741 | 120638 | pNext = pReader->aNode; |
| 119742 | 120639 | } |
| 120640 | + |
| 120641 | + assert( !fts3SegReaderIsPending(pReader) ); |
| 120642 | + |
| 120643 | + rc = fts3SegReaderRequire(pReader, pNext, FTS3_VARINT_MAX*2); |
| 120644 | + if( rc!=SQLITE_OK ) return rc; |
| 119743 | 120645 | |
| 119744 | 120646 | /* Because of the FTS3_NODE_PADDING bytes of padding, the following is |
| 119745 | | - ** safe (no risk of overread) even if the node data is corrupted. |
| 119746 | | - */ |
| 120647 | + ** safe (no risk of overread) even if the node data is corrupted. */ |
| 119747 | 120648 | pNext += sqlite3Fts3GetVarint32(pNext, &nPrefix); |
| 119748 | 120649 | pNext += sqlite3Fts3GetVarint32(pNext, &nSuffix); |
| 119749 | 120650 | if( nPrefix<0 || nSuffix<=0 |
| 119750 | 120651 | || &pNext[nSuffix]>&pReader->aNode[pReader->nNode] |
| 119751 | 120652 | ){ |
| | @@ -119759,10 +120660,14 @@ |
| 119759 | 120660 | return SQLITE_NOMEM; |
| 119760 | 120661 | } |
| 119761 | 120662 | pReader->zTerm = zNew; |
| 119762 | 120663 | pReader->nTermAlloc = nNew; |
| 119763 | 120664 | } |
| 120665 | + |
| 120666 | + rc = fts3SegReaderRequire(pReader, pNext, nSuffix+FTS3_VARINT_MAX); |
| 120667 | + if( rc!=SQLITE_OK ) return rc; |
| 120668 | + |
| 119764 | 120669 | memcpy(&pReader->zTerm[nPrefix], pNext, nSuffix); |
| 119765 | 120670 | pReader->nTerm = nPrefix+nSuffix; |
| 119766 | 120671 | pNext += nSuffix; |
| 119767 | 120672 | pNext += sqlite3Fts3GetVarint32(pNext, &pReader->nDoclist); |
| 119768 | 120673 | pReader->aDoclist = pNext; |
| | @@ -119771,11 +120676,11 @@ |
| 119771 | 120676 | /* Check that the doclist does not appear to extend past the end of the |
| 119772 | 120677 | ** b-tree node. And that the final byte of the doclist is 0x00. If either |
| 119773 | 120678 | ** of these statements is untrue, then the data structure is corrupt. |
| 119774 | 120679 | */ |
| 119775 | 120680 | if( &pReader->aDoclist[pReader->nDoclist]>&pReader->aNode[pReader->nNode] |
| 119776 | | - || pReader->aDoclist[pReader->nDoclist-1] |
| 120681 | + || (pReader->nPopulate==0 && pReader->aDoclist[pReader->nDoclist-1]) |
| 119777 | 120682 | ){ |
| 119778 | 120683 | return SQLITE_CORRUPT_VTAB; |
| 119779 | 120684 | } |
| 119780 | 120685 | return SQLITE_OK; |
| 119781 | 120686 | } |
| | @@ -119782,16 +120687,30 @@ |
| 119782 | 120687 | |
| 119783 | 120688 | /* |
| 119784 | 120689 | ** Set the SegReader to point to the first docid in the doclist associated |
| 119785 | 120690 | ** with the current term. |
| 119786 | 120691 | */ |
| 119787 | | -static void fts3SegReaderFirstDocid(Fts3SegReader *pReader){ |
| 119788 | | - int n; |
| 120692 | +static int fts3SegReaderFirstDocid(Fts3Table *pTab, Fts3SegReader *pReader){ |
| 120693 | + int rc = SQLITE_OK; |
| 119789 | 120694 | assert( pReader->aDoclist ); |
| 119790 | 120695 | assert( !pReader->pOffsetList ); |
| 119791 | | - n = sqlite3Fts3GetVarint(pReader->aDoclist, &pReader->iDocid); |
| 119792 | | - pReader->pOffsetList = &pReader->aDoclist[n]; |
| 120696 | + if( pTab->bDescIdx && fts3SegReaderIsPending(pReader) ){ |
| 120697 | + u8 bEof = 0; |
| 120698 | + pReader->iDocid = 0; |
| 120699 | + pReader->nOffsetList = 0; |
| 120700 | + sqlite3Fts3DoclistPrev(0, |
| 120701 | + pReader->aDoclist, pReader->nDoclist, &pReader->pOffsetList, |
| 120702 | + &pReader->iDocid, &pReader->nOffsetList, &bEof |
| 120703 | + ); |
| 120704 | + }else{ |
| 120705 | + rc = fts3SegReaderRequire(pReader, pReader->aDoclist, FTS3_VARINT_MAX); |
| 120706 | + if( rc==SQLITE_OK ){ |
| 120707 | + int n = sqlite3Fts3GetVarint(pReader->aDoclist, &pReader->iDocid); |
| 120708 | + pReader->pOffsetList = &pReader->aDoclist[n]; |
| 120709 | + } |
| 120710 | + } |
| 120711 | + return rc; |
| 119793 | 120712 | } |
| 119794 | 120713 | |
| 119795 | 120714 | /* |
| 119796 | 120715 | ** Advance the SegReader to point to the next docid in the doclist |
| 119797 | 120716 | ** associated with the current term. |
| | @@ -119800,132 +120719,126 @@ |
| 119800 | 120719 | ** *ppOffsetList is set to point to the first column-offset list |
| 119801 | 120720 | ** in the doclist entry (i.e. immediately past the docid varint). |
| 119802 | 120721 | ** *pnOffsetList is set to the length of the set of column-offset |
| 119803 | 120722 | ** lists, not including the nul-terminator byte. For example: |
| 119804 | 120723 | */ |
| 119805 | | -static void fts3SegReaderNextDocid( |
| 119806 | | - Fts3SegReader *pReader, |
| 119807 | | - char **ppOffsetList, |
| 119808 | | - int *pnOffsetList |
| 120724 | +static int fts3SegReaderNextDocid( |
| 120725 | + Fts3Table *pTab, |
| 120726 | + Fts3SegReader *pReader, /* Reader to advance to next docid */ |
| 120727 | + char **ppOffsetList, /* OUT: Pointer to current position-list */ |
| 120728 | + int *pnOffsetList /* OUT: Length of *ppOffsetList in bytes */ |
| 119809 | 120729 | ){ |
| 120730 | + int rc = SQLITE_OK; |
| 119810 | 120731 | char *p = pReader->pOffsetList; |
| 119811 | 120732 | char c = 0; |
| 119812 | 120733 | |
| 119813 | | - /* Pointer p currently points at the first byte of an offset list. The |
| 119814 | | - ** following two lines advance it to point one byte past the end of |
| 119815 | | - ** the same offset list. |
| 119816 | | - */ |
| 119817 | | - while( *p | c ) c = *p++ & 0x80; |
| 119818 | | - p++; |
| 119819 | | - |
| 119820 | | - /* If required, populate the output variables with a pointer to and the |
| 119821 | | - ** size of the previous offset-list. |
| 119822 | | - */ |
| 119823 | | - if( ppOffsetList ){ |
| 119824 | | - *ppOffsetList = pReader->pOffsetList; |
| 119825 | | - *pnOffsetList = (int)(p - pReader->pOffsetList - 1); |
| 119826 | | - } |
| 119827 | | - |
| 119828 | | - /* If there are no more entries in the doclist, set pOffsetList to |
| 119829 | | - ** NULL. Otherwise, set Fts3SegReader.iDocid to the next docid and |
| 119830 | | - ** Fts3SegReader.pOffsetList to point to the next offset list before |
| 119831 | | - ** returning. |
| 119832 | | - */ |
| 119833 | | - if( p>=&pReader->aDoclist[pReader->nDoclist] ){ |
| 119834 | | - pReader->pOffsetList = 0; |
| 120734 | + assert( p ); |
| 120735 | + |
| 120736 | + if( pTab->bDescIdx && fts3SegReaderIsPending(pReader) ){ |
| 120737 | + /* A pending-terms seg-reader for an FTS4 table that uses order=desc. |
| 120738 | + ** Pending-terms doclists are always built up in ascending order, so |
| 120739 | + ** we have to iterate through them backwards here. */ |
| 120740 | + u8 bEof = 0; |
| 120741 | + if( ppOffsetList ){ |
| 120742 | + *ppOffsetList = pReader->pOffsetList; |
| 120743 | + *pnOffsetList = pReader->nOffsetList - 1; |
| 120744 | + } |
| 120745 | + sqlite3Fts3DoclistPrev(0, |
| 120746 | + pReader->aDoclist, pReader->nDoclist, &p, &pReader->iDocid, |
| 120747 | + &pReader->nOffsetList, &bEof |
| 120748 | + ); |
| 120749 | + if( bEof ){ |
| 120750 | + pReader->pOffsetList = 0; |
| 120751 | + }else{ |
| 120752 | + pReader->pOffsetList = p; |
| 120753 | + } |
| 119835 | 120754 | }else{ |
| 119836 | | - sqlite3_int64 iDelta; |
| 119837 | | - pReader->pOffsetList = p + sqlite3Fts3GetVarint(p, &iDelta); |
| 119838 | | - pReader->iDocid += iDelta; |
| 119839 | | - } |
| 119840 | | -} |
| 119841 | | - |
| 119842 | | -/* |
| 119843 | | -** This function is called to estimate the amount of data that will be |
| 119844 | | -** loaded from the disk If SegReaderIterate() is called on this seg-reader, |
| 119845 | | -** in units of average document size. |
| 119846 | | -** |
| 119847 | | -** This can be used as follows: If the caller has a small doclist that |
| 119848 | | -** contains references to N documents, and is considering merging it with |
| 119849 | | -** a large doclist (size X "average documents"), it may opt not to load |
| 119850 | | -** the large doclist if X>N. |
| 119851 | | -*/ |
| 119852 | | -SQLITE_PRIVATE int sqlite3Fts3SegReaderCost( |
| 119853 | | - Fts3Cursor *pCsr, /* FTS3 cursor handle */ |
| 119854 | | - Fts3SegReader *pReader, /* Segment-reader handle */ |
| 119855 | | - int *pnCost /* IN/OUT: Number of bytes read */ |
| 120755 | + |
| 120756 | + /* Pointer p currently points at the first byte of an offset list. The |
| 120757 | + ** following block advances it to point one byte past the end of |
| 120758 | + ** the same offset list. */ |
| 120759 | + while( 1 ){ |
| 120760 | + |
| 120761 | + /* The following line of code (and the "p++" below the while() loop) is |
| 120762 | + ** normally all that is required to move pointer p to the desired |
| 120763 | + ** position. The exception is if this node is being loaded from disk |
| 120764 | + ** incrementally and pointer "p" now points to the first byte passed |
| 120765 | + ** the populated part of pReader->aNode[]. |
| 120766 | + */ |
| 120767 | + while( *p | c ) c = *p++ & 0x80; |
| 120768 | + assert( *p==0 ); |
| 120769 | + |
| 120770 | + if( pReader->pBlob==0 || p<&pReader->aNode[pReader->nPopulate] ) break; |
| 120771 | + rc = fts3SegReaderIncrRead(pReader); |
| 120772 | + if( rc!=SQLITE_OK ) return rc; |
| 120773 | + } |
| 120774 | + p++; |
| 120775 | + |
| 120776 | + /* If required, populate the output variables with a pointer to and the |
| 120777 | + ** size of the previous offset-list. |
| 120778 | + */ |
| 120779 | + if( ppOffsetList ){ |
| 120780 | + *ppOffsetList = pReader->pOffsetList; |
| 120781 | + *pnOffsetList = (int)(p - pReader->pOffsetList - 1); |
| 120782 | + } |
| 120783 | + |
| 120784 | + /* If there are no more entries in the doclist, set pOffsetList to |
| 120785 | + ** NULL. Otherwise, set Fts3SegReader.iDocid to the next docid and |
| 120786 | + ** Fts3SegReader.pOffsetList to point to the next offset list before |
| 120787 | + ** returning. |
| 120788 | + */ |
| 120789 | + if( p>=&pReader->aDoclist[pReader->nDoclist] ){ |
| 120790 | + pReader->pOffsetList = 0; |
| 120791 | + }else{ |
| 120792 | + rc = fts3SegReaderRequire(pReader, p, FTS3_VARINT_MAX); |
| 120793 | + if( rc==SQLITE_OK ){ |
| 120794 | + sqlite3_int64 iDelta; |
| 120795 | + pReader->pOffsetList = p + sqlite3Fts3GetVarint(p, &iDelta); |
| 120796 | + if( pTab->bDescIdx ){ |
| 120797 | + pReader->iDocid -= iDelta; |
| 120798 | + }else{ |
| 120799 | + pReader->iDocid += iDelta; |
| 120800 | + } |
| 120801 | + } |
| 120802 | + } |
| 120803 | + } |
| 120804 | + |
| 120805 | + return SQLITE_OK; |
| 120806 | +} |
| 120807 | + |
| 120808 | + |
| 120809 | +SQLITE_PRIVATE int sqlite3Fts3MsrOvfl( |
| 120810 | + Fts3Cursor *pCsr, |
| 120811 | + Fts3MultiSegReader *pMsr, |
| 120812 | + int *pnOvfl |
| 119856 | 120813 | ){ |
| 119857 | 120814 | Fts3Table *p = (Fts3Table*)pCsr->base.pVtab; |
| 119858 | | - int rc = SQLITE_OK; /* Return code */ |
| 119859 | | - int nCost = 0; /* Cost in bytes to return */ |
| 119860 | | - int pgsz = p->nPgsz; /* Database page size */ |
| 119861 | | - |
| 119862 | | - /* If this seg-reader is reading the pending-terms table, or if all data |
| 119863 | | - ** for the segment is stored on the root page of the b-tree, then the cost |
| 119864 | | - ** is zero. In this case all required data is already in main memory. |
| 119865 | | - */ |
| 119866 | | - if( p->bHasStat |
| 119867 | | - && !fts3SegReaderIsPending(pReader) |
| 119868 | | - && !fts3SegReaderIsRootOnly(pReader) |
| 119869 | | - ){ |
| 119870 | | - int nBlob = 0; |
| 119871 | | - sqlite3_int64 iBlock; |
| 119872 | | - |
| 119873 | | - if( pCsr->nRowAvg==0 ){ |
| 119874 | | - /* The average document size, which is required to calculate the cost |
| 119875 | | - ** of each doclist, has not yet been determined. Read the required |
| 119876 | | - ** data from the %_stat table to calculate it. |
| 119877 | | - ** |
| 119878 | | - ** Entry 0 of the %_stat table is a blob containing (nCol+1) FTS3 |
| 119879 | | - ** varints, where nCol is the number of columns in the FTS3 table. |
| 119880 | | - ** The first varint is the number of documents currently stored in |
| 119881 | | - ** the table. The following nCol varints contain the total amount of |
| 119882 | | - ** data stored in all rows of each column of the table, from left |
| 119883 | | - ** to right. |
| 119884 | | - */ |
| 119885 | | - sqlite3_stmt *pStmt; |
| 119886 | | - sqlite3_int64 nDoc = 0; |
| 119887 | | - sqlite3_int64 nByte = 0; |
| 119888 | | - const char *pEnd; |
| 119889 | | - const char *a; |
| 119890 | | - |
| 119891 | | - rc = sqlite3Fts3SelectDoctotal(p, &pStmt); |
| 119892 | | - if( rc!=SQLITE_OK ) return rc; |
| 119893 | | - a = sqlite3_column_blob(pStmt, 0); |
| 119894 | | - assert( a ); |
| 119895 | | - |
| 119896 | | - pEnd = &a[sqlite3_column_bytes(pStmt, 0)]; |
| 119897 | | - a += sqlite3Fts3GetVarint(a, &nDoc); |
| 119898 | | - while( a<pEnd ){ |
| 119899 | | - a += sqlite3Fts3GetVarint(a, &nByte); |
| 119900 | | - } |
| 119901 | | - if( nDoc==0 || nByte==0 ){ |
| 119902 | | - sqlite3_reset(pStmt); |
| 119903 | | - return SQLITE_CORRUPT_VTAB; |
| 119904 | | - } |
| 119905 | | - |
| 119906 | | - pCsr->nRowAvg = (int)(((nByte / nDoc) + pgsz) / pgsz); |
| 119907 | | - assert( pCsr->nRowAvg>0 ); |
| 119908 | | - rc = sqlite3_reset(pStmt); |
| 119909 | | - if( rc!=SQLITE_OK ) return rc; |
| 119910 | | - } |
| 119911 | | - |
| 119912 | | - /* Assume that a blob flows over onto overflow pages if it is larger |
| 119913 | | - ** than (pgsz-35) bytes in size (the file-format documentation |
| 119914 | | - ** confirms this). |
| 119915 | | - */ |
| 119916 | | - for(iBlock=pReader->iStartBlock; iBlock<=pReader->iLeafEndBlock; iBlock++){ |
| 119917 | | - rc = sqlite3Fts3ReadBlock(p, iBlock, 0, &nBlob); |
| 119918 | | - if( rc!=SQLITE_OK ) break; |
| 119919 | | - if( (nBlob+35)>pgsz ){ |
| 119920 | | - int nOvfl = (nBlob + 34)/pgsz; |
| 119921 | | - nCost += ((nOvfl + pCsr->nRowAvg - 1)/pCsr->nRowAvg); |
| 119922 | | - } |
| 119923 | | - } |
| 119924 | | - } |
| 119925 | | - |
| 119926 | | - *pnCost += nCost; |
| 120815 | + int nOvfl = 0; |
| 120816 | + int ii; |
| 120817 | + int rc = SQLITE_OK; |
| 120818 | + int pgsz = p->nPgsz; |
| 120819 | + |
| 120820 | + assert( p->bHasStat ); |
| 120821 | + assert( pgsz>0 ); |
| 120822 | + |
| 120823 | + for(ii=0; rc==SQLITE_OK && ii<pMsr->nSegment; ii++){ |
| 120824 | + Fts3SegReader *pReader = pMsr->apSegment[ii]; |
| 120825 | + if( !fts3SegReaderIsPending(pReader) |
| 120826 | + && !fts3SegReaderIsRootOnly(pReader) |
| 120827 | + ){ |
| 120828 | + int jj; |
| 120829 | + for(jj=pReader->iStartBlock; jj<=pReader->iLeafEndBlock; jj++){ |
| 120830 | + int nBlob; |
| 120831 | + rc = sqlite3Fts3ReadBlock(p, jj, 0, &nBlob, 0); |
| 120832 | + if( rc!=SQLITE_OK ) break; |
| 120833 | + if( (nBlob+35)>pgsz ){ |
| 120834 | + nOvfl += (nBlob + 34)/pgsz; |
| 120835 | + } |
| 120836 | + } |
| 120837 | + } |
| 120838 | + } |
| 120839 | + *pnOvfl = nOvfl; |
| 119927 | 120840 | return rc; |
| 119928 | 120841 | } |
| 119929 | 120842 | |
| 119930 | 120843 | /* |
| 119931 | 120844 | ** Free all allocations associated with the iterator passed as the |
| | @@ -119934,10 +120847,11 @@ |
| 119934 | 120847 | SQLITE_PRIVATE void sqlite3Fts3SegReaderFree(Fts3SegReader *pReader){ |
| 119935 | 120848 | if( pReader && !fts3SegReaderIsPending(pReader) ){ |
| 119936 | 120849 | sqlite3_free(pReader->zTerm); |
| 119937 | 120850 | if( !fts3SegReaderIsRootOnly(pReader) ){ |
| 119938 | 120851 | sqlite3_free(pReader->aNode); |
| 120852 | + sqlite3_blob_close(pReader->pBlob); |
| 119939 | 120853 | } |
| 119940 | 120854 | } |
| 119941 | 120855 | sqlite3_free(pReader); |
| 119942 | 120856 | } |
| 119943 | 120857 | |
| | @@ -120010,28 +120924,46 @@ |
| 120010 | 120924 | } |
| 120011 | 120925 | |
| 120012 | 120926 | /* |
| 120013 | 120927 | ** This function is used to allocate an Fts3SegReader that iterates through |
| 120014 | 120928 | ** a subset of the terms stored in the Fts3Table.pendingTerms array. |
| 120929 | +** |
| 120930 | +** If the isPrefixIter parameter is zero, then the returned SegReader iterates |
| 120931 | +** through each term in the pending-terms table. Or, if isPrefixIter is |
| 120932 | +** non-zero, it iterates through each term and its prefixes. For example, if |
| 120933 | +** the pending terms hash table contains the terms "sqlite", "mysql" and |
| 120934 | +** "firebird", then the iterator visits the following 'terms' (in the order |
| 120935 | +** shown): |
| 120936 | +** |
| 120937 | +** f fi fir fire fireb firebi firebir firebird |
| 120938 | +** m my mys mysq mysql |
| 120939 | +** s sq sql sqli sqlit sqlite |
| 120940 | +** |
| 120941 | +** Whereas if isPrefixIter is zero, the terms visited are: |
| 120942 | +** |
| 120943 | +** firebird mysql sqlite |
| 120015 | 120944 | */ |
| 120016 | 120945 | SQLITE_PRIVATE int sqlite3Fts3SegReaderPending( |
| 120017 | 120946 | Fts3Table *p, /* Virtual table handle */ |
| 120947 | + int iIndex, /* Index for p->aIndex */ |
| 120018 | 120948 | const char *zTerm, /* Term to search for */ |
| 120019 | 120949 | int nTerm, /* Size of buffer zTerm */ |
| 120020 | | - int isPrefix, /* True for a term-prefix query */ |
| 120950 | + int bPrefix, /* True for a prefix iterator */ |
| 120021 | 120951 | Fts3SegReader **ppReader /* OUT: SegReader for pending-terms */ |
| 120022 | 120952 | ){ |
| 120023 | 120953 | Fts3SegReader *pReader = 0; /* Fts3SegReader object to return */ |
| 120024 | 120954 | Fts3HashElem **aElem = 0; /* Array of term hash entries to scan */ |
| 120025 | 120955 | int nElem = 0; /* Size of array at aElem */ |
| 120026 | 120956 | int rc = SQLITE_OK; /* Return Code */ |
| 120957 | + Fts3Hash *pHash; |
| 120027 | 120958 | |
| 120028 | | - if( isPrefix ){ |
| 120959 | + pHash = &p->aIndex[iIndex].hPending; |
| 120960 | + if( bPrefix ){ |
| 120029 | 120961 | int nAlloc = 0; /* Size of allocated array at aElem */ |
| 120030 | 120962 | Fts3HashElem *pE = 0; /* Iterator variable */ |
| 120031 | 120963 | |
| 120032 | | - for(pE=fts3HashFirst(&p->pendingTerms); pE; pE=fts3HashNext(pE)){ |
| 120964 | + for(pE=fts3HashFirst(pHash); pE; pE=fts3HashNext(pE)){ |
| 120033 | 120965 | char *zKey = (char *)fts3HashKey(pE); |
| 120034 | 120966 | int nKey = fts3HashKeysize(pE); |
| 120035 | 120967 | if( nTerm==0 || (nKey>=nTerm && 0==memcmp(zKey, zTerm, nTerm)) ){ |
| 120036 | 120968 | if( nElem==nAlloc ){ |
| 120037 | 120969 | Fts3HashElem **aElem2; |
| | @@ -120044,10 +120976,11 @@ |
| 120044 | 120976 | nElem = 0; |
| 120045 | 120977 | break; |
| 120046 | 120978 | } |
| 120047 | 120979 | aElem = aElem2; |
| 120048 | 120980 | } |
| 120981 | + |
| 120049 | 120982 | aElem[nElem++] = pE; |
| 120050 | 120983 | } |
| 120051 | 120984 | } |
| 120052 | 120985 | |
| 120053 | 120986 | /* If more than one term matches the prefix, sort the Fts3HashElem |
| | @@ -120057,11 +120990,13 @@ |
| 120057 | 120990 | if( nElem>1 ){ |
| 120058 | 120991 | qsort(aElem, nElem, sizeof(Fts3HashElem *), fts3CompareElemByTerm); |
| 120059 | 120992 | } |
| 120060 | 120993 | |
| 120061 | 120994 | }else{ |
| 120062 | | - Fts3HashElem *pE = fts3HashFindElem(&p->pendingTerms, zTerm, nTerm); |
| 120995 | + /* The query is a simple term lookup that matches at most one term in |
| 120996 | + ** the index. All that is required is a straight hash-lookup. */ |
| 120997 | + Fts3HashElem *pE = fts3HashFindElem(pHash, zTerm, nTerm); |
| 120063 | 120998 | if( pE ){ |
| 120064 | 120999 | aElem = &pE; |
| 120065 | 121000 | nElem = 1; |
| 120066 | 121001 | } |
| 120067 | 121002 | } |
| | @@ -120077,11 +121012,11 @@ |
| 120077 | 121012 | pReader->ppNextElem = (Fts3HashElem **)&pReader[1]; |
| 120078 | 121013 | memcpy(pReader->ppNextElem, aElem, nElem*sizeof(Fts3HashElem *)); |
| 120079 | 121014 | } |
| 120080 | 121015 | } |
| 120081 | 121016 | |
| 120082 | | - if( isPrefix ){ |
| 121017 | + if( bPrefix ){ |
| 120083 | 121018 | sqlite3_free(aElem); |
| 120084 | 121019 | } |
| 120085 | 121020 | *ppReader = pReader; |
| 120086 | 121021 | return rc; |
| 120087 | 121022 | } |
| | @@ -120137,10 +121072,22 @@ |
| 120137 | 121072 | if( pLhs->iDocid==pRhs->iDocid ){ |
| 120138 | 121073 | rc = pRhs->iIdx - pLhs->iIdx; |
| 120139 | 121074 | }else{ |
| 120140 | 121075 | rc = (pLhs->iDocid > pRhs->iDocid) ? 1 : -1; |
| 120141 | 121076 | } |
| 121077 | + } |
| 121078 | + assert( pLhs->aNode && pRhs->aNode ); |
| 121079 | + return rc; |
| 121080 | +} |
| 121081 | +static int fts3SegReaderDoclistCmpRev(Fts3SegReader *pLhs, Fts3SegReader *pRhs){ |
| 121082 | + int rc = (pLhs->pOffsetList==0)-(pRhs->pOffsetList==0); |
| 121083 | + if( rc==0 ){ |
| 121084 | + if( pLhs->iDocid==pRhs->iDocid ){ |
| 121085 | + rc = pRhs->iIdx - pLhs->iIdx; |
| 121086 | + }else{ |
| 121087 | + rc = (pLhs->iDocid < pRhs->iDocid) ? 1 : -1; |
| 121088 | + } |
| 120142 | 121089 | } |
| 120143 | 121090 | assert( pLhs->aNode && pRhs->aNode ); |
| 120144 | 121091 | return rc; |
| 120145 | 121092 | } |
| 120146 | 121093 | |
| | @@ -120689,25 +121636,34 @@ |
| 120689 | 121636 | } |
| 120690 | 121637 | return rc; |
| 120691 | 121638 | } |
| 120692 | 121639 | |
| 120693 | 121640 | /* |
| 120694 | | -** Set *pnSegment to the total number of segments in the database. Set |
| 120695 | | -** *pnMax to the largest segment level in the database (segment levels |
| 120696 | | -** are stored in the 'level' column of the %_segdir table). |
| 121641 | +** Set *pnMax to the largest segment level in the database for the index |
| 121642 | +** iIndex. |
| 121643 | +** |
| 121644 | +** Segment levels are stored in the 'level' column of the %_segdir table. |
| 120697 | 121645 | ** |
| 120698 | 121646 | ** Return SQLITE_OK if successful, or an SQLite error code if not. |
| 120699 | 121647 | */ |
| 120700 | | -static int fts3SegmentCountMax(Fts3Table *p, int *pnSegment, int *pnMax){ |
| 121648 | +static int fts3SegmentMaxLevel(Fts3Table *p, int iIndex, int *pnMax){ |
| 120701 | 121649 | sqlite3_stmt *pStmt; |
| 120702 | 121650 | int rc; |
| 121651 | + assert( iIndex>=0 && iIndex<p->nIndex ); |
| 120703 | 121652 | |
| 120704 | | - rc = fts3SqlStmt(p, SQL_SELECT_SEGDIR_COUNT_MAX, &pStmt, 0); |
| 121653 | + /* Set pStmt to the compiled version of: |
| 121654 | + ** |
| 121655 | + ** SELECT max(level) FROM %Q.'%q_segdir' WHERE level BETWEEN ? AND ? |
| 121656 | + ** |
| 121657 | + ** (1024 is actually the value of macro FTS3_SEGDIR_PREFIXLEVEL_STR). |
| 121658 | + */ |
| 121659 | + rc = fts3SqlStmt(p, SQL_SELECT_SEGDIR_MAX_LEVEL, &pStmt, 0); |
| 120705 | 121660 | if( rc!=SQLITE_OK ) return rc; |
| 121661 | + sqlite3_bind_int(pStmt, 1, iIndex*FTS3_SEGDIR_MAXLEVEL); |
| 121662 | + sqlite3_bind_int(pStmt, 2, (iIndex+1)*FTS3_SEGDIR_MAXLEVEL - 1); |
| 120706 | 121663 | if( SQLITE_ROW==sqlite3_step(pStmt) ){ |
| 120707 | | - *pnSegment = sqlite3_column_int(pStmt, 0); |
| 120708 | | - *pnMax = sqlite3_column_int(pStmt, 1); |
| 121664 | + *pnMax = sqlite3_column_int(pStmt, 0); |
| 120709 | 121665 | } |
| 120710 | 121666 | return sqlite3_reset(pStmt); |
| 120711 | 121667 | } |
| 120712 | 121668 | |
| 120713 | 121669 | /* |
| | @@ -120724,10 +121680,11 @@ |
| 120724 | 121680 | ** |
| 120725 | 121681 | ** SQLITE_OK is returned if successful, otherwise an SQLite error code. |
| 120726 | 121682 | */ |
| 120727 | 121683 | static int fts3DeleteSegdir( |
| 120728 | 121684 | Fts3Table *p, /* Virtual table handle */ |
| 121685 | + int iIndex, /* Index for p->aIndex */ |
| 120729 | 121686 | int iLevel, /* Level of %_segdir entries to delete */ |
| 120730 | 121687 | Fts3SegReader **apSegment, /* Array of SegReader objects */ |
| 120731 | 121688 | int nReader /* Size of array apSegment */ |
| 120732 | 121689 | ){ |
| 120733 | 121690 | int rc; /* Return Code */ |
| | @@ -120746,23 +121703,28 @@ |
| 120746 | 121703 | } |
| 120747 | 121704 | if( rc!=SQLITE_OK ){ |
| 120748 | 121705 | return rc; |
| 120749 | 121706 | } |
| 120750 | 121707 | |
| 121708 | + assert( iLevel>=0 || iLevel==FTS3_SEGCURSOR_ALL ); |
| 120751 | 121709 | if( iLevel==FTS3_SEGCURSOR_ALL ){ |
| 120752 | | - fts3SqlExec(&rc, p, SQL_DELETE_ALL_SEGDIR, 0); |
| 120753 | | - }else if( iLevel==FTS3_SEGCURSOR_PENDING ){ |
| 120754 | | - sqlite3Fts3PendingTermsClear(p); |
| 121710 | + rc = fts3SqlStmt(p, SQL_DELETE_SEGDIR_RANGE, &pDelete, 0); |
| 121711 | + if( rc==SQLITE_OK ){ |
| 121712 | + sqlite3_bind_int(pDelete, 1, iIndex*FTS3_SEGDIR_MAXLEVEL); |
| 121713 | + sqlite3_bind_int(pDelete, 2, (iIndex+1) * FTS3_SEGDIR_MAXLEVEL - 1); |
| 121714 | + } |
| 120755 | 121715 | }else{ |
| 120756 | | - assert( iLevel>=0 ); |
| 120757 | | - rc = fts3SqlStmt(p, SQL_DELETE_SEGDIR_BY_LEVEL, &pDelete, 0); |
| 121716 | + rc = fts3SqlStmt(p, SQL_DELETE_SEGDIR_LEVEL, &pDelete, 0); |
| 120758 | 121717 | if( rc==SQLITE_OK ){ |
| 120759 | | - sqlite3_bind_int(pDelete, 1, iLevel); |
| 120760 | | - sqlite3_step(pDelete); |
| 120761 | | - rc = sqlite3_reset(pDelete); |
| 121718 | + sqlite3_bind_int(pDelete, 1, iIndex*FTS3_SEGDIR_MAXLEVEL + iLevel); |
| 120762 | 121719 | } |
| 120763 | 121720 | } |
| 121721 | + |
| 121722 | + if( rc==SQLITE_OK ){ |
| 121723 | + sqlite3_step(pDelete); |
| 121724 | + rc = sqlite3_reset(pDelete); |
| 121725 | + } |
| 120764 | 121726 | |
| 120765 | 121727 | return rc; |
| 120766 | 121728 | } |
| 120767 | 121729 | |
| 120768 | 121730 | /* |
| | @@ -120805,14 +121767,124 @@ |
| 120805 | 121767 | } |
| 120806 | 121768 | |
| 120807 | 121769 | *ppList = pList; |
| 120808 | 121770 | *pnList = nList; |
| 120809 | 121771 | } |
| 121772 | + |
| 121773 | +SQLITE_PRIVATE int sqlite3Fts3MsrIncrStart( |
| 121774 | + Fts3Table *p, /* Virtual table handle */ |
| 121775 | + Fts3MultiSegReader *pCsr, /* Cursor object */ |
| 121776 | + int iCol, /* Column to match on. */ |
| 121777 | + const char *zTerm, /* Term to iterate through a doclist for */ |
| 121778 | + int nTerm /* Number of bytes in zTerm */ |
| 121779 | +){ |
| 121780 | + int i; |
| 121781 | + int nSegment = pCsr->nSegment; |
| 121782 | + int (*xCmp)(Fts3SegReader *, Fts3SegReader *) = ( |
| 121783 | + p->bDescIdx ? fts3SegReaderDoclistCmpRev : fts3SegReaderDoclistCmp |
| 121784 | + ); |
| 121785 | + |
| 121786 | + assert( pCsr->pFilter==0 ); |
| 121787 | + assert( zTerm && nTerm>0 ); |
| 121788 | + |
| 121789 | + /* Advance each segment iterator until it points to the term zTerm/nTerm. */ |
| 121790 | + for(i=0; i<nSegment; i++){ |
| 121791 | + Fts3SegReader *pSeg = pCsr->apSegment[i]; |
| 121792 | + do { |
| 121793 | + int rc = fts3SegReaderNext(p, pSeg, 1); |
| 121794 | + if( rc!=SQLITE_OK ) return rc; |
| 121795 | + }while( fts3SegReaderTermCmp(pSeg, zTerm, nTerm)<0 ); |
| 121796 | + } |
| 121797 | + fts3SegReaderSort(pCsr->apSegment, nSegment, nSegment, fts3SegReaderCmp); |
| 121798 | + |
| 121799 | + /* Determine how many of the segments actually point to zTerm/nTerm. */ |
| 121800 | + for(i=0; i<nSegment; i++){ |
| 121801 | + Fts3SegReader *pSeg = pCsr->apSegment[i]; |
| 121802 | + if( !pSeg->aNode || fts3SegReaderTermCmp(pSeg, zTerm, nTerm) ){ |
| 121803 | + break; |
| 121804 | + } |
| 121805 | + } |
| 121806 | + pCsr->nAdvance = i; |
| 121807 | + |
| 121808 | + /* Advance each of the segments to point to the first docid. */ |
| 121809 | + for(i=0; i<pCsr->nAdvance; i++){ |
| 121810 | + int rc = fts3SegReaderFirstDocid(p, pCsr->apSegment[i]); |
| 121811 | + if( rc!=SQLITE_OK ) return rc; |
| 121812 | + } |
| 121813 | + fts3SegReaderSort(pCsr->apSegment, i, i, xCmp); |
| 121814 | + |
| 121815 | + assert( iCol<0 || iCol<p->nColumn ); |
| 121816 | + pCsr->iColFilter = iCol; |
| 121817 | + |
| 121818 | + return SQLITE_OK; |
| 121819 | +} |
| 121820 | + |
| 121821 | +SQLITE_PRIVATE int sqlite3Fts3MsrIncrNext( |
| 121822 | + Fts3Table *p, /* Virtual table handle */ |
| 121823 | + Fts3MultiSegReader *pMsr, /* Multi-segment-reader handle */ |
| 121824 | + sqlite3_int64 *piDocid, /* OUT: Docid value */ |
| 121825 | + char **paPoslist, /* OUT: Pointer to position list */ |
| 121826 | + int *pnPoslist /* OUT: Size of position list in bytes */ |
| 121827 | +){ |
| 121828 | + int nMerge = pMsr->nAdvance; |
| 121829 | + Fts3SegReader **apSegment = pMsr->apSegment; |
| 121830 | + int (*xCmp)(Fts3SegReader *, Fts3SegReader *) = ( |
| 121831 | + p->bDescIdx ? fts3SegReaderDoclistCmpRev : fts3SegReaderDoclistCmp |
| 121832 | + ); |
| 121833 | + |
| 121834 | + if( nMerge==0 ){ |
| 121835 | + *paPoslist = 0; |
| 121836 | + return SQLITE_OK; |
| 121837 | + } |
| 121838 | + |
| 121839 | + while( 1 ){ |
| 121840 | + Fts3SegReader *pSeg; |
| 121841 | + pSeg = pMsr->apSegment[0]; |
| 121842 | + |
| 121843 | + if( pSeg->pOffsetList==0 ){ |
| 121844 | + *paPoslist = 0; |
| 121845 | + break; |
| 121846 | + }else{ |
| 121847 | + int rc; |
| 121848 | + char *pList; |
| 121849 | + int nList; |
| 121850 | + int j; |
| 121851 | + sqlite3_int64 iDocid = apSegment[0]->iDocid; |
| 121852 | + |
| 121853 | + rc = fts3SegReaderNextDocid(p, apSegment[0], &pList, &nList); |
| 121854 | + j = 1; |
| 121855 | + while( rc==SQLITE_OK |
| 121856 | + && j<nMerge |
| 121857 | + && apSegment[j]->pOffsetList |
| 121858 | + && apSegment[j]->iDocid==iDocid |
| 121859 | + ){ |
| 121860 | + rc = fts3SegReaderNextDocid(p, apSegment[j], 0, 0); |
| 121861 | + j++; |
| 121862 | + } |
| 121863 | + if( rc!=SQLITE_OK ) return rc; |
| 121864 | + fts3SegReaderSort(pMsr->apSegment, nMerge, j, xCmp); |
| 121865 | + |
| 121866 | + if( pMsr->iColFilter>=0 ){ |
| 121867 | + fts3ColumnFilter(pMsr->iColFilter, &pList, &nList); |
| 121868 | + } |
| 121869 | + |
| 121870 | + if( nList>0 ){ |
| 121871 | + *piDocid = iDocid; |
| 121872 | + *paPoslist = pList; |
| 121873 | + *pnPoslist = nList; |
| 121874 | + break; |
| 121875 | + } |
| 121876 | + } |
| 121877 | + |
| 121878 | + } |
| 121879 | + |
| 121880 | + return SQLITE_OK; |
| 121881 | +} |
| 120810 | 121882 | |
| 120811 | 121883 | SQLITE_PRIVATE int sqlite3Fts3SegReaderStart( |
| 120812 | 121884 | Fts3Table *p, /* Virtual table handle */ |
| 120813 | | - Fts3SegReaderCursor *pCsr, /* Cursor object */ |
| 121885 | + Fts3MultiSegReader *pCsr, /* Cursor object */ |
| 120814 | 121886 | Fts3SegFilter *pFilter /* Restrictions on range of iteration */ |
| 120815 | 121887 | ){ |
| 120816 | 121888 | int i; |
| 120817 | 121889 | |
| 120818 | 121890 | /* Initialize the cursor object */ |
| | @@ -120827,11 +121899,11 @@ |
| 120827 | 121899 | for(i=0; i<pCsr->nSegment; i++){ |
| 120828 | 121900 | int nTerm = pFilter->nTerm; |
| 120829 | 121901 | const char *zTerm = pFilter->zTerm; |
| 120830 | 121902 | Fts3SegReader *pSeg = pCsr->apSegment[i]; |
| 120831 | 121903 | do { |
| 120832 | | - int rc = fts3SegReaderNext(p, pSeg); |
| 121904 | + int rc = fts3SegReaderNext(p, pSeg, 0); |
| 120833 | 121905 | if( rc!=SQLITE_OK ) return rc; |
| 120834 | 121906 | }while( zTerm && fts3SegReaderTermCmp(pSeg, zTerm, nTerm)<0 ); |
| 120835 | 121907 | } |
| 120836 | 121908 | fts3SegReaderSort( |
| 120837 | 121909 | pCsr->apSegment, pCsr->nSegment, pCsr->nSegment, fts3SegReaderCmp); |
| | @@ -120839,11 +121911,11 @@ |
| 120839 | 121911 | return SQLITE_OK; |
| 120840 | 121912 | } |
| 120841 | 121913 | |
| 120842 | 121914 | SQLITE_PRIVATE int sqlite3Fts3SegReaderStep( |
| 120843 | 121915 | Fts3Table *p, /* Virtual table handle */ |
| 120844 | | - Fts3SegReaderCursor *pCsr /* Cursor object */ |
| 121916 | + Fts3MultiSegReader *pCsr /* Cursor object */ |
| 120845 | 121917 | ){ |
| 120846 | 121918 | int rc = SQLITE_OK; |
| 120847 | 121919 | |
| 120848 | 121920 | int isIgnoreEmpty = (pCsr->pFilter->flags & FTS3_SEGMENT_IGNORE_EMPTY); |
| 120849 | 121921 | int isRequirePos = (pCsr->pFilter->flags & FTS3_SEGMENT_REQUIRE_POS); |
| | @@ -120852,10 +121924,13 @@ |
| 120852 | 121924 | int isScan = (pCsr->pFilter->flags & FTS3_SEGMENT_SCAN); |
| 120853 | 121925 | |
| 120854 | 121926 | Fts3SegReader **apSegment = pCsr->apSegment; |
| 120855 | 121927 | int nSegment = pCsr->nSegment; |
| 120856 | 121928 | Fts3SegFilter *pFilter = pCsr->pFilter; |
| 121929 | + int (*xCmp)(Fts3SegReader *, Fts3SegReader *) = ( |
| 121930 | + p->bDescIdx ? fts3SegReaderDoclistCmpRev : fts3SegReaderDoclistCmp |
| 121931 | + ); |
| 120857 | 121932 | |
| 120858 | 121933 | if( pCsr->nSegment==0 ) return SQLITE_OK; |
| 120859 | 121934 | |
| 120860 | 121935 | do { |
| 120861 | 121936 | int nMerge; |
| | @@ -120863,11 +121938,11 @@ |
| 120863 | 121938 | |
| 120864 | 121939 | /* Advance the first pCsr->nAdvance entries in the apSegment[] array |
| 120865 | 121940 | ** forward. Then sort the list in order of current term again. |
| 120866 | 121941 | */ |
| 120867 | 121942 | for(i=0; i<pCsr->nAdvance; i++){ |
| 120868 | | - rc = fts3SegReaderNext(p, apSegment[i]); |
| 121943 | + rc = fts3SegReaderNext(p, apSegment[i], 0); |
| 120869 | 121944 | if( rc!=SQLITE_OK ) return rc; |
| 120870 | 121945 | } |
| 120871 | 121946 | fts3SegReaderSort(apSegment, nSegment, pCsr->nAdvance, fts3SegReaderCmp); |
| 120872 | 121947 | pCsr->nAdvance = 0; |
| 120873 | 121948 | |
| | @@ -120902,11 +121977,14 @@ |
| 120902 | 121977 | ){ |
| 120903 | 121978 | nMerge++; |
| 120904 | 121979 | } |
| 120905 | 121980 | |
| 120906 | 121981 | assert( isIgnoreEmpty || (isRequirePos && !isColFilter) ); |
| 120907 | | - if( nMerge==1 && !isIgnoreEmpty ){ |
| 121982 | + if( nMerge==1 |
| 121983 | + && !isIgnoreEmpty |
| 121984 | + && (p->bDescIdx==0 || fts3SegReaderIsPending(apSegment[0])==0) |
| 121985 | + ){ |
| 120908 | 121986 | pCsr->aDoclist = apSegment[0]->aDoclist; |
| 120909 | 121987 | pCsr->nDoclist = apSegment[0]->nDoclist; |
| 120910 | 121988 | rc = SQLITE_ROW; |
| 120911 | 121989 | }else{ |
| 120912 | 121990 | int nDoclist = 0; /* Size of doclist */ |
| | @@ -120915,56 +121993,66 @@ |
| 120915 | 121993 | /* The current term of the first nMerge entries in the array |
| 120916 | 121994 | ** of Fts3SegReader objects is the same. The doclists must be merged |
| 120917 | 121995 | ** and a single term returned with the merged doclist. |
| 120918 | 121996 | */ |
| 120919 | 121997 | for(i=0; i<nMerge; i++){ |
| 120920 | | - fts3SegReaderFirstDocid(apSegment[i]); |
| 121998 | + fts3SegReaderFirstDocid(p, apSegment[i]); |
| 120921 | 121999 | } |
| 120922 | | - fts3SegReaderSort(apSegment, nMerge, nMerge, fts3SegReaderDoclistCmp); |
| 122000 | + fts3SegReaderSort(apSegment, nMerge, nMerge, xCmp); |
| 120923 | 122001 | while( apSegment[0]->pOffsetList ){ |
| 120924 | 122002 | int j; /* Number of segments that share a docid */ |
| 120925 | 122003 | char *pList; |
| 120926 | 122004 | int nList; |
| 120927 | 122005 | int nByte; |
| 120928 | 122006 | sqlite3_int64 iDocid = apSegment[0]->iDocid; |
| 120929 | | - fts3SegReaderNextDocid(apSegment[0], &pList, &nList); |
| 122007 | + fts3SegReaderNextDocid(p, apSegment[0], &pList, &nList); |
| 120930 | 122008 | j = 1; |
| 120931 | 122009 | while( j<nMerge |
| 120932 | 122010 | && apSegment[j]->pOffsetList |
| 120933 | 122011 | && apSegment[j]->iDocid==iDocid |
| 120934 | 122012 | ){ |
| 120935 | | - fts3SegReaderNextDocid(apSegment[j], 0, 0); |
| 122013 | + fts3SegReaderNextDocid(p, apSegment[j], 0, 0); |
| 120936 | 122014 | j++; |
| 120937 | 122015 | } |
| 120938 | 122016 | |
| 120939 | 122017 | if( isColFilter ){ |
| 120940 | 122018 | fts3ColumnFilter(pFilter->iCol, &pList, &nList); |
| 120941 | 122019 | } |
| 120942 | 122020 | |
| 120943 | 122021 | if( !isIgnoreEmpty || nList>0 ){ |
| 120944 | | - nByte = sqlite3Fts3VarintLen(iDocid-iPrev) + (isRequirePos?nList+1:0); |
| 122022 | + |
| 122023 | + /* Calculate the 'docid' delta value to write into the merged |
| 122024 | + ** doclist. */ |
| 122025 | + sqlite3_int64 iDelta; |
| 122026 | + if( p->bDescIdx && nDoclist>0 ){ |
| 122027 | + iDelta = iPrev - iDocid; |
| 122028 | + }else{ |
| 122029 | + iDelta = iDocid - iPrev; |
| 122030 | + } |
| 122031 | + assert( iDelta>0 || (nDoclist==0 && iDelta==iDocid) ); |
| 122032 | + assert( nDoclist>0 || iDelta==iDocid ); |
| 122033 | + |
| 122034 | + nByte = sqlite3Fts3VarintLen(iDelta) + (isRequirePos?nList+1:0); |
| 120945 | 122035 | if( nDoclist+nByte>pCsr->nBuffer ){ |
| 120946 | 122036 | char *aNew; |
| 120947 | 122037 | pCsr->nBuffer = (nDoclist+nByte)*2; |
| 120948 | 122038 | aNew = sqlite3_realloc(pCsr->aBuffer, pCsr->nBuffer); |
| 120949 | 122039 | if( !aNew ){ |
| 120950 | 122040 | return SQLITE_NOMEM; |
| 120951 | 122041 | } |
| 120952 | 122042 | pCsr->aBuffer = aNew; |
| 120953 | 122043 | } |
| 120954 | | - nDoclist += sqlite3Fts3PutVarint( |
| 120955 | | - &pCsr->aBuffer[nDoclist], iDocid-iPrev |
| 120956 | | - ); |
| 122044 | + nDoclist += sqlite3Fts3PutVarint(&pCsr->aBuffer[nDoclist], iDelta); |
| 120957 | 122045 | iPrev = iDocid; |
| 120958 | 122046 | if( isRequirePos ){ |
| 120959 | 122047 | memcpy(&pCsr->aBuffer[nDoclist], pList, nList); |
| 120960 | 122048 | nDoclist += nList; |
| 120961 | 122049 | pCsr->aBuffer[nDoclist++] = '\0'; |
| 120962 | 122050 | } |
| 120963 | 122051 | } |
| 120964 | 122052 | |
| 120965 | | - fts3SegReaderSort(apSegment, nMerge, j, fts3SegReaderDoclistCmp); |
| 122053 | + fts3SegReaderSort(apSegment, nMerge, j, xCmp); |
| 120966 | 122054 | } |
| 120967 | 122055 | if( nDoclist>0 ){ |
| 120968 | 122056 | pCsr->aDoclist = pCsr->aBuffer; |
| 120969 | 122057 | pCsr->nDoclist = nDoclist; |
| 120970 | 122058 | rc = SQLITE_ROW; |
| | @@ -120973,13 +122061,14 @@ |
| 120973 | 122061 | pCsr->nAdvance = nMerge; |
| 120974 | 122062 | }while( rc==SQLITE_OK ); |
| 120975 | 122063 | |
| 120976 | 122064 | return rc; |
| 120977 | 122065 | } |
| 122066 | + |
| 120978 | 122067 | |
| 120979 | 122068 | SQLITE_PRIVATE void sqlite3Fts3SegReaderFinish( |
| 120980 | | - Fts3SegReaderCursor *pCsr /* Cursor object */ |
| 122069 | + Fts3MultiSegReader *pCsr /* Cursor object */ |
| 120981 | 122070 | ){ |
| 120982 | 122071 | if( pCsr ){ |
| 120983 | 122072 | int i; |
| 120984 | 122073 | for(i=0; i<pCsr->nSegment; i++){ |
| 120985 | 122074 | sqlite3Fts3SegReaderFree(pCsr->apSegment[i]); |
| | @@ -121002,47 +122091,60 @@ |
| 121002 | 122091 | ** If this function is called with iLevel<0, but there is only one |
| 121003 | 122092 | ** segment in the database, SQLITE_DONE is returned immediately. |
| 121004 | 122093 | ** Otherwise, if successful, SQLITE_OK is returned. If an error occurs, |
| 121005 | 122094 | ** an SQLite error code is returned. |
| 121006 | 122095 | */ |
| 121007 | | -static int fts3SegmentMerge(Fts3Table *p, int iLevel){ |
| 122096 | +static int fts3SegmentMerge(Fts3Table *p, int iIndex, int iLevel){ |
| 121008 | 122097 | int rc; /* Return code */ |
| 121009 | 122098 | int iIdx = 0; /* Index of new segment */ |
| 121010 | | - int iNewLevel = 0; /* Level to create new segment at */ |
| 122099 | + int iNewLevel = 0; /* Level/index to create new segment at */ |
| 121011 | 122100 | SegmentWriter *pWriter = 0; /* Used to write the new, merged, segment */ |
| 121012 | 122101 | Fts3SegFilter filter; /* Segment term filter condition */ |
| 121013 | | - Fts3SegReaderCursor csr; /* Cursor to iterate through level(s) */ |
| 122102 | + Fts3MultiSegReader csr; /* Cursor to iterate through level(s) */ |
| 122103 | + int bIgnoreEmpty = 0; /* True to ignore empty segments */ |
| 121014 | 122104 | |
| 121015 | | - rc = sqlite3Fts3SegReaderCursor(p, iLevel, 0, 0, 1, 0, &csr); |
| 122105 | + assert( iLevel==FTS3_SEGCURSOR_ALL |
| 122106 | + || iLevel==FTS3_SEGCURSOR_PENDING |
| 122107 | + || iLevel>=0 |
| 122108 | + ); |
| 122109 | + assert( iLevel<FTS3_SEGDIR_MAXLEVEL ); |
| 122110 | + assert( iIndex>=0 && iIndex<p->nIndex ); |
| 122111 | + |
| 122112 | + rc = sqlite3Fts3SegReaderCursor(p, iIndex, iLevel, 0, 0, 1, 0, &csr); |
| 121016 | 122113 | if( rc!=SQLITE_OK || csr.nSegment==0 ) goto finished; |
| 121017 | 122114 | |
| 121018 | 122115 | if( iLevel==FTS3_SEGCURSOR_ALL ){ |
| 121019 | 122116 | /* This call is to merge all segments in the database to a single |
| 121020 | 122117 | ** segment. The level of the new segment is equal to the the numerically |
| 121021 | | - ** greatest segment level currently present in the database. The index |
| 121022 | | - ** of the new segment is always 0. */ |
| 121023 | | - int nDummy; /* TODO: Remove this */ |
| 122118 | + ** greatest segment level currently present in the database for this |
| 122119 | + ** index. The idx of the new segment is always 0. */ |
| 121024 | 122120 | if( csr.nSegment==1 ){ |
| 121025 | 122121 | rc = SQLITE_DONE; |
| 121026 | 122122 | goto finished; |
| 121027 | 122123 | } |
| 121028 | | - rc = fts3SegmentCountMax(p, &nDummy, &iNewLevel); |
| 122124 | + rc = fts3SegmentMaxLevel(p, iIndex, &iNewLevel); |
| 122125 | + bIgnoreEmpty = 1; |
| 122126 | + |
| 122127 | + }else if( iLevel==FTS3_SEGCURSOR_PENDING ){ |
| 122128 | + iNewLevel = iIndex * FTS3_SEGDIR_MAXLEVEL; |
| 122129 | + rc = fts3AllocateSegdirIdx(p, iIndex, 0, &iIdx); |
| 121029 | 122130 | }else{ |
| 121030 | | - /* This call is to merge all segments at level iLevel. Find the next |
| 122131 | + /* This call is to merge all segments at level iLevel. find the next |
| 121031 | 122132 | ** available segment index at level iLevel+1. The call to |
| 121032 | 122133 | ** fts3AllocateSegdirIdx() will merge the segments at level iLevel+1 to |
| 121033 | 122134 | ** a single iLevel+2 segment if necessary. */ |
| 121034 | | - iNewLevel = iLevel+1; |
| 121035 | | - rc = fts3AllocateSegdirIdx(p, iNewLevel, &iIdx); |
| 122135 | + rc = fts3AllocateSegdirIdx(p, iIndex, iLevel+1, &iIdx); |
| 122136 | + iNewLevel = iIndex * FTS3_SEGDIR_MAXLEVEL + iLevel+1; |
| 121036 | 122137 | } |
| 121037 | 122138 | if( rc!=SQLITE_OK ) goto finished; |
| 121038 | 122139 | assert( csr.nSegment>0 ); |
| 121039 | | - assert( iNewLevel>=0 ); |
| 122140 | + assert( iNewLevel>=(iIndex*FTS3_SEGDIR_MAXLEVEL) ); |
| 122141 | + assert( iNewLevel<((iIndex+1)*FTS3_SEGDIR_MAXLEVEL) ); |
| 121040 | 122142 | |
| 121041 | 122143 | memset(&filter, 0, sizeof(Fts3SegFilter)); |
| 121042 | 122144 | filter.flags = FTS3_SEGMENT_REQUIRE_POS; |
| 121043 | | - filter.flags |= (iLevel==FTS3_SEGCURSOR_ALL ? FTS3_SEGMENT_IGNORE_EMPTY : 0); |
| 122145 | + filter.flags |= (bIgnoreEmpty ? FTS3_SEGMENT_IGNORE_EMPTY : 0); |
| 121044 | 122146 | |
| 121045 | 122147 | rc = sqlite3Fts3SegReaderStart(p, &csr, &filter); |
| 121046 | 122148 | while( SQLITE_OK==rc ){ |
| 121047 | 122149 | rc = sqlite3Fts3SegReaderStep(p, &csr); |
| 121048 | 122150 | if( rc!=SQLITE_ROW ) break; |
| | @@ -121050,12 +122152,14 @@ |
| 121050 | 122152 | csr.zTerm, csr.nTerm, csr.aDoclist, csr.nDoclist); |
| 121051 | 122153 | } |
| 121052 | 122154 | if( rc!=SQLITE_OK ) goto finished; |
| 121053 | 122155 | assert( pWriter ); |
| 121054 | 122156 | |
| 121055 | | - rc = fts3DeleteSegdir(p, iLevel, csr.apSegment, csr.nSegment); |
| 121056 | | - if( rc!=SQLITE_OK ) goto finished; |
| 122157 | + if( iLevel!=FTS3_SEGCURSOR_PENDING ){ |
| 122158 | + rc = fts3DeleteSegdir(p, iIndex, iLevel, csr.apSegment, csr.nSegment); |
| 122159 | + if( rc!=SQLITE_OK ) goto finished; |
| 122160 | + } |
| 121057 | 122161 | rc = fts3SegWriterFlush(p, pWriter, iNewLevel, iIdx); |
| 121058 | 122162 | |
| 121059 | 122163 | finished: |
| 121060 | 122164 | fts3SegWriterFree(pWriter); |
| 121061 | 122165 | sqlite3Fts3SegReaderFinish(&csr); |
| | @@ -121062,14 +122166,21 @@ |
| 121062 | 122166 | return rc; |
| 121063 | 122167 | } |
| 121064 | 122168 | |
| 121065 | 122169 | |
| 121066 | 122170 | /* |
| 121067 | | -** Flush the contents of pendingTerms to a level 0 segment. |
| 122171 | +** Flush the contents of pendingTerms to level 0 segments. |
| 121068 | 122172 | */ |
| 121069 | 122173 | SQLITE_PRIVATE int sqlite3Fts3PendingTermsFlush(Fts3Table *p){ |
| 121070 | | - return fts3SegmentMerge(p, FTS3_SEGCURSOR_PENDING); |
| 122174 | + int rc = SQLITE_OK; |
| 122175 | + int i; |
| 122176 | + for(i=0; rc==SQLITE_OK && i<p->nIndex; i++){ |
| 122177 | + rc = fts3SegmentMerge(p, i, FTS3_SEGCURSOR_PENDING); |
| 122178 | + if( rc==SQLITE_DONE ) rc = SQLITE_OK; |
| 122179 | + } |
| 122180 | + sqlite3Fts3PendingTermsClear(p); |
| 122181 | + return rc; |
| 121071 | 122182 | } |
| 121072 | 122183 | |
| 121073 | 122184 | /* |
| 121074 | 122185 | ** Encode N integers as varints into a blob. |
| 121075 | 122186 | */ |
| | @@ -121215,10 +122326,27 @@ |
| 121215 | 122326 | sqlite3_bind_blob(pStmt, 1, pBlob, nBlob, SQLITE_STATIC); |
| 121216 | 122327 | sqlite3_step(pStmt); |
| 121217 | 122328 | *pRC = sqlite3_reset(pStmt); |
| 121218 | 122329 | sqlite3_free(a); |
| 121219 | 122330 | } |
| 122331 | + |
| 122332 | +static int fts3DoOptimize(Fts3Table *p, int bReturnDone){ |
| 122333 | + int i; |
| 122334 | + int bSeenDone = 0; |
| 122335 | + int rc = SQLITE_OK; |
| 122336 | + for(i=0; rc==SQLITE_OK && i<p->nIndex; i++){ |
| 122337 | + rc = fts3SegmentMerge(p, i, FTS3_SEGCURSOR_ALL); |
| 122338 | + if( rc==SQLITE_DONE ){ |
| 122339 | + bSeenDone = 1; |
| 122340 | + rc = SQLITE_OK; |
| 122341 | + } |
| 122342 | + } |
| 122343 | + sqlite3Fts3SegmentsClose(p); |
| 122344 | + sqlite3Fts3PendingTermsClear(p); |
| 122345 | + |
| 122346 | + return (rc==SQLITE_OK && bReturnDone && bSeenDone) ? SQLITE_DONE : rc; |
| 122347 | +} |
| 121220 | 122348 | |
| 121221 | 122349 | /* |
| 121222 | 122350 | ** Handle a 'special' INSERT of the form: |
| 121223 | 122351 | ** |
| 121224 | 122352 | ** "INSERT INTO tbl(tbl) VALUES(<expr>)" |
| | @@ -121232,16 +122360,11 @@ |
| 121232 | 122360 | int nVal = sqlite3_value_bytes(pVal); |
| 121233 | 122361 | |
| 121234 | 122362 | if( !zVal ){ |
| 121235 | 122363 | return SQLITE_NOMEM; |
| 121236 | 122364 | }else if( nVal==8 && 0==sqlite3_strnicmp(zVal, "optimize", 8) ){ |
| 121237 | | - rc = fts3SegmentMerge(p, FTS3_SEGCURSOR_ALL); |
| 121238 | | - if( rc==SQLITE_DONE ){ |
| 121239 | | - rc = SQLITE_OK; |
| 121240 | | - }else{ |
| 121241 | | - sqlite3Fts3PendingTermsClear(p); |
| 121242 | | - } |
| 122365 | + rc = fts3DoOptimize(p, 0); |
| 121243 | 122366 | #ifdef SQLITE_TEST |
| 121244 | 122367 | }else if( nVal>9 && 0==sqlite3_strnicmp(zVal, "nodesize=", 9) ){ |
| 121245 | 122368 | p->nNodeSize = atoi(&zVal[9]); |
| 121246 | 122369 | rc = SQLITE_OK; |
| 121247 | 122370 | }else if( nVal>11 && 0==sqlite3_strnicmp(zVal, "maxpending=", 9) ){ |
| | @@ -121250,61 +122373,23 @@ |
| 121250 | 122373 | #endif |
| 121251 | 122374 | }else{ |
| 121252 | 122375 | rc = SQLITE_ERROR; |
| 121253 | 122376 | } |
| 121254 | 122377 | |
| 121255 | | - sqlite3Fts3SegmentsClose(p); |
| 121256 | 122378 | return rc; |
| 121257 | 122379 | } |
| 121258 | 122380 | |
| 121259 | | -/* |
| 121260 | | -** Return the deferred doclist associated with deferred token pDeferred. |
| 121261 | | -** This function assumes that sqlite3Fts3CacheDeferredDoclists() has already |
| 121262 | | -** been called to allocate and populate the doclist. |
| 121263 | | -*/ |
| 121264 | | -SQLITE_PRIVATE char *sqlite3Fts3DeferredDoclist(Fts3DeferredToken *pDeferred, int *pnByte){ |
| 121265 | | - if( pDeferred->pList ){ |
| 121266 | | - *pnByte = pDeferred->pList->nData; |
| 121267 | | - return pDeferred->pList->aData; |
| 121268 | | - } |
| 121269 | | - *pnByte = 0; |
| 121270 | | - return 0; |
| 121271 | | -} |
| 121272 | | - |
| 121273 | | -/* |
| 121274 | | -** Helper fucntion for FreeDeferredDoclists(). This function removes all |
| 121275 | | -** references to deferred doclists from within the tree of Fts3Expr |
| 121276 | | -** structures headed by |
| 121277 | | -*/ |
| 121278 | | -static void fts3DeferredDoclistClear(Fts3Expr *pExpr){ |
| 121279 | | - if( pExpr ){ |
| 121280 | | - fts3DeferredDoclistClear(pExpr->pLeft); |
| 121281 | | - fts3DeferredDoclistClear(pExpr->pRight); |
| 121282 | | - if( pExpr->isLoaded ){ |
| 121283 | | - sqlite3_free(pExpr->aDoclist); |
| 121284 | | - pExpr->isLoaded = 0; |
| 121285 | | - pExpr->aDoclist = 0; |
| 121286 | | - pExpr->nDoclist = 0; |
| 121287 | | - pExpr->pCurrent = 0; |
| 121288 | | - pExpr->iCurrent = 0; |
| 121289 | | - } |
| 121290 | | - } |
| 121291 | | -} |
| 121292 | | - |
| 121293 | 122381 | /* |
| 121294 | 122382 | ** Delete all cached deferred doclists. Deferred doclists are cached |
| 121295 | 122383 | ** (allocated) by the sqlite3Fts3CacheDeferredDoclists() function. |
| 121296 | 122384 | */ |
| 121297 | 122385 | SQLITE_PRIVATE void sqlite3Fts3FreeDeferredDoclists(Fts3Cursor *pCsr){ |
| 121298 | 122386 | Fts3DeferredToken *pDef; |
| 121299 | 122387 | for(pDef=pCsr->pDeferred; pDef; pDef=pDef->pNext){ |
| 121300 | | - sqlite3_free(pDef->pList); |
| 122388 | + fts3PendingListDelete(pDef->pList); |
| 121301 | 122389 | pDef->pList = 0; |
| 121302 | 122390 | } |
| 121303 | | - if( pCsr->pDeferred ){ |
| 121304 | | - fts3DeferredDoclistClear(pCsr->pExpr); |
| 121305 | | - } |
| 121306 | 122391 | } |
| 121307 | 122392 | |
| 121308 | 122393 | /* |
| 121309 | 122394 | ** Free all entries in the pCsr->pDeffered list. Entries are added to |
| 121310 | 122395 | ** this list using sqlite3Fts3DeferToken(). |
| | @@ -121312,11 +122397,11 @@ |
| 121312 | 122397 | SQLITE_PRIVATE void sqlite3Fts3FreeDeferredTokens(Fts3Cursor *pCsr){ |
| 121313 | 122398 | Fts3DeferredToken *pDef; |
| 121314 | 122399 | Fts3DeferredToken *pNext; |
| 121315 | 122400 | for(pDef=pCsr->pDeferred; pDef; pDef=pNext){ |
| 121316 | 122401 | pNext = pDef->pNext; |
| 121317 | | - sqlite3_free(pDef->pList); |
| 122402 | + fts3PendingListDelete(pDef->pList); |
| 121318 | 122403 | sqlite3_free(pDef); |
| 121319 | 122404 | } |
| 121320 | 122405 | pCsr->pDeferred = 0; |
| 121321 | 122406 | } |
| 121322 | 122407 | |
| | @@ -121376,10 +122461,37 @@ |
| 121376 | 122461 | } |
| 121377 | 122462 | } |
| 121378 | 122463 | |
| 121379 | 122464 | return rc; |
| 121380 | 122465 | } |
| 122466 | + |
| 122467 | +SQLITE_PRIVATE int sqlite3Fts3DeferredTokenList( |
| 122468 | + Fts3DeferredToken *p, |
| 122469 | + char **ppData, |
| 122470 | + int *pnData |
| 122471 | +){ |
| 122472 | + char *pRet; |
| 122473 | + int nSkip; |
| 122474 | + sqlite3_int64 dummy; |
| 122475 | + |
| 122476 | + *ppData = 0; |
| 122477 | + *pnData = 0; |
| 122478 | + |
| 122479 | + if( p->pList==0 ){ |
| 122480 | + return SQLITE_OK; |
| 122481 | + } |
| 122482 | + |
| 122483 | + pRet = (char *)sqlite3_malloc(p->pList->nData); |
| 122484 | + if( !pRet ) return SQLITE_NOMEM; |
| 122485 | + |
| 122486 | + nSkip = sqlite3Fts3GetVarint(p->pList->aData, &dummy); |
| 122487 | + *pnData = p->pList->nData - nSkip; |
| 122488 | + *ppData = pRet; |
| 122489 | + |
| 122490 | + memcpy(pRet, &p->pList->aData[nSkip], *pnData); |
| 122491 | + return SQLITE_OK; |
| 122492 | +} |
| 121381 | 122493 | |
| 121382 | 122494 | /* |
| 121383 | 122495 | ** Add an entry for token pToken to the pCsr->pDeferred list. |
| 121384 | 122496 | */ |
| 121385 | 122497 | SQLITE_PRIVATE int sqlite3Fts3DeferToken( |
| | @@ -121451,11 +122563,11 @@ |
| 121451 | 122563 | ){ |
| 121452 | 122564 | Fts3Table *p = (Fts3Table *)pVtab; |
| 121453 | 122565 | int rc = SQLITE_OK; /* Return Code */ |
| 121454 | 122566 | int isRemove = 0; /* True for an UPDATE or DELETE */ |
| 121455 | 122567 | sqlite3_int64 iRemove = 0; /* Rowid removed by UPDATE or DELETE */ |
| 121456 | | - u32 *aSzIns; /* Sizes of inserted documents */ |
| 122568 | + u32 *aSzIns = 0; /* Sizes of inserted documents */ |
| 121457 | 122569 | u32 *aSzDel; /* Sizes of deleted documents */ |
| 121458 | 122570 | int nChng = 0; /* Net change in number of documents */ |
| 121459 | 122571 | int bInsertDone = 0; |
| 121460 | 122572 | |
| 121461 | 122573 | assert( p->pSegments==0 ); |
| | @@ -121466,16 +122578,20 @@ |
| 121466 | 122578 | */ |
| 121467 | 122579 | if( nArg>1 |
| 121468 | 122580 | && sqlite3_value_type(apVal[0])==SQLITE_NULL |
| 121469 | 122581 | && sqlite3_value_type(apVal[p->nColumn+2])!=SQLITE_NULL |
| 121470 | 122582 | ){ |
| 121471 | | - return fts3SpecialInsert(p, apVal[p->nColumn+2]); |
| 122583 | + rc = fts3SpecialInsert(p, apVal[p->nColumn+2]); |
| 122584 | + goto update_out; |
| 121472 | 122585 | } |
| 121473 | 122586 | |
| 121474 | 122587 | /* Allocate space to hold the change in document sizes */ |
| 121475 | 122588 | aSzIns = sqlite3_malloc( sizeof(aSzIns[0])*(p->nColumn+1)*2 ); |
| 121476 | | - if( aSzIns==0 ) return SQLITE_NOMEM; |
| 122589 | + if( aSzIns==0 ){ |
| 122590 | + rc = SQLITE_NOMEM; |
| 122591 | + goto update_out; |
| 122592 | + } |
| 121477 | 122593 | aSzDel = &aSzIns[p->nColumn+1]; |
| 121478 | 122594 | memset(aSzIns, 0, sizeof(aSzIns[0])*(p->nColumn+1)*2); |
| 121479 | 122595 | |
| 121480 | 122596 | /* If this is an INSERT operation, or an UPDATE that modifies the rowid |
| 121481 | 122597 | ** value, then this operation requires constraint handling. |
| | @@ -121521,12 +122637,11 @@ |
| 121521 | 122637 | bInsertDone = 1; |
| 121522 | 122638 | } |
| 121523 | 122639 | } |
| 121524 | 122640 | } |
| 121525 | 122641 | if( rc!=SQLITE_OK ){ |
| 121526 | | - sqlite3_free(aSzIns); |
| 121527 | | - return rc; |
| 122642 | + goto update_out; |
| 121528 | 122643 | } |
| 121529 | 122644 | |
| 121530 | 122645 | /* If this is a DELETE or UPDATE operation, remove the old record. */ |
| 121531 | 122646 | if( sqlite3_value_type(apVal[0])!=SQLITE_NULL ){ |
| 121532 | 122647 | assert( sqlite3_value_type(apVal[0])==SQLITE_INTEGER ); |
| | @@ -121555,10 +122670,11 @@ |
| 121555 | 122670 | |
| 121556 | 122671 | if( p->bHasStat ){ |
| 121557 | 122672 | fts3UpdateDocTotals(&rc, p, aSzIns, aSzDel, nChng); |
| 121558 | 122673 | } |
| 121559 | 122674 | |
| 122675 | + update_out: |
| 121560 | 122676 | sqlite3_free(aSzIns); |
| 121561 | 122677 | sqlite3Fts3SegmentsClose(p); |
| 121562 | 122678 | return rc; |
| 121563 | 122679 | } |
| 121564 | 122680 | |
| | @@ -121569,16 +122685,14 @@ |
| 121569 | 122685 | */ |
| 121570 | 122686 | SQLITE_PRIVATE int sqlite3Fts3Optimize(Fts3Table *p){ |
| 121571 | 122687 | int rc; |
| 121572 | 122688 | rc = sqlite3_exec(p->db, "SAVEPOINT fts3", 0, 0, 0); |
| 121573 | 122689 | if( rc==SQLITE_OK ){ |
| 121574 | | - rc = fts3SegmentMerge(p, FTS3_SEGCURSOR_ALL); |
| 121575 | | - if( rc==SQLITE_OK ){ |
| 121576 | | - rc = sqlite3_exec(p->db, "RELEASE fts3", 0, 0, 0); |
| 121577 | | - if( rc==SQLITE_OK ){ |
| 121578 | | - sqlite3Fts3PendingTermsClear(p); |
| 121579 | | - } |
| 122690 | + rc = fts3DoOptimize(p, 1); |
| 122691 | + if( rc==SQLITE_OK || rc==SQLITE_DONE ){ |
| 122692 | + int rc2 = sqlite3_exec(p->db, "RELEASE fts3", 0, 0, 0); |
| 122693 | + if( rc2!=SQLITE_OK ) rc = rc2; |
| 121580 | 122694 | }else{ |
| 121581 | 122695 | sqlite3_exec(p->db, "ROLLBACK TO fts3", 0, 0, 0); |
| 121582 | 122696 | sqlite3_exec(p->db, "RELEASE fts3", 0, 0, 0); |
| 121583 | 122697 | } |
| 121584 | 122698 | } |
| | @@ -121763,76 +122877,24 @@ |
| 121763 | 122877 | ){ |
| 121764 | 122878 | int iPhrase = 0; /* Variable used as the phrase counter */ |
| 121765 | 122879 | return fts3ExprIterate2(pExpr, &iPhrase, x, pCtx); |
| 121766 | 122880 | } |
| 121767 | 122881 | |
| 121768 | | -/* |
| 121769 | | -** The argument to this function is always a phrase node. Its doclist |
| 121770 | | -** (Fts3Expr.aDoclist[]) and the doclists associated with all phrase nodes |
| 121771 | | -** to the left of this one in the query tree have already been loaded. |
| 121772 | | -** |
| 121773 | | -** If this phrase node is part of a series of phrase nodes joined by |
| 121774 | | -** NEAR operators (and is not the left-most of said series), then elements are |
| 121775 | | -** removed from the phrases doclist consistent with the NEAR restriction. If |
| 121776 | | -** required, elements may be removed from the doclists of phrases to the |
| 121777 | | -** left of this one that are part of the same series of NEAR operator |
| 121778 | | -** connected phrases. |
| 121779 | | -** |
| 121780 | | -** If an OOM error occurs, SQLITE_NOMEM is returned. Otherwise, SQLITE_OK. |
| 121781 | | -*/ |
| 121782 | | -static int fts3ExprNearTrim(Fts3Expr *pExpr){ |
| 121783 | | - int rc = SQLITE_OK; |
| 121784 | | - Fts3Expr *pParent = pExpr->pParent; |
| 121785 | | - |
| 121786 | | - assert( pExpr->eType==FTSQUERY_PHRASE ); |
| 121787 | | - while( rc==SQLITE_OK |
| 121788 | | - && pParent |
| 121789 | | - && pParent->eType==FTSQUERY_NEAR |
| 121790 | | - && pParent->pRight==pExpr |
| 121791 | | - ){ |
| 121792 | | - /* This expression (pExpr) is the right-hand-side of a NEAR operator. |
| 121793 | | - ** Find the expression to the left of the same operator. |
| 121794 | | - */ |
| 121795 | | - int nNear = pParent->nNear; |
| 121796 | | - Fts3Expr *pLeft = pParent->pLeft; |
| 121797 | | - |
| 121798 | | - if( pLeft->eType!=FTSQUERY_PHRASE ){ |
| 121799 | | - assert( pLeft->eType==FTSQUERY_NEAR ); |
| 121800 | | - assert( pLeft->pRight->eType==FTSQUERY_PHRASE ); |
| 121801 | | - pLeft = pLeft->pRight; |
| 121802 | | - } |
| 121803 | | - |
| 121804 | | - rc = sqlite3Fts3ExprNearTrim(pLeft, pExpr, nNear); |
| 121805 | | - |
| 121806 | | - pExpr = pLeft; |
| 121807 | | - pParent = pExpr->pParent; |
| 121808 | | - } |
| 121809 | | - |
| 121810 | | - return rc; |
| 121811 | | -} |
| 121812 | | - |
| 121813 | 122882 | /* |
| 121814 | 122883 | ** This is an fts3ExprIterate() callback used while loading the doclists |
| 121815 | 122884 | ** for each phrase into Fts3Expr.aDoclist[]/nDoclist. See also |
| 121816 | 122885 | ** fts3ExprLoadDoclists(). |
| 121817 | 122886 | */ |
| 121818 | 122887 | static int fts3ExprLoadDoclistsCb(Fts3Expr *pExpr, int iPhrase, void *ctx){ |
| 121819 | 122888 | int rc = SQLITE_OK; |
| 122889 | + Fts3Phrase *pPhrase = pExpr->pPhrase; |
| 121820 | 122890 | LoadDoclistCtx *p = (LoadDoclistCtx *)ctx; |
| 121821 | 122891 | |
| 121822 | 122892 | UNUSED_PARAMETER(iPhrase); |
| 121823 | 122893 | |
| 121824 | 122894 | p->nPhrase++; |
| 121825 | | - p->nToken += pExpr->pPhrase->nToken; |
| 121826 | | - |
| 121827 | | - if( pExpr->isLoaded==0 ){ |
| 121828 | | - rc = sqlite3Fts3ExprLoadDoclist(p->pCsr, pExpr); |
| 121829 | | - pExpr->isLoaded = 1; |
| 121830 | | - if( rc==SQLITE_OK ){ |
| 121831 | | - rc = fts3ExprNearTrim(pExpr); |
| 121832 | | - } |
| 121833 | | - } |
| 122895 | + p->nToken += pPhrase->nToken; |
| 121834 | 122896 | |
| 121835 | 122897 | return rc; |
| 121836 | 122898 | } |
| 121837 | 122899 | |
| 121838 | 122900 | /* |
| | @@ -122002,11 +123064,11 @@ |
| 122002 | 123064 | SnippetPhrase *pPhrase = &p->aPhrase[iPhrase]; |
| 122003 | 123065 | char *pCsr; |
| 122004 | 123066 | |
| 122005 | 123067 | pPhrase->nToken = pExpr->pPhrase->nToken; |
| 122006 | 123068 | |
| 122007 | | - pCsr = sqlite3Fts3FindPositions(p->pCsr, pExpr, p->pCsr->iPrevId, p->iCol); |
| 123069 | + pCsr = sqlite3Fts3EvalPhrasePoslist(p->pCsr, pExpr, p->iCol); |
| 122008 | 123070 | if( pCsr ){ |
| 122009 | 123071 | int iFirst = 0; |
| 122010 | 123072 | pPhrase->pList = pCsr; |
| 122011 | 123073 | fts3GetDeltaPosition(&pCsr, &iFirst); |
| 122012 | 123074 | pPhrase->pHead = pCsr; |
| | @@ -122359,30 +123421,10 @@ |
| 122359 | 123421 | |
| 122360 | 123422 | *ppCollist = pEnd; |
| 122361 | 123423 | return nEntry; |
| 122362 | 123424 | } |
| 122363 | 123425 | |
| 122364 | | -static void fts3LoadColumnlistCounts(char **pp, u32 *aOut, int isGlobal){ |
| 122365 | | - char *pCsr = *pp; |
| 122366 | | - while( *pCsr ){ |
| 122367 | | - int nHit; |
| 122368 | | - sqlite3_int64 iCol = 0; |
| 122369 | | - if( *pCsr==0x01 ){ |
| 122370 | | - pCsr++; |
| 122371 | | - pCsr += sqlite3Fts3GetVarint(pCsr, &iCol); |
| 122372 | | - } |
| 122373 | | - nHit = fts3ColumnlistCount(&pCsr); |
| 122374 | | - assert( nHit>0 ); |
| 122375 | | - if( isGlobal ){ |
| 122376 | | - aOut[iCol*3+1]++; |
| 122377 | | - } |
| 122378 | | - aOut[iCol*3] += nHit; |
| 122379 | | - } |
| 122380 | | - pCsr++; |
| 122381 | | - *pp = pCsr; |
| 122382 | | -} |
| 122383 | | - |
| 122384 | 123426 | /* |
| 122385 | 123427 | ** fts3ExprIterate() callback used to collect the "global" matchinfo stats |
| 122386 | 123428 | ** for a single query. |
| 122387 | 123429 | ** |
| 122388 | 123430 | ** fts3ExprIterate() callback to load the 'global' elements of a |
| | @@ -122412,52 +123454,13 @@ |
| 122412 | 123454 | Fts3Expr *pExpr, /* Phrase expression node */ |
| 122413 | 123455 | int iPhrase, /* Phrase number (numbered from zero) */ |
| 122414 | 123456 | void *pCtx /* Pointer to MatchInfo structure */ |
| 122415 | 123457 | ){ |
| 122416 | 123458 | MatchInfo *p = (MatchInfo *)pCtx; |
| 122417 | | - Fts3Cursor *pCsr = p->pCursor; |
| 122418 | | - char *pIter; |
| 122419 | | - char *pEnd; |
| 122420 | | - char *pFree = 0; |
| 122421 | | - u32 *aOut = &p->aMatchinfo[3*iPhrase*p->nCol]; |
| 122422 | | - |
| 122423 | | - assert( pExpr->isLoaded ); |
| 122424 | | - assert( pExpr->eType==FTSQUERY_PHRASE ); |
| 122425 | | - |
| 122426 | | - if( pCsr->pDeferred ){ |
| 122427 | | - Fts3Phrase *pPhrase = pExpr->pPhrase; |
| 122428 | | - int ii; |
| 122429 | | - for(ii=0; ii<pPhrase->nToken; ii++){ |
| 122430 | | - if( pPhrase->aToken[ii].bFulltext ) break; |
| 122431 | | - } |
| 122432 | | - if( ii<pPhrase->nToken ){ |
| 122433 | | - int nFree = 0; |
| 122434 | | - int rc = sqlite3Fts3ExprLoadFtDoclist(pCsr, pExpr, &pFree, &nFree); |
| 122435 | | - if( rc!=SQLITE_OK ) return rc; |
| 122436 | | - pIter = pFree; |
| 122437 | | - pEnd = &pFree[nFree]; |
| 122438 | | - }else{ |
| 122439 | | - int iCol; /* Column index */ |
| 122440 | | - for(iCol=0; iCol<p->nCol; iCol++){ |
| 122441 | | - aOut[iCol*3 + 1] = (u32)p->nDoc; |
| 122442 | | - aOut[iCol*3 + 2] = (u32)p->nDoc; |
| 122443 | | - } |
| 122444 | | - return SQLITE_OK; |
| 122445 | | - } |
| 122446 | | - }else{ |
| 122447 | | - pIter = pExpr->aDoclist; |
| 122448 | | - pEnd = &pExpr->aDoclist[pExpr->nDoclist]; |
| 122449 | | - } |
| 122450 | | - |
| 122451 | | - /* Fill in the global hit count matrix row for this phrase. */ |
| 122452 | | - while( pIter<pEnd ){ |
| 122453 | | - while( *pIter++ & 0x80 ); /* Skip past docid. */ |
| 122454 | | - fts3LoadColumnlistCounts(&pIter, &aOut[1], 1); |
| 122455 | | - } |
| 122456 | | - |
| 122457 | | - sqlite3_free(pFree); |
| 122458 | | - return SQLITE_OK; |
| 123459 | + return sqlite3Fts3EvalPhraseStats( |
| 123460 | + p->pCursor, pExpr, &p->aMatchinfo[3*iPhrase*p->nCol] |
| 123461 | + ); |
| 122459 | 123462 | } |
| 122460 | 123463 | |
| 122461 | 123464 | /* |
| 122462 | 123465 | ** fts3ExprIterate() callback used to collect the "local" part of the |
| 122463 | 123466 | ** FTS3_MATCHINFO_HITS array. The local stats are those elements of the |
| | @@ -122470,18 +123473,17 @@ |
| 122470 | 123473 | ){ |
| 122471 | 123474 | MatchInfo *p = (MatchInfo *)pCtx; |
| 122472 | 123475 | int iStart = iPhrase * p->nCol * 3; |
| 122473 | 123476 | int i; |
| 122474 | 123477 | |
| 122475 | | - for(i=0; i<p->nCol; i++) p->aMatchinfo[iStart+i*3] = 0; |
| 122476 | | - |
| 122477 | | - if( pExpr->aDoclist ){ |
| 123478 | + for(i=0; i<p->nCol; i++){ |
| 122478 | 123479 | char *pCsr; |
| 122479 | | - |
| 122480 | | - pCsr = sqlite3Fts3FindPositions(p->pCursor, pExpr, p->pCursor->iPrevId, -1); |
| 123480 | + pCsr = sqlite3Fts3EvalPhrasePoslist(p->pCursor, pExpr, i); |
| 122481 | 123481 | if( pCsr ){ |
| 122482 | | - fts3LoadColumnlistCounts(&pCsr, &p->aMatchinfo[iStart], 0); |
| 123482 | + p->aMatchinfo[iStart+i*3] = fts3ColumnlistCount(&pCsr); |
| 123483 | + }else{ |
| 123484 | + p->aMatchinfo[iStart+i*3] = 0; |
| 122483 | 123485 | } |
| 122484 | 123486 | } |
| 122485 | 123487 | |
| 122486 | 123488 | return SQLITE_OK; |
| 122487 | 123489 | } |
| | @@ -122563,13 +123565,12 @@ |
| 122563 | 123565 | ** values for a matchinfo() FTS3_MATCHINFO_LCS request. |
| 122564 | 123566 | */ |
| 122565 | 123567 | typedef struct LcsIterator LcsIterator; |
| 122566 | 123568 | struct LcsIterator { |
| 122567 | 123569 | Fts3Expr *pExpr; /* Pointer to phrase expression */ |
| 122568 | | - char *pRead; /* Cursor used to iterate through aDoclist */ |
| 122569 | 123570 | int iPosOffset; /* Tokens count up to end of this phrase */ |
| 122570 | | - int iCol; /* Current column number */ |
| 123571 | + char *pRead; /* Cursor used to iterate through aDoclist */ |
| 122571 | 123572 | int iPos; /* Current position */ |
| 122572 | 123573 | }; |
| 122573 | 123574 | |
| 122574 | 123575 | /* |
| 122575 | 123576 | ** If LcsIterator.iCol is set to the following value, the iterator has |
| | @@ -122596,21 +123597,14 @@ |
| 122596 | 123597 | char *pRead = pIter->pRead; |
| 122597 | 123598 | sqlite3_int64 iRead; |
| 122598 | 123599 | int rc = 0; |
| 122599 | 123600 | |
| 122600 | 123601 | pRead += sqlite3Fts3GetVarint(pRead, &iRead); |
| 122601 | | - if( iRead==0 ){ |
| 122602 | | - pIter->iCol = LCS_ITERATOR_FINISHED; |
| 123602 | + if( iRead==0 || iRead==1 ){ |
| 123603 | + pRead = 0; |
| 122603 | 123604 | rc = 1; |
| 122604 | 123605 | }else{ |
| 122605 | | - if( iRead==1 ){ |
| 122606 | | - pRead += sqlite3Fts3GetVarint(pRead, &iRead); |
| 122607 | | - pIter->iCol = (int)iRead; |
| 122608 | | - pIter->iPos = pIter->iPosOffset; |
| 122609 | | - pRead += sqlite3Fts3GetVarint(pRead, &iRead); |
| 122610 | | - rc = 1; |
| 122611 | | - } |
| 122612 | 123606 | pIter->iPos += (int)(iRead-2); |
| 122613 | 123607 | } |
| 122614 | 123608 | |
| 122615 | 123609 | pIter->pRead = pRead; |
| 122616 | 123610 | return rc; |
| | @@ -122638,46 +123632,38 @@ |
| 122638 | 123632 | **/ |
| 122639 | 123633 | aIter = sqlite3_malloc(sizeof(LcsIterator) * pCsr->nPhrase); |
| 122640 | 123634 | if( !aIter ) return SQLITE_NOMEM; |
| 122641 | 123635 | memset(aIter, 0, sizeof(LcsIterator) * pCsr->nPhrase); |
| 122642 | 123636 | (void)fts3ExprIterate(pCsr->pExpr, fts3MatchinfoLcsCb, (void*)aIter); |
| 123637 | + |
| 122643 | 123638 | for(i=0; i<pInfo->nPhrase; i++){ |
| 122644 | 123639 | LcsIterator *pIter = &aIter[i]; |
| 122645 | 123640 | nToken -= pIter->pExpr->pPhrase->nToken; |
| 122646 | 123641 | pIter->iPosOffset = nToken; |
| 122647 | | - pIter->pRead = sqlite3Fts3FindPositions(pCsr,pIter->pExpr,pCsr->iPrevId,-1); |
| 122648 | | - if( pIter->pRead ){ |
| 122649 | | - pIter->iPos = pIter->iPosOffset; |
| 122650 | | - fts3LcsIteratorAdvance(&aIter[i]); |
| 122651 | | - }else{ |
| 122652 | | - pIter->iCol = LCS_ITERATOR_FINISHED; |
| 122653 | | - } |
| 122654 | 123642 | } |
| 122655 | 123643 | |
| 122656 | 123644 | for(iCol=0; iCol<pInfo->nCol; iCol++){ |
| 122657 | 123645 | int nLcs = 0; /* LCS value for this column */ |
| 122658 | 123646 | int nLive = 0; /* Number of iterators in aIter not at EOF */ |
| 122659 | 123647 | |
| 122660 | | - /* Loop through the iterators in aIter[]. Set nLive to the number of |
| 122661 | | - ** iterators that point to a position-list corresponding to column iCol. |
| 122662 | | - */ |
| 122663 | 123648 | for(i=0; i<pInfo->nPhrase; i++){ |
| 122664 | | - assert( aIter[i].iCol>=iCol ); |
| 122665 | | - if( aIter[i].iCol==iCol ) nLive++; |
| 123649 | + LcsIterator *pIt = &aIter[i]; |
| 123650 | + pIt->pRead = sqlite3Fts3EvalPhrasePoslist(pCsr, pIt->pExpr, iCol); |
| 123651 | + if( pIt->pRead ){ |
| 123652 | + pIt->iPos = pIt->iPosOffset; |
| 123653 | + fts3LcsIteratorAdvance(&aIter[i]); |
| 123654 | + nLive++; |
| 123655 | + } |
| 122666 | 123656 | } |
| 122667 | 123657 | |
| 122668 | | - /* The following loop runs until all iterators in aIter[] have finished |
| 122669 | | - ** iterating through positions in column iCol. Exactly one of the |
| 122670 | | - ** iterators is advanced each time the body of the loop is run. |
| 122671 | | - */ |
| 122672 | 123658 | while( nLive>0 ){ |
| 122673 | 123659 | LcsIterator *pAdv = 0; /* The iterator to advance by one position */ |
| 122674 | 123660 | int nThisLcs = 0; /* LCS for the current iterator positions */ |
| 122675 | 123661 | |
| 122676 | 123662 | for(i=0; i<pInfo->nPhrase; i++){ |
| 122677 | 123663 | LcsIterator *pIter = &aIter[i]; |
| 122678 | | - if( iCol!=pIter->iCol ){ |
| 123664 | + if( pIter->pRead==0 ){ |
| 122679 | 123665 | /* This iterator is already at EOF for this column. */ |
| 122680 | 123666 | nThisLcs = 0; |
| 122681 | 123667 | }else{ |
| 122682 | 123668 | if( pAdv==0 || pIter->iPos<pAdv->iPos ){ |
| 122683 | 123669 | pAdv = pIter; |
| | @@ -123013,11 +123999,11 @@ |
| 123013 | 123999 | int iTerm; /* For looping through nTerm phrase terms */ |
| 123014 | 124000 | char *pList; /* Pointer to position list for phrase */ |
| 123015 | 124001 | int iPos = 0; /* First position in position-list */ |
| 123016 | 124002 | |
| 123017 | 124003 | UNUSED_PARAMETER(iPhrase); |
| 123018 | | - pList = sqlite3Fts3FindPositions(p->pCsr, pExpr, p->iDocid, p->iCol); |
| 124004 | + pList = sqlite3Fts3EvalPhrasePoslist(p->pCsr, pExpr, p->iCol); |
| 123019 | 124005 | nTerm = pExpr->pPhrase->nToken; |
| 123020 | 124006 | if( pList ){ |
| 123021 | 124007 | fts3GetDeltaPosition(&pList, &iPos); |
| 123022 | 124008 | assert( iPos>=0 ); |
| 123023 | 124009 | } |
| 123024 | 124010 | |