Fossil SCM

Add the configure script and merge in the latest trunk changes.

drh 2011-06-15 15:33 UTC autoconf merge
Commit 57ca24966f88c1f36fd433eb3ecef48e665786c7
+51
--- a/configure
+++ b/configure
@@ -0,0 +1,51 @@
1
+#! /bin/sh
2
+# Guess values for system-dependent variables and create Makefiles.
3
+# Ge2erated by GNU Autoconf 2.61 for fossil 20110528.
4
+#
5
+# Report bugs to <[email protected]>.
6
+#
7
+# Copyright (C) 1992, 1993, 1994, 1995, 1996, 1998, 1999, 2000, 2001,
8
+# 2, 2007, 2008 Free Software Foundation06 Free Software Foundation, Inc.
9
+# This configure script is free software; the Free Software Foundation
10
+# gives unlimited permission to copy, distribute and modify it.
11
+## --------------------- ##
12
+## M4sh Initialization. ##
13
+## --------------------- ##
14
+
15
+# Be more Bourne compatible
16
+DUALCASE=1; export DUALCASE # for MKS sh
17
+if test -n "${ZSH_VERSION+set}" && (emulate sh) >/dev/null 2>&1; then
18
+ Pre-4.2 versions of Zsh do word splitting on ${1+"$@"}, which
19
+ # is contrary to our usage. Disable this feature.
20
+ alias -g '${1+"$@"}'='"$@"'
21
+ setopt NO_GLOB_SUBST
22
+else
23
+ case `(set -o) 2>/dev/null` in
24
+ *posix*) set -o posix ;;
25
+esac
26
+
27
+fi
28
+
29
+
30
+
31
+
32
+# PATH needs CR
33
+# Avoid depending upon Character Ranges.
34
+as_cr_letters='abcdefghijklmnopqrstuvwxyz'
35
+as_cr_LETTERS='ABCDEFGHIJKLMNOPQRSTUVWXYZ'
36
+as_cr_Letters=$as_cr_letters$as_cr_LETTERS
37
+as_cr_digits='0123456789'
38
+as_cr_alnum=$as_cr_Letters$as_cr_digits
39
+
40
+as_nl='
41
+'
42
+export as_nl
43
+# Printing a long string crashes Solaris 7 /usr/bin/printf.
44
+as_echo='\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\'
45
+as_echo=$as_echo$as_echo$as_echo$as_echo$as_echo
46
+as_echo=$as_echo$as_echo$as_echo$as_echo$as_echo$as_echo
47
+if (test "X`printf %s $as_echo`" = "X$as_echo
48
+ IFS=$as_save_IFS
49
+ testecho='printf %s
50
+else
51
+ if test
--- a/configure
+++ b/configure
@@ -0,0 +1,51 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
--- a/configure
+++ b/configure
@@ -0,0 +1,51 @@
1 #! /bin/sh
2 # Guess values for system-dependent variables and create Makefiles.
3 # Ge2erated by GNU Autoconf 2.61 for fossil 20110528.
4 #
5 # Report bugs to <[email protected]>.
6 #
7 # Copyright (C) 1992, 1993, 1994, 1995, 1996, 1998, 1999, 2000, 2001,
8 # 2, 2007, 2008 Free Software Foundation06 Free Software Foundation, Inc.
9 # This configure script is free software; the Free Software Foundation
10 # gives unlimited permission to copy, distribute and modify it.
11 ## --------------------- ##
12 ## M4sh Initialization. ##
13 ## --------------------- ##
14
15 # Be more Bourne compatible
16 DUALCASE=1; export DUALCASE # for MKS sh
17 if test -n "${ZSH_VERSION+set}" && (emulate sh) >/dev/null 2>&1; then
18 Pre-4.2 versions of Zsh do word splitting on ${1+"$@"}, which
19 # is contrary to our usage. Disable this feature.
20 alias -g '${1+"$@"}'='"$@"'
21 setopt NO_GLOB_SUBST
22 else
23 case `(set -o) 2>/dev/null` in
24 *posix*) set -o posix ;;
25 esac
26
27 fi
28
29
30
31
32 # PATH needs CR
33 # Avoid depending upon Character Ranges.
34 as_cr_letters='abcdefghijklmnopqrstuvwxyz'
35 as_cr_LETTERS='ABCDEFGHIJKLMNOPQRSTUVWXYZ'
36 as_cr_Letters=$as_cr_letters$as_cr_LETTERS
37 as_cr_digits='0123456789'
38 as_cr_alnum=$as_cr_Letters$as_cr_digits
39
40 as_nl='
41 '
42 export as_nl
43 # Printing a long string crashes Solaris 7 /usr/bin/printf.
44 as_echo='\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\'
45 as_echo=$as_echo$as_echo$as_echo$as_echo$as_echo
46 as_echo=$as_echo$as_echo$as_echo$as_echo$as_echo$as_echo
47 if (test "X`printf %s $as_echo`" = "X$as_echo
48 IFS=$as_save_IFS
49 testecho='printf %s
50 else
51 if test
+51
--- a/configure
+++ b/configure
@@ -0,0 +1,51 @@
1
+#! /bin/sh
2
+# Guess values for system-dependent variables and create Makefiles.
3
+# Ge2erated by GNU Autoconf 2.61 for fossil 20110528.
4
+#
5
+# Report bugs to <[email protected]>.
6
+#
7
+# Copyright (C) 1992, 1993, 1994, 1995, 1996, 1998, 1999, 2000, 2001,
8
+# 2, 2007, 2008 Free Software Foundation06 Free Software Foundation, Inc.
9
+# This configure script is free software; the Free Software Foundation
10
+# gives unlimited permission to copy, distribute and modify it.
11
+## --------------------- ##
12
+## M4sh Initialization. ##
13
+## --------------------- ##
14
+
15
+# Be more Bourne compatible
16
+DUALCASE=1; export DUALCASE # for MKS sh
17
+if test -n "${ZSH_VERSION+set}" && (emulate sh) >/dev/null 2>&1; then
18
+ Pre-4.2 versions of Zsh do word splitting on ${1+"$@"}, which
19
+ # is contrary to our usage. Disable this feature.
20
+ alias -g '${1+"$@"}'='"$@"'
21
+ setopt NO_GLOB_SUBST
22
+else
23
+ case `(set -o) 2>/dev/null` in
24
+ *posix*) set -o posix ;;
25
+esac
26
+
27
+fi
28
+
29
+
30
+
31
+
32
+# PATH needs CR
33
+# Avoid depending upon Character Ranges.
34
+as_cr_letters='abcdefghijklmnopqrstuvwxyz'
35
+as_cr_LETTERS='ABCDEFGHIJKLMNOPQRSTUVWXYZ'
36
+as_cr_Letters=$as_cr_letters$as_cr_LETTERS
37
+as_cr_digits='0123456789'
38
+as_cr_alnum=$as_cr_Letters$as_cr_digits
39
+
40
+as_nl='
41
+'
42
+export as_nl
43
+# Printing a long string crashes Solaris 7 /usr/bin/printf.
44
+as_echo='\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\'
45
+as_echo=$as_echo$as_echo$as_echo$as_echo$as_echo
46
+as_echo=$as_echo$as_echo$as_echo$as_echo$as_echo$as_echo
47
+if (test "X`printf %s $as_echo`" = "X$as_echo
48
+ IFS=$as_save_IFS
49
+ testecho='printf %s
50
+else
51
+ if test
--- a/configure
+++ b/configure
@@ -0,0 +1,51 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
--- a/configure
+++ b/configure
@@ -0,0 +1,51 @@
1 #! /bin/sh
2 # Guess values for system-dependent variables and create Makefiles.
3 # Ge2erated by GNU Autoconf 2.61 for fossil 20110528.
4 #
5 # Report bugs to <[email protected]>.
6 #
7 # Copyright (C) 1992, 1993, 1994, 1995, 1996, 1998, 1999, 2000, 2001,
8 # 2, 2007, 2008 Free Software Foundation06 Free Software Foundation, Inc.
9 # This configure script is free software; the Free Software Foundation
10 # gives unlimited permission to copy, distribute and modify it.
11 ## --------------------- ##
12 ## M4sh Initialization. ##
13 ## --------------------- ##
14
15 # Be more Bourne compatible
16 DUALCASE=1; export DUALCASE # for MKS sh
17 if test -n "${ZSH_VERSION+set}" && (emulate sh) >/dev/null 2>&1; then
18 Pre-4.2 versions of Zsh do word splitting on ${1+"$@"}, which
19 # is contrary to our usage. Disable this feature.
20 alias -g '${1+"$@"}'='"$@"'
21 setopt NO_GLOB_SUBST
22 else
23 case `(set -o) 2>/dev/null` in
24 *posix*) set -o posix ;;
25 esac
26
27 fi
28
29
30
31
32 # PATH needs CR
33 # Avoid depending upon Character Ranges.
34 as_cr_letters='abcdefghijklmnopqrstuvwxyz'
35 as_cr_LETTERS='ABCDEFGHIJKLMNOPQRSTUVWXYZ'
36 as_cr_Letters=$as_cr_letters$as_cr_LETTERS
37 as_cr_digits='0123456789'
38 as_cr_alnum=$as_cr_Letters$as_cr_digits
39
40 as_nl='
41 '
42 export as_nl
43 # Printing a long string crashes Solaris 7 /usr/bin/printf.
44 as_echo='\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\'
45 as_echo=$as_echo$as_echo$as_echo$as_echo$as_echo
46 as_echo=$as_echo$as_echo$as_echo$as_echo$as_echo$as_echo
47 if (test "X`printf %s $as_echo`" = "X$as_echo
48 IFS=$as_save_IFS
49 testecho='printf %s
50 else
51 if test
+3 -3
--- src/info.c
+++ src/info.c
@@ -190,11 +190,11 @@
190190
" (SELECT uuid FROM blob WHERE rid=tagxref.srcid AND rid!=%d),"
191191
" value, datetime(tagxref.mtime,'localtime'), tagtype,"
192192
" (SELECT uuid FROM blob WHERE rid=tagxref.origid AND rid!=%d)"
193193
" FROM tagxref JOIN tag ON tagxref.tagid=tag.tagid"
194194
" WHERE tagxref.rid=%d AND tagname NOT GLOB '%s'"
195
- " ORDER BY tagname", rid, rid, rid, zNotGlob
195
+ " ORDER BY tagname /*sort*/", rid, rid, rid, zNotGlob
196196
);
197197
while( db_step(&q)==SQLITE_ROW ){
198198
const char *zTagname = db_column_text(&q, 1);
199199
const char *zSrcUuid = db_column_text(&q, 2);
200200
const char *zValue = db_column_text(&q, 3);
@@ -525,11 +525,11 @@
525525
" (SELECT uuid FROM blob WHERE rid=mlink.pid),"
526526
" (SELECT uuid FROM blob WHERE rid=mlink.fid),"
527527
" (SELECT name FROM filename WHERE filename.fnid=mlink.pfnid)"
528528
" FROM mlink JOIN filename ON filename.fnid=mlink.fnid"
529529
" WHERE mlink.mid=%d"
530
- " ORDER BY name",
530
+ " ORDER BY name /*sort*/",
531531
rid
532532
);
533533
while( db_step(&q)==SQLITE_ROW ){
534534
const char *zName = db_column_text(&q,0);
535535
int mperm = db_column_int(&q, 1);
@@ -1790,11 +1790,11 @@
17901790
@ <input type="text" style="width:15;" name="tagname" value="%h(zNewTag)" />
17911791
db_prepare(&q,
17921792
"SELECT tag.tagid, tagname FROM tagxref, tag"
17931793
" WHERE tagxref.rid=%d AND tagtype>0 AND tagxref.tagid=tag.tagid"
17941794
" ORDER BY CASE WHEN tagname GLOB 'sym-*' THEN substr(tagname,5)"
1795
- " ELSE tagname END",
1795
+ " ELSE tagname END /*sort*/",
17961796
rid
17971797
);
17981798
while( db_step(&q)==SQLITE_ROW ){
17991799
int tagid = db_column_int(&q, 0);
18001800
const char *zTagName = db_column_text(&q, 1);
18011801
--- src/info.c
+++ src/info.c
@@ -190,11 +190,11 @@
190 " (SELECT uuid FROM blob WHERE rid=tagxref.srcid AND rid!=%d),"
191 " value, datetime(tagxref.mtime,'localtime'), tagtype,"
192 " (SELECT uuid FROM blob WHERE rid=tagxref.origid AND rid!=%d)"
193 " FROM tagxref JOIN tag ON tagxref.tagid=tag.tagid"
194 " WHERE tagxref.rid=%d AND tagname NOT GLOB '%s'"
195 " ORDER BY tagname", rid, rid, rid, zNotGlob
196 );
197 while( db_step(&q)==SQLITE_ROW ){
198 const char *zTagname = db_column_text(&q, 1);
199 const char *zSrcUuid = db_column_text(&q, 2);
200 const char *zValue = db_column_text(&q, 3);
@@ -525,11 +525,11 @@
525 " (SELECT uuid FROM blob WHERE rid=mlink.pid),"
526 " (SELECT uuid FROM blob WHERE rid=mlink.fid),"
527 " (SELECT name FROM filename WHERE filename.fnid=mlink.pfnid)"
528 " FROM mlink JOIN filename ON filename.fnid=mlink.fnid"
529 " WHERE mlink.mid=%d"
530 " ORDER BY name",
531 rid
532 );
533 while( db_step(&q)==SQLITE_ROW ){
534 const char *zName = db_column_text(&q,0);
535 int mperm = db_column_int(&q, 1);
@@ -1790,11 +1790,11 @@
1790 @ <input type="text" style="width:15;" name="tagname" value="%h(zNewTag)" />
1791 db_prepare(&q,
1792 "SELECT tag.tagid, tagname FROM tagxref, tag"
1793 " WHERE tagxref.rid=%d AND tagtype>0 AND tagxref.tagid=tag.tagid"
1794 " ORDER BY CASE WHEN tagname GLOB 'sym-*' THEN substr(tagname,5)"
1795 " ELSE tagname END",
1796 rid
1797 );
1798 while( db_step(&q)==SQLITE_ROW ){
1799 int tagid = db_column_int(&q, 0);
1800 const char *zTagName = db_column_text(&q, 1);
1801
--- src/info.c
+++ src/info.c
@@ -190,11 +190,11 @@
190 " (SELECT uuid FROM blob WHERE rid=tagxref.srcid AND rid!=%d),"
191 " value, datetime(tagxref.mtime,'localtime'), tagtype,"
192 " (SELECT uuid FROM blob WHERE rid=tagxref.origid AND rid!=%d)"
193 " FROM tagxref JOIN tag ON tagxref.tagid=tag.tagid"
194 " WHERE tagxref.rid=%d AND tagname NOT GLOB '%s'"
195 " ORDER BY tagname /*sort*/", rid, rid, rid, zNotGlob
196 );
197 while( db_step(&q)==SQLITE_ROW ){
198 const char *zTagname = db_column_text(&q, 1);
199 const char *zSrcUuid = db_column_text(&q, 2);
200 const char *zValue = db_column_text(&q, 3);
@@ -525,11 +525,11 @@
525 " (SELECT uuid FROM blob WHERE rid=mlink.pid),"
526 " (SELECT uuid FROM blob WHERE rid=mlink.fid),"
527 " (SELECT name FROM filename WHERE filename.fnid=mlink.pfnid)"
528 " FROM mlink JOIN filename ON filename.fnid=mlink.fnid"
529 " WHERE mlink.mid=%d"
530 " ORDER BY name /*sort*/",
531 rid
532 );
533 while( db_step(&q)==SQLITE_ROW ){
534 const char *zName = db_column_text(&q,0);
535 int mperm = db_column_int(&q, 1);
@@ -1790,11 +1790,11 @@
1790 @ <input type="text" style="width:15;" name="tagname" value="%h(zNewTag)" />
1791 db_prepare(&q,
1792 "SELECT tag.tagid, tagname FROM tagxref, tag"
1793 " WHERE tagxref.rid=%d AND tagtype>0 AND tagxref.tagid=tag.tagid"
1794 " ORDER BY CASE WHEN tagname GLOB 'sym-*' THEN substr(tagname,5)"
1795 " ELSE tagname END /*sort*/",
1796 rid
1797 );
1798 while( db_step(&q)==SQLITE_ROW ){
1799 int tagid = db_column_int(&q, 0);
1800 const char *zTagName = db_column_text(&q, 1);
1801
+1 -1
--- src/main.c
+++ src/main.c
@@ -2,11 +2,11 @@
22
** Copyright (c) 2006 D. Richard Hipp
33
**
44
** This program is free software; you can redistribute it and/or
55
** modify it under the terms of the Simplified BSD License (also
66
** known as the "2-Clause License" or "FreeBSD License".)
7
-
7
+**
88
** This program is distributed in the hope that it will be useful,
99
** but without any warranty; without even the implied warranty of
1010
** merchantability or fitness for a particular purpose.
1111
**
1212
** Author contact information:
1313
--- src/main.c
+++ src/main.c
@@ -2,11 +2,11 @@
2 ** Copyright (c) 2006 D. Richard Hipp
3 **
4 ** This program is free software; you can redistribute it and/or
5 ** modify it under the terms of the Simplified BSD License (also
6 ** known as the "2-Clause License" or "FreeBSD License".)
7
8 ** This program is distributed in the hope that it will be useful,
9 ** but without any warranty; without even the implied warranty of
10 ** merchantability or fitness for a particular purpose.
11 **
12 ** Author contact information:
13
--- src/main.c
+++ src/main.c
@@ -2,11 +2,11 @@
2 ** Copyright (c) 2006 D. Richard Hipp
3 **
4 ** This program is free software; you can redistribute it and/or
5 ** modify it under the terms of the Simplified BSD License (also
6 ** known as the "2-Clause License" or "FreeBSD License".)
7 **
8 ** This program is distributed in the hope that it will be useful,
9 ** but without any warranty; without even the implied warranty of
10 ** merchantability or fitness for a particular purpose.
11 **
12 ** Author contact information:
13
+1 -1
--- src/path.c
+++ src/path.c
@@ -366,11 +366,11 @@
366366
367367
*pnChng = 0;
368368
*aiChng = 0;
369369
if( iFrom==iTo ) return;
370370
path_reset();
371
- p = path_shortest(iFrom, iTo, 0);
371
+ p = path_shortest(iFrom, iTo, 1);
372372
if( p==0 ) return;
373373
path_reverse_path();
374374
db_prepare(&q1,
375375
"SELECT pfnid, fnid FROM mlink WHERE mid=:mid AND pfnid>0"
376376
);
377377
--- src/path.c
+++ src/path.c
@@ -366,11 +366,11 @@
366
367 *pnChng = 0;
368 *aiChng = 0;
369 if( iFrom==iTo ) return;
370 path_reset();
371 p = path_shortest(iFrom, iTo, 0);
372 if( p==0 ) return;
373 path_reverse_path();
374 db_prepare(&q1,
375 "SELECT pfnid, fnid FROM mlink WHERE mid=:mid AND pfnid>0"
376 );
377
--- src/path.c
+++ src/path.c
@@ -366,11 +366,11 @@
366
367 *pnChng = 0;
368 *aiChng = 0;
369 if( iFrom==iTo ) return;
370 path_reset();
371 p = path_shortest(iFrom, iTo, 1);
372 if( p==0 ) return;
373 path_reverse_path();
374 db_prepare(&q1,
375 "SELECT pfnid, fnid FROM mlink WHERE mid=:mid AND pfnid>0"
376 );
377
+2822 -1836
--- src/sqlite3.c
+++ src/sqlite3.c
@@ -650,11 +650,11 @@
650650
** [sqlite3_libversion_number()], [sqlite3_sourceid()],
651651
** [sqlite_version()] and [sqlite_source_id()].
652652
*/
653653
#define SQLITE_VERSION "3.7.7"
654654
#define SQLITE_VERSION_NUMBER 3007007
655
-#define SQLITE_SOURCE_ID "2011-06-03 14:19:10 0206bc6f87bb9393218a380fc5b18039d334a8d8"
655
+#define SQLITE_SOURCE_ID "2011-06-15 13:11:06 f9750870ee04935f338e4d808900fee5a8b2b389"
656656
657657
/*
658658
** CAPI3REF: Run-Time Library Version Numbers
659659
** KEYWORDS: sqlite3_version, sqlite3_sourceid
660660
**
@@ -8483,10 +8483,11 @@
84838483
SQLITE_PRIVATE int sqlite3VdbeAddOp2(Vdbe*,int,int,int);
84848484
SQLITE_PRIVATE int sqlite3VdbeAddOp3(Vdbe*,int,int,int,int);
84858485
SQLITE_PRIVATE int sqlite3VdbeAddOp4(Vdbe*,int,int,int,int,const char *zP4,int);
84868486
SQLITE_PRIVATE int sqlite3VdbeAddOp4Int(Vdbe*,int,int,int,int,int);
84878487
SQLITE_PRIVATE int sqlite3VdbeAddOpList(Vdbe*, int nOp, VdbeOpList const *aOp);
8488
+SQLITE_PRIVATE void sqlite3VdbeAddParseSchemaOp(Vdbe*,int,char*);
84888489
SQLITE_PRIVATE void sqlite3VdbeChangeP1(Vdbe*, int addr, int P1);
84898490
SQLITE_PRIVATE void sqlite3VdbeChangeP2(Vdbe*, int addr, int P2);
84908491
SQLITE_PRIVATE void sqlite3VdbeChangeP3(Vdbe*, int addr, int P3);
84918492
SQLITE_PRIVATE void sqlite3VdbeChangeP5(Vdbe*, u8 P5);
84928493
SQLITE_PRIVATE void sqlite3VdbeJumpHere(Vdbe*, int addr);
@@ -9282,11 +9283,11 @@
92829283
** sqlite3_close().
92839284
*
92849285
** A thread must be holding a mutex on the corresponding Btree in order
92859286
** to access Schema content. This implies that the thread must also be
92869287
** holding a mutex on the sqlite3 connection pointer that owns the Btree.
9287
-** For a TEMP Schema, on the connection mutex is required.
9288
+** For a TEMP Schema, only the connection mutex is required.
92889289
*/
92899290
struct Schema {
92909291
int schema_cookie; /* Database schema version number for this file */
92919292
int iGeneration; /* Generation counter. Incremented with each change */
92929293
Hash tblHash; /* All tables indexed by name */
@@ -11479,11 +11480,11 @@
1147911480
SQLITE_PRIVATE int sqlite3AtoF(const char *z, double*, int, u8);
1148011481
SQLITE_PRIVATE int sqlite3GetInt32(const char *, int*);
1148111482
SQLITE_PRIVATE int sqlite3Atoi(const char*);
1148211483
SQLITE_PRIVATE int sqlite3Utf16ByteLen(const void *pData, int nChar);
1148311484
SQLITE_PRIVATE int sqlite3Utf8CharLen(const char *pData, int nByte);
11484
-SQLITE_PRIVATE int sqlite3Utf8Read(const u8*, const u8**);
11485
+SQLITE_PRIVATE u32 sqlite3Utf8Read(const u8*, const u8**);
1148511486
1148611487
/*
1148711488
** Routines to read and write variable-length integers. These used to
1148811489
** be defined locally, but now we use the varint routines in the util.c
1148911490
** file. Code should use the MACRO forms below, as the Varint32 versions
@@ -20086,11 +20087,11 @@
2008620087
} \
2008720088
if( c<0x80 \
2008820089
|| (c&0xFFFFF800)==0xD800 \
2008920090
|| (c&0xFFFFFFFE)==0xFFFE ){ c = 0xFFFD; } \
2009020091
}
20091
-SQLITE_PRIVATE int sqlite3Utf8Read(
20092
+SQLITE_PRIVATE u32 sqlite3Utf8Read(
2009220093
const unsigned char *zIn, /* First byte of UTF-8 character */
2009320094
const unsigned char **pzNext /* Write first byte past UTF-8 char here */
2009420095
){
2009520096
unsigned int c;
2009620097
@@ -35805,11 +35806,11 @@
3580535806
memset(pCache, 0, sz);
3580635807
if( separateCache ){
3580735808
pGroup = (PGroup*)&pCache[1];
3580835809
pGroup->mxPinned = 10;
3580935810
}else{
35810
- pGroup = &pcache1_g.grp;
35811
+ pGroup = &pcache1.grp;
3581135812
}
3581235813
pCache->pGroup = pGroup;
3581335814
pCache->szPage = szPage;
3581435815
pCache->bPurgeable = (bPurgeable ? 1 : 0);
3581535816
if( bPurgeable ){
@@ -48324,10 +48325,12 @@
4832448325
**
4832548326
** This routine works only for pages that do not contain overflow cells.
4832648327
*/
4832748328
#define findCell(P,I) \
4832848329
((P)->aData + ((P)->maskPage & get2byte(&(P)->aData[(P)->cellOffset+2*(I)])))
48330
+#define findCellv2(D,M,O,I) (D+(M&get2byte(D+(O+2*(I)))))
48331
+
4832948332
4833048333
/*
4833148334
** This a more complex version of findCell() that works for
4833248335
** pages that do contain overflow cells.
4833348336
*/
@@ -51918,11 +51921,11 @@
5191851921
assert( pCur->apPage[pCur->iPage]->nCell==0 );
5191951922
return SQLITE_OK;
5192051923
}
5192151924
assert( pCur->apPage[0]->intKey || pIdxKey );
5192251925
for(;;){
51923
- int lwr, upr;
51926
+ int lwr, upr, idx;
5192451927
Pgno chldPg;
5192551928
MemPage *pPage = pCur->apPage[pCur->iPage];
5192651929
int c;
5192751930
5192851931
/* pPage->nCell must be greater than zero. If this is the root-page
@@ -51934,18 +51937,18 @@
5193451937
assert( pPage->nCell>0 );
5193551938
assert( pPage->intKey==(pIdxKey==0) );
5193651939
lwr = 0;
5193751940
upr = pPage->nCell-1;
5193851941
if( biasRight ){
51939
- pCur->aiIdx[pCur->iPage] = (u16)upr;
51942
+ pCur->aiIdx[pCur->iPage] = (u16)(idx = upr);
5194051943
}else{
51941
- pCur->aiIdx[pCur->iPage] = (u16)((upr+lwr)/2);
51944
+ pCur->aiIdx[pCur->iPage] = (u16)(idx = (upr+lwr)/2);
5194251945
}
5194351946
for(;;){
51944
- int idx = pCur->aiIdx[pCur->iPage]; /* Index of current cell in pPage */
5194551947
u8 *pCell; /* Pointer to current cell in pPage */
5194651948
51949
+ assert( idx==pCur->aiIdx[pCur->iPage] );
5194751950
pCur->info.nSize = 0;
5194851951
pCell = findCell(pPage, idx) + pPage->childPtrSize;
5194951952
if( pPage->intKey ){
5195051953
i64 nCellKey;
5195151954
if( pPage->hasData ){
@@ -52024,11 +52027,11 @@
5202452027
upr = idx-1;
5202552028
}
5202652029
if( lwr>upr ){
5202752030
break;
5202852031
}
52029
- pCur->aiIdx[pCur->iPage] = (u16)((lwr+upr)/2);
52032
+ pCur->aiIdx[pCur->iPage] = (u16)(idx = (lwr+upr)/2);
5203052033
}
5203152034
assert( lwr==upr+1 );
5203252035
assert( pPage->isInit );
5203352036
if( pPage->leaf ){
5203452037
chldPg = 0;
@@ -52886,13 +52889,13 @@
5288652889
if( rc ){
5288752890
*pRC = rc;
5288852891
return;
5288952892
}
5289052893
endPtr = &data[pPage->cellOffset + 2*pPage->nCell - 2];
52894
+ assert( (SQLITE_PTR_TO_INT(ptr)&1)==0 ); /* ptr is always 2-byte aligned */
5289152895
while( ptr<endPtr ){
52892
- ptr[0] = ptr[2];
52893
- ptr[1] = ptr[3];
52896
+ *(u16*)ptr = *(u16*)&ptr[2];
5289452897
ptr += 2;
5289552898
}
5289652899
pPage->nCell--;
5289752900
put2byte(&data[hdr+3], pPage->nCell);
5289852901
pPage->nFree += 2;
@@ -52929,10 +52932,11 @@
5292952932
int end; /* First byte past the last cell pointer in data[] */
5293052933
int ins; /* Index in data[] where new cell pointer is inserted */
5293152934
int cellOffset; /* Address of first cell pointer in data[] */
5293252935
u8 *data; /* The content of the whole page */
5293352936
u8 *ptr; /* Used for moving information around in data[] */
52937
+ u8 *endPtr; /* End of the loop */
5293452938
5293552939
int nSkip = (iChild ? 4 : 0);
5293652940
5293752941
if( *pRC ) return;
5293852942
@@ -52979,13 +52983,16 @@
5297952983
pPage->nFree -= (u16)(2 + sz);
5298052984
memcpy(&data[idx+nSkip], pCell+nSkip, sz-nSkip);
5298152985
if( iChild ){
5298252986
put4byte(&data[idx], iChild);
5298352987
}
52984
- for(j=end, ptr=&data[j]; j>ins; j-=2, ptr-=2){
52985
- ptr[0] = ptr[-2];
52986
- ptr[1] = ptr[-1];
52988
+ ptr = &data[end];
52989
+ endPtr = &data[ins];
52990
+ assert( (SQLITE_PTR_TO_INT(ptr)&1)==0 ); /* ptr is always 2-byte aligned */
52991
+ while( ptr>endPtr ){
52992
+ *(u16*)ptr = *(u16*)&ptr[-2];
52993
+ ptr -= 2;
5298752994
}
5298852995
put2byte(&data[ins], idx);
5298952996
put2byte(&data[pPage->hdrOffset+3], pPage->nCell);
5299052997
#ifndef SQLITE_OMIT_AUTOVACUUM
5299152998
if( pPage->pBt->autoVacuum ){
@@ -53026,14 +53033,15 @@
5302653033
assert( get2byteNotZero(&data[hdr+5])==nUsable );
5302753034
5302853035
pCellptr = &data[pPage->cellOffset + nCell*2];
5302953036
cellbody = nUsable;
5303053037
for(i=nCell-1; i>=0; i--){
53038
+ u16 sz = aSize[i];
5303153039
pCellptr -= 2;
53032
- cellbody -= aSize[i];
53040
+ cellbody -= sz;
5303353041
put2byte(pCellptr, cellbody);
53034
- memcpy(&data[cellbody], apCell[i], aSize[i]);
53042
+ memcpy(&data[cellbody], apCell[i], sz);
5303553043
}
5303653044
put2byte(&data[hdr+3], nCell);
5303753045
put2byte(&data[hdr+5], cellbody);
5303853046
pPage->nFree -= (nCell*2 + nUsable - cellbody);
5303953047
pPage->nCell = (u16)nCell;
@@ -53483,16 +53491,28 @@
5348353491
memcpy(pOld, apOld[i], sizeof(MemPage));
5348453492
pOld->aData = (void*)&pOld[1];
5348553493
memcpy(pOld->aData, apOld[i]->aData, pBt->pageSize);
5348653494
5348753495
limit = pOld->nCell+pOld->nOverflow;
53488
- for(j=0; j<limit; j++){
53489
- assert( nCell<nMaxCells );
53490
- apCell[nCell] = findOverflowCell(pOld, j);
53491
- szCell[nCell] = cellSizePtr(pOld, apCell[nCell]);
53492
- nCell++;
53493
- }
53496
+ if( pOld->nOverflow>0 ){
53497
+ for(j=0; j<limit; j++){
53498
+ assert( nCell<nMaxCells );
53499
+ apCell[nCell] = findOverflowCell(pOld, j);
53500
+ szCell[nCell] = cellSizePtr(pOld, apCell[nCell]);
53501
+ nCell++;
53502
+ }
53503
+ }else{
53504
+ u8 *aData = pOld->aData;
53505
+ u16 maskPage = pOld->maskPage;
53506
+ u16 cellOffset = pOld->cellOffset;
53507
+ for(j=0; j<limit; j++){
53508
+ assert( nCell<nMaxCells );
53509
+ apCell[nCell] = findCellv2(aData, maskPage, cellOffset, j);
53510
+ szCell[nCell] = cellSizePtr(pOld, apCell[nCell]);
53511
+ nCell++;
53512
+ }
53513
+ }
5349453514
if( i<nOld-1 && !leafData){
5349553515
u16 sz = (u16)szNew[i];
5349653516
u8 *pTemp;
5349753517
assert( nCell<nMaxCells );
5349853518
szCell[nCell] = sz;
@@ -57637,17 +57657,10 @@
5763757657
pOp->p1 = p1;
5763857658
pOp->p2 = p2;
5763957659
pOp->p3 = p3;
5764057660
pOp->p4.p = 0;
5764157661
pOp->p4type = P4_NOTUSED;
57642
- p->expired = 0;
57643
- if( op==OP_ParseSchema ){
57644
- /* Any program that uses the OP_ParseSchema opcode needs to lock
57645
- ** all btrees. */
57646
- int j;
57647
- for(j=0; j<p->db->nDb; j++) sqlite3VdbeUsesBtree(p, j);
57648
- }
5764957662
#ifdef SQLITE_DEBUG
5765057663
pOp->zComment = 0;
5765157664
if( sqlite3VdbeAddopTrace ) sqlite3VdbePrintOp(0, i, &p->aOp[i]);
5765257665
#endif
5765357666
#ifdef VDBE_PROFILE
@@ -57681,10 +57694,24 @@
5768157694
){
5768257695
int addr = sqlite3VdbeAddOp3(p, op, p1, p2, p3);
5768357696
sqlite3VdbeChangeP4(p, addr, zP4, p4type);
5768457697
return addr;
5768557698
}
57699
+
57700
+/*
57701
+** Add an OP_ParseSchema opcode. This routine is broken out from
57702
+** sqlite3VdbeAddOp4() since it needs to also local all btrees.
57703
+**
57704
+** The zWhere string must have been obtained from sqlite3_malloc().
57705
+** This routine will take ownership of the allocated memory.
57706
+*/
57707
+SQLITE_PRIVATE void sqlite3VdbeAddParseSchemaOp(Vdbe *p, int iDb, char *zWhere){
57708
+ int j;
57709
+ int addr = sqlite3VdbeAddOp3(p, OP_ParseSchema, iDb, 0, 0);
57710
+ sqlite3VdbeChangeP4(p, addr, zWhere, P4_DYNAMIC);
57711
+ for(j=0; j<p->db->nDb; j++) sqlite3VdbeUsesBtree(p, j);
57712
+}
5768657713
5768757714
/*
5768857715
** Add an opcode that includes the p4 value as an integer.
5768957716
*/
5769057717
SQLITE_PRIVATE int sqlite3VdbeAddOp4Int(
@@ -64083,10 +64110,20 @@
6408364110
u.ag.ctx.pColl = pOp[-1].p4.pColl;
6408464111
}
6408564112
db->lastRowid = lastRowid;
6408664113
(*u.ag.ctx.pFunc->xFunc)(&u.ag.ctx, u.ag.n, u.ag.apVal); /* IMP: R-24505-23230 */
6408764114
lastRowid = db->lastRowid;
64115
+
64116
+ /* If any auxiliary data functions have been called by this user function,
64117
+ ** immediately call the destructor for any non-static values.
64118
+ */
64119
+ if( u.ag.ctx.pVdbeFunc ){
64120
+ sqlite3VdbeDeleteAuxData(u.ag.ctx.pVdbeFunc, pOp->p1);
64121
+ pOp->p4.pVdbeFunc = u.ag.ctx.pVdbeFunc;
64122
+ pOp->p4type = P4_VDBEFUNC;
64123
+ }
64124
+
6408864125
if( db->mallocFailed ){
6408964126
/* Even though a malloc() has failed, the implementation of the
6409064127
** user function may have called an sqlite3_result_XXX() function
6409164128
** to return a value. The following call releases any resources
6409264129
** associated with such a value.
@@ -64093,19 +64130,10 @@
6409364130
*/
6409464131
sqlite3VdbeMemRelease(&u.ag.ctx.s);
6409564132
goto no_mem;
6409664133
}
6409764134
64098
- /* If any auxiliary data functions have been called by this user function,
64099
- ** immediately call the destructor for any non-static values.
64100
- */
64101
- if( u.ag.ctx.pVdbeFunc ){
64102
- sqlite3VdbeDeleteAuxData(u.ag.ctx.pVdbeFunc, pOp->p1);
64103
- pOp->p4.pVdbeFunc = u.ag.ctx.pVdbeFunc;
64104
- pOp->p4type = P4_VDBEFUNC;
64105
- }
64106
-
6410764135
/* If the function returned an error, throw an exception */
6410864136
if( u.ag.ctx.isError ){
6410964137
sqlite3SetString(&p->zErrMsg, db, "%s", sqlite3_value_text(&u.ag.ctx.s));
6411064138
rc = u.ag.ctx.isError;
6411164139
}
@@ -75255,18 +75283,18 @@
7525575283
sqlite3VdbeAddOp4(v, OP_DropTable, iDb, 0, 0, pTab->zName, 0);
7525675284
7525775285
/* Reload the table, index and permanent trigger schemas. */
7525875286
zWhere = sqlite3MPrintf(pParse->db, "tbl_name=%Q", zName);
7525975287
if( !zWhere ) return;
75260
- sqlite3VdbeAddOp4(v, OP_ParseSchema, iDb, 0, 0, zWhere, P4_DYNAMIC);
75288
+ sqlite3VdbeAddParseSchemaOp(v, iDb, zWhere);
7526175289
7526275290
#ifndef SQLITE_OMIT_TRIGGER
7526375291
/* Now, if the table is not stored in the temp database, reload any temp
7526475292
** triggers. Don't use IN(...) in case SQLITE_OMIT_SUBQUERY is defined.
7526575293
*/
7526675294
if( (zWhere=whereTempTriggers(pParse, pTab))!=0 ){
75267
- sqlite3VdbeAddOp4(v, OP_ParseSchema, 1, 0, 0, zWhere, P4_DYNAMIC);
75295
+ sqlite3VdbeAddParseSchemaOp(v, 1, zWhere);
7526875296
}
7526975297
#endif
7527075298
}
7527175299
7527275300
/*
@@ -78875,12 +78903,12 @@
7887578903
}
7887678904
}
7887778905
#endif
7887878906
7887978907
/* Reparse everything to update our internal data structures */
78880
- sqlite3VdbeAddOp4(v, OP_ParseSchema, iDb, 0, 0,
78881
- sqlite3MPrintf(db, "tbl_name='%q'",p->zName), P4_DYNAMIC);
78908
+ sqlite3VdbeAddParseSchemaOp(v, iDb,
78909
+ sqlite3MPrintf(db, "tbl_name='%q'", p->zName));
7888278910
}
7888378911
7888478912
7888578913
/* Add the table to the in-memory representation of the database.
7888678914
*/
@@ -80073,13 +80101,12 @@
8007380101
** to invalidate all pre-compiled statements.
8007480102
*/
8007580103
if( pTblName ){
8007680104
sqlite3RefillIndex(pParse, pIndex, iMem);
8007780105
sqlite3ChangeCookie(pParse, iDb);
80078
- sqlite3VdbeAddOp4(v, OP_ParseSchema, iDb, 0, 0,
80079
- sqlite3MPrintf(db, "name='%q' AND type='index'", pIndex->zName),
80080
- P4_DYNAMIC);
80106
+ sqlite3VdbeAddParseSchemaOp(v, iDb,
80107
+ sqlite3MPrintf(db, "name='%q' AND type='index'", pIndex->zName));
8008180108
sqlite3VdbeAddOp1(v, OP_Expire, 0);
8008280109
}
8008380110
}
8008480111
8008580112
/* When adding an index to the list of indices for a table, make
@@ -82621,14 +82648,14 @@
8262182648
** character is exactly one byte in size. Also, all characters are
8262282649
** able to participate in upper-case-to-lower-case mappings in EBCDIC
8262382650
** whereas only characters less than 0x80 do in ASCII.
8262482651
*/
8262582652
#if defined(SQLITE_EBCDIC)
82626
-# define sqlite3Utf8Read(A,C) (*(A++))
82627
-# define GlogUpperToLower(A) A = sqlite3UpperToLower[A]
82653
+# define sqlite3Utf8Read(A,C) (*(A++))
82654
+# define GlogUpperToLower(A) A = sqlite3UpperToLower[A]
8262882655
#else
82629
-# define GlogUpperToLower(A) if( A<0x80 ){ A = sqlite3UpperToLower[A]; }
82656
+# define GlogUpperToLower(A) if( !((A)&~0x7f) ){ A = sqlite3UpperToLower[A]; }
8263082657
#endif
8263182658
8263282659
static const struct compareInfo globInfo = { '*', '?', '[', 0 };
8263382660
/* The correct SQL-92 behavior is for the LIKE operator to ignore
8263482661
** case. Thus 'a' LIKE 'A' would be true. */
@@ -82667,13 +82694,13 @@
8266782694
*/
8266882695
static int patternCompare(
8266982696
const u8 *zPattern, /* The glob pattern */
8267082697
const u8 *zString, /* The string to compare against the glob */
8267182698
const struct compareInfo *pInfo, /* Information about how to do the compare */
82672
- const int esc /* The escape character */
82699
+ u32 esc /* The escape character */
8267382700
){
82674
- int c, c2;
82701
+ u32 c, c2;
8267582702
int invert;
8267682703
int seen;
8267782704
u8 matchOne = pInfo->matchOne;
8267882705
u8 matchAll = pInfo->matchAll;
8267982706
u8 matchSet = pInfo->matchSet;
@@ -82723,11 +82750,11 @@
8272382750
}else if( !prevEscape && c==matchOne ){
8272482751
if( sqlite3Utf8Read(zString, &zString)==0 ){
8272582752
return 0;
8272682753
}
8272782754
}else if( c==matchSet ){
82728
- int prior_c = 0;
82755
+ u32 prior_c = 0;
8272982756
assert( esc==0 ); /* This only occurs for GLOB, not LIKE */
8273082757
seen = 0;
8273182758
invert = 0;
8273282759
c = sqlite3Utf8Read(zString, &zString);
8273382760
if( c==0 ) return 0;
@@ -82799,11 +82826,11 @@
8279982826
sqlite3_context *context,
8280082827
int argc,
8280182828
sqlite3_value **argv
8280282829
){
8280382830
const unsigned char *zA, *zB;
82804
- int escape = 0;
82831
+ u32 escape = 0;
8280582832
int nPat;
8280682833
sqlite3 *db = sqlite3_context_db_handle(context);
8280782834
8280882835
zB = sqlite3_value_text(argv[0]);
8280982836
zA = sqlite3_value_text(argv[1]);
@@ -84110,17 +84137,29 @@
8411084137
}
8411184138
8411284139
/* If the parent table is the same as the child table, and we are about
8411384140
** to increment the constraint-counter (i.e. this is an INSERT operation),
8411484141
** then check if the row being inserted matches itself. If so, do not
84115
- ** increment the constraint-counter. */
84142
+ ** increment the constraint-counter.
84143
+ **
84144
+ ** If any of the parent-key values are NULL, then the row cannot match
84145
+ ** itself. So set JUMPIFNULL to make sure we do the OP_Found if any
84146
+ ** of the parent-key values are NULL (at this point it is known that
84147
+ ** none of the child key values are).
84148
+ */
8411684149
if( pTab==pFKey->pFrom && nIncr==1 ){
8411784150
int iJump = sqlite3VdbeCurrentAddr(v) + nCol + 1;
8411884151
for(i=0; i<nCol; i++){
8411984152
int iChild = aiCol[i]+1+regData;
8412084153
int iParent = pIdx->aiColumn[i]+1+regData;
84154
+ assert( aiCol[i]!=pTab->iPKey );
84155
+ if( pIdx->aiColumn[i]==pTab->iPKey ){
84156
+ /* The parent key is a composite key that includes the IPK column */
84157
+ iParent = regData;
84158
+ }
8412184159
sqlite3VdbeAddOp3(v, OP_Ne, iChild, iJump, iParent);
84160
+ sqlite3VdbeChangeP5(v, SQLITE_JUMPIFNULL);
8412284161
}
8412384162
sqlite3VdbeAddOp2(v, OP_Goto, 0, iOk);
8412484163
}
8412584164
8412684165
sqlite3VdbeAddOp3(v, OP_MakeRecord, regTemp, nCol, regRec);
@@ -95336,13 +95375,12 @@
9533695375
"INSERT INTO %Q.%s VALUES('trigger',%Q,%Q,0,'CREATE TRIGGER %q')",
9533795376
db->aDb[iDb].zName, SCHEMA_TABLE(iDb), zName,
9533895377
pTrig->table, z);
9533995378
sqlite3DbFree(db, z);
9534095379
sqlite3ChangeCookie(pParse, iDb);
95341
- sqlite3VdbeAddOp4(v, OP_ParseSchema, iDb, 0, 0, sqlite3MPrintf(
95342
- db, "type='trigger' AND name='%q'", zName), P4_DYNAMIC
95343
- );
95380
+ sqlite3VdbeAddParseSchemaOp(v, iDb,
95381
+ sqlite3MPrintf(db, "type='trigger' AND name='%q'", zName));
9534495382
}
9534595383
9534695384
if( db->init.busy ){
9534795385
Trigger *pLink = pTrig;
9534895386
Hash *pHash = &db->aDb[iDb].pSchema->trigHash;
@@ -96392,11 +96430,11 @@
9639296430
aRegIdx = sqlite3DbMallocRaw(db, sizeof(Index*) * nIdx );
9639396431
if( aRegIdx==0 ) goto update_cleanup;
9639496432
}
9639596433
for(j=0, pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext, j++){
9639696434
int reg;
96397
- if( chngRowid ){
96435
+ if( hasFK || chngRowid ){
9639896436
reg = ++pParse->nMem;
9639996437
}else{
9640096438
reg = 0;
9640196439
for(i=0; i<pIdx->nColumn; i++){
9640296440
if( aXRef[pIdx->aiColumn[i]]>=0 ){
@@ -97547,11 +97585,11 @@
9754797585
v = sqlite3GetVdbe(pParse);
9754897586
sqlite3ChangeCookie(pParse, iDb);
9754997587
9755097588
sqlite3VdbeAddOp2(v, OP_Expire, 0, 0);
9755197589
zWhere = sqlite3MPrintf(db, "name='%q' AND type='table'", pTab->zName);
97552
- sqlite3VdbeAddOp4(v, OP_ParseSchema, iDb, 0, 0, zWhere, P4_DYNAMIC);
97590
+ sqlite3VdbeAddParseSchemaOp(v, iDb, zWhere);
9755397591
sqlite3VdbeAddOp4(v, OP_VCreate, iDb, 0, 0,
9755497592
pTab->zName, sqlite3Strlen30(pTab->zName) + 1);
9755597593
}
9755697594
9755797595
/* If we are rereading the sqlite_master table create the in-memory
@@ -107274,17 +107312,16 @@
107274107312
#ifndef SQLITE_OMIT_BLOB_LITERAL
107275107313
case 'x': case 'X': {
107276107314
testcase( z[0]=='x' ); testcase( z[0]=='X' );
107277107315
if( z[1]=='\'' ){
107278107316
*tokenType = TK_BLOB;
107279
- for(i=2; (c=z[i])!=0 && c!='\''; i++){
107280
- if( !sqlite3Isxdigit(c) ){
107281
- *tokenType = TK_ILLEGAL;
107282
- }
107317
+ for(i=2; sqlite3Isxdigit(z[i]); i++){}
107318
+ if( z[i]!='\'' || i%2 ){
107319
+ *tokenType = TK_ILLEGAL;
107320
+ while( z[i] && z[i]!='\'' ){ i++; }
107283107321
}
107284
- if( i%2 || !c ) *tokenType = TK_ILLEGAL;
107285
- if( c ) i++;
107322
+ if( z[i] ) i++;
107286107323
return i;
107287107324
}
107288107325
/* Otherwise fall through to the next case */
107289107326
}
107290107327
#endif
@@ -111710,16 +111747,39 @@
111710111747
** similar macro called ArraySize(). Use a different name to avoid
111711111748
** a collision when building an amalgamation with built-in FTS3.
111712111749
*/
111713111750
#define SizeofArray(X) ((int)(sizeof(X)/sizeof(X[0])))
111714111751
111752
+
111753
+#ifndef MIN
111754
+# define MIN(x,y) ((x)<(y)?(x):(y))
111755
+#endif
111756
+
111715111757
/*
111716111758
** Maximum length of a varint encoded integer. The varint format is different
111717111759
** from that used by SQLite, so the maximum length is 10, not 9.
111718111760
*/
111719111761
#define FTS3_VARINT_MAX 10
111720111762
111763
+/*
111764
+** FTS4 virtual tables may maintain multiple indexes - one index of all terms
111765
+** in the document set and zero or more prefix indexes. All indexes are stored
111766
+** as one or more b+-trees in the %_segments and %_segdir tables.
111767
+**
111768
+** It is possible to determine which index a b+-tree belongs to based on the
111769
+** value stored in the "%_segdir.level" column. Given this value L, the index
111770
+** that the b+-tree belongs to is (L<<10). In other words, all b+-trees with
111771
+** level values between 0 and 1023 (inclusive) belong to index 0, all levels
111772
+** between 1024 and 2047 to index 1, and so on.
111773
+**
111774
+** It is considered impossible for an index to use more than 1024 levels. In
111775
+** theory though this may happen, but only after at least
111776
+** (FTS3_MERGE_COUNT^1024) separate flushes of the pending-terms tables.
111777
+*/
111778
+#define FTS3_SEGDIR_MAXLEVEL 1024
111779
+#define FTS3_SEGDIR_MAXLEVEL_STR "1024"
111780
+
111721111781
/*
111722111782
** The testcase() macro is only used by the amalgamation. If undefined,
111723111783
** make it a no-op.
111724111784
*/
111725111785
#ifndef testcase
@@ -111787,14 +111847,15 @@
111787111847
typedef struct Fts3Cursor Fts3Cursor;
111788111848
typedef struct Fts3Expr Fts3Expr;
111789111849
typedef struct Fts3Phrase Fts3Phrase;
111790111850
typedef struct Fts3PhraseToken Fts3PhraseToken;
111791111851
111852
+typedef struct Fts3Doclist Fts3Doclist;
111792111853
typedef struct Fts3SegFilter Fts3SegFilter;
111793111854
typedef struct Fts3DeferredToken Fts3DeferredToken;
111794111855
typedef struct Fts3SegReader Fts3SegReader;
111795
-typedef struct Fts3SegReaderCursor Fts3SegReaderCursor;
111856
+typedef struct Fts3MultiSegReader Fts3MultiSegReader;
111796111857
111797111858
/*
111798111859
** A connection to a fulltext index is an instance of the following
111799111860
** structure. The xCreate and xConnect methods create an instance
111800111861
** of this structure and xDestroy and xDisconnect free that instance.
@@ -111811,33 +111872,45 @@
111811111872
sqlite3_tokenizer *pTokenizer; /* tokenizer for inserts and queries */
111812111873
111813111874
/* Precompiled statements used by the implementation. Each of these
111814111875
** statements is run and reset within a single virtual table API call.
111815111876
*/
111816
- sqlite3_stmt *aStmt[24];
111877
+ sqlite3_stmt *aStmt[27];
111817111878
111818111879
char *zReadExprlist;
111819111880
char *zWriteExprlist;
111820111881
111821111882
int nNodeSize; /* Soft limit for node size */
111822111883
u8 bHasStat; /* True if %_stat table exists */
111823111884
u8 bHasDocsize; /* True if %_docsize table exists */
111885
+ u8 bDescIdx; /* True if doclists are in reverse order */
111824111886
int nPgsz; /* Page size for host database */
111825111887
char *zSegmentsTbl; /* Name of %_segments table */
111826111888
sqlite3_blob *pSegments; /* Blob handle open on %_segments table */
111827111889
111828
- /* The following hash table is used to buffer pending index updates during
111890
+ /* TODO: Fix the first paragraph of this comment.
111891
+ **
111892
+ ** The following hash table is used to buffer pending index updates during
111829111893
** transactions. Variable nPendingData estimates the memory size of the
111830111894
** pending data, including hash table overhead, but not malloc overhead.
111831111895
** When nPendingData exceeds nMaxPendingData, the buffer is flushed
111832111896
** automatically. Variable iPrevDocid is the docid of the most recently
111833111897
** inserted record.
111898
+ **
111899
+ ** A single FTS4 table may have multiple full-text indexes. For each index
111900
+ ** there is an entry in the aIndex[] array. Index 0 is an index of all the
111901
+ ** terms that appear in the document set. Each subsequent index in aIndex[]
111902
+ ** is an index of prefixes of a specific length.
111834111903
*/
111835
- int nMaxPendingData;
111836
- int nPendingData;
111837
- sqlite_int64 iPrevDocid;
111838
- Fts3Hash pendingTerms;
111904
+ int nIndex; /* Size of aIndex[] */
111905
+ struct Fts3Index {
111906
+ int nPrefix; /* Prefix length (0 for main terms index) */
111907
+ Fts3Hash hPending; /* Pending terms table for this index */
111908
+ } *aIndex;
111909
+ int nMaxPendingData; /* Max pending data before flush to disk */
111910
+ int nPendingData; /* Current bytes of pending data */
111911
+ sqlite_int64 iPrevDocid; /* Docid of most recently inserted document */
111839111912
111840111913
#if defined(SQLITE_DEBUG)
111841111914
/* State variables used for validating that the transaction control
111842111915
** methods of the virtual table are called at appropriate times. These
111843111916
** values do not contribution to the FTS computation; they are used for
@@ -111864,13 +111937,14 @@
111864111937
Fts3DeferredToken *pDeferred; /* Deferred search tokens, if any */
111865111938
sqlite3_int64 iPrevId; /* Previous id read from aDoclist */
111866111939
char *pNextId; /* Pointer into the body of aDoclist */
111867111940
char *aDoclist; /* List of docids for full-text queries */
111868111941
int nDoclist; /* Size of buffer at aDoclist */
111869
- int desc; /* True to sort in descending order */
111942
+ u8 bDesc; /* True to sort in descending order */
111870111943
int eEvalmode; /* An FTS3_EVAL_XX constant */
111871111944
int nRowAvg; /* Average size of database rows, in pages */
111945
+ int nDoc; /* Documents in table */
111872111946
111873111947
int isMatchinfoNeeded; /* True when aMatchinfo[] needs filling in */
111874111948
u32 *aMatchinfo; /* Information about most recent match */
111875111949
int nMatchinfo; /* Number of elements in aMatchinfo[] */
111876111950
char *zMatchinfo; /* Matchinfo specification */
@@ -111897,66 +111971,90 @@
111897111971
*/
111898111972
#define FTS3_FULLSCAN_SEARCH 0 /* Linear scan of %_content table */
111899111973
#define FTS3_DOCID_SEARCH 1 /* Lookup by rowid on %_content table */
111900111974
#define FTS3_FULLTEXT_SEARCH 2 /* Full-text index search */
111901111975
111976
+
111977
+struct Fts3Doclist {
111978
+ char *aAll; /* Array containing doclist (or NULL) */
111979
+ int nAll; /* Size of a[] in bytes */
111980
+ char *pNextDocid; /* Pointer to next docid */
111981
+
111982
+ sqlite3_int64 iDocid; /* Current docid (if pList!=0) */
111983
+ int bFreeList; /* True if pList should be sqlite3_free()d */
111984
+ char *pList; /* Pointer to position list following iDocid */
111985
+ int nList; /* Length of position list */
111986
+} doclist;
111987
+
111902111988
/*
111903111989
** A "phrase" is a sequence of one or more tokens that must match in
111904111990
** sequence. A single token is the base case and the most common case.
111905111991
** For a sequence of tokens contained in double-quotes (i.e. "one two three")
111906111992
** nToken will be the number of tokens in the string.
111907
-**
111908
-** The nDocMatch and nMatch variables contain data that may be used by the
111909
-** matchinfo() function. They are populated when the full-text index is
111910
-** queried for hits on the phrase. If one or more tokens in the phrase
111911
-** are deferred, the nDocMatch and nMatch variables are populated based
111912
-** on the assumption that the
111913111993
*/
111914111994
struct Fts3PhraseToken {
111915111995
char *z; /* Text of the token */
111916111996
int n; /* Number of bytes in buffer z */
111917111997
int isPrefix; /* True if token ends with a "*" character */
111998
+
111999
+ /* Variables above this point are populated when the expression is
112000
+ ** parsed (by code in fts3_expr.c). Below this point the variables are
112001
+ ** used when evaluating the expression. */
111918112002
int bFulltext; /* True if full-text index was used */
111919
- Fts3SegReaderCursor *pSegcsr; /* Segment-reader for this token */
111920112003
Fts3DeferredToken *pDeferred; /* Deferred token object for this token */
112004
+ Fts3MultiSegReader *pSegcsr; /* Segment-reader for this token */
111921112005
};
111922112006
111923112007
struct Fts3Phrase {
111924
- /* Variables populated by fts3_expr.c when parsing a MATCH expression */
112008
+ /* Cache of doclist for this phrase. */
112009
+ Fts3Doclist doclist;
112010
+ int bIncr; /* True if doclist is loaded incrementally */
112011
+
112012
+ /* Variables below this point are populated by fts3_expr.c when parsing
112013
+ ** a MATCH expression. Everything above is part of the evaluation phase.
112014
+ */
111925112015
int nToken; /* Number of tokens in the phrase */
111926112016
int iColumn; /* Index of column this phrase must match */
111927
- int isNot; /* Phrase prefixed by unary not (-) operator */
111928112017
Fts3PhraseToken aToken[1]; /* One entry for each token in the phrase */
111929112018
};
111930112019
111931112020
/*
111932112021
** A tree of these objects forms the RHS of a MATCH operator.
111933112022
**
111934
-** If Fts3Expr.eType is either FTSQUERY_NEAR or FTSQUERY_PHRASE and isLoaded
111935
-** is true, then aDoclist points to a malloced buffer, size nDoclist bytes,
111936
-** containing the results of the NEAR or phrase query in FTS3 doclist
111937
-** format. As usual, the initial "Length" field found in doclists stored
111938
-** on disk is omitted from this buffer.
112023
+** If Fts3Expr.eType is FTSQUERY_PHRASE and isLoaded is true, then aDoclist
112024
+** points to a malloced buffer, size nDoclist bytes, containing the results
112025
+** of this phrase query in FTS3 doclist format. As usual, the initial
112026
+** "Length" field found in doclists stored on disk is omitted from this
112027
+** buffer.
111939112028
**
111940
-** Variable pCurrent always points to the start of a docid field within
111941
-** aDoclist. Since the doclist is usually scanned in docid order, this can
111942
-** be used to accelerate seeking to the required docid within the doclist.
112029
+** Variable aMI is used only for FTSQUERY_NEAR nodes to store the global
112030
+** matchinfo data. If it is not NULL, it points to an array of size nCol*3,
112031
+** where nCol is the number of columns in the queried FTS table. The array
112032
+** is populated as follows:
112033
+**
112034
+** aMI[iCol*3 + 0] = Undefined
112035
+** aMI[iCol*3 + 1] = Number of occurrences
112036
+** aMI[iCol*3 + 2] = Number of rows containing at least one instance
112037
+**
112038
+** The aMI array is allocated using sqlite3_malloc(). It should be freed
112039
+** when the expression node is.
111943112040
*/
111944112041
struct Fts3Expr {
111945112042
int eType; /* One of the FTSQUERY_XXX values defined below */
111946112043
int nNear; /* Valid if eType==FTSQUERY_NEAR */
111947112044
Fts3Expr *pParent; /* pParent->pLeft==this or pParent->pRight==this */
111948112045
Fts3Expr *pLeft; /* Left operand */
111949112046
Fts3Expr *pRight; /* Right operand */
111950112047
Fts3Phrase *pPhrase; /* Valid if eType==FTSQUERY_PHRASE */
111951112048
111952
- int isLoaded; /* True if aDoclist/nDoclist are initialized. */
111953
- char *aDoclist; /* Buffer containing doclist */
111954
- int nDoclist; /* Size of aDoclist in bytes */
112049
+ /* The following are used by the fts3_eval.c module. */
112050
+ sqlite3_int64 iDocid; /* Current docid */
112051
+ u8 bEof; /* True this expression is at EOF already */
112052
+ u8 bStart; /* True if iDocid is valid */
112053
+ u8 bDeferred; /* True if this expression is entirely deferred */
111955112054
111956
- sqlite3_int64 iCurrent;
111957
- char *pCurrent;
112055
+ u32 *aMI;
111958112056
};
111959112057
111960112058
/*
111961112059
** Candidate values for Fts3Query.eType. Note that the order of the first
111962112060
** four values is in order of precedence when parsing expressions. For
@@ -111980,35 +112078,36 @@
111980112078
SQLITE_PRIVATE int sqlite3Fts3PendingTermsFlush(Fts3Table *);
111981112079
SQLITE_PRIVATE void sqlite3Fts3PendingTermsClear(Fts3Table *);
111982112080
SQLITE_PRIVATE int sqlite3Fts3Optimize(Fts3Table *);
111983112081
SQLITE_PRIVATE int sqlite3Fts3SegReaderNew(int, sqlite3_int64,
111984112082
sqlite3_int64, sqlite3_int64, const char *, int, Fts3SegReader**);
111985
-SQLITE_PRIVATE int sqlite3Fts3SegReaderPending(Fts3Table*,const char*,int,int,Fts3SegReader**);
112083
+SQLITE_PRIVATE int sqlite3Fts3SegReaderPending(
112084
+ Fts3Table*,int,const char*,int,int,Fts3SegReader**);
111986112085
SQLITE_PRIVATE void sqlite3Fts3SegReaderFree(Fts3SegReader *);
111987
-SQLITE_PRIVATE int sqlite3Fts3SegReaderCost(Fts3Cursor *, Fts3SegReader *, int *);
111988
-SQLITE_PRIVATE int sqlite3Fts3AllSegdirs(Fts3Table*, int, sqlite3_stmt **);
112086
+SQLITE_PRIVATE int sqlite3Fts3AllSegdirs(Fts3Table*, int, int, sqlite3_stmt **);
111989112087
SQLITE_PRIVATE int sqlite3Fts3ReadLock(Fts3Table *);
111990
-SQLITE_PRIVATE int sqlite3Fts3ReadBlock(Fts3Table*, sqlite3_int64, char **, int*);
112088
+SQLITE_PRIVATE int sqlite3Fts3ReadBlock(Fts3Table*, sqlite3_int64, char **, int*, int*);
111991112089
111992112090
SQLITE_PRIVATE int sqlite3Fts3SelectDoctotal(Fts3Table *, sqlite3_stmt **);
111993112091
SQLITE_PRIVATE int sqlite3Fts3SelectDocsize(Fts3Table *, sqlite3_int64, sqlite3_stmt **);
111994112092
111995112093
SQLITE_PRIVATE void sqlite3Fts3FreeDeferredTokens(Fts3Cursor *);
111996112094
SQLITE_PRIVATE int sqlite3Fts3DeferToken(Fts3Cursor *, Fts3PhraseToken *, int);
111997112095
SQLITE_PRIVATE int sqlite3Fts3CacheDeferredDoclists(Fts3Cursor *);
111998112096
SQLITE_PRIVATE void sqlite3Fts3FreeDeferredDoclists(Fts3Cursor *);
111999
-SQLITE_PRIVATE char *sqlite3Fts3DeferredDoclist(Fts3DeferredToken *, int *);
112000112097
SQLITE_PRIVATE void sqlite3Fts3SegmentsClose(Fts3Table *);
112001112098
112002
-#define FTS3_SEGCURSOR_PENDING -1
112003
-#define FTS3_SEGCURSOR_ALL -2
112099
+/* Special values interpreted by sqlite3SegReaderCursor() */
112100
+#define FTS3_SEGCURSOR_PENDING -1
112101
+#define FTS3_SEGCURSOR_ALL -2
112004112102
112005
-SQLITE_PRIVATE int sqlite3Fts3SegReaderStart(Fts3Table*, Fts3SegReaderCursor*, Fts3SegFilter*);
112006
-SQLITE_PRIVATE int sqlite3Fts3SegReaderStep(Fts3Table *, Fts3SegReaderCursor *);
112007
-SQLITE_PRIVATE void sqlite3Fts3SegReaderFinish(Fts3SegReaderCursor *);
112103
+SQLITE_PRIVATE int sqlite3Fts3SegReaderStart(Fts3Table*, Fts3MultiSegReader*, Fts3SegFilter*);
112104
+SQLITE_PRIVATE int sqlite3Fts3SegReaderStep(Fts3Table *, Fts3MultiSegReader *);
112105
+SQLITE_PRIVATE void sqlite3Fts3SegReaderFinish(Fts3MultiSegReader *);
112106
+
112008112107
SQLITE_PRIVATE int sqlite3Fts3SegReaderCursor(
112009
- Fts3Table *, int, const char *, int, int, int, Fts3SegReaderCursor *);
112108
+ Fts3Table *, int, int, const char *, int, int, int, Fts3MultiSegReader *);
112010112109
112011112110
/* Flags allowed as part of the 4th argument to SegmentReaderIterate() */
112012112111
#define FTS3_SEGMENT_REQUIRE_POS 0x00000001
112013112112
#define FTS3_SEGMENT_IGNORE_EMPTY 0x00000002
112014112113
#define FTS3_SEGMENT_COLUMN_FILTER 0x00000004
@@ -112021,21 +112120,24 @@
112021112120
int nTerm;
112022112121
int iCol;
112023112122
int flags;
112024112123
};
112025112124
112026
-struct Fts3SegReaderCursor {
112125
+struct Fts3MultiSegReader {
112027112126
/* Used internally by sqlite3Fts3SegReaderXXX() calls */
112028112127
Fts3SegReader **apSegment; /* Array of Fts3SegReader objects */
112029112128
int nSegment; /* Size of apSegment array */
112030112129
int nAdvance; /* How many seg-readers to advance */
112031112130
Fts3SegFilter *pFilter; /* Pointer to filter object */
112032112131
char *aBuffer; /* Buffer to merge doclists in */
112033112132
int nBuffer; /* Allocated size of aBuffer[] in bytes */
112034112133
112035
- /* Cost of running this iterator. Used by fts3.c only. */
112036
- int nCost;
112134
+ int iColFilter; /* If >=0, filter for this column */
112135
+
112136
+ /* Used by fts3.c only. */
112137
+ int nCost; /* Cost of running iterator */
112138
+ int bLookup; /* True if a lookup of a single entry. */
112037112139
112038112140
/* Output values. Valid only after Fts3SegReaderStep() returns SQLITE_ROW. */
112039112141
char *zTerm; /* Pointer to term buffer */
112040112142
int nTerm; /* Size of zTerm in bytes */
112041112143
char *aDoclist; /* Pointer to doclist buffer */
@@ -112046,15 +112148,13 @@
112046112148
SQLITE_PRIVATE int sqlite3Fts3PutVarint(char *, sqlite3_int64);
112047112149
SQLITE_PRIVATE int sqlite3Fts3GetVarint(const char *, sqlite_int64 *);
112048112150
SQLITE_PRIVATE int sqlite3Fts3GetVarint32(const char *, int *);
112049112151
SQLITE_PRIVATE int sqlite3Fts3VarintLen(sqlite3_uint64);
112050112152
SQLITE_PRIVATE void sqlite3Fts3Dequote(char *);
112153
+SQLITE_PRIVATE void sqlite3Fts3DoclistPrev(int,char*,int,char**,sqlite3_int64*,int*,u8*);
112051112154
112052
-SQLITE_PRIVATE char *sqlite3Fts3FindPositions(Fts3Cursor *, Fts3Expr *, sqlite3_int64, int);
112053
-SQLITE_PRIVATE int sqlite3Fts3ExprLoadDoclist(Fts3Cursor *, Fts3Expr *);
112054
-SQLITE_PRIVATE int sqlite3Fts3ExprLoadFtDoclist(Fts3Cursor *, Fts3Expr *, char **, int *);
112055
-SQLITE_PRIVATE int sqlite3Fts3ExprNearTrim(Fts3Expr *, Fts3Expr *, int);
112155
+SQLITE_PRIVATE int sqlite3Fts3EvalPhraseStats(Fts3Cursor *, Fts3Expr *, u32 *);
112056112156
112057112157
/* fts3_tokenizer.c */
112058112158
SQLITE_PRIVATE const char *sqlite3Fts3NextToken(const char *, int *);
112059112159
SQLITE_PRIVATE int sqlite3Fts3InitHashTable(sqlite3 *, Fts3Hash *, const char *);
112060112160
SQLITE_PRIVATE int sqlite3Fts3InitTokenizer(Fts3Hash *pHash, const char *,
@@ -112079,10 +112179,33 @@
112079112179
SQLITE_PRIVATE int sqlite3Fts3InitTerm(sqlite3 *db);
112080112180
#endif
112081112181
112082112182
/* fts3_aux.c */
112083112183
SQLITE_PRIVATE int sqlite3Fts3InitAux(sqlite3 *db);
112184
+
112185
+SQLITE_PRIVATE int sqlite3Fts3TermSegReaderCursor(
112186
+ Fts3Cursor *pCsr, /* Virtual table cursor handle */
112187
+ const char *zTerm, /* Term to query for */
112188
+ int nTerm, /* Size of zTerm in bytes */
112189
+ int isPrefix, /* True for a prefix search */
112190
+ Fts3MultiSegReader **ppSegcsr /* OUT: Allocated seg-reader cursor */
112191
+);
112192
+
112193
+SQLITE_PRIVATE void sqlite3Fts3EvalPhraseCleanup(Fts3Phrase *);
112194
+
112195
+SQLITE_PRIVATE int sqlite3Fts3EvalStart(Fts3Cursor *, Fts3Expr *, int);
112196
+SQLITE_PRIVATE int sqlite3Fts3EvalNext(Fts3Cursor *pCsr);
112197
+
112198
+SQLITE_PRIVATE int sqlite3Fts3MsrIncrStart(
112199
+ Fts3Table*, Fts3MultiSegReader*, int, const char*, int);
112200
+SQLITE_PRIVATE int sqlite3Fts3MsrIncrNext(
112201
+ Fts3Table *, Fts3MultiSegReader *, sqlite3_int64 *, char **, int *);
112202
+SQLITE_PRIVATE char *sqlite3Fts3EvalPhrasePoslist(Fts3Cursor *, Fts3Expr *, int iCol);
112203
+SQLITE_PRIVATE int sqlite3Fts3MsrOvfl(Fts3Cursor *, Fts3MultiSegReader *, int *);
112204
+
112205
+SQLITE_PRIVATE int sqlite3Fts3DeferredTokenList(Fts3DeferredToken *, char **, int *);
112206
+
112084112207
112085112208
#endif /* _FTSINT_H */
112086112209
112087112210
/************** End of fts3Int.h *********************************************/
112088112211
/************** Continuing where we left off in fts3.c ***********************/
@@ -112200,16 +112323,16 @@
112200112323
112201112324
/*
112202112325
** When this function is called, *pp points to the first byte following a
112203112326
** varint that is part of a doclist (or position-list, or any other list
112204112327
** of varints). This function moves *pp to point to the start of that varint,
112205
-** and decrements the value stored in *pVal by the varint value.
112328
+** and sets *pVal by the varint value.
112206112329
**
112207112330
** Argument pStart points to the first byte of the doclist that the
112208112331
** varint is part of.
112209112332
*/
112210
-static void fts3GetReverseDeltaVarint(
112333
+static void fts3GetReverseVarint(
112211112334
char **pp,
112212112335
char *pStart,
112213112336
sqlite3_int64 *pVal
112214112337
){
112215112338
sqlite3_int64 iVal;
@@ -112221,25 +112344,11 @@
112221112344
for(p = (*pp)-2; p>=pStart && *p&0x80; p--);
112222112345
p++;
112223112346
*pp = p;
112224112347
112225112348
sqlite3Fts3GetVarint(p, &iVal);
112226
- *pVal -= iVal;
112227
-}
112228
-
112229
-/*
112230
-** As long as *pp has not reached its end (pEnd), then do the same
112231
-** as fts3GetDeltaVarint(): read a single varint and add it to *pVal.
112232
-** But if we have reached the end of the varint, just set *pp=0 and
112233
-** leave *pVal unchanged.
112234
-*/
112235
-static void fts3GetDeltaVarint2(char **pp, char *pEnd, sqlite3_int64 *pVal){
112236
- if( *pp>=pEnd ){
112237
- *pp = 0;
112238
- }else{
112239
- fts3GetDeltaVarint(pp, pVal);
112240
- }
112349
+ *pVal = iVal;
112241112350
}
112242112351
112243112352
/*
112244112353
** The xDisconnect() virtual table method.
112245112354
*/
@@ -112608,10 +112717,62 @@
112608112717
fts3Appendf(pRc, &zRet, ",%s(?)", zFunction);
112609112718
}
112610112719
sqlite3_free(zFree);
112611112720
return zRet;
112612112721
}
112722
+
112723
+static int fts3GobbleInt(const char **pp, int *pnOut){
112724
+ const char *p = *pp;
112725
+ int nInt = 0;
112726
+ for(p=*pp; p[0]>='0' && p[0]<='9'; p++){
112727
+ nInt = nInt * 10 + (p[0] - '0');
112728
+ }
112729
+ if( p==*pp ) return SQLITE_ERROR;
112730
+ *pnOut = nInt;
112731
+ *pp = p;
112732
+ return SQLITE_OK;
112733
+}
112734
+
112735
+
112736
+static int fts3PrefixParameter(
112737
+ const char *zParam, /* ABC in prefix=ABC parameter to parse */
112738
+ int *pnIndex, /* OUT: size of *apIndex[] array */
112739
+ struct Fts3Index **apIndex, /* OUT: Array of indexes for this table */
112740
+ struct Fts3Index **apFree /* OUT: Free this with sqlite3_free() */
112741
+){
112742
+ struct Fts3Index *aIndex;
112743
+ int nIndex = 1;
112744
+
112745
+ if( zParam && zParam[0] ){
112746
+ const char *p;
112747
+ nIndex++;
112748
+ for(p=zParam; *p; p++){
112749
+ if( *p==',' ) nIndex++;
112750
+ }
112751
+ }
112752
+
112753
+ aIndex = sqlite3_malloc(sizeof(struct Fts3Index) * nIndex);
112754
+ *apIndex = *apFree = aIndex;
112755
+ *pnIndex = nIndex;
112756
+ if( !aIndex ){
112757
+ return SQLITE_NOMEM;
112758
+ }
112759
+
112760
+ memset(aIndex, 0, sizeof(struct Fts3Index) * nIndex);
112761
+ if( zParam ){
112762
+ const char *p = zParam;
112763
+ int i;
112764
+ for(i=1; i<nIndex; i++){
112765
+ int nPrefix;
112766
+ if( fts3GobbleInt(&p, &nPrefix) ) return SQLITE_ERROR;
112767
+ aIndex[i].nPrefix = nPrefix;
112768
+ p++;
112769
+ }
112770
+ }
112771
+
112772
+ return SQLITE_OK;
112773
+}
112613112774
112614112775
/*
112615112776
** This function is the implementation of both the xConnect and xCreate
112616112777
** methods of the FTS3 virtual table.
112617112778
**
@@ -112641,16 +112802,23 @@
112641112802
int nCol = 0; /* Number of columns in the FTS table */
112642112803
char *zCsr; /* Space for holding column names */
112643112804
int nDb; /* Bytes required to hold database name */
112644112805
int nName; /* Bytes required to hold table name */
112645112806
int isFts4 = (argv[0][3]=='4'); /* True for FTS4, false for FTS3 */
112646
- int bNoDocsize = 0; /* True to omit %_docsize table */
112647112807
const char **aCol; /* Array of column names */
112648112808
sqlite3_tokenizer *pTokenizer = 0; /* Tokenizer for this table */
112649112809
112650
- char *zCompress = 0;
112651
- char *zUncompress = 0;
112810
+ int nIndex; /* Size of aIndex[] array */
112811
+ struct Fts3Index *aIndex; /* Array of indexes for this table */
112812
+ struct Fts3Index *aFree = 0; /* Free this before returning */
112813
+
112814
+ /* The results of parsing supported FTS4 key=value options: */
112815
+ int bNoDocsize = 0; /* True to omit %_docsize table */
112816
+ int bDescIdx = 0; /* True to store descending indexes */
112817
+ char *zPrefix = 0; /* Prefix parameter value (or NULL) */
112818
+ char *zCompress = 0; /* compress=? parameter (or NULL) */
112819
+ char *zUncompress = 0; /* uncompress=? parameter (or NULL) */
112652112820
112653112821
assert( strlen(argv[0])==4 );
112654112822
assert( (sqlite3_strnicmp(argv[0], "fts4", 4)==0 && isFts4)
112655112823
|| (sqlite3_strnicmp(argv[0], "fts3", 4)==0 && !isFts4)
112656112824
);
@@ -112687,32 +112855,76 @@
112687112855
rc = sqlite3Fts3InitTokenizer(pHash, &z[9], &pTokenizer, pzErr);
112688112856
}
112689112857
112690112858
/* Check if it is an FTS4 special argument. */
112691112859
else if( isFts4 && fts3IsSpecialColumn(z, &nKey, &zVal) ){
112860
+ struct Fts4Option {
112861
+ const char *zOpt;
112862
+ int nOpt;
112863
+ char **pzVar;
112864
+ } aFts4Opt[] = {
112865
+ { "matchinfo", 9, 0 }, /* 0 -> MATCHINFO */
112866
+ { "prefix", 6, 0 }, /* 1 -> PREFIX */
112867
+ { "compress", 8, 0 }, /* 2 -> COMPRESS */
112868
+ { "uncompress", 10, 0 }, /* 3 -> UNCOMPRESS */
112869
+ { "order", 5, 0 } /* 4 -> ORDER */
112870
+ };
112871
+
112872
+ int iOpt;
112692112873
if( !zVal ){
112693112874
rc = SQLITE_NOMEM;
112694
- goto fts3_init_out;
112695
- }
112696
- if( nKey==9 && 0==sqlite3_strnicmp(z, "matchinfo", 9) ){
112697
- if( strlen(zVal)==4 && 0==sqlite3_strnicmp(zVal, "fts3", 4) ){
112698
- bNoDocsize = 1;
112699
- }else{
112700
- *pzErr = sqlite3_mprintf("unrecognized matchinfo: %s", zVal);
112701
- rc = SQLITE_ERROR;
112702
- }
112703
- }else if( nKey==8 && 0==sqlite3_strnicmp(z, "compress", 8) ){
112704
- zCompress = zVal;
112705
- zVal = 0;
112706
- }else if( nKey==10 && 0==sqlite3_strnicmp(z, "uncompress", 10) ){
112707
- zUncompress = zVal;
112708
- zVal = 0;
112709
- }else{
112710
- *pzErr = sqlite3_mprintf("unrecognized parameter: %s", z);
112711
- rc = SQLITE_ERROR;
112712
- }
112713
- sqlite3_free(zVal);
112875
+ }else{
112876
+ for(iOpt=0; iOpt<SizeofArray(aFts4Opt); iOpt++){
112877
+ struct Fts4Option *pOp = &aFts4Opt[iOpt];
112878
+ if( nKey==pOp->nOpt && !sqlite3_strnicmp(z, pOp->zOpt, pOp->nOpt) ){
112879
+ break;
112880
+ }
112881
+ }
112882
+ if( iOpt==SizeofArray(aFts4Opt) ){
112883
+ *pzErr = sqlite3_mprintf("unrecognized parameter: %s", z);
112884
+ rc = SQLITE_ERROR;
112885
+ }else{
112886
+ switch( iOpt ){
112887
+ case 0: /* MATCHINFO */
112888
+ if( strlen(zVal)!=4 || sqlite3_strnicmp(zVal, "fts3", 4) ){
112889
+ *pzErr = sqlite3_mprintf("unrecognized matchinfo: %s", zVal);
112890
+ rc = SQLITE_ERROR;
112891
+ }
112892
+ bNoDocsize = 1;
112893
+ break;
112894
+
112895
+ case 1: /* PREFIX */
112896
+ sqlite3_free(zPrefix);
112897
+ zPrefix = zVal;
112898
+ zVal = 0;
112899
+ break;
112900
+
112901
+ case 2: /* COMPRESS */
112902
+ sqlite3_free(zCompress);
112903
+ zCompress = zVal;
112904
+ zVal = 0;
112905
+ break;
112906
+
112907
+ case 3: /* UNCOMPRESS */
112908
+ sqlite3_free(zUncompress);
112909
+ zUncompress = zVal;
112910
+ zVal = 0;
112911
+ break;
112912
+
112913
+ case 4: /* ORDER */
112914
+ if( (strlen(zVal)!=3 || sqlite3_strnicmp(zVal, "asc", 3))
112915
+ && (strlen(zVal)!=4 || sqlite3_strnicmp(zVal, "desc", 3))
112916
+ ){
112917
+ *pzErr = sqlite3_mprintf("unrecognized order: %s", zVal);
112918
+ rc = SQLITE_ERROR;
112919
+ }
112920
+ bDescIdx = (zVal[0]=='d' || zVal[0]=='D');
112921
+ break;
112922
+ }
112923
+ }
112924
+ sqlite3_free(zVal);
112925
+ }
112714112926
}
112715112927
112716112928
/* Otherwise, the argument is a column name. */
112717112929
else {
112718112930
nString += (int)(strlen(z) + 1);
@@ -112732,14 +112944,21 @@
112732112944
rc = sqlite3Fts3InitTokenizer(pHash, "simple", &pTokenizer, pzErr);
112733112945
if( rc!=SQLITE_OK ) goto fts3_init_out;
112734112946
}
112735112947
assert( pTokenizer );
112736112948
112949
+ rc = fts3PrefixParameter(zPrefix, &nIndex, &aIndex, &aFree);
112950
+ if( rc==SQLITE_ERROR ){
112951
+ assert( zPrefix );
112952
+ *pzErr = sqlite3_mprintf("error parsing prefix parameter: %s", zPrefix);
112953
+ }
112954
+ if( rc!=SQLITE_OK ) goto fts3_init_out;
112737112955
112738112956
/* Allocate and populate the Fts3Table structure. */
112739
- nByte = sizeof(Fts3Table) + /* Fts3Table */
112957
+ nByte = sizeof(Fts3Table) + /* Fts3Table */
112740112958
nCol * sizeof(char *) + /* azColumn */
112959
+ nIndex * sizeof(struct Fts3Index) + /* aIndex */
112741112960
nName + /* zName */
112742112961
nDb + /* zDb */
112743112962
nString; /* Space for azColumn strings */
112744112963
p = (Fts3Table*)sqlite3_malloc(nByte);
112745112964
if( p==0 ){
@@ -112750,20 +112969,26 @@
112750112969
p->db = db;
112751112970
p->nColumn = nCol;
112752112971
p->nPendingData = 0;
112753112972
p->azColumn = (char **)&p[1];
112754112973
p->pTokenizer = pTokenizer;
112755
- p->nNodeSize = 1000;
112756112974
p->nMaxPendingData = FTS3_MAX_PENDING_DATA;
112757112975
p->bHasDocsize = (isFts4 && bNoDocsize==0);
112758112976
p->bHasStat = isFts4;
112977
+ p->bDescIdx = bDescIdx;
112759112978
TESTONLY( p->inTransaction = -1 );
112760112979
TESTONLY( p->mxSavepoint = -1 );
112761
- fts3HashInit(&p->pendingTerms, FTS3_HASH_STRING, 1);
112980
+
112981
+ p->aIndex = (struct Fts3Index *)&p->azColumn[nCol];
112982
+ memcpy(p->aIndex, aIndex, sizeof(struct Fts3Index) * nIndex);
112983
+ p->nIndex = nIndex;
112984
+ for(i=0; i<nIndex; i++){
112985
+ fts3HashInit(&p->aIndex[i].hPending, FTS3_HASH_STRING, 1);
112986
+ }
112762112987
112763112988
/* Fill in the zName and zDb fields of the vtab structure. */
112764
- zCsr = (char *)&p->azColumn[nCol];
112989
+ zCsr = (char *)&p->aIndex[nIndex];
112765112990
p->zName = zCsr;
112766112991
memcpy(zCsr, argv[2], nName);
112767112992
zCsr += nName;
112768112993
p->zDb = zCsr;
112769112994
memcpy(zCsr, argv[1], nDb);
@@ -112797,19 +113022,20 @@
112797113022
if( isCreate ){
112798113023
rc = fts3CreateTables(p);
112799113024
}
112800113025
112801113026
/* Figure out the page-size for the database. This is required in order to
112802
- ** estimate the cost of loading large doclists from the database (see
112803
- ** function sqlite3Fts3SegReaderCost() for details).
112804
- */
113027
+ ** estimate the cost of loading large doclists from the database. */
112805113028
fts3DatabasePageSize(&rc, p);
113029
+ p->nNodeSize = p->nPgsz-35;
112806113030
112807113031
/* Declare the table schema to SQLite. */
112808113032
fts3DeclareVtab(&rc, p);
112809113033
112810113034
fts3_init_out:
113035
+ sqlite3_free(zPrefix);
113036
+ sqlite3_free(aFree);
112811113037
sqlite3_free(zCompress);
112812113038
sqlite3_free(zUncompress);
112813113039
sqlite3_free((void *)aCol);
112814113040
if( rc!=SQLITE_OK ){
112815113041
if( p ){
@@ -112816,10 +113042,11 @@
112816113042
fts3DisconnectMethod((sqlite3_vtab *)p);
112817113043
}else if( pTokenizer ){
112818113044
pTokenizer->pModule->xDestroy(pTokenizer);
112819113045
}
112820113046
}else{
113047
+ assert( p->pSegments==0 );
112821113048
*ppVTab = &p->base;
112822113049
}
112823113050
return rc;
112824113051
}
112825113052
@@ -112913,14 +113140,15 @@
112913113140
if( pOrder->desc ){
112914113141
pInfo->idxStr = "DESC";
112915113142
}else{
112916113143
pInfo->idxStr = "ASC";
112917113144
}
113145
+ pInfo->orderByConsumed = 1;
112918113146
}
112919
- pInfo->orderByConsumed = 1;
112920113147
}
112921113148
113149
+ assert( p->pSegments==0 );
112922113150
return SQLITE_OK;
112923113151
}
112924113152
112925113153
/*
112926113154
** Implementation of xOpen method.
@@ -112952,10 +113180,11 @@
112952113180
sqlite3_finalize(pCsr->pStmt);
112953113181
sqlite3Fts3ExprFree(pCsr->pExpr);
112954113182
sqlite3Fts3FreeDeferredTokens(pCsr);
112955113183
sqlite3_free(pCsr->aDoclist);
112956113184
sqlite3_free(pCsr->aMatchinfo);
113185
+ assert( ((Fts3Table *)pCsr->base.pVtab)->pSegments==0 );
112957113186
sqlite3_free(pCsr);
112958113187
return SQLITE_OK;
112959113188
}
112960113189
112961113190
/*
@@ -112963,12 +113192,12 @@
112963113192
** of the %_content table that contains the last match. Return
112964113193
** SQLITE_OK on success.
112965113194
*/
112966113195
static int fts3CursorSeek(sqlite3_context *pContext, Fts3Cursor *pCsr){
112967113196
if( pCsr->isRequireSeek ){
112968
- pCsr->isRequireSeek = 0;
112969113197
sqlite3_bind_int64(pCsr->pStmt, 1, pCsr->iPrevId);
113198
+ pCsr->isRequireSeek = 0;
112970113199
if( SQLITE_ROW==sqlite3_step(pCsr->pStmt) ){
112971113200
return SQLITE_OK;
112972113201
}else{
112973113202
int rc = sqlite3_reset(pCsr->pStmt);
112974113203
if( rc==SQLITE_OK ){
@@ -113145,21 +113374,21 @@
113145113374
if( rc==SQLITE_OK && iHeight>1 ){
113146113375
char *zBlob = 0; /* Blob read from %_segments table */
113147113376
int nBlob; /* Size of zBlob in bytes */
113148113377
113149113378
if( piLeaf && piLeaf2 && (*piLeaf!=*piLeaf2) ){
113150
- rc = sqlite3Fts3ReadBlock(p, *piLeaf, &zBlob, &nBlob);
113379
+ rc = sqlite3Fts3ReadBlock(p, *piLeaf, &zBlob, &nBlob, 0);
113151113380
if( rc==SQLITE_OK ){
113152113381
rc = fts3SelectLeaf(p, zTerm, nTerm, zBlob, nBlob, piLeaf, 0);
113153113382
}
113154113383
sqlite3_free(zBlob);
113155113384
piLeaf = 0;
113156113385
zBlob = 0;
113157113386
}
113158113387
113159113388
if( rc==SQLITE_OK ){
113160
- rc = sqlite3Fts3ReadBlock(p, piLeaf ? *piLeaf : *piLeaf2, &zBlob, &nBlob);
113389
+ rc = sqlite3Fts3ReadBlock(p, piLeaf?*piLeaf:*piLeaf2, &zBlob, &nBlob, 0);
113161113390
}
113162113391
if( rc==SQLITE_OK ){
113163113392
rc = fts3SelectLeaf(p, zTerm, nTerm, zBlob, nBlob, piLeaf, piLeaf2);
113164113393
}
113165113394
sqlite3_free(zBlob);
@@ -113531,11 +113760,23 @@
113531113760
*pp = p;
113532113761
return 1;
113533113762
}
113534113763
113535113764
/*
113536
-** Merge two position-lists as required by the NEAR operator.
113765
+** Merge two position-lists as required by the NEAR operator. The argument
113766
+** position lists correspond to the left and right phrases of an expression
113767
+** like:
113768
+**
113769
+** "phrase 1" NEAR "phrase number 2"
113770
+**
113771
+** Position list *pp1 corresponds to the left-hand side of the NEAR
113772
+** expression and *pp2 to the right. As usual, the indexes in the position
113773
+** lists are the offsets of the last token in each phrase (tokens "1" and "2"
113774
+** in the example above).
113775
+**
113776
+** The output position list - written to *pp - is a copy of *pp2 with those
113777
+** entries that are not sufficiently NEAR entries in *pp1 removed.
113537113778
*/
113538113779
static int fts3PoslistNearMerge(
113539113780
char **pp, /* Output buffer */
113540113781
char *aTmp, /* Temporary buffer space */
113541113782
int nRight, /* Maximum difference in token positions */
@@ -113544,218 +113785,31 @@
113544113785
char **pp2 /* IN/OUT: Right input list */
113545113786
){
113546113787
char *p1 = *pp1;
113547113788
char *p2 = *pp2;
113548113789
113549
- if( !pp ){
113550
- if( fts3PoslistPhraseMerge(0, nRight, 0, 0, pp1, pp2) ) return 1;
113551
- *pp1 = p1;
113552
- *pp2 = p2;
113553
- return fts3PoslistPhraseMerge(0, nLeft, 0, 0, pp2, pp1);
113554
- }else{
113555
- char *pTmp1 = aTmp;
113556
- char *pTmp2;
113557
- char *aTmp2;
113558
- int res = 1;
113559
-
113560
- fts3PoslistPhraseMerge(&pTmp1, nRight, 0, 0, pp1, pp2);
113561
- aTmp2 = pTmp2 = pTmp1;
113562
- *pp1 = p1;
113563
- *pp2 = p2;
113564
- fts3PoslistPhraseMerge(&pTmp2, nLeft, 1, 0, pp2, pp1);
113565
- if( pTmp1!=aTmp && pTmp2!=aTmp2 ){
113566
- fts3PoslistMerge(pp, &aTmp, &aTmp2);
113567
- }else if( pTmp1!=aTmp ){
113568
- fts3PoslistCopy(pp, &aTmp);
113569
- }else if( pTmp2!=aTmp2 ){
113570
- fts3PoslistCopy(pp, &aTmp2);
113571
- }else{
113572
- res = 0;
113573
- }
113574
-
113575
- return res;
113576
- }
113577
-}
113578
-
113579
-/*
113580
-** Values that may be used as the first parameter to fts3DoclistMerge().
113581
-*/
113582
-#define MERGE_NOT 2 /* D + D -> D */
113583
-#define MERGE_AND 3 /* D + D -> D */
113584
-#define MERGE_OR 4 /* D + D -> D */
113585
-#define MERGE_POS_OR 5 /* P + P -> P */
113586
-#define MERGE_PHRASE 6 /* P + P -> D */
113587
-#define MERGE_POS_PHRASE 7 /* P + P -> P */
113588
-#define MERGE_NEAR 8 /* P + P -> D */
113589
-#define MERGE_POS_NEAR 9 /* P + P -> P */
113590
-
113591
-/*
113592
-** Merge the two doclists passed in buffer a1 (size n1 bytes) and a2
113593
-** (size n2 bytes). The output is written to pre-allocated buffer aBuffer,
113594
-** which is guaranteed to be large enough to hold the results. The number
113595
-** of bytes written to aBuffer is stored in *pnBuffer before returning.
113596
-**
113597
-** If successful, SQLITE_OK is returned. Otherwise, if a malloc error
113598
-** occurs while allocating a temporary buffer as part of the merge operation,
113599
-** SQLITE_NOMEM is returned.
113600
-*/
113601
-static int fts3DoclistMerge(
113602
- int mergetype, /* One of the MERGE_XXX constants */
113603
- int nParam1, /* Used by MERGE_NEAR and MERGE_POS_NEAR */
113604
- int nParam2, /* Used by MERGE_NEAR and MERGE_POS_NEAR */
113605
- char *aBuffer, /* Pre-allocated output buffer */
113606
- int *pnBuffer, /* OUT: Bytes written to aBuffer */
113607
- char *a1, /* Buffer containing first doclist */
113608
- int n1, /* Size of buffer a1 */
113609
- char *a2, /* Buffer containing second doclist */
113610
- int n2, /* Size of buffer a2 */
113611
- int *pnDoc /* OUT: Number of docids in output */
113612
-){
113613
- sqlite3_int64 i1 = 0;
113614
- sqlite3_int64 i2 = 0;
113615
- sqlite3_int64 iPrev = 0;
113616
-
113617
- char *p = aBuffer;
113618
- char *p1 = a1;
113619
- char *p2 = a2;
113620
- char *pEnd1 = &a1[n1];
113621
- char *pEnd2 = &a2[n2];
113622
- int nDoc = 0;
113623
-
113624
- assert( mergetype==MERGE_OR || mergetype==MERGE_POS_OR
113625
- || mergetype==MERGE_AND || mergetype==MERGE_NOT
113626
- || mergetype==MERGE_PHRASE || mergetype==MERGE_POS_PHRASE
113627
- || mergetype==MERGE_NEAR || mergetype==MERGE_POS_NEAR
113628
- );
113629
-
113630
- if( !aBuffer ){
113631
- *pnBuffer = 0;
113632
- return SQLITE_NOMEM;
113633
- }
113634
-
113635
- /* Read the first docid from each doclist */
113636
- fts3GetDeltaVarint2(&p1, pEnd1, &i1);
113637
- fts3GetDeltaVarint2(&p2, pEnd2, &i2);
113638
-
113639
- switch( mergetype ){
113640
- case MERGE_OR:
113641
- case MERGE_POS_OR:
113642
- while( p1 || p2 ){
113643
- if( p2 && p1 && i1==i2 ){
113644
- fts3PutDeltaVarint(&p, &iPrev, i1);
113645
- if( mergetype==MERGE_POS_OR ) fts3PoslistMerge(&p, &p1, &p2);
113646
- fts3GetDeltaVarint2(&p1, pEnd1, &i1);
113647
- fts3GetDeltaVarint2(&p2, pEnd2, &i2);
113648
- }else if( !p2 || (p1 && i1<i2) ){
113649
- fts3PutDeltaVarint(&p, &iPrev, i1);
113650
- if( mergetype==MERGE_POS_OR ) fts3PoslistCopy(&p, &p1);
113651
- fts3GetDeltaVarint2(&p1, pEnd1, &i1);
113652
- }else{
113653
- fts3PutDeltaVarint(&p, &iPrev, i2);
113654
- if( mergetype==MERGE_POS_OR ) fts3PoslistCopy(&p, &p2);
113655
- fts3GetDeltaVarint2(&p2, pEnd2, &i2);
113656
- }
113657
- }
113658
- break;
113659
-
113660
- case MERGE_AND:
113661
- while( p1 && p2 ){
113662
- if( i1==i2 ){
113663
- fts3PutDeltaVarint(&p, &iPrev, i1);
113664
- fts3GetDeltaVarint2(&p1, pEnd1, &i1);
113665
- fts3GetDeltaVarint2(&p2, pEnd2, &i2);
113666
- nDoc++;
113667
- }else if( i1<i2 ){
113668
- fts3GetDeltaVarint2(&p1, pEnd1, &i1);
113669
- }else{
113670
- fts3GetDeltaVarint2(&p2, pEnd2, &i2);
113671
- }
113672
- }
113673
- break;
113674
-
113675
- case MERGE_NOT:
113676
- while( p1 ){
113677
- if( p2 && i1==i2 ){
113678
- fts3GetDeltaVarint2(&p1, pEnd1, &i1);
113679
- fts3GetDeltaVarint2(&p2, pEnd2, &i2);
113680
- }else if( !p2 || i1<i2 ){
113681
- fts3PutDeltaVarint(&p, &iPrev, i1);
113682
- fts3GetDeltaVarint2(&p1, pEnd1, &i1);
113683
- }else{
113684
- fts3GetDeltaVarint2(&p2, pEnd2, &i2);
113685
- }
113686
- }
113687
- break;
113688
-
113689
- case MERGE_POS_PHRASE:
113690
- case MERGE_PHRASE: {
113691
- char **ppPos = (mergetype==MERGE_PHRASE ? 0 : &p);
113692
- while( p1 && p2 ){
113693
- if( i1==i2 ){
113694
- char *pSave = p;
113695
- sqlite3_int64 iPrevSave = iPrev;
113696
- fts3PutDeltaVarint(&p, &iPrev, i1);
113697
- if( 0==fts3PoslistPhraseMerge(ppPos, nParam1, 0, 1, &p1, &p2) ){
113698
- p = pSave;
113699
- iPrev = iPrevSave;
113700
- }else{
113701
- nDoc++;
113702
- }
113703
- fts3GetDeltaVarint2(&p1, pEnd1, &i1);
113704
- fts3GetDeltaVarint2(&p2, pEnd2, &i2);
113705
- }else if( i1<i2 ){
113706
- fts3PoslistCopy(0, &p1);
113707
- fts3GetDeltaVarint2(&p1, pEnd1, &i1);
113708
- }else{
113709
- fts3PoslistCopy(0, &p2);
113710
- fts3GetDeltaVarint2(&p2, pEnd2, &i2);
113711
- }
113712
- }
113713
- break;
113714
- }
113715
-
113716
- default: assert( mergetype==MERGE_POS_NEAR || mergetype==MERGE_NEAR ); {
113717
- char *aTmp = 0;
113718
- char **ppPos = 0;
113719
-
113720
- if( mergetype==MERGE_POS_NEAR ){
113721
- ppPos = &p;
113722
- aTmp = sqlite3_malloc(2*(n1+n2+1));
113723
- if( !aTmp ){
113724
- return SQLITE_NOMEM;
113725
- }
113726
- }
113727
-
113728
- while( p1 && p2 ){
113729
- if( i1==i2 ){
113730
- char *pSave = p;
113731
- sqlite3_int64 iPrevSave = iPrev;
113732
- fts3PutDeltaVarint(&p, &iPrev, i1);
113733
-
113734
- if( !fts3PoslistNearMerge(ppPos, aTmp, nParam1, nParam2, &p1, &p2) ){
113735
- iPrev = iPrevSave;
113736
- p = pSave;
113737
- }
113738
-
113739
- fts3GetDeltaVarint2(&p1, pEnd1, &i1);
113740
- fts3GetDeltaVarint2(&p2, pEnd2, &i2);
113741
- }else if( i1<i2 ){
113742
- fts3PoslistCopy(0, &p1);
113743
- fts3GetDeltaVarint2(&p1, pEnd1, &i1);
113744
- }else{
113745
- fts3PoslistCopy(0, &p2);
113746
- fts3GetDeltaVarint2(&p2, pEnd2, &i2);
113747
- }
113748
- }
113749
- sqlite3_free(aTmp);
113750
- break;
113751
- }
113752
- }
113753
-
113754
- if( pnDoc ) *pnDoc = nDoc;
113755
- *pnBuffer = (int)(p-aBuffer);
113756
- return SQLITE_OK;
113790
+ char *pTmp1 = aTmp;
113791
+ char *pTmp2;
113792
+ char *aTmp2;
113793
+ int res = 1;
113794
+
113795
+ fts3PoslistPhraseMerge(&pTmp1, nRight, 0, 0, pp1, pp2);
113796
+ aTmp2 = pTmp2 = pTmp1;
113797
+ *pp1 = p1;
113798
+ *pp2 = p2;
113799
+ fts3PoslistPhraseMerge(&pTmp2, nLeft, 1, 0, pp2, pp1);
113800
+ if( pTmp1!=aTmp && pTmp2!=aTmp2 ){
113801
+ fts3PoslistMerge(pp, &aTmp, &aTmp2);
113802
+ }else if( pTmp1!=aTmp ){
113803
+ fts3PoslistCopy(pp, &aTmp);
113804
+ }else if( pTmp2!=aTmp2 ){
113805
+ fts3PoslistCopy(pp, &aTmp2);
113806
+ }else{
113807
+ res = 0;
113808
+ }
113809
+
113810
+ return res;
113757113811
}
113758113812
113759113813
/*
113760113814
** A pointer to an instance of this structure is used as the context
113761113815
** argument to sqlite3Fts3SegReaderIterate()
@@ -113764,10 +113818,152 @@
113764113818
struct TermSelect {
113765113819
int isReqPos;
113766113820
char *aaOutput[16]; /* Malloc'd output buffer */
113767113821
int anOutput[16]; /* Size of output in bytes */
113768113822
};
113823
+
113824
+
113825
+static void fts3GetDeltaVarint3(
113826
+ char **pp,
113827
+ char *pEnd,
113828
+ int bDescIdx,
113829
+ sqlite3_int64 *pVal
113830
+){
113831
+ if( *pp>=pEnd ){
113832
+ *pp = 0;
113833
+ }else{
113834
+ sqlite3_int64 iVal;
113835
+ *pp += sqlite3Fts3GetVarint(*pp, &iVal);
113836
+ if( bDescIdx ){
113837
+ *pVal -= iVal;
113838
+ }else{
113839
+ *pVal += iVal;
113840
+ }
113841
+ }
113842
+}
113843
+
113844
+static void fts3PutDeltaVarint3(
113845
+ char **pp, /* IN/OUT: Output pointer */
113846
+ int bDescIdx, /* True for descending docids */
113847
+ sqlite3_int64 *piPrev, /* IN/OUT: Previous value written to list */
113848
+ int *pbFirst, /* IN/OUT: True after first int written */
113849
+ sqlite3_int64 iVal /* Write this value to the list */
113850
+){
113851
+ sqlite3_int64 iWrite;
113852
+ if( bDescIdx==0 || *pbFirst==0 ){
113853
+ iWrite = iVal - *piPrev;
113854
+ }else{
113855
+ iWrite = *piPrev - iVal;
113856
+ }
113857
+ assert( *pbFirst || *piPrev==0 );
113858
+ assert( *pbFirst==0 || iWrite>0 );
113859
+ *pp += sqlite3Fts3PutVarint(*pp, iWrite);
113860
+ *piPrev = iVal;
113861
+ *pbFirst = 1;
113862
+}
113863
+
113864
+#define COMPARE_DOCID(i1, i2) ((bDescIdx?-1:1) * (i1-i2))
113865
+
113866
+static int fts3DoclistOrMerge(
113867
+ int bDescIdx, /* True if arguments are desc */
113868
+ char *a1, int n1, /* First doclist */
113869
+ char *a2, int n2, /* Second doclist */
113870
+ char **paOut, int *pnOut /* OUT: Malloc'd doclist */
113871
+){
113872
+ sqlite3_int64 i1 = 0;
113873
+ sqlite3_int64 i2 = 0;
113874
+ sqlite3_int64 iPrev = 0;
113875
+ char *pEnd1 = &a1[n1];
113876
+ char *pEnd2 = &a2[n2];
113877
+ char *p1 = a1;
113878
+ char *p2 = a2;
113879
+ char *p;
113880
+ char *aOut;
113881
+ int bFirstOut = 0;
113882
+
113883
+ *paOut = 0;
113884
+ *pnOut = 0;
113885
+ aOut = sqlite3_malloc(n1+n2);
113886
+ if( !aOut ) return SQLITE_NOMEM;
113887
+
113888
+ p = aOut;
113889
+ fts3GetDeltaVarint3(&p1, pEnd1, 0, &i1);
113890
+ fts3GetDeltaVarint3(&p2, pEnd2, 0, &i2);
113891
+ while( p1 || p2 ){
113892
+ sqlite3_int64 iDiff = COMPARE_DOCID(i1, i2);
113893
+
113894
+ if( p2 && p1 && iDiff==0 ){
113895
+ fts3PutDeltaVarint3(&p, bDescIdx, &iPrev, &bFirstOut, i1);
113896
+ fts3PoslistMerge(&p, &p1, &p2);
113897
+ fts3GetDeltaVarint3(&p1, pEnd1, bDescIdx, &i1);
113898
+ fts3GetDeltaVarint3(&p2, pEnd2, bDescIdx, &i2);
113899
+ }else if( !p2 || (p1 && iDiff<0) ){
113900
+ fts3PutDeltaVarint3(&p, bDescIdx, &iPrev, &bFirstOut, i1);
113901
+ fts3PoslistCopy(&p, &p1);
113902
+ fts3GetDeltaVarint3(&p1, pEnd1, bDescIdx, &i1);
113903
+ }else{
113904
+ fts3PutDeltaVarint3(&p, bDescIdx, &iPrev, &bFirstOut, i2);
113905
+ fts3PoslistCopy(&p, &p2);
113906
+ fts3GetDeltaVarint3(&p2, pEnd2, bDescIdx, &i2);
113907
+ }
113908
+ }
113909
+
113910
+ *paOut = aOut;
113911
+ *pnOut = (p-aOut);
113912
+ return SQLITE_OK;
113913
+}
113914
+
113915
+static void fts3DoclistPhraseMerge(
113916
+ int bDescIdx, /* True if arguments are desc */
113917
+ int nDist, /* Distance from left to right (1=adjacent) */
113918
+ char *aLeft, int nLeft, /* Left doclist */
113919
+ char *aRight, int *pnRight /* IN/OUT: Right/output doclist */
113920
+){
113921
+ sqlite3_int64 i1 = 0;
113922
+ sqlite3_int64 i2 = 0;
113923
+ sqlite3_int64 iPrev = 0;
113924
+ char *pEnd1 = &aLeft[nLeft];
113925
+ char *pEnd2 = &aRight[*pnRight];
113926
+ char *p1 = aLeft;
113927
+ char *p2 = aRight;
113928
+ char *p;
113929
+ int bFirstOut = 0;
113930
+ char *aOut = aRight;
113931
+
113932
+ assert( nDist>0 );
113933
+
113934
+ p = aOut;
113935
+ fts3GetDeltaVarint3(&p1, pEnd1, 0, &i1);
113936
+ fts3GetDeltaVarint3(&p2, pEnd2, 0, &i2);
113937
+
113938
+ while( p1 && p2 ){
113939
+ sqlite3_int64 iDiff = COMPARE_DOCID(i1, i2);
113940
+ if( iDiff==0 ){
113941
+ char *pSave = p;
113942
+ sqlite3_int64 iPrevSave = iPrev;
113943
+ int bFirstOutSave = bFirstOut;
113944
+
113945
+ fts3PutDeltaVarint3(&p, bDescIdx, &iPrev, &bFirstOut, i1);
113946
+ if( 0==fts3PoslistPhraseMerge(&p, nDist, 0, 1, &p1, &p2) ){
113947
+ p = pSave;
113948
+ iPrev = iPrevSave;
113949
+ bFirstOut = bFirstOutSave;
113950
+ }
113951
+ fts3GetDeltaVarint3(&p1, pEnd1, bDescIdx, &i1);
113952
+ fts3GetDeltaVarint3(&p2, pEnd2, bDescIdx, &i2);
113953
+ }else if( iDiff<0 ){
113954
+ fts3PoslistCopy(0, &p1);
113955
+ fts3GetDeltaVarint3(&p1, pEnd1, bDescIdx, &i1);
113956
+ }else{
113957
+ fts3PoslistCopy(0, &p2);
113958
+ fts3GetDeltaVarint3(&p2, pEnd2, bDescIdx, &i2);
113959
+ }
113960
+ }
113961
+
113962
+ *pnRight = p - aOut;
113963
+}
113964
+
113769113965
113770113966
/*
113771113967
** Merge all doclists in the TermSelect.aaOutput[] array into a single
113772113968
** doclist stored in TermSelect.aaOutput[0]. If successful, delete all
113773113969
** other doclists (except the aaOutput[0] one) and return SQLITE_OK.
@@ -113774,12 +113970,11 @@
113774113970
**
113775113971
** If an OOM error occurs, return SQLITE_NOMEM. In this case it is
113776113972
** the responsibility of the caller to free any doclists left in the
113777113973
** TermSelect.aaOutput[] array.
113778113974
*/
113779
-static int fts3TermSelectMerge(TermSelect *pTS){
113780
- int mergetype = (pTS->isReqPos ? MERGE_POS_OR : MERGE_OR);
113975
+static int fts3TermSelectMerge(Fts3Table *p, TermSelect *pTS){
113781113976
char *aOut = 0;
113782113977
int nOut = 0;
113783113978
int i;
113784113979
113785113980
/* Loop through the doclists in the aaOutput[] array. Merge them all
@@ -113790,19 +113985,21 @@
113790113985
if( !aOut ){
113791113986
aOut = pTS->aaOutput[i];
113792113987
nOut = pTS->anOutput[i];
113793113988
pTS->aaOutput[i] = 0;
113794113989
}else{
113795
- int nNew = nOut + pTS->anOutput[i];
113796
- char *aNew = sqlite3_malloc(nNew);
113797
- if( !aNew ){
113990
+ int nNew;
113991
+ char *aNew;
113992
+
113993
+ int rc = fts3DoclistOrMerge(p->bDescIdx,
113994
+ pTS->aaOutput[i], pTS->anOutput[i], aOut, nOut, &aNew, &nNew
113995
+ );
113996
+ if( rc!=SQLITE_OK ){
113798113997
sqlite3_free(aOut);
113799
- return SQLITE_NOMEM;
113998
+ return rc;
113800113999
}
113801
- fts3DoclistMerge(mergetype, 0, 0,
113802
- aNew, &nNew, pTS->aaOutput[i], pTS->anOutput[i], aOut, nOut, 0
113803
- );
114000
+
113804114001
sqlite3_free(pTS->aaOutput[i]);
113805114002
sqlite3_free(aOut);
113806114003
pTS->aaOutput[i] = 0;
113807114004
aOut = aNew;
113808114005
nOut = nNew;
@@ -113834,217 +114031,241 @@
113834114031
UNUSED_PARAMETER(zTerm);
113835114032
UNUSED_PARAMETER(nTerm);
113836114033
113837114034
if( pTS->aaOutput[0]==0 ){
113838114035
/* If this is the first term selected, copy the doclist to the output
113839
- ** buffer using memcpy(). TODO: Add a way to transfer control of the
113840
- ** aDoclist buffer from the caller so as to avoid the memcpy().
113841
- */
114036
+ ** buffer using memcpy(). */
113842114037
pTS->aaOutput[0] = sqlite3_malloc(nDoclist);
113843114038
pTS->anOutput[0] = nDoclist;
113844114039
if( pTS->aaOutput[0] ){
113845114040
memcpy(pTS->aaOutput[0], aDoclist, nDoclist);
113846114041
}else{
113847114042
return SQLITE_NOMEM;
113848114043
}
113849114044
}else{
113850
- int mergetype = (pTS->isReqPos ? MERGE_POS_OR : MERGE_OR);
113851114045
char *aMerge = aDoclist;
113852114046
int nMerge = nDoclist;
113853114047
int iOut;
113854114048
113855114049
for(iOut=0; iOut<SizeofArray(pTS->aaOutput); iOut++){
113856
- char *aNew;
113857
- int nNew;
113858114050
if( pTS->aaOutput[iOut]==0 ){
113859114051
assert( iOut>0 );
113860114052
pTS->aaOutput[iOut] = aMerge;
113861114053
pTS->anOutput[iOut] = nMerge;
113862114054
break;
113863
- }
113864
-
113865
- nNew = nMerge + pTS->anOutput[iOut];
113866
- aNew = sqlite3_malloc(nNew);
113867
- if( !aNew ){
113868
- if( aMerge!=aDoclist ){
113869
- sqlite3_free(aMerge);
113870
- }
113871
- return SQLITE_NOMEM;
113872
- }
113873
- fts3DoclistMerge(mergetype, 0, 0, aNew, &nNew,
113874
- pTS->aaOutput[iOut], pTS->anOutput[iOut], aMerge, nMerge, 0
113875
- );
113876
-
113877
- if( iOut>0 ) sqlite3_free(aMerge);
113878
- sqlite3_free(pTS->aaOutput[iOut]);
113879
- pTS->aaOutput[iOut] = 0;
113880
-
113881
- aMerge = aNew;
113882
- nMerge = nNew;
113883
- if( (iOut+1)==SizeofArray(pTS->aaOutput) ){
113884
- pTS->aaOutput[iOut] = aMerge;
113885
- pTS->anOutput[iOut] = nMerge;
113886
- }
113887
- }
113888
- }
113889
- return SQLITE_OK;
113890
-}
113891
-
113892
-static int fts3DeferredTermSelect(
113893
- Fts3DeferredToken *pToken, /* Phrase token */
113894
- int isTermPos, /* True to include positions */
113895
- int *pnOut, /* OUT: Size of list */
113896
- char **ppOut /* OUT: Body of list */
113897
-){
113898
- char *aSource;
113899
- int nSource;
113900
-
113901
- aSource = sqlite3Fts3DeferredDoclist(pToken, &nSource);
113902
- if( !aSource ){
113903
- *pnOut = 0;
113904
- *ppOut = 0;
113905
- }else if( isTermPos ){
113906
- *ppOut = sqlite3_malloc(nSource);
113907
- if( !*ppOut ) return SQLITE_NOMEM;
113908
- memcpy(*ppOut, aSource, nSource);
113909
- *pnOut = nSource;
113910
- }else{
113911
- sqlite3_int64 docid;
113912
- *pnOut = sqlite3Fts3GetVarint(aSource, &docid);
113913
- *ppOut = sqlite3_malloc(*pnOut);
113914
- if( !*ppOut ) return SQLITE_NOMEM;
113915
- sqlite3Fts3PutVarint(*ppOut, docid);
113916
- }
113917
-
113918
- return SQLITE_OK;
113919
-}
113920
-
113921
-SQLITE_PRIVATE int sqlite3Fts3SegReaderCursor(
114055
+ }else{
114056
+ char *aNew;
114057
+ int nNew;
114058
+
114059
+ int rc = fts3DoclistOrMerge(p->bDescIdx, aMerge, nMerge,
114060
+ pTS->aaOutput[iOut], pTS->anOutput[iOut], &aNew, &nNew
114061
+ );
114062
+ if( rc!=SQLITE_OK ){
114063
+ if( aMerge!=aDoclist ) sqlite3_free(aMerge);
114064
+ return rc;
114065
+ }
114066
+
114067
+ if( aMerge!=aDoclist ) sqlite3_free(aMerge);
114068
+ sqlite3_free(pTS->aaOutput[iOut]);
114069
+ pTS->aaOutput[iOut] = 0;
114070
+
114071
+ aMerge = aNew;
114072
+ nMerge = nNew;
114073
+ if( (iOut+1)==SizeofArray(pTS->aaOutput) ){
114074
+ pTS->aaOutput[iOut] = aMerge;
114075
+ pTS->anOutput[iOut] = nMerge;
114076
+ }
114077
+ }
114078
+ }
114079
+ }
114080
+ return SQLITE_OK;
114081
+}
114082
+
114083
+/*
114084
+** Append SegReader object pNew to the end of the pCsr->apSegment[] array.
114085
+*/
114086
+static int fts3SegReaderCursorAppend(
114087
+ Fts3MultiSegReader *pCsr,
114088
+ Fts3SegReader *pNew
114089
+){
114090
+ if( (pCsr->nSegment%16)==0 ){
114091
+ Fts3SegReader **apNew;
114092
+ int nByte = (pCsr->nSegment + 16)*sizeof(Fts3SegReader*);
114093
+ apNew = (Fts3SegReader **)sqlite3_realloc(pCsr->apSegment, nByte);
114094
+ if( !apNew ){
114095
+ sqlite3Fts3SegReaderFree(pNew);
114096
+ return SQLITE_NOMEM;
114097
+ }
114098
+ pCsr->apSegment = apNew;
114099
+ }
114100
+ pCsr->apSegment[pCsr->nSegment++] = pNew;
114101
+ return SQLITE_OK;
114102
+}
114103
+
114104
+static int fts3SegReaderCursor(
113922114105
Fts3Table *p, /* FTS3 table handle */
114106
+ int iIndex, /* Index to search (from 0 to p->nIndex-1) */
113923114107
int iLevel, /* Level of segments to scan */
113924114108
const char *zTerm, /* Term to query for */
113925114109
int nTerm, /* Size of zTerm in bytes */
113926114110
int isPrefix, /* True for a prefix search */
113927114111
int isScan, /* True to scan from zTerm to EOF */
113928
- Fts3SegReaderCursor *pCsr /* Cursor object to populate */
114112
+ Fts3MultiSegReader *pCsr /* Cursor object to populate */
113929114113
){
113930114114
int rc = SQLITE_OK;
113931114115
int rc2;
113932
- int iAge = 0;
113933114116
sqlite3_stmt *pStmt = 0;
113934
- Fts3SegReader *pPending = 0;
113935
-
113936
- assert( iLevel==FTS3_SEGCURSOR_ALL
113937
- || iLevel==FTS3_SEGCURSOR_PENDING
113938
- || iLevel>=0
113939
- );
113940
- assert( FTS3_SEGCURSOR_PENDING<0 );
113941
- assert( FTS3_SEGCURSOR_ALL<0 );
113942
- assert( iLevel==FTS3_SEGCURSOR_ALL || (zTerm==0 && isPrefix==1) );
113943
- assert( isPrefix==0 || isScan==0 );
113944
-
113945
-
113946
- memset(pCsr, 0, sizeof(Fts3SegReaderCursor));
113947
-
113948
- /* If iLevel is less than 0, include a seg-reader for the pending-terms. */
113949
- assert( isScan==0 || fts3HashCount(&p->pendingTerms)==0 );
113950
- if( iLevel<0 && isScan==0 ){
113951
- rc = sqlite3Fts3SegReaderPending(p, zTerm, nTerm, isPrefix, &pPending);
113952
- if( rc==SQLITE_OK && pPending ){
113953
- int nByte = (sizeof(Fts3SegReader *) * 16);
113954
- pCsr->apSegment = (Fts3SegReader **)sqlite3_malloc(nByte);
113955
- if( pCsr->apSegment==0 ){
113956
- rc = SQLITE_NOMEM;
113957
- }else{
113958
- pCsr->apSegment[0] = pPending;
113959
- pCsr->nSegment = 1;
113960
- pPending = 0;
113961
- }
114117
+
114118
+ /* If iLevel is less than 0 and this is not a scan, include a seg-reader
114119
+ ** for the pending-terms. If this is a scan, then this call must be being
114120
+ ** made by an fts4aux module, not an FTS table. In this case calling
114121
+ ** Fts3SegReaderPending might segfault, as the data structures used by
114122
+ ** fts4aux are not completely populated. So it's easiest to filter these
114123
+ ** calls out here. */
114124
+ if( iLevel<0 && p->aIndex ){
114125
+ Fts3SegReader *pSeg = 0;
114126
+ rc = sqlite3Fts3SegReaderPending(p, iIndex, zTerm, nTerm, isPrefix, &pSeg);
114127
+ if( rc==SQLITE_OK && pSeg ){
114128
+ rc = fts3SegReaderCursorAppend(pCsr, pSeg);
113962114129
}
113963114130
}
113964114131
113965114132
if( iLevel!=FTS3_SEGCURSOR_PENDING ){
113966114133
if( rc==SQLITE_OK ){
113967
- rc = sqlite3Fts3AllSegdirs(p, iLevel, &pStmt);
114134
+ rc = sqlite3Fts3AllSegdirs(p, iIndex, iLevel, &pStmt);
113968114135
}
114136
+
113969114137
while( rc==SQLITE_OK && SQLITE_ROW==(rc = sqlite3_step(pStmt)) ){
114138
+ Fts3SegReader *pSeg = 0;
113970114139
113971114140
/* Read the values returned by the SELECT into local variables. */
113972114141
sqlite3_int64 iStartBlock = sqlite3_column_int64(pStmt, 1);
113973114142
sqlite3_int64 iLeavesEndBlock = sqlite3_column_int64(pStmt, 2);
113974114143
sqlite3_int64 iEndBlock = sqlite3_column_int64(pStmt, 3);
113975114144
int nRoot = sqlite3_column_bytes(pStmt, 4);
113976114145
char const *zRoot = sqlite3_column_blob(pStmt, 4);
113977114146
113978
- /* If nSegment is a multiple of 16 the array needs to be extended. */
113979
- if( (pCsr->nSegment%16)==0 ){
113980
- Fts3SegReader **apNew;
113981
- int nByte = (pCsr->nSegment + 16)*sizeof(Fts3SegReader*);
113982
- apNew = (Fts3SegReader **)sqlite3_realloc(pCsr->apSegment, nByte);
113983
- if( !apNew ){
113984
- rc = SQLITE_NOMEM;
113985
- goto finished;
113986
- }
113987
- pCsr->apSegment = apNew;
113988
- }
113989
-
113990114147
/* If zTerm is not NULL, and this segment is not stored entirely on its
113991114148
** root node, the range of leaves scanned can be reduced. Do this. */
113992114149
if( iStartBlock && zTerm ){
113993114150
sqlite3_int64 *pi = (isPrefix ? &iLeavesEndBlock : 0);
113994114151
rc = fts3SelectLeaf(p, zTerm, nTerm, zRoot, nRoot, &iStartBlock, pi);
113995114152
if( rc!=SQLITE_OK ) goto finished;
113996114153
if( isPrefix==0 && isScan==0 ) iLeavesEndBlock = iStartBlock;
113997114154
}
113998114155
113999
- rc = sqlite3Fts3SegReaderNew(iAge, iStartBlock, iLeavesEndBlock,
114000
- iEndBlock, zRoot, nRoot, &pCsr->apSegment[pCsr->nSegment]
114156
+ rc = sqlite3Fts3SegReaderNew(pCsr->nSegment+1,
114157
+ iStartBlock, iLeavesEndBlock, iEndBlock, zRoot, nRoot, &pSeg
114001114158
);
114002114159
if( rc!=SQLITE_OK ) goto finished;
114003
- pCsr->nSegment++;
114004
- iAge++;
114160
+ rc = fts3SegReaderCursorAppend(pCsr, pSeg);
114005114161
}
114006114162
}
114007114163
114008114164
finished:
114009114165
rc2 = sqlite3_reset(pStmt);
114010114166
if( rc==SQLITE_DONE ) rc = rc2;
114011
- sqlite3Fts3SegReaderFree(pPending);
114012114167
114013114168
return rc;
114014114169
}
114015114170
114171
+/*
114172
+** Set up a cursor object for iterating through a full-text index or a
114173
+** single level therein.
114174
+*/
114175
+SQLITE_PRIVATE int sqlite3Fts3SegReaderCursor(
114176
+ Fts3Table *p, /* FTS3 table handle */
114177
+ int iIndex, /* Index to search (from 0 to p->nIndex-1) */
114178
+ int iLevel, /* Level of segments to scan */
114179
+ const char *zTerm, /* Term to query for */
114180
+ int nTerm, /* Size of zTerm in bytes */
114181
+ int isPrefix, /* True for a prefix search */
114182
+ int isScan, /* True to scan from zTerm to EOF */
114183
+ Fts3MultiSegReader *pCsr /* Cursor object to populate */
114184
+){
114185
+ assert( iIndex>=0 && iIndex<p->nIndex );
114186
+ assert( iLevel==FTS3_SEGCURSOR_ALL
114187
+ || iLevel==FTS3_SEGCURSOR_PENDING
114188
+ || iLevel>=0
114189
+ );
114190
+ assert( iLevel<FTS3_SEGDIR_MAXLEVEL );
114191
+ assert( FTS3_SEGCURSOR_ALL<0 && FTS3_SEGCURSOR_PENDING<0 );
114192
+ assert( isPrefix==0 || isScan==0 );
114016114193
114017
-static int fts3TermSegReaderCursor(
114194
+ /* "isScan" is only set to true by the ft4aux module, an ordinary
114195
+ ** full-text tables. */
114196
+ assert( isScan==0 || p->aIndex==0 );
114197
+
114198
+ memset(pCsr, 0, sizeof(Fts3MultiSegReader));
114199
+
114200
+ return fts3SegReaderCursor(
114201
+ p, iIndex, iLevel, zTerm, nTerm, isPrefix, isScan, pCsr
114202
+ );
114203
+}
114204
+
114205
+static int fts3SegReaderCursorAddZero(
114206
+ Fts3Table *p,
114207
+ const char *zTerm,
114208
+ int nTerm,
114209
+ Fts3MultiSegReader *pCsr
114210
+){
114211
+ return fts3SegReaderCursor(p, 0, FTS3_SEGCURSOR_ALL, zTerm, nTerm, 0, 0,pCsr);
114212
+}
114213
+
114214
+
114215
+SQLITE_PRIVATE int sqlite3Fts3TermSegReaderCursor(
114018114216
Fts3Cursor *pCsr, /* Virtual table cursor handle */
114019114217
const char *zTerm, /* Term to query for */
114020114218
int nTerm, /* Size of zTerm in bytes */
114021114219
int isPrefix, /* True for a prefix search */
114022
- Fts3SegReaderCursor **ppSegcsr /* OUT: Allocated seg-reader cursor */
114220
+ Fts3MultiSegReader **ppSegcsr /* OUT: Allocated seg-reader cursor */
114023114221
){
114024
- Fts3SegReaderCursor *pSegcsr; /* Object to allocate and return */
114222
+ Fts3MultiSegReader *pSegcsr; /* Object to allocate and return */
114025114223
int rc = SQLITE_NOMEM; /* Return code */
114026114224
114027
- pSegcsr = sqlite3_malloc(sizeof(Fts3SegReaderCursor));
114225
+ pSegcsr = sqlite3_malloc(sizeof(Fts3MultiSegReader));
114028114226
if( pSegcsr ){
114227
+ int i;
114228
+ int bFound = 0; /* True once an index has been found */
114029114229
Fts3Table *p = (Fts3Table *)pCsr->base.pVtab;
114030
- int i;
114031
- int nCost = 0;
114032
- rc = sqlite3Fts3SegReaderCursor(
114033
- p, FTS3_SEGCURSOR_ALL, zTerm, nTerm, isPrefix, 0, pSegcsr);
114034
-
114035
- for(i=0; rc==SQLITE_OK && i<pSegcsr->nSegment; i++){
114036
- rc = sqlite3Fts3SegReaderCost(pCsr, pSegcsr->apSegment[i], &nCost);
114037
- }
114038
- pSegcsr->nCost = nCost;
114230
+
114231
+ if( isPrefix ){
114232
+ for(i=1; bFound==0 && i<p->nIndex; i++){
114233
+ if( p->aIndex[i].nPrefix==nTerm ){
114234
+ bFound = 1;
114235
+ rc = sqlite3Fts3SegReaderCursor(
114236
+ p, i, FTS3_SEGCURSOR_ALL, zTerm, nTerm, 0, 0, pSegcsr);
114237
+ pSegcsr->bLookup = 1;
114238
+ }
114239
+ }
114240
+
114241
+ for(i=1; bFound==0 && i<p->nIndex; i++){
114242
+ if( p->aIndex[i].nPrefix==nTerm+1 ){
114243
+ bFound = 1;
114244
+ rc = sqlite3Fts3SegReaderCursor(
114245
+ p, i, FTS3_SEGCURSOR_ALL, zTerm, nTerm, 1, 0, pSegcsr
114246
+ );
114247
+ if( rc==SQLITE_OK ){
114248
+ rc = fts3SegReaderCursorAddZero(p, zTerm, nTerm, pSegcsr);
114249
+ }
114250
+ }
114251
+ }
114252
+ }
114253
+
114254
+ if( bFound==0 ){
114255
+ rc = sqlite3Fts3SegReaderCursor(
114256
+ p, 0, FTS3_SEGCURSOR_ALL, zTerm, nTerm, isPrefix, 0, pSegcsr
114257
+ );
114258
+ pSegcsr->bLookup = !isPrefix;
114259
+ }
114039114260
}
114040114261
114041114262
*ppSegcsr = pSegcsr;
114042114263
return rc;
114043114264
}
114044114265
114045
-static void fts3SegReaderCursorFree(Fts3SegReaderCursor *pSegcsr){
114266
+static void fts3SegReaderCursorFree(Fts3MultiSegReader *pSegcsr){
114046114267
sqlite3Fts3SegReaderFinish(pSegcsr);
114047114268
sqlite3_free(pSegcsr);
114048114269
}
114049114270
114050114271
/*
@@ -114065,11 +114286,11 @@
114065114286
int isReqPos, /* True to include position lists in output */
114066114287
int *pnOut, /* OUT: Size of buffer at *ppOut */
114067114288
char **ppOut /* OUT: Malloced result buffer */
114068114289
){
114069114290
int rc; /* Return code */
114070
- Fts3SegReaderCursor *pSegcsr; /* Seg-reader cursor for this term */
114291
+ Fts3MultiSegReader *pSegcsr; /* Seg-reader cursor for this term */
114071114292
TermSelect tsc; /* Context object for fts3TermSelectCb() */
114072114293
Fts3SegFilter filter; /* Segment term filter configuration */
114073114294
114074114295
pSegcsr = pTok->pSegcsr;
114075114296
memset(&tsc, 0, sizeof(TermSelect));
@@ -114091,11 +114312,11 @@
114091114312
pSegcsr->zTerm, pSegcsr->nTerm, pSegcsr->aDoclist, pSegcsr->nDoclist
114092114313
);
114093114314
}
114094114315
114095114316
if( rc==SQLITE_OK ){
114096
- rc = fts3TermSelectMerge(&tsc);
114317
+ rc = fts3TermSelectMerge(p, &tsc);
114097114318
}
114098114319
if( rc==SQLITE_OK ){
114099114320
*ppOut = tsc.aaOutput[0];
114100114321
*pnOut = tsc.anOutput[0];
114101114322
}else{
@@ -114141,664 +114362,10 @@
114141114362
}
114142114363
114143114364
return nDoc;
114144114365
}
114145114366
114146
-/*
114147
-** Call sqlite3Fts3DeferToken() for each token in the expression pExpr.
114148
-*/
114149
-static int fts3DeferExpression(Fts3Cursor *pCsr, Fts3Expr *pExpr){
114150
- int rc = SQLITE_OK;
114151
- if( pExpr ){
114152
- rc = fts3DeferExpression(pCsr, pExpr->pLeft);
114153
- if( rc==SQLITE_OK ){
114154
- rc = fts3DeferExpression(pCsr, pExpr->pRight);
114155
- }
114156
- if( pExpr->eType==FTSQUERY_PHRASE ){
114157
- int iCol = pExpr->pPhrase->iColumn;
114158
- int i;
114159
- for(i=0; rc==SQLITE_OK && i<pExpr->pPhrase->nToken; i++){
114160
- Fts3PhraseToken *pToken = &pExpr->pPhrase->aToken[i];
114161
- if( pToken->pDeferred==0 ){
114162
- rc = sqlite3Fts3DeferToken(pCsr, pToken, iCol);
114163
- }
114164
- }
114165
- }
114166
- }
114167
- return rc;
114168
-}
114169
-
114170
-/*
114171
-** This function removes the position information from a doclist. When
114172
-** called, buffer aList (size *pnList bytes) contains a doclist that includes
114173
-** position information. This function removes the position information so
114174
-** that aList contains only docids, and adjusts *pnList to reflect the new
114175
-** (possibly reduced) size of the doclist.
114176
-*/
114177
-static void fts3DoclistStripPositions(
114178
- char *aList, /* IN/OUT: Buffer containing doclist */
114179
- int *pnList /* IN/OUT: Size of doclist in bytes */
114180
-){
114181
- if( aList ){
114182
- char *aEnd = &aList[*pnList]; /* Pointer to one byte after EOF */
114183
- char *p = aList; /* Input cursor */
114184
- char *pOut = aList; /* Output cursor */
114185
-
114186
- while( p<aEnd ){
114187
- sqlite3_int64 delta;
114188
- p += sqlite3Fts3GetVarint(p, &delta);
114189
- fts3PoslistCopy(0, &p);
114190
- pOut += sqlite3Fts3PutVarint(pOut, delta);
114191
- }
114192
-
114193
- *pnList = (int)(pOut - aList);
114194
- }
114195
-}
114196
-
114197
-/*
114198
-** Return a DocList corresponding to the phrase *pPhrase.
114199
-**
114200
-** If this function returns SQLITE_OK, but *pnOut is set to a negative value,
114201
-** then no tokens in the phrase were looked up in the full-text index. This
114202
-** is only possible when this function is called from within xFilter(). The
114203
-** caller should assume that all documents match the phrase. The actual
114204
-** filtering will take place in xNext().
114205
-*/
114206
-static int fts3PhraseSelect(
114207
- Fts3Cursor *pCsr, /* Virtual table cursor handle */
114208
- Fts3Phrase *pPhrase, /* Phrase to return a doclist for */
114209
- int isReqPos, /* True if output should contain positions */
114210
- char **paOut, /* OUT: Pointer to malloc'd result buffer */
114211
- int *pnOut /* OUT: Size of buffer at *paOut */
114212
-){
114213
- char *pOut = 0;
114214
- int nOut = 0;
114215
- int rc = SQLITE_OK;
114216
- int ii;
114217
- int iCol = pPhrase->iColumn;
114218
- int isTermPos = (pPhrase->nToken>1 || isReqPos);
114219
- Fts3Table *p = (Fts3Table *)pCsr->base.pVtab;
114220
- int isFirst = 1;
114221
-
114222
- int iPrevTok = 0;
114223
- int nDoc = 0;
114224
-
114225
- /* If this is an xFilter() evaluation, create a segment-reader for each
114226
- ** phrase token. Or, if this is an xNext() or snippet/offsets/matchinfo
114227
- ** evaluation, only create segment-readers if there are no Fts3DeferredToken
114228
- ** objects attached to the phrase-tokens.
114229
- */
114230
- for(ii=0; ii<pPhrase->nToken; ii++){
114231
- Fts3PhraseToken *pTok = &pPhrase->aToken[ii];
114232
- if( pTok->pSegcsr==0 ){
114233
- if( (pCsr->eEvalmode==FTS3_EVAL_FILTER)
114234
- || (pCsr->eEvalmode==FTS3_EVAL_NEXT && pCsr->pDeferred==0)
114235
- || (pCsr->eEvalmode==FTS3_EVAL_MATCHINFO && pTok->bFulltext)
114236
- ){
114237
- rc = fts3TermSegReaderCursor(
114238
- pCsr, pTok->z, pTok->n, pTok->isPrefix, &pTok->pSegcsr
114239
- );
114240
- if( rc!=SQLITE_OK ) return rc;
114241
- }
114242
- }
114243
- }
114244
-
114245
- for(ii=0; ii<pPhrase->nToken; ii++){
114246
- Fts3PhraseToken *pTok; /* Token to find doclist for */
114247
- int iTok = 0; /* The token being queried this iteration */
114248
- char *pList = 0; /* Pointer to token doclist */
114249
- int nList = 0; /* Size of buffer at pList */
114250
-
114251
- /* Select a token to process. If this is an xFilter() call, then tokens
114252
- ** are processed in order from least to most costly. Otherwise, tokens
114253
- ** are processed in the order in which they occur in the phrase.
114254
- */
114255
- if( pCsr->eEvalmode==FTS3_EVAL_MATCHINFO ){
114256
- assert( isReqPos );
114257
- iTok = ii;
114258
- pTok = &pPhrase->aToken[iTok];
114259
- if( pTok->bFulltext==0 ) continue;
114260
- }else if( pCsr->eEvalmode==FTS3_EVAL_NEXT || isReqPos ){
114261
- iTok = ii;
114262
- pTok = &pPhrase->aToken[iTok];
114263
- }else{
114264
- int nMinCost = 0x7FFFFFFF;
114265
- int jj;
114266
-
114267
- /* Find the remaining token with the lowest cost. */
114268
- for(jj=0; jj<pPhrase->nToken; jj++){
114269
- Fts3SegReaderCursor *pSegcsr = pPhrase->aToken[jj].pSegcsr;
114270
- if( pSegcsr && pSegcsr->nCost<nMinCost ){
114271
- iTok = jj;
114272
- nMinCost = pSegcsr->nCost;
114273
- }
114274
- }
114275
- pTok = &pPhrase->aToken[iTok];
114276
-
114277
- /* This branch is taken if it is determined that loading the doclist
114278
- ** for the next token would require more IO than loading all documents
114279
- ** currently identified by doclist pOut/nOut. No further doclists will
114280
- ** be loaded from the full-text index for this phrase.
114281
- */
114282
- if( nMinCost>nDoc && ii>0 ){
114283
- rc = fts3DeferExpression(pCsr, pCsr->pExpr);
114284
- break;
114285
- }
114286
- }
114287
-
114288
- if( pCsr->eEvalmode==FTS3_EVAL_NEXT && pTok->pDeferred ){
114289
- rc = fts3DeferredTermSelect(pTok->pDeferred, isTermPos, &nList, &pList);
114290
- }else{
114291
- if( pTok->pSegcsr ){
114292
- rc = fts3TermSelect(p, pTok, iCol, isTermPos, &nList, &pList);
114293
- }
114294
- pTok->bFulltext = 1;
114295
- }
114296
- assert( rc!=SQLITE_OK || pCsr->eEvalmode || pTok->pSegcsr==0 );
114297
- if( rc!=SQLITE_OK ) break;
114298
-
114299
- if( isFirst ){
114300
- pOut = pList;
114301
- nOut = nList;
114302
- if( pCsr->eEvalmode==FTS3_EVAL_FILTER && pPhrase->nToken>1 ){
114303
- nDoc = fts3DoclistCountDocids(1, pOut, nOut);
114304
- }
114305
- isFirst = 0;
114306
- iPrevTok = iTok;
114307
- }else{
114308
- /* Merge the new term list and the current output. */
114309
- char *aLeft, *aRight;
114310
- int nLeft, nRight;
114311
- int nDist;
114312
- int mt;
114313
-
114314
- /* If this is the final token of the phrase, and positions were not
114315
- ** requested by the caller, use MERGE_PHRASE instead of POS_PHRASE.
114316
- ** This drops the position information from the output list.
114317
- */
114318
- mt = MERGE_POS_PHRASE;
114319
- if( ii==pPhrase->nToken-1 && !isReqPos ) mt = MERGE_PHRASE;
114320
-
114321
- assert( iPrevTok!=iTok );
114322
- if( iPrevTok<iTok ){
114323
- aLeft = pOut;
114324
- nLeft = nOut;
114325
- aRight = pList;
114326
- nRight = nList;
114327
- nDist = iTok-iPrevTok;
114328
- iPrevTok = iTok;
114329
- }else{
114330
- aRight = pOut;
114331
- nRight = nOut;
114332
- aLeft = pList;
114333
- nLeft = nList;
114334
- nDist = iPrevTok-iTok;
114335
- }
114336
- pOut = aRight;
114337
- fts3DoclistMerge(
114338
- mt, nDist, 0, pOut, &nOut, aLeft, nLeft, aRight, nRight, &nDoc
114339
- );
114340
- sqlite3_free(aLeft);
114341
- }
114342
- assert( nOut==0 || pOut!=0 );
114343
- }
114344
-
114345
- if( rc==SQLITE_OK ){
114346
- if( ii!=pPhrase->nToken ){
114347
- assert( pCsr->eEvalmode==FTS3_EVAL_FILTER && isReqPos==0 );
114348
- fts3DoclistStripPositions(pOut, &nOut);
114349
- }
114350
- *paOut = pOut;
114351
- *pnOut = nOut;
114352
- }else{
114353
- sqlite3_free(pOut);
114354
- }
114355
- return rc;
114356
-}
114357
-
114358
-/*
114359
-** This function merges two doclists according to the requirements of a
114360
-** NEAR operator.
114361
-**
114362
-** Both input doclists must include position information. The output doclist
114363
-** includes position information if the first argument to this function
114364
-** is MERGE_POS_NEAR, or does not if it is MERGE_NEAR.
114365
-*/
114366
-static int fts3NearMerge(
114367
- int mergetype, /* MERGE_POS_NEAR or MERGE_NEAR */
114368
- int nNear, /* Parameter to NEAR operator */
114369
- int nTokenLeft, /* Number of tokens in LHS phrase arg */
114370
- char *aLeft, /* Doclist for LHS (incl. positions) */
114371
- int nLeft, /* Size of LHS doclist in bytes */
114372
- int nTokenRight, /* As nTokenLeft */
114373
- char *aRight, /* As aLeft */
114374
- int nRight, /* As nRight */
114375
- char **paOut, /* OUT: Results of merge (malloced) */
114376
- int *pnOut /* OUT: Sized of output buffer */
114377
-){
114378
- char *aOut; /* Buffer to write output doclist to */
114379
- int rc; /* Return code */
114380
-
114381
- assert( mergetype==MERGE_POS_NEAR || MERGE_NEAR );
114382
-
114383
- aOut = sqlite3_malloc(nLeft+nRight+1);
114384
- if( aOut==0 ){
114385
- rc = SQLITE_NOMEM;
114386
- }else{
114387
- rc = fts3DoclistMerge(mergetype, nNear+nTokenRight, nNear+nTokenLeft,
114388
- aOut, pnOut, aLeft, nLeft, aRight, nRight, 0
114389
- );
114390
- if( rc!=SQLITE_OK ){
114391
- sqlite3_free(aOut);
114392
- aOut = 0;
114393
- }
114394
- }
114395
-
114396
- *paOut = aOut;
114397
- return rc;
114398
-}
114399
-
114400
-/*
114401
-** This function is used as part of the processing for the snippet() and
114402
-** offsets() functions.
114403
-**
114404
-** Both pLeft and pRight are expression nodes of type FTSQUERY_PHRASE. Both
114405
-** have their respective doclists (including position information) loaded
114406
-** in Fts3Expr.aDoclist/nDoclist. This function removes all entries from
114407
-** each doclist that are not within nNear tokens of a corresponding entry
114408
-** in the other doclist.
114409
-*/
114410
-SQLITE_PRIVATE int sqlite3Fts3ExprNearTrim(Fts3Expr *pLeft, Fts3Expr *pRight, int nNear){
114411
- int rc; /* Return code */
114412
-
114413
- assert( pLeft->eType==FTSQUERY_PHRASE );
114414
- assert( pRight->eType==FTSQUERY_PHRASE );
114415
- assert( pLeft->isLoaded && pRight->isLoaded );
114416
-
114417
- if( pLeft->aDoclist==0 || pRight->aDoclist==0 ){
114418
- sqlite3_free(pLeft->aDoclist);
114419
- sqlite3_free(pRight->aDoclist);
114420
- pRight->aDoclist = 0;
114421
- pLeft->aDoclist = 0;
114422
- rc = SQLITE_OK;
114423
- }else{
114424
- char *aOut; /* Buffer in which to assemble new doclist */
114425
- int nOut; /* Size of buffer aOut in bytes */
114426
-
114427
- rc = fts3NearMerge(MERGE_POS_NEAR, nNear,
114428
- pLeft->pPhrase->nToken, pLeft->aDoclist, pLeft->nDoclist,
114429
- pRight->pPhrase->nToken, pRight->aDoclist, pRight->nDoclist,
114430
- &aOut, &nOut
114431
- );
114432
- if( rc!=SQLITE_OK ) return rc;
114433
- sqlite3_free(pRight->aDoclist);
114434
- pRight->aDoclist = aOut;
114435
- pRight->nDoclist = nOut;
114436
-
114437
- rc = fts3NearMerge(MERGE_POS_NEAR, nNear,
114438
- pRight->pPhrase->nToken, pRight->aDoclist, pRight->nDoclist,
114439
- pLeft->pPhrase->nToken, pLeft->aDoclist, pLeft->nDoclist,
114440
- &aOut, &nOut
114441
- );
114442
- sqlite3_free(pLeft->aDoclist);
114443
- pLeft->aDoclist = aOut;
114444
- pLeft->nDoclist = nOut;
114445
- }
114446
- return rc;
114447
-}
114448
-
114449
-
114450
-/*
114451
-** Allocate an Fts3SegReaderArray for each token in the expression pExpr.
114452
-** The allocated objects are stored in the Fts3PhraseToken.pArray member
114453
-** variables of each token structure.
114454
-*/
114455
-static int fts3ExprAllocateSegReaders(
114456
- Fts3Cursor *pCsr, /* FTS3 table */
114457
- Fts3Expr *pExpr, /* Expression to create seg-readers for */
114458
- int *pnExpr /* OUT: Number of AND'd expressions */
114459
-){
114460
- int rc = SQLITE_OK; /* Return code */
114461
-
114462
- assert( pCsr->eEvalmode==FTS3_EVAL_FILTER );
114463
- if( pnExpr && pExpr->eType!=FTSQUERY_AND ){
114464
- (*pnExpr)++;
114465
- pnExpr = 0;
114466
- }
114467
-
114468
- if( pExpr->eType==FTSQUERY_PHRASE ){
114469
- Fts3Phrase *pPhrase = pExpr->pPhrase;
114470
- int ii;
114471
-
114472
- for(ii=0; rc==SQLITE_OK && ii<pPhrase->nToken; ii++){
114473
- Fts3PhraseToken *pTok = &pPhrase->aToken[ii];
114474
- if( pTok->pSegcsr==0 ){
114475
- rc = fts3TermSegReaderCursor(
114476
- pCsr, pTok->z, pTok->n, pTok->isPrefix, &pTok->pSegcsr
114477
- );
114478
- }
114479
- }
114480
- }else{
114481
- rc = fts3ExprAllocateSegReaders(pCsr, pExpr->pLeft, pnExpr);
114482
- if( rc==SQLITE_OK ){
114483
- rc = fts3ExprAllocateSegReaders(pCsr, pExpr->pRight, pnExpr);
114484
- }
114485
- }
114486
- return rc;
114487
-}
114488
-
114489
-/*
114490
-** Free the Fts3SegReaderArray objects associated with each token in the
114491
-** expression pExpr. In other words, this function frees the resources
114492
-** allocated by fts3ExprAllocateSegReaders().
114493
-*/
114494
-static void fts3ExprFreeSegReaders(Fts3Expr *pExpr){
114495
- if( pExpr ){
114496
- Fts3Phrase *pPhrase = pExpr->pPhrase;
114497
- if( pPhrase ){
114498
- int kk;
114499
- for(kk=0; kk<pPhrase->nToken; kk++){
114500
- fts3SegReaderCursorFree(pPhrase->aToken[kk].pSegcsr);
114501
- pPhrase->aToken[kk].pSegcsr = 0;
114502
- }
114503
- }
114504
- fts3ExprFreeSegReaders(pExpr->pLeft);
114505
- fts3ExprFreeSegReaders(pExpr->pRight);
114506
- }
114507
-}
114508
-
114509
-/*
114510
-** Return the sum of the costs of all tokens in the expression pExpr. This
114511
-** function must be called after Fts3SegReaderArrays have been allocated
114512
-** for all tokens using fts3ExprAllocateSegReaders().
114513
-*/
114514
-static int fts3ExprCost(Fts3Expr *pExpr){
114515
- int nCost; /* Return value */
114516
- if( pExpr->eType==FTSQUERY_PHRASE ){
114517
- Fts3Phrase *pPhrase = pExpr->pPhrase;
114518
- int ii;
114519
- nCost = 0;
114520
- for(ii=0; ii<pPhrase->nToken; ii++){
114521
- Fts3SegReaderCursor *pSegcsr = pPhrase->aToken[ii].pSegcsr;
114522
- if( pSegcsr ) nCost += pSegcsr->nCost;
114523
- }
114524
- }else{
114525
- nCost = fts3ExprCost(pExpr->pLeft) + fts3ExprCost(pExpr->pRight);
114526
- }
114527
- return nCost;
114528
-}
114529
-
114530
-/*
114531
-** The following is a helper function (and type) for fts3EvalExpr(). It
114532
-** must be called after Fts3SegReaders have been allocated for every token
114533
-** in the expression. See the context it is called from in fts3EvalExpr()
114534
-** for further explanation.
114535
-*/
114536
-typedef struct ExprAndCost ExprAndCost;
114537
-struct ExprAndCost {
114538
- Fts3Expr *pExpr;
114539
- int nCost;
114540
-};
114541
-static void fts3ExprAssignCosts(
114542
- Fts3Expr *pExpr, /* Expression to create seg-readers for */
114543
- ExprAndCost **ppExprCost /* OUT: Write to *ppExprCost */
114544
-){
114545
- if( pExpr->eType==FTSQUERY_AND ){
114546
- fts3ExprAssignCosts(pExpr->pLeft, ppExprCost);
114547
- fts3ExprAssignCosts(pExpr->pRight, ppExprCost);
114548
- }else{
114549
- (*ppExprCost)->pExpr = pExpr;
114550
- (*ppExprCost)->nCost = fts3ExprCost(pExpr);
114551
- (*ppExprCost)++;
114552
- }
114553
-}
114554
-
114555
-/*
114556
-** Evaluate the full-text expression pExpr against FTS3 table pTab. Store
114557
-** the resulting doclist in *paOut and *pnOut. This routine mallocs for
114558
-** the space needed to store the output. The caller is responsible for
114559
-** freeing the space when it has finished.
114560
-**
114561
-** This function is called in two distinct contexts:
114562
-**
114563
-** * From within the virtual table xFilter() method. In this case, the
114564
-** output doclist contains entries for all rows in the table, based on
114565
-** data read from the full-text index.
114566
-**
114567
-** In this case, if the query expression contains one or more tokens that
114568
-** are very common, then the returned doclist may contain a superset of
114569
-** the documents that actually match the expression.
114570
-**
114571
-** * From within the virtual table xNext() method. This call is only made
114572
-** if the call from within xFilter() found that there were very common
114573
-** tokens in the query expression and did return a superset of the
114574
-** matching documents. In this case the returned doclist contains only
114575
-** entries that correspond to the current row of the table. Instead of
114576
-** reading the data for each token from the full-text index, the data is
114577
-** already available in-memory in the Fts3PhraseToken.pDeferred structures.
114578
-** See fts3EvalDeferred() for how it gets there.
114579
-**
114580
-** In the first case above, Fts3Cursor.doDeferred==0. In the second (if it is
114581
-** required) Fts3Cursor.doDeferred==1.
114582
-**
114583
-** If the SQLite invokes the snippet(), offsets() or matchinfo() function
114584
-** as part of a SELECT on an FTS3 table, this function is called on each
114585
-** individual phrase expression in the query. If there were very common tokens
114586
-** found in the xFilter() call, then this function is called once for phrase
114587
-** for each row visited, and the returned doclist contains entries for the
114588
-** current row only. Otherwise, if there were no very common tokens, then this
114589
-** function is called once only for each phrase in the query and the returned
114590
-** doclist contains entries for all rows of the table.
114591
-**
114592
-** Fts3Cursor.doDeferred==1 when this function is called on phrases as a
114593
-** result of a snippet(), offsets() or matchinfo() invocation.
114594
-*/
114595
-static int fts3EvalExpr(
114596
- Fts3Cursor *p, /* Virtual table cursor handle */
114597
- Fts3Expr *pExpr, /* Parsed fts3 expression */
114598
- char **paOut, /* OUT: Pointer to malloc'd result buffer */
114599
- int *pnOut, /* OUT: Size of buffer at *paOut */
114600
- int isReqPos /* Require positions in output buffer */
114601
-){
114602
- int rc = SQLITE_OK; /* Return code */
114603
-
114604
- /* Zero the output parameters. */
114605
- *paOut = 0;
114606
- *pnOut = 0;
114607
-
114608
- if( pExpr ){
114609
- assert( pExpr->eType==FTSQUERY_NEAR || pExpr->eType==FTSQUERY_OR
114610
- || pExpr->eType==FTSQUERY_AND || pExpr->eType==FTSQUERY_NOT
114611
- || pExpr->eType==FTSQUERY_PHRASE
114612
- );
114613
- assert( pExpr->eType==FTSQUERY_PHRASE || isReqPos==0 );
114614
-
114615
- if( pExpr->eType==FTSQUERY_PHRASE ){
114616
- rc = fts3PhraseSelect(p, pExpr->pPhrase,
114617
- isReqPos || (pExpr->pParent && pExpr->pParent->eType==FTSQUERY_NEAR),
114618
- paOut, pnOut
114619
- );
114620
- fts3ExprFreeSegReaders(pExpr);
114621
- }else if( p->eEvalmode==FTS3_EVAL_FILTER && pExpr->eType==FTSQUERY_AND ){
114622
- ExprAndCost *aExpr = 0; /* Array of AND'd expressions and costs */
114623
- int nExpr = 0; /* Size of aExpr[] */
114624
- char *aRet = 0; /* Doclist to return to caller */
114625
- int nRet = 0; /* Length of aRet[] in bytes */
114626
- int nDoc = 0x7FFFFFFF;
114627
-
114628
- assert( !isReqPos );
114629
-
114630
- rc = fts3ExprAllocateSegReaders(p, pExpr, &nExpr);
114631
- if( rc==SQLITE_OK ){
114632
- assert( nExpr>1 );
114633
- aExpr = sqlite3_malloc(sizeof(ExprAndCost) * nExpr);
114634
- if( !aExpr ) rc = SQLITE_NOMEM;
114635
- }
114636
- if( rc==SQLITE_OK ){
114637
- int ii; /* Used to iterate through expressions */
114638
-
114639
- fts3ExprAssignCosts(pExpr, &aExpr);
114640
- aExpr -= nExpr;
114641
- for(ii=0; ii<nExpr; ii++){
114642
- char *aNew;
114643
- int nNew;
114644
- int jj;
114645
- ExprAndCost *pBest = 0;
114646
-
114647
- for(jj=0; jj<nExpr; jj++){
114648
- ExprAndCost *pCand = &aExpr[jj];
114649
- if( pCand->pExpr && (pBest==0 || pCand->nCost<pBest->nCost) ){
114650
- pBest = pCand;
114651
- }
114652
- }
114653
-
114654
- if( pBest->nCost>nDoc ){
114655
- rc = fts3DeferExpression(p, p->pExpr);
114656
- break;
114657
- }else{
114658
- rc = fts3EvalExpr(p, pBest->pExpr, &aNew, &nNew, 0);
114659
- if( rc!=SQLITE_OK ) break;
114660
- pBest->pExpr = 0;
114661
- if( ii==0 ){
114662
- aRet = aNew;
114663
- nRet = nNew;
114664
- nDoc = fts3DoclistCountDocids(0, aRet, nRet);
114665
- }else{
114666
- fts3DoclistMerge(
114667
- MERGE_AND, 0, 0, aRet, &nRet, aRet, nRet, aNew, nNew, &nDoc
114668
- );
114669
- sqlite3_free(aNew);
114670
- }
114671
- }
114672
- }
114673
- }
114674
-
114675
- if( rc==SQLITE_OK ){
114676
- *paOut = aRet;
114677
- *pnOut = nRet;
114678
- }else{
114679
- assert( *paOut==0 );
114680
- sqlite3_free(aRet);
114681
- }
114682
- sqlite3_free(aExpr);
114683
- fts3ExprFreeSegReaders(pExpr);
114684
-
114685
- }else{
114686
- char *aLeft;
114687
- char *aRight;
114688
- int nLeft;
114689
- int nRight;
114690
-
114691
- assert( pExpr->eType==FTSQUERY_NEAR
114692
- || pExpr->eType==FTSQUERY_OR
114693
- || pExpr->eType==FTSQUERY_NOT
114694
- || (pExpr->eType==FTSQUERY_AND && p->eEvalmode==FTS3_EVAL_NEXT)
114695
- );
114696
-
114697
- if( 0==(rc = fts3EvalExpr(p, pExpr->pRight, &aRight, &nRight, isReqPos))
114698
- && 0==(rc = fts3EvalExpr(p, pExpr->pLeft, &aLeft, &nLeft, isReqPos))
114699
- ){
114700
- switch( pExpr->eType ){
114701
- case FTSQUERY_NEAR: {
114702
- Fts3Expr *pLeft;
114703
- Fts3Expr *pRight;
114704
- int mergetype = MERGE_NEAR;
114705
- if( pExpr->pParent && pExpr->pParent->eType==FTSQUERY_NEAR ){
114706
- mergetype = MERGE_POS_NEAR;
114707
- }
114708
- pLeft = pExpr->pLeft;
114709
- while( pLeft->eType==FTSQUERY_NEAR ){
114710
- pLeft=pLeft->pRight;
114711
- }
114712
- pRight = pExpr->pRight;
114713
- assert( pRight->eType==FTSQUERY_PHRASE );
114714
- assert( pLeft->eType==FTSQUERY_PHRASE );
114715
-
114716
- rc = fts3NearMerge(mergetype, pExpr->nNear,
114717
- pLeft->pPhrase->nToken, aLeft, nLeft,
114718
- pRight->pPhrase->nToken, aRight, nRight,
114719
- paOut, pnOut
114720
- );
114721
- sqlite3_free(aLeft);
114722
- break;
114723
- }
114724
-
114725
- case FTSQUERY_OR: {
114726
- /* Allocate a buffer for the output. The maximum size is the
114727
- ** sum of the sizes of the two input buffers. The +1 term is
114728
- ** so that a buffer of zero bytes is never allocated - this can
114729
- ** cause fts3DoclistMerge() to incorrectly return SQLITE_NOMEM.
114730
- */
114731
- char *aBuffer = sqlite3_malloc(nRight+nLeft+1);
114732
- rc = fts3DoclistMerge(MERGE_OR, 0, 0, aBuffer, pnOut,
114733
- aLeft, nLeft, aRight, nRight, 0
114734
- );
114735
- *paOut = aBuffer;
114736
- sqlite3_free(aLeft);
114737
- break;
114738
- }
114739
-
114740
- default: {
114741
- assert( FTSQUERY_NOT==MERGE_NOT && FTSQUERY_AND==MERGE_AND );
114742
- fts3DoclistMerge(pExpr->eType, 0, 0, aLeft, pnOut,
114743
- aLeft, nLeft, aRight, nRight, 0
114744
- );
114745
- *paOut = aLeft;
114746
- break;
114747
- }
114748
- }
114749
- }
114750
- sqlite3_free(aRight);
114751
- }
114752
- }
114753
-
114754
- assert( rc==SQLITE_OK || *paOut==0 );
114755
- return rc;
114756
-}
114757
-
114758
-/*
114759
-** This function is called from within xNext() for each row visited by
114760
-** an FTS3 query. If evaluating the FTS3 query expression within xFilter()
114761
-** was able to determine the exact set of matching rows, this function sets
114762
-** *pbRes to true and returns SQLITE_IO immediately.
114763
-**
114764
-** Otherwise, if evaluating the query expression within xFilter() returned a
114765
-** superset of the matching documents instead of an exact set (this happens
114766
-** when the query includes very common tokens and it is deemed too expensive to
114767
-** load their doclists from disk), this function tests if the current row
114768
-** really does match the FTS3 query.
114769
-**
114770
-** If an error occurs, an SQLite error code is returned. Otherwise, SQLITE_OK
114771
-** is returned and *pbRes is set to true if the current row matches the
114772
-** FTS3 query (and should be included in the results returned to SQLite), or
114773
-** false otherwise.
114774
-*/
114775
-static int fts3EvalDeferred(
114776
- Fts3Cursor *pCsr, /* FTS3 cursor pointing at row to test */
114777
- int *pbRes /* OUT: Set to true if row is a match */
114778
-){
114779
- int rc = SQLITE_OK;
114780
- if( pCsr->pDeferred==0 ){
114781
- *pbRes = 1;
114782
- }else{
114783
- rc = fts3CursorSeek(0, pCsr);
114784
- if( rc==SQLITE_OK ){
114785
- sqlite3Fts3FreeDeferredDoclists(pCsr);
114786
- rc = sqlite3Fts3CacheDeferredDoclists(pCsr);
114787
- }
114788
- if( rc==SQLITE_OK ){
114789
- char *a = 0;
114790
- int n = 0;
114791
- rc = fts3EvalExpr(pCsr, pCsr->pExpr, &a, &n, 0);
114792
- assert( n>=0 );
114793
- *pbRes = (n>0);
114794
- sqlite3_free(a);
114795
- }
114796
- }
114797
- return rc;
114798
-}
114799
-
114800114367
/*
114801114368
** Advance the cursor to the next row in the %_content table that
114802114369
** matches the search criteria. For a MATCH search, this will be
114803114370
** the next row that matches. For a full-table scan, this will be
114804114371
** simply the next row in the %_content table. For a docid lookup,
@@ -114807,43 +114374,24 @@
114807114374
** Return SQLITE_OK if nothing goes wrong. SQLITE_OK is returned
114808114375
** even if we reach end-of-file. The fts3EofMethod() will be called
114809114376
** subsequently to determine whether or not an EOF was hit.
114810114377
*/
114811114378
static int fts3NextMethod(sqlite3_vtab_cursor *pCursor){
114812
- int res;
114813
- int rc = SQLITE_OK; /* Return code */
114379
+ int rc;
114814114380
Fts3Cursor *pCsr = (Fts3Cursor *)pCursor;
114815
-
114816
- pCsr->eEvalmode = FTS3_EVAL_NEXT;
114817
- do {
114818
- if( pCsr->aDoclist==0 ){
114819
- if( SQLITE_ROW!=sqlite3_step(pCsr->pStmt) ){
114820
- pCsr->isEof = 1;
114821
- rc = sqlite3_reset(pCsr->pStmt);
114822
- break;
114823
- }
114381
+ if( pCsr->eSearch==FTS3_DOCID_SEARCH || pCsr->eSearch==FTS3_FULLSCAN_SEARCH ){
114382
+ if( SQLITE_ROW!=sqlite3_step(pCsr->pStmt) ){
114383
+ pCsr->isEof = 1;
114384
+ rc = sqlite3_reset(pCsr->pStmt);
114385
+ }else{
114824114386
pCsr->iPrevId = sqlite3_column_int64(pCsr->pStmt, 0);
114825
- }else{
114826
- if( pCsr->desc==0 ){
114827
- if( pCsr->pNextId>=&pCsr->aDoclist[pCsr->nDoclist] ){
114828
- pCsr->isEof = 1;
114829
- break;
114830
- }
114831
- fts3GetDeltaVarint(&pCsr->pNextId, &pCsr->iPrevId);
114832
- }else{
114833
- fts3GetReverseDeltaVarint(&pCsr->pNextId,pCsr->aDoclist,&pCsr->iPrevId);
114834
- if( pCsr->pNextId<=pCsr->aDoclist ){
114835
- pCsr->isEof = 1;
114836
- break;
114837
- }
114838
- }
114839
- sqlite3_reset(pCsr->pStmt);
114840
- pCsr->isRequireSeek = 1;
114841
- pCsr->isMatchinfoNeeded = 1;
114842
- }
114843
- }while( SQLITE_OK==(rc = fts3EvalDeferred(pCsr, &res)) && res==0 );
114844
-
114387
+ rc = SQLITE_OK;
114388
+ }
114389
+ }else{
114390
+ rc = sqlite3Fts3EvalNext((Fts3Cursor *)pCursor);
114391
+ }
114392
+ assert( ((Fts3Table *)pCsr->base.pVtab)->pSegments==0 );
114845114393
return rc;
114846114394
}
114847114395
114848114396
/*
114849114397
** This is the xFilter interface for the virtual table. See
@@ -114866,15 +114414,11 @@
114866114414
int idxNum, /* Strategy index */
114867114415
const char *idxStr, /* Unused */
114868114416
int nVal, /* Number of elements in apVal */
114869114417
sqlite3_value **apVal /* Arguments for the indexing scheme */
114870114418
){
114871
- const char *azSql[] = {
114872
- "SELECT %s FROM %Q.'%q_content' AS x WHERE docid = ?", /* non-full-scan */
114873
- "SELECT %s FROM %Q.'%q_content' AS x ORDER BY docid %s", /* full-scan */
114874
- };
114875
- int rc; /* Return code */
114419
+ int rc;
114876114420
char *zSql; /* SQL statement used to access %_content */
114877114421
Fts3Table *p = (Fts3Table *)pCursor->pVtab;
114878114422
Fts3Cursor *pCsr = (Fts3Cursor *)pCursor;
114879114423
114880114424
UNUSED_PARAMETER(idxStr);
@@ -114889,10 +114433,17 @@
114889114433
sqlite3_finalize(pCsr->pStmt);
114890114434
sqlite3_free(pCsr->aDoclist);
114891114435
sqlite3Fts3ExprFree(pCsr->pExpr);
114892114436
memset(&pCursor[1], 0, sizeof(Fts3Cursor)-sizeof(sqlite3_vtab_cursor));
114893114437
114438
+ if( idxStr ){
114439
+ pCsr->bDesc = (idxStr[0]=='D');
114440
+ }else{
114441
+ pCsr->bDesc = p->bDescIdx;
114442
+ }
114443
+ pCsr->eSearch = (i16)idxNum;
114444
+
114894114445
if( idxNum!=FTS3_DOCID_SEARCH && idxNum!=FTS3_FULLSCAN_SEARCH ){
114895114446
int iCol = idxNum-FTS3_FULLTEXT_SEARCH;
114896114447
const char *zQuery = (const char *)sqlite3_value_text(apVal[0]);
114897114448
114898114449
if( zQuery==0 && sqlite3_value_type(apVal[0])!=SQLITE_NULL ){
@@ -114902,20 +114453,21 @@
114902114453
rc = sqlite3Fts3ExprParse(p->pTokenizer, p->azColumn, p->nColumn,
114903114454
iCol, zQuery, -1, &pCsr->pExpr
114904114455
);
114905114456
if( rc!=SQLITE_OK ){
114906114457
if( rc==SQLITE_ERROR ){
114907
- p->base.zErrMsg = sqlite3_mprintf("malformed MATCH expression: [%s]",
114908
- zQuery);
114458
+ static const char *zErr = "malformed MATCH expression: [%s]";
114459
+ p->base.zErrMsg = sqlite3_mprintf(zErr, zQuery);
114909114460
}
114910114461
return rc;
114911114462
}
114912114463
114913114464
rc = sqlite3Fts3ReadLock(p);
114914114465
if( rc!=SQLITE_OK ) return rc;
114915114466
114916
- rc = fts3EvalExpr(pCsr, pCsr->pExpr, &pCsr->aDoclist, &pCsr->nDoclist, 0);
114467
+ rc = sqlite3Fts3EvalStart(pCsr, pCsr->pExpr, 1);
114468
+
114917114469
sqlite3Fts3SegmentsClose(p);
114918114470
if( rc!=SQLITE_OK ) return rc;
114919114471
pCsr->pNextId = pCsr->aDoclist;
114920114472
pCsr->iPrevId = 0;
114921114473
}
@@ -114923,41 +114475,28 @@
114923114475
/* Compile a SELECT statement for this cursor. For a full-table-scan, the
114924114476
** statement loops through all rows of the %_content table. For a
114925114477
** full-text query or docid lookup, the statement retrieves a single
114926114478
** row by docid.
114927114479
*/
114928
- zSql = (char *)azSql[idxNum==FTS3_FULLSCAN_SEARCH];
114929
- zSql = sqlite3_mprintf(
114930
- zSql, p->zReadExprlist, p->zDb, p->zName, (idxStr ? idxStr : "ASC")
114931
- );
114932
- if( !zSql ){
114933
- rc = SQLITE_NOMEM;
114480
+ if( idxNum==FTS3_FULLSCAN_SEARCH ){
114481
+ const char *zSort = (pCsr->bDesc ? "DESC" : "ASC");
114482
+ const char *zTmpl = "SELECT %s FROM %Q.'%q_content' AS x ORDER BY docid %s";
114483
+ zSql = sqlite3_mprintf(zTmpl, p->zReadExprlist, p->zDb, p->zName, zSort);
114934114484
}else{
114935
- rc = sqlite3_prepare_v2(p->db, zSql, -1, &pCsr->pStmt, 0);
114936
- sqlite3_free(zSql);
114485
+ const char *zTmpl = "SELECT %s FROM %Q.'%q_content' AS x WHERE docid = ?";
114486
+ zSql = sqlite3_mprintf(zTmpl, p->zReadExprlist, p->zDb, p->zName);
114937114487
}
114938
- if( rc==SQLITE_OK && idxNum==FTS3_DOCID_SEARCH ){
114488
+ if( !zSql ) return SQLITE_NOMEM;
114489
+ rc = sqlite3_prepare_v2(p->db, zSql, -1, &pCsr->pStmt, 0);
114490
+ sqlite3_free(zSql);
114491
+ if( rc!=SQLITE_OK ) return rc;
114492
+
114493
+ if( idxNum==FTS3_DOCID_SEARCH ){
114939114494
rc = sqlite3_bind_value(pCsr->pStmt, 1, apVal[0]);
114940
- }
114941
- pCsr->eSearch = (i16)idxNum;
114942
-
114943
- assert( pCsr->desc==0 );
114944
- if( rc!=SQLITE_OK ) return rc;
114945
- if( rc==SQLITE_OK && pCsr->nDoclist>0 && idxStr && idxStr[0]=='D' ){
114946
- sqlite3_int64 iDocid = 0;
114947
- char *csr = pCsr->aDoclist;
114948
- while( csr<&pCsr->aDoclist[pCsr->nDoclist] ){
114949
- fts3GetDeltaVarint(&csr, &iDocid);
114950
- }
114951
- pCsr->pNextId = csr;
114952
- pCsr->iPrevId = iDocid;
114953
- pCsr->desc = 1;
114954
- pCsr->isRequireSeek = 1;
114955
- pCsr->isMatchinfoNeeded = 1;
114956
- pCsr->eEvalmode = FTS3_EVAL_NEXT;
114957
- return SQLITE_OK;
114958
- }
114495
+ if( rc!=SQLITE_OK ) return rc;
114496
+ }
114497
+
114959114498
return fts3NextMethod(pCursor);
114960114499
}
114961114500
114962114501
/*
114963114502
** This is the xEof method of the virtual table. SQLite calls this
@@ -114973,20 +114512,11 @@
114973114512
** exposes %_content.docid as the rowid for the virtual table. The
114974114513
** rowid should be written to *pRowid.
114975114514
*/
114976114515
static int fts3RowidMethod(sqlite3_vtab_cursor *pCursor, sqlite_int64 *pRowid){
114977114516
Fts3Cursor *pCsr = (Fts3Cursor *) pCursor;
114978
- if( pCsr->aDoclist ){
114979
- *pRowid = pCsr->iPrevId;
114980
- }else{
114981
- /* This branch runs if the query is implemented using a full-table scan
114982
- ** (not using the full-text index). In this case grab the rowid from the
114983
- ** SELECT statement.
114984
- */
114985
- assert( pCsr->isRequireSeek==0 );
114986
- *pRowid = sqlite3_column_int64(pCsr->pStmt, 0);
114987
- }
114517
+ *pRowid = pCsr->iPrevId;
114988114518
return SQLITE_OK;
114989114519
}
114990114520
114991114521
/*
114992114522
** This is the xColumn method, called by SQLite to request a value from
@@ -114995,11 +114525,11 @@
114995114525
static int fts3ColumnMethod(
114996114526
sqlite3_vtab_cursor *pCursor, /* Cursor to retrieve value from */
114997114527
sqlite3_context *pContext, /* Context for sqlite3_result_xxx() calls */
114998114528
int iCol /* Index of column to read value from */
114999114529
){
115000
- int rc; /* Return Code */
114530
+ int rc = SQLITE_OK; /* Return Code */
115001114531
Fts3Cursor *pCsr = (Fts3Cursor *) pCursor;
115002114532
Fts3Table *p = (Fts3Table *)pCursor->pVtab;
115003114533
115004114534
/* The column value supplied by SQLite must be in range. */
115005114535
assert( iCol>=0 && iCol<=p->nColumn+1 );
@@ -115006,25 +114536,24 @@
115006114536
115007114537
if( iCol==p->nColumn+1 ){
115008114538
/* This call is a request for the "docid" column. Since "docid" is an
115009114539
** alias for "rowid", use the xRowid() method to obtain the value.
115010114540
*/
115011
- sqlite3_int64 iRowid;
115012
- rc = fts3RowidMethod(pCursor, &iRowid);
115013
- sqlite3_result_int64(pContext, iRowid);
114541
+ sqlite3_result_int64(pContext, pCsr->iPrevId);
115014114542
}else if( iCol==p->nColumn ){
115015114543
/* The extra column whose name is the same as the table.
115016114544
** Return a blob which is a pointer to the cursor.
115017114545
*/
115018114546
sqlite3_result_blob(pContext, &pCsr, sizeof(pCsr), SQLITE_TRANSIENT);
115019
- rc = SQLITE_OK;
115020114547
}else{
115021114548
rc = fts3CursorSeek(0, pCsr);
115022114549
if( rc==SQLITE_OK ){
115023114550
sqlite3_result_value(pContext, sqlite3_column_value(pCsr->pStmt, iCol+1));
115024114551
}
115025114552
}
114553
+
114554
+ assert( ((Fts3Table *)pCsr->base.pVtab)->pSegments==0 );
115026114555
return rc;
115027114556
}
115028114557
115029114558
/*
115030114559
** This function is the implementation of the xUpdate callback used by
@@ -115054,10 +114583,11 @@
115054114583
** Implementation of xBegin() method. This is a no-op.
115055114584
*/
115056114585
static int fts3BeginMethod(sqlite3_vtab *pVtab){
115057114586
UNUSED_PARAMETER(pVtab);
115058114587
TESTONLY( Fts3Table *p = (Fts3Table*)pVtab );
114588
+ assert( p->pSegments==0 );
115059114589
assert( p->nPendingData==0 );
115060114590
assert( p->inTransaction!=1 );
115061114591
TESTONLY( p->inTransaction = 1 );
115062114592
TESTONLY( p->mxSavepoint = -1; );
115063114593
return SQLITE_OK;
@@ -115071,10 +114601,11 @@
115071114601
static int fts3CommitMethod(sqlite3_vtab *pVtab){
115072114602
UNUSED_PARAMETER(pVtab);
115073114603
TESTONLY( Fts3Table *p = (Fts3Table*)pVtab );
115074114604
assert( p->nPendingData==0 );
115075114605
assert( p->inTransaction!=0 );
114606
+ assert( p->pSegments==0 );
115076114607
TESTONLY( p->inTransaction = 0 );
115077114608
TESTONLY( p->mxSavepoint = -1; );
115078114609
return SQLITE_OK;
115079114610
}
115080114611
@@ -115088,132 +114619,30 @@
115088114619
assert( p->inTransaction!=0 );
115089114620
TESTONLY( p->inTransaction = 0 );
115090114621
TESTONLY( p->mxSavepoint = -1; );
115091114622
return SQLITE_OK;
115092114623
}
115093
-
115094
-/*
115095
-** Load the doclist associated with expression pExpr to pExpr->aDoclist.
115096
-** The loaded doclist contains positions as well as the document ids.
115097
-** This is used by the matchinfo(), snippet() and offsets() auxillary
115098
-** functions.
115099
-*/
115100
-SQLITE_PRIVATE int sqlite3Fts3ExprLoadDoclist(Fts3Cursor *pCsr, Fts3Expr *pExpr){
115101
- int rc;
115102
- assert( pExpr->eType==FTSQUERY_PHRASE && pExpr->pPhrase );
115103
- assert( pCsr->eEvalmode==FTS3_EVAL_NEXT );
115104
- rc = fts3EvalExpr(pCsr, pExpr, &pExpr->aDoclist, &pExpr->nDoclist, 1);
115105
- return rc;
115106
-}
115107
-
115108
-SQLITE_PRIVATE int sqlite3Fts3ExprLoadFtDoclist(
115109
- Fts3Cursor *pCsr,
115110
- Fts3Expr *pExpr,
115111
- char **paDoclist,
115112
- int *pnDoclist
115113
-){
115114
- int rc;
115115
- assert( pCsr->eEvalmode==FTS3_EVAL_NEXT );
115116
- assert( pExpr->eType==FTSQUERY_PHRASE && pExpr->pPhrase );
115117
- pCsr->eEvalmode = FTS3_EVAL_MATCHINFO;
115118
- rc = fts3EvalExpr(pCsr, pExpr, paDoclist, pnDoclist, 1);
115119
- pCsr->eEvalmode = FTS3_EVAL_NEXT;
115120
- return rc;
115121
-}
115122
-
115123114624
115124114625
/*
115125114626
** When called, *ppPoslist must point to the byte immediately following the
115126114627
** end of a position-list. i.e. ( (*ppPoslist)[-1]==POS_END ). This function
115127114628
** moves *ppPoslist so that it instead points to the first byte of the
115128114629
** same position list.
115129114630
*/
115130114631
static void fts3ReversePoslist(char *pStart, char **ppPoslist){
115131
- char *p = &(*ppPoslist)[-3];
115132
- char c = p[1];
114632
+ char *p = &(*ppPoslist)[-2];
114633
+ char c;
114634
+
114635
+ while( p>pStart && (c=*p--)==0 );
115133114636
while( p>pStart && (*p & 0x80) | c ){
115134114637
c = *p--;
115135114638
}
115136114639
if( p>pStart ){ p = &p[2]; }
115137114640
while( *p++&0x80 );
115138114641
*ppPoslist = p;
115139114642
}
115140114643
115141
-
115142
-/*
115143
-** After ExprLoadDoclist() (see above) has been called, this function is
115144
-** used to iterate/search through the position lists that make up the doclist
115145
-** stored in pExpr->aDoclist.
115146
-*/
115147
-SQLITE_PRIVATE char *sqlite3Fts3FindPositions(
115148
- Fts3Cursor *pCursor, /* Associate FTS3 cursor */
115149
- Fts3Expr *pExpr, /* Access this expressions doclist */
115150
- sqlite3_int64 iDocid, /* Docid associated with requested pos-list */
115151
- int iCol /* Column of requested pos-list */
115152
-){
115153
- assert( pExpr->isLoaded );
115154
- if( pExpr->aDoclist ){
115155
- char *pEnd = &pExpr->aDoclist[pExpr->nDoclist];
115156
- char *pCsr;
115157
-
115158
- if( pExpr->pCurrent==0 ){
115159
- if( pCursor->desc==0 ){
115160
- pExpr->pCurrent = pExpr->aDoclist;
115161
- pExpr->iCurrent = 0;
115162
- fts3GetDeltaVarint(&pExpr->pCurrent, &pExpr->iCurrent);
115163
- }else{
115164
- pCsr = pExpr->aDoclist;
115165
- while( pCsr<pEnd ){
115166
- fts3GetDeltaVarint(&pCsr, &pExpr->iCurrent);
115167
- fts3PoslistCopy(0, &pCsr);
115168
- }
115169
- fts3ReversePoslist(pExpr->aDoclist, &pCsr);
115170
- pExpr->pCurrent = pCsr;
115171
- }
115172
- }
115173
- pCsr = pExpr->pCurrent;
115174
- assert( pCsr );
115175
-
115176
- while( (pCursor->desc==0 && pCsr<pEnd)
115177
- || (pCursor->desc && pCsr>pExpr->aDoclist)
115178
- ){
115179
- if( pCursor->desc==0 && pExpr->iCurrent<iDocid ){
115180
- fts3PoslistCopy(0, &pCsr);
115181
- if( pCsr<pEnd ){
115182
- fts3GetDeltaVarint(&pCsr, &pExpr->iCurrent);
115183
- }
115184
- pExpr->pCurrent = pCsr;
115185
- }else if( pCursor->desc && pExpr->iCurrent>iDocid ){
115186
- fts3GetReverseDeltaVarint(&pCsr, pExpr->aDoclist, &pExpr->iCurrent);
115187
- fts3ReversePoslist(pExpr->aDoclist, &pCsr);
115188
- pExpr->pCurrent = pCsr;
115189
- }else{
115190
- if( pExpr->iCurrent==iDocid ){
115191
- int iThis = 0;
115192
- if( iCol<0 ){
115193
- /* If iCol is negative, return a pointer to the start of the
115194
- ** position-list (instead of a pointer to the start of a list
115195
- ** of offsets associated with a specific column).
115196
- */
115197
- return pCsr;
115198
- }
115199
- while( iThis<iCol ){
115200
- fts3ColumnlistCopy(0, &pCsr);
115201
- if( *pCsr==0x00 ) return 0;
115202
- pCsr++;
115203
- pCsr += sqlite3Fts3GetVarint32(pCsr, &iThis);
115204
- }
115205
- if( iCol==iThis && (*pCsr&0xFE) ) return pCsr;
115206
- }
115207
- return 0;
115208
- }
115209
- }
115210
- }
115211
-
115212
- return 0;
115213
-}
115214
-
115215114644
/*
115216114645
** Helper function used by the implementation of the overloaded snippet(),
115217114646
** offsets() and optimize() SQL functions.
115218114647
**
115219114648
** If the value passed as the third argument is a blob of size
@@ -115441,16 +114870,15 @@
115441114870
);
115442114871
return rc;
115443114872
}
115444114873
115445114874
static int fts3SavepointMethod(sqlite3_vtab *pVtab, int iSavepoint){
115446
- Fts3Table *p = (Fts3Table*)pVtab;
115447114875
UNUSED_PARAMETER(iSavepoint);
115448
- assert( p->inTransaction );
115449
- assert( p->mxSavepoint < iSavepoint );
115450
- TESTONLY( p->mxSavepoint = iSavepoint );
115451
- return sqlite3Fts3PendingTermsFlush(p);
114876
+ assert( ((Fts3Table *)pVtab)->inTransaction );
114877
+ assert( ((Fts3Table *)pVtab)->mxSavepoint < iSavepoint );
114878
+ TESTONLY( ((Fts3Table *)pVtab)->mxSavepoint = iSavepoint );
114879
+ return fts3SyncMethod(pVtab);
115452114880
}
115453114881
static int fts3ReleaseMethod(sqlite3_vtab *pVtab, int iSavepoint){
115454114882
TESTONLY( Fts3Table *p = (Fts3Table*)pVtab );
115455114883
UNUSED_PARAMETER(iSavepoint);
115456114884
UNUSED_PARAMETER(pVtab);
@@ -115617,10 +115045,1282 @@
115617115045
SQLITE_EXTENSION_INIT2(pApi)
115618115046
return sqlite3Fts3Init(db);
115619115047
}
115620115048
#endif
115621115049
115050
+
115051
+/*
115052
+** Allocate an Fts3MultiSegReader for each token in the expression headed
115053
+** by pExpr.
115054
+**
115055
+** An Fts3SegReader object is a cursor that can seek or scan a range of
115056
+** entries within a single segment b-tree. An Fts3MultiSegReader uses multiple
115057
+** Fts3SegReader objects internally to provide an interface to seek or scan
115058
+** within the union of all segments of a b-tree. Hence the name.
115059
+**
115060
+** If the allocated Fts3MultiSegReader just seeks to a single entry in a
115061
+** segment b-tree (if the term is not a prefix or it is a prefix for which
115062
+** there exists prefix b-tree of the right length) then it may be traversed
115063
+** and merged incrementally. Otherwise, it has to be merged into an in-memory
115064
+** doclist and then traversed.
115065
+*/
115066
+static void fts3EvalAllocateReaders(
115067
+ Fts3Cursor *pCsr,
115068
+ Fts3Expr *pExpr,
115069
+ int *pnToken, /* OUT: Total number of tokens in phrase. */
115070
+ int *pnOr, /* OUT: Total number of OR nodes in expr. */
115071
+ int *pRc
115072
+){
115073
+ if( pExpr && SQLITE_OK==*pRc ){
115074
+ if( pExpr->eType==FTSQUERY_PHRASE ){
115075
+ int i;
115076
+ int nToken = pExpr->pPhrase->nToken;
115077
+ *pnToken += nToken;
115078
+ for(i=0; i<nToken; i++){
115079
+ Fts3PhraseToken *pToken = &pExpr->pPhrase->aToken[i];
115080
+ int rc = sqlite3Fts3TermSegReaderCursor(pCsr,
115081
+ pToken->z, pToken->n, pToken->isPrefix, &pToken->pSegcsr
115082
+ );
115083
+ if( rc!=SQLITE_OK ){
115084
+ *pRc = rc;
115085
+ return;
115086
+ }
115087
+ }
115088
+ }else{
115089
+ *pnOr += (pExpr->eType==FTSQUERY_OR);
115090
+ fts3EvalAllocateReaders(pCsr, pExpr->pLeft, pnToken, pnOr, pRc);
115091
+ fts3EvalAllocateReaders(pCsr, pExpr->pRight, pnToken, pnOr, pRc);
115092
+ }
115093
+ }
115094
+}
115095
+
115096
+static int fts3EvalPhraseLoad(
115097
+ Fts3Cursor *pCsr,
115098
+ Fts3Phrase *p
115099
+){
115100
+ Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab;
115101
+ int iToken;
115102
+ int rc = SQLITE_OK;
115103
+
115104
+ char *aDoclist = 0;
115105
+ int nDoclist = 0;
115106
+ int iPrev = -1;
115107
+
115108
+ for(iToken=0; rc==SQLITE_OK && iToken<p->nToken; iToken++){
115109
+ Fts3PhraseToken *pToken = &p->aToken[iToken];
115110
+ assert( pToken->pSegcsr || pToken->pDeferred );
115111
+
115112
+ if( pToken->pDeferred==0 ){
115113
+ int nThis = 0;
115114
+ char *pThis = 0;
115115
+ rc = fts3TermSelect(pTab, pToken, p->iColumn, 1, &nThis, &pThis);
115116
+ if( rc==SQLITE_OK ){
115117
+ if( pThis==0 ){
115118
+ sqlite3_free(aDoclist);
115119
+ aDoclist = 0;
115120
+ nDoclist = 0;
115121
+ break;
115122
+ }else if( aDoclist==0 ){
115123
+ aDoclist = pThis;
115124
+ nDoclist = nThis;
115125
+ }else{
115126
+ assert( iPrev>=0 );
115127
+ fts3DoclistPhraseMerge(pTab->bDescIdx,
115128
+ iToken-iPrev, aDoclist, nDoclist, pThis, &nThis
115129
+ );
115130
+ sqlite3_free(aDoclist);
115131
+ aDoclist = pThis;
115132
+ nDoclist = nThis;
115133
+ }
115134
+ iPrev = iToken;
115135
+ }
115136
+ }
115137
+ }
115138
+
115139
+ if( rc==SQLITE_OK ){
115140
+ p->doclist.aAll = aDoclist;
115141
+ p->doclist.nAll = nDoclist;
115142
+ }else{
115143
+ sqlite3_free(aDoclist);
115144
+ }
115145
+ return rc;
115146
+}
115147
+
115148
+static int fts3EvalDeferredPhrase(Fts3Cursor *pCsr, Fts3Phrase *pPhrase){
115149
+ int iToken;
115150
+ int rc = SQLITE_OK;
115151
+
115152
+ int nMaxUndeferred = -1;
115153
+ char *aPoslist = 0;
115154
+ int nPoslist = 0;
115155
+ int iPrev = -1;
115156
+
115157
+ assert( pPhrase->doclist.bFreeList==0 );
115158
+
115159
+ for(iToken=0; rc==SQLITE_OK && iToken<pPhrase->nToken; iToken++){
115160
+ Fts3PhraseToken *pToken = &pPhrase->aToken[iToken];
115161
+ Fts3DeferredToken *pDeferred = pToken->pDeferred;
115162
+
115163
+ if( pDeferred ){
115164
+ char *pList;
115165
+ int nList;
115166
+ rc = sqlite3Fts3DeferredTokenList(pDeferred, &pList, &nList);
115167
+ if( rc!=SQLITE_OK ) return rc;
115168
+
115169
+ if( pList==0 ){
115170
+ sqlite3_free(aPoslist);
115171
+ pPhrase->doclist.pList = 0;
115172
+ pPhrase->doclist.nList = 0;
115173
+ return SQLITE_OK;
115174
+
115175
+ }else if( aPoslist==0 ){
115176
+ aPoslist = pList;
115177
+ nPoslist = nList;
115178
+
115179
+ }else{
115180
+ char *aOut = pList;
115181
+ char *p1 = aPoslist;
115182
+ char *p2 = aOut;
115183
+
115184
+ assert( iPrev>=0 );
115185
+ fts3PoslistPhraseMerge(&aOut, iToken-iPrev, 0, 1, &p1, &p2);
115186
+ sqlite3_free(aPoslist);
115187
+ aPoslist = pList;
115188
+ nPoslist = aOut - aPoslist;
115189
+ if( nPoslist==0 ){
115190
+ sqlite3_free(aPoslist);
115191
+ pPhrase->doclist.pList = 0;
115192
+ pPhrase->doclist.nList = 0;
115193
+ return SQLITE_OK;
115194
+ }
115195
+ }
115196
+ iPrev = iToken;
115197
+ }else{
115198
+ nMaxUndeferred = iToken;
115199
+ }
115200
+ }
115201
+
115202
+ if( iPrev>=0 ){
115203
+ if( nMaxUndeferred<0 ){
115204
+ pPhrase->doclist.pList = aPoslist;
115205
+ pPhrase->doclist.nList = nPoslist;
115206
+ pPhrase->doclist.iDocid = pCsr->iPrevId;
115207
+ pPhrase->doclist.bFreeList = 1;
115208
+ }else{
115209
+ int nDistance;
115210
+ char *p1;
115211
+ char *p2;
115212
+ char *aOut;
115213
+
115214
+ if( nMaxUndeferred>iPrev ){
115215
+ p1 = aPoslist;
115216
+ p2 = pPhrase->doclist.pList;
115217
+ nDistance = nMaxUndeferred - iPrev;
115218
+ }else{
115219
+ p1 = pPhrase->doclist.pList;
115220
+ p2 = aPoslist;
115221
+ nDistance = iPrev - nMaxUndeferred;
115222
+ }
115223
+
115224
+ aOut = (char *)sqlite3_malloc(nPoslist+8);
115225
+ if( !aOut ){
115226
+ sqlite3_free(aPoslist);
115227
+ return SQLITE_NOMEM;
115228
+ }
115229
+
115230
+ pPhrase->doclist.pList = aOut;
115231
+ if( fts3PoslistPhraseMerge(&aOut, nDistance, 0, 1, &p1, &p2) ){
115232
+ pPhrase->doclist.bFreeList = 1;
115233
+ pPhrase->doclist.nList = (aOut - pPhrase->doclist.pList);
115234
+ }else{
115235
+ sqlite3_free(aOut);
115236
+ pPhrase->doclist.pList = 0;
115237
+ pPhrase->doclist.nList = 0;
115238
+ }
115239
+ sqlite3_free(aPoslist);
115240
+ }
115241
+ }
115242
+
115243
+ return SQLITE_OK;
115244
+}
115245
+
115246
+/*
115247
+** This function is called for each Fts3Phrase in a full-text query
115248
+** expression to initialize the mechanism for returning rows. Once this
115249
+** function has been called successfully on an Fts3Phrase, it may be
115250
+** used with fts3EvalPhraseNext() to iterate through the matching docids.
115251
+*/
115252
+static int fts3EvalPhraseStart(Fts3Cursor *pCsr, int bOptOk, Fts3Phrase *p){
115253
+ int rc;
115254
+ Fts3PhraseToken *pFirst = &p->aToken[0];
115255
+ Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab;
115256
+
115257
+ assert( p->doclist.aAll==0 );
115258
+ if( pCsr->bDesc==pTab->bDescIdx && bOptOk==1 && p->nToken==1
115259
+ && pFirst->pSegcsr && pFirst->pSegcsr->bLookup
115260
+ ){
115261
+ /* Use the incremental approach. */
115262
+ int iCol = (p->iColumn >= pTab->nColumn ? -1 : p->iColumn);
115263
+ rc = sqlite3Fts3MsrIncrStart(
115264
+ pTab, pFirst->pSegcsr, iCol, pFirst->z, pFirst->n);
115265
+ p->bIncr = 1;
115266
+
115267
+ }else{
115268
+ /* Load the full doclist for the phrase into memory. */
115269
+ rc = fts3EvalPhraseLoad(pCsr, p);
115270
+ p->bIncr = 0;
115271
+ }
115272
+
115273
+ assert( rc!=SQLITE_OK || p->nToken<1 || p->aToken[0].pSegcsr==0 || p->bIncr );
115274
+ return rc;
115275
+}
115276
+
115277
+/*
115278
+** This function is used to iterate backwards (from the end to start)
115279
+** through doclists.
115280
+*/
115281
+SQLITE_PRIVATE void sqlite3Fts3DoclistPrev(
115282
+ int bDescIdx, /* True if the doclist is desc */
115283
+ char *aDoclist, /* Pointer to entire doclist */
115284
+ int nDoclist, /* Length of aDoclist in bytes */
115285
+ char **ppIter, /* IN/OUT: Iterator pointer */
115286
+ sqlite3_int64 *piDocid, /* IN/OUT: Docid pointer */
115287
+ int *pnList, /* IN/OUT: List length pointer */
115288
+ u8 *pbEof /* OUT: End-of-file flag */
115289
+){
115290
+ char *p = *ppIter;
115291
+
115292
+ assert( nDoclist>0 );
115293
+ assert( *pbEof==0 );
115294
+ assert( p || *piDocid==0 );
115295
+ assert( !p || (p>aDoclist && p<&aDoclist[nDoclist]) );
115296
+
115297
+ if( p==0 ){
115298
+ sqlite3_int64 iDocid = 0;
115299
+ char *pNext = 0;
115300
+ char *pDocid = aDoclist;
115301
+ char *pEnd = &aDoclist[nDoclist];
115302
+ int iMul = 1;
115303
+
115304
+ while( pDocid<pEnd ){
115305
+ sqlite3_int64 iDelta;
115306
+ pDocid += sqlite3Fts3GetVarint(pDocid, &iDelta);
115307
+ iDocid += (iMul * iDelta);
115308
+ pNext = pDocid;
115309
+ fts3PoslistCopy(0, &pDocid);
115310
+ while( pDocid<pEnd && *pDocid==0 ) pDocid++;
115311
+ iMul = (bDescIdx ? -1 : 1);
115312
+ }
115313
+
115314
+ *pnList = pEnd - pNext;
115315
+ *ppIter = pNext;
115316
+ *piDocid = iDocid;
115317
+ }else{
115318
+ int iMul = (bDescIdx ? -1 : 1);
115319
+ sqlite3_int64 iDelta;
115320
+ fts3GetReverseVarint(&p, aDoclist, &iDelta);
115321
+ *piDocid -= (iMul * iDelta);
115322
+
115323
+ if( p==aDoclist ){
115324
+ *pbEof = 1;
115325
+ }else{
115326
+ char *pSave = p;
115327
+ fts3ReversePoslist(aDoclist, &p);
115328
+ *pnList = (pSave - p);
115329
+ }
115330
+ *ppIter = p;
115331
+ }
115332
+}
115333
+
115334
+/*
115335
+** Attempt to move the phrase iterator to point to the next matching docid.
115336
+** If an error occurs, return an SQLite error code. Otherwise, return
115337
+** SQLITE_OK.
115338
+**
115339
+** If there is no "next" entry and no error occurs, then *pbEof is set to
115340
+** 1 before returning. Otherwise, if no error occurs and the iterator is
115341
+** successfully advanced, *pbEof is set to 0.
115342
+*/
115343
+static int fts3EvalPhraseNext(
115344
+ Fts3Cursor *pCsr,
115345
+ Fts3Phrase *p,
115346
+ u8 *pbEof
115347
+){
115348
+ int rc = SQLITE_OK;
115349
+ Fts3Doclist *pDL = &p->doclist;
115350
+ Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab;
115351
+
115352
+ if( p->bIncr ){
115353
+ assert( p->nToken==1 );
115354
+ assert( pDL->pNextDocid==0 );
115355
+ rc = sqlite3Fts3MsrIncrNext(pTab, p->aToken[0].pSegcsr,
115356
+ &pDL->iDocid, &pDL->pList, &pDL->nList
115357
+ );
115358
+ if( rc==SQLITE_OK && !pDL->pList ){
115359
+ *pbEof = 1;
115360
+ }
115361
+ }else if( pCsr->bDesc!=pTab->bDescIdx && pDL->nAll ){
115362
+ sqlite3Fts3DoclistPrev(pTab->bDescIdx, pDL->aAll, pDL->nAll,
115363
+ &pDL->pNextDocid, &pDL->iDocid, &pDL->nList, pbEof
115364
+ );
115365
+ pDL->pList = pDL->pNextDocid;
115366
+ }else{
115367
+ char *pIter; /* Used to iterate through aAll */
115368
+ char *pEnd = &pDL->aAll[pDL->nAll]; /* 1 byte past end of aAll */
115369
+ if( pDL->pNextDocid ){
115370
+ pIter = pDL->pNextDocid;
115371
+ }else{
115372
+ pIter = pDL->aAll;
115373
+ }
115374
+
115375
+ if( pIter>=pEnd ){
115376
+ /* We have already reached the end of this doclist. EOF. */
115377
+ *pbEof = 1;
115378
+ }else{
115379
+ sqlite3_int64 iDelta;
115380
+ pIter += sqlite3Fts3GetVarint(pIter, &iDelta);
115381
+ if( pTab->bDescIdx==0 || pDL->pNextDocid==0 ){
115382
+ pDL->iDocid += iDelta;
115383
+ }else{
115384
+ pDL->iDocid -= iDelta;
115385
+ }
115386
+ pDL->pList = pIter;
115387
+ fts3PoslistCopy(0, &pIter);
115388
+ pDL->nList = (pIter - pDL->pList);
115389
+
115390
+ /* pIter now points just past the 0x00 that terminates the position-
115391
+ ** list for document pDL->iDocid. However, if this position-list was
115392
+ ** edited in place by fts3EvalNearTrim2(), then pIter may not actually
115393
+ ** point to the start of the next docid value. The following line deals
115394
+ ** with this case by advancing pIter past the zero-padding added by
115395
+ ** fts3EvalNearTrim2(). */
115396
+ while( pIter<pEnd && *pIter==0 ) pIter++;
115397
+
115398
+ pDL->pNextDocid = pIter;
115399
+ assert( *pIter || pIter>=&pDL->aAll[pDL->nAll] );
115400
+ *pbEof = 0;
115401
+ }
115402
+ }
115403
+
115404
+ return rc;
115405
+}
115406
+
115407
+static void fts3EvalStartReaders(
115408
+ Fts3Cursor *pCsr,
115409
+ Fts3Expr *pExpr,
115410
+ int bOptOk,
115411
+ int *pRc
115412
+){
115413
+ if( pExpr && SQLITE_OK==*pRc ){
115414
+ if( pExpr->eType==FTSQUERY_PHRASE ){
115415
+ int i;
115416
+ int nToken = pExpr->pPhrase->nToken;
115417
+ for(i=0; i<nToken; i++){
115418
+ if( pExpr->pPhrase->aToken[i].pDeferred==0 ) break;
115419
+ }
115420
+ pExpr->bDeferred = (i==nToken);
115421
+ *pRc = fts3EvalPhraseStart(pCsr, bOptOk, pExpr->pPhrase);
115422
+ }else{
115423
+ fts3EvalStartReaders(pCsr, pExpr->pLeft, bOptOk, pRc);
115424
+ fts3EvalStartReaders(pCsr, pExpr->pRight, bOptOk, pRc);
115425
+ pExpr->bDeferred = (pExpr->pLeft->bDeferred && pExpr->pRight->bDeferred);
115426
+ }
115427
+ }
115428
+}
115429
+
115430
+
115431
+typedef struct Fts3TokenAndCost Fts3TokenAndCost;
115432
+struct Fts3TokenAndCost {
115433
+ Fts3PhraseToken *pToken;
115434
+ Fts3Expr *pRoot;
115435
+ int nOvfl;
115436
+ int iCol;
115437
+};
115438
+
115439
+static void fts3EvalTokenCosts(
115440
+ Fts3Cursor *pCsr,
115441
+ Fts3Expr *pRoot,
115442
+ Fts3Expr *pExpr,
115443
+ Fts3TokenAndCost **ppTC,
115444
+ Fts3Expr ***ppOr,
115445
+ int *pRc
115446
+){
115447
+ if( *pRc==SQLITE_OK && pExpr ){
115448
+ if( pExpr->eType==FTSQUERY_PHRASE ){
115449
+ Fts3Phrase *pPhrase = pExpr->pPhrase;
115450
+ int i;
115451
+ for(i=0; *pRc==SQLITE_OK && i<pPhrase->nToken; i++){
115452
+ Fts3TokenAndCost *pTC = (*ppTC)++;
115453
+ pTC->pRoot = pRoot;
115454
+ pTC->pToken = &pPhrase->aToken[i];
115455
+ pTC->iCol = pPhrase->iColumn;
115456
+ *pRc = sqlite3Fts3MsrOvfl(pCsr, pTC->pToken->pSegcsr, &pTC->nOvfl);
115457
+ }
115458
+ }else if( pExpr->eType!=FTSQUERY_NOT ){
115459
+ if( pExpr->eType==FTSQUERY_OR ){
115460
+ pRoot = pExpr->pLeft;
115461
+ **ppOr = pRoot;
115462
+ (*ppOr)++;
115463
+ }
115464
+ fts3EvalTokenCosts(pCsr, pRoot, pExpr->pLeft, ppTC, ppOr, pRc);
115465
+ if( pExpr->eType==FTSQUERY_OR ){
115466
+ pRoot = pExpr->pRight;
115467
+ **ppOr = pRoot;
115468
+ (*ppOr)++;
115469
+ }
115470
+ fts3EvalTokenCosts(pCsr, pRoot, pExpr->pRight, ppTC, ppOr, pRc);
115471
+ }
115472
+ }
115473
+}
115474
+
115475
+static int fts3EvalAverageDocsize(Fts3Cursor *pCsr, int *pnPage){
115476
+ if( pCsr->nRowAvg==0 ){
115477
+ /* The average document size, which is required to calculate the cost
115478
+ ** of each doclist, has not yet been determined. Read the required
115479
+ ** data from the %_stat table to calculate it.
115480
+ **
115481
+ ** Entry 0 of the %_stat table is a blob containing (nCol+1) FTS3
115482
+ ** varints, where nCol is the number of columns in the FTS3 table.
115483
+ ** The first varint is the number of documents currently stored in
115484
+ ** the table. The following nCol varints contain the total amount of
115485
+ ** data stored in all rows of each column of the table, from left
115486
+ ** to right.
115487
+ */
115488
+ int rc;
115489
+ Fts3Table *p = (Fts3Table*)pCsr->base.pVtab;
115490
+ sqlite3_stmt *pStmt;
115491
+ sqlite3_int64 nDoc = 0;
115492
+ sqlite3_int64 nByte = 0;
115493
+ const char *pEnd;
115494
+ const char *a;
115495
+
115496
+ rc = sqlite3Fts3SelectDoctotal(p, &pStmt);
115497
+ if( rc!=SQLITE_OK ) return rc;
115498
+ a = sqlite3_column_blob(pStmt, 0);
115499
+ assert( a );
115500
+
115501
+ pEnd = &a[sqlite3_column_bytes(pStmt, 0)];
115502
+ a += sqlite3Fts3GetVarint(a, &nDoc);
115503
+ while( a<pEnd ){
115504
+ a += sqlite3Fts3GetVarint(a, &nByte);
115505
+ }
115506
+ if( nDoc==0 || nByte==0 ){
115507
+ sqlite3_reset(pStmt);
115508
+ return SQLITE_CORRUPT_VTAB;
115509
+ }
115510
+
115511
+ pCsr->nDoc = nDoc;
115512
+ pCsr->nRowAvg = (int)(((nByte / nDoc) + p->nPgsz) / p->nPgsz);
115513
+ assert( pCsr->nRowAvg>0 );
115514
+ rc = sqlite3_reset(pStmt);
115515
+ if( rc!=SQLITE_OK ) return rc;
115516
+ }
115517
+
115518
+ *pnPage = pCsr->nRowAvg;
115519
+ return SQLITE_OK;
115520
+}
115521
+
115522
+static int fts3EvalSelectDeferred(
115523
+ Fts3Cursor *pCsr,
115524
+ Fts3Expr *pRoot,
115525
+ Fts3TokenAndCost *aTC,
115526
+ int nTC
115527
+){
115528
+ int nDocSize = 0;
115529
+ int nDocEst = 0;
115530
+ int rc = SQLITE_OK;
115531
+ Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab;
115532
+ int ii;
115533
+
115534
+ int nOvfl = 0;
115535
+ int nTerm = 0;
115536
+
115537
+ for(ii=0; ii<nTC; ii++){
115538
+ if( aTC[ii].pRoot==pRoot ){
115539
+ nOvfl += aTC[ii].nOvfl;
115540
+ nTerm++;
115541
+ }
115542
+ }
115543
+ if( nOvfl==0 || nTerm<2 ) return SQLITE_OK;
115544
+
115545
+ rc = fts3EvalAverageDocsize(pCsr, &nDocSize);
115546
+
115547
+ for(ii=0; ii<nTerm && rc==SQLITE_OK; ii++){
115548
+ int jj;
115549
+ Fts3TokenAndCost *pTC = 0;
115550
+
115551
+ for(jj=0; jj<nTC; jj++){
115552
+ if( aTC[jj].pToken && aTC[jj].pRoot==pRoot
115553
+ && (!pTC || aTC[jj].nOvfl<pTC->nOvfl)
115554
+ ){
115555
+ pTC = &aTC[jj];
115556
+ }
115557
+ }
115558
+ assert( pTC );
115559
+
115560
+ /* At this point pTC points to the cheapest remaining token. */
115561
+ if( ii==0 ){
115562
+ if( pTC->nOvfl ){
115563
+ nDocEst = (pTC->nOvfl * pTab->nPgsz + pTab->nPgsz) / 10;
115564
+ }else{
115565
+ /* TODO: Fix this so that the doclist need not be read twice. */
115566
+ Fts3PhraseToken *pToken = pTC->pToken;
115567
+ int nList = 0;
115568
+ char *pList = 0;
115569
+ rc = fts3TermSelect(pTab, pToken, pTC->iCol, 1, &nList, &pList);
115570
+ if( rc==SQLITE_OK ){
115571
+ nDocEst = fts3DoclistCountDocids(1, pList, nList);
115572
+ }
115573
+ sqlite3_free(pList);
115574
+ if( rc==SQLITE_OK ){
115575
+ rc = sqlite3Fts3TermSegReaderCursor(pCsr,
115576
+ pToken->z, pToken->n, pToken->isPrefix, &pToken->pSegcsr
115577
+ );
115578
+ }
115579
+ }
115580
+ }else{
115581
+ if( pTC->nOvfl>=(nDocEst*nDocSize) ){
115582
+ Fts3PhraseToken *pToken = pTC->pToken;
115583
+ rc = sqlite3Fts3DeferToken(pCsr, pToken, pTC->iCol);
115584
+ fts3SegReaderCursorFree(pToken->pSegcsr);
115585
+ pToken->pSegcsr = 0;
115586
+ }
115587
+ nDocEst = 1 + (nDocEst/4);
115588
+ }
115589
+ pTC->pToken = 0;
115590
+ }
115591
+
115592
+ return rc;
115593
+}
115594
+
115595
+SQLITE_PRIVATE int sqlite3Fts3EvalStart(Fts3Cursor *pCsr, Fts3Expr *pExpr, int bOptOk){
115596
+ Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab;
115597
+ int rc = SQLITE_OK;
115598
+ int nToken = 0;
115599
+ int nOr = 0;
115600
+
115601
+ /* Allocate a MultiSegReader for each token in the expression. */
115602
+ fts3EvalAllocateReaders(pCsr, pExpr, &nToken, &nOr, &rc);
115603
+
115604
+ /* Call fts3EvalPhraseStart() on all phrases in the expression. TODO:
115605
+ ** This call will eventually also be responsible for determining which
115606
+ ** tokens are 'deferred' until the document text is loaded into memory.
115607
+ **
115608
+ ** Each token in each phrase is dealt with using one of the following
115609
+ ** three strategies:
115610
+ **
115611
+ ** 1. Entire doclist loaded into memory as part of the
115612
+ ** fts3EvalStartReaders() call.
115613
+ **
115614
+ ** 2. Doclist loaded into memory incrementally, as part of each
115615
+ ** sqlite3Fts3EvalNext() call.
115616
+ **
115617
+ ** 3. Token doclist is never loaded. Instead, documents are loaded into
115618
+ ** memory and scanned for the token as part of the sqlite3Fts3EvalNext()
115619
+ ** call. This is known as a "deferred" token.
115620
+ */
115621
+
115622
+ /* If bOptOk is true, check if there are any tokens that should be deferred.
115623
+ */
115624
+ if( rc==SQLITE_OK && bOptOk && nToken>1 && pTab->bHasStat ){
115625
+ Fts3TokenAndCost *aTC;
115626
+ Fts3Expr **apOr;
115627
+ aTC = (Fts3TokenAndCost *)sqlite3_malloc(
115628
+ sizeof(Fts3TokenAndCost) * nToken
115629
+ + sizeof(Fts3Expr *) * nOr * 2
115630
+ );
115631
+ apOr = (Fts3Expr **)&aTC[nToken];
115632
+
115633
+ if( !aTC ){
115634
+ rc = SQLITE_NOMEM;
115635
+ }else{
115636
+ int ii;
115637
+ Fts3TokenAndCost *pTC = aTC;
115638
+ Fts3Expr **ppOr = apOr;
115639
+
115640
+ fts3EvalTokenCosts(pCsr, 0, pExpr, &pTC, &ppOr, &rc);
115641
+ nToken = pTC-aTC;
115642
+ nOr = ppOr-apOr;
115643
+
115644
+ if( rc==SQLITE_OK ){
115645
+ rc = fts3EvalSelectDeferred(pCsr, 0, aTC, nToken);
115646
+ for(ii=0; rc==SQLITE_OK && ii<nOr; ii++){
115647
+ rc = fts3EvalSelectDeferred(pCsr, apOr[ii], aTC, nToken);
115648
+ }
115649
+ }
115650
+
115651
+ sqlite3_free(aTC);
115652
+ }
115653
+ }
115654
+
115655
+ fts3EvalStartReaders(pCsr, pExpr, bOptOk, &rc);
115656
+ return rc;
115657
+}
115658
+
115659
+static void fts3EvalZeroPoslist(Fts3Phrase *pPhrase){
115660
+ if( pPhrase->doclist.bFreeList ){
115661
+ sqlite3_free(pPhrase->doclist.pList);
115662
+ }
115663
+ pPhrase->doclist.pList = 0;
115664
+ pPhrase->doclist.nList = 0;
115665
+ pPhrase->doclist.bFreeList = 0;
115666
+}
115667
+
115668
+static int fts3EvalNearTrim2(
115669
+ int nNear,
115670
+ char *aTmp, /* Temporary space to use */
115671
+ char **paPoslist, /* IN/OUT: Position list */
115672
+ int *pnToken, /* IN/OUT: Tokens in phrase of *paPoslist */
115673
+ Fts3Phrase *pPhrase /* The phrase object to trim the doclist of */
115674
+){
115675
+ int nParam1 = nNear + pPhrase->nToken;
115676
+ int nParam2 = nNear + *pnToken;
115677
+ int nNew;
115678
+ char *p2;
115679
+ char *pOut;
115680
+ int res;
115681
+
115682
+ assert( pPhrase->doclist.pList );
115683
+
115684
+ p2 = pOut = pPhrase->doclist.pList;
115685
+ res = fts3PoslistNearMerge(
115686
+ &pOut, aTmp, nParam1, nParam2, paPoslist, &p2
115687
+ );
115688
+ if( res ){
115689
+ nNew = (pOut - pPhrase->doclist.pList) - 1;
115690
+ assert( pPhrase->doclist.pList[nNew]=='\0' );
115691
+ assert( nNew<=pPhrase->doclist.nList && nNew>0 );
115692
+ memset(&pPhrase->doclist.pList[nNew], 0, pPhrase->doclist.nList - nNew);
115693
+ pPhrase->doclist.nList = nNew;
115694
+ *paPoslist = pPhrase->doclist.pList;
115695
+ *pnToken = pPhrase->nToken;
115696
+ }
115697
+
115698
+ return res;
115699
+}
115700
+
115701
+static int fts3EvalNearTest(Fts3Expr *pExpr, int *pRc){
115702
+ int res = 1;
115703
+
115704
+ /* The following block runs if pExpr is the root of a NEAR query.
115705
+ ** For example, the query:
115706
+ **
115707
+ ** "w" NEAR "x" NEAR "y" NEAR "z"
115708
+ **
115709
+ ** which is represented in tree form as:
115710
+ **
115711
+ ** |
115712
+ ** +--NEAR--+ <-- root of NEAR query
115713
+ ** | |
115714
+ ** +--NEAR--+ "z"
115715
+ ** | |
115716
+ ** +--NEAR--+ "y"
115717
+ ** | |
115718
+ ** "w" "x"
115719
+ **
115720
+ ** The right-hand child of a NEAR node is always a phrase. The
115721
+ ** left-hand child may be either a phrase or a NEAR node. There are
115722
+ ** no exceptions to this.
115723
+ */
115724
+ if( *pRc==SQLITE_OK
115725
+ && pExpr->eType==FTSQUERY_NEAR
115726
+ && pExpr->bEof==0
115727
+ && (pExpr->pParent==0 || pExpr->pParent->eType!=FTSQUERY_NEAR)
115728
+ ){
115729
+ Fts3Expr *p;
115730
+ int nTmp = 0; /* Bytes of temp space */
115731
+ char *aTmp; /* Temp space for PoslistNearMerge() */
115732
+
115733
+ /* Allocate temporary working space. */
115734
+ for(p=pExpr; p->pLeft; p=p->pLeft){
115735
+ nTmp += p->pRight->pPhrase->doclist.nList;
115736
+ }
115737
+ nTmp += p->pPhrase->doclist.nList;
115738
+ aTmp = sqlite3_malloc(nTmp*2);
115739
+ if( !aTmp ){
115740
+ *pRc = SQLITE_NOMEM;
115741
+ res = 0;
115742
+ }else{
115743
+ char *aPoslist = p->pPhrase->doclist.pList;
115744
+ int nToken = p->pPhrase->nToken;
115745
+
115746
+ for(p=p->pParent;res && p && p->eType==FTSQUERY_NEAR; p=p->pParent){
115747
+ Fts3Phrase *pPhrase = p->pRight->pPhrase;
115748
+ int nNear = p->nNear;
115749
+ res = fts3EvalNearTrim2(nNear, aTmp, &aPoslist, &nToken, pPhrase);
115750
+ }
115751
+
115752
+ aPoslist = pExpr->pRight->pPhrase->doclist.pList;
115753
+ nToken = pExpr->pRight->pPhrase->nToken;
115754
+ for(p=pExpr->pLeft; p && res; p=p->pLeft){
115755
+ int nNear = p->pParent->nNear;
115756
+ Fts3Phrase *pPhrase = (
115757
+ p->eType==FTSQUERY_NEAR ? p->pRight->pPhrase : p->pPhrase
115758
+ );
115759
+ res = fts3EvalNearTrim2(nNear, aTmp, &aPoslist, &nToken, pPhrase);
115760
+ }
115761
+ }
115762
+
115763
+ sqlite3_free(aTmp);
115764
+ }
115765
+
115766
+ return res;
115767
+}
115768
+
115769
+/*
115770
+** This macro is used by the fts3EvalNext() function. The two arguments are
115771
+** 64-bit docid values. If the current query is "ORDER BY docid ASC", then
115772
+** the macro returns (i1 - i2). Or if it is "ORDER BY docid DESC", then
115773
+** it returns (i2 - i1). This allows the same code to be used for merging
115774
+** doclists in ascending or descending order.
115775
+*/
115776
+#define DOCID_CMP(i1, i2) ((pCsr->bDesc?-1:1) * (i1-i2))
115777
+
115778
+static void fts3EvalNext(
115779
+ Fts3Cursor *pCsr,
115780
+ Fts3Expr *pExpr,
115781
+ int *pRc
115782
+){
115783
+ if( *pRc==SQLITE_OK ){
115784
+ assert( pExpr->bEof==0 );
115785
+ pExpr->bStart = 1;
115786
+
115787
+ switch( pExpr->eType ){
115788
+ case FTSQUERY_NEAR:
115789
+ case FTSQUERY_AND: {
115790
+ Fts3Expr *pLeft = pExpr->pLeft;
115791
+ Fts3Expr *pRight = pExpr->pRight;
115792
+ assert( !pLeft->bDeferred || !pRight->bDeferred );
115793
+ if( pLeft->bDeferred ){
115794
+ fts3EvalNext(pCsr, pRight, pRc);
115795
+ pExpr->iDocid = pRight->iDocid;
115796
+ pExpr->bEof = pRight->bEof;
115797
+ }else if( pRight->bDeferred ){
115798
+ fts3EvalNext(pCsr, pLeft, pRc);
115799
+ pExpr->iDocid = pLeft->iDocid;
115800
+ pExpr->bEof = pLeft->bEof;
115801
+ }else{
115802
+ fts3EvalNext(pCsr, pLeft, pRc);
115803
+ fts3EvalNext(pCsr, pRight, pRc);
115804
+
115805
+ while( !pLeft->bEof && !pRight->bEof && *pRc==SQLITE_OK ){
115806
+ sqlite3_int64 iDiff = DOCID_CMP(pLeft->iDocid, pRight->iDocid);
115807
+ if( iDiff==0 ) break;
115808
+ if( iDiff<0 ){
115809
+ fts3EvalNext(pCsr, pLeft, pRc);
115810
+ }else{
115811
+ fts3EvalNext(pCsr, pRight, pRc);
115812
+ }
115813
+ }
115814
+
115815
+ pExpr->iDocid = pLeft->iDocid;
115816
+ pExpr->bEof = (pLeft->bEof || pRight->bEof);
115817
+ }
115818
+ break;
115819
+ }
115820
+
115821
+ case FTSQUERY_OR: {
115822
+ Fts3Expr *pLeft = pExpr->pLeft;
115823
+ Fts3Expr *pRight = pExpr->pRight;
115824
+ sqlite3_int64 iCmp = DOCID_CMP(pLeft->iDocid, pRight->iDocid);
115825
+
115826
+ assert( pLeft->bStart || pLeft->iDocid==pRight->iDocid );
115827
+ assert( pRight->bStart || pLeft->iDocid==pRight->iDocid );
115828
+
115829
+ if( pRight->bEof || (pLeft->bEof==0 && iCmp<0) ){
115830
+ fts3EvalNext(pCsr, pLeft, pRc);
115831
+ }else if( pLeft->bEof || (pRight->bEof==0 && iCmp>0) ){
115832
+ fts3EvalNext(pCsr, pRight, pRc);
115833
+ }else{
115834
+ fts3EvalNext(pCsr, pLeft, pRc);
115835
+ fts3EvalNext(pCsr, pRight, pRc);
115836
+ }
115837
+
115838
+ pExpr->bEof = (pLeft->bEof && pRight->bEof);
115839
+ iCmp = DOCID_CMP(pLeft->iDocid, pRight->iDocid);
115840
+ if( pRight->bEof || (pLeft->bEof==0 && iCmp<0) ){
115841
+ pExpr->iDocid = pLeft->iDocid;
115842
+ }else{
115843
+ pExpr->iDocid = pRight->iDocid;
115844
+ }
115845
+
115846
+ break;
115847
+ }
115848
+
115849
+ case FTSQUERY_NOT: {
115850
+ Fts3Expr *pLeft = pExpr->pLeft;
115851
+ Fts3Expr *pRight = pExpr->pRight;
115852
+
115853
+ if( pRight->bStart==0 ){
115854
+ fts3EvalNext(pCsr, pRight, pRc);
115855
+ assert( *pRc!=SQLITE_OK || pRight->bStart );
115856
+ }
115857
+
115858
+ fts3EvalNext(pCsr, pLeft, pRc);
115859
+ if( pLeft->bEof==0 ){
115860
+ while( !*pRc
115861
+ && !pRight->bEof
115862
+ && DOCID_CMP(pLeft->iDocid, pRight->iDocid)>0
115863
+ ){
115864
+ fts3EvalNext(pCsr, pRight, pRc);
115865
+ }
115866
+ }
115867
+ pExpr->iDocid = pLeft->iDocid;
115868
+ pExpr->bEof = pLeft->bEof;
115869
+ break;
115870
+ }
115871
+
115872
+ default: {
115873
+ Fts3Phrase *pPhrase = pExpr->pPhrase;
115874
+ fts3EvalZeroPoslist(pPhrase);
115875
+ *pRc = fts3EvalPhraseNext(pCsr, pPhrase, &pExpr->bEof);
115876
+ pExpr->iDocid = pPhrase->doclist.iDocid;
115877
+ break;
115878
+ }
115879
+ }
115880
+ }
115881
+}
115882
+
115883
+static int fts3EvalDeferredTest(Fts3Cursor *pCsr, Fts3Expr *pExpr, int *pRc){
115884
+ int bHit = 1;
115885
+ if( *pRc==SQLITE_OK ){
115886
+ switch( pExpr->eType ){
115887
+ case FTSQUERY_NEAR:
115888
+ case FTSQUERY_AND:
115889
+ bHit = (
115890
+ fts3EvalDeferredTest(pCsr, pExpr->pLeft, pRc)
115891
+ && fts3EvalDeferredTest(pCsr, pExpr->pRight, pRc)
115892
+ && fts3EvalNearTest(pExpr, pRc)
115893
+ );
115894
+
115895
+ /* If the NEAR expression does not match any rows, zero the doclist for
115896
+ ** all phrases involved in the NEAR. This is because the snippet(),
115897
+ ** offsets() and matchinfo() functions are not supposed to recognize
115898
+ ** any instances of phrases that are part of unmatched NEAR queries.
115899
+ ** For example if this expression:
115900
+ **
115901
+ ** ... MATCH 'a OR (b NEAR c)'
115902
+ **
115903
+ ** is matched against a row containing:
115904
+ **
115905
+ ** 'a b d e'
115906
+ **
115907
+ ** then any snippet() should ony highlight the "a" term, not the "b"
115908
+ ** (as "b" is part of a non-matching NEAR clause).
115909
+ */
115910
+ if( bHit==0
115911
+ && pExpr->eType==FTSQUERY_NEAR
115912
+ && (pExpr->pParent==0 || pExpr->pParent->eType!=FTSQUERY_NEAR)
115913
+ ){
115914
+ Fts3Expr *p;
115915
+ for(p=pExpr; p->pPhrase==0; p=p->pLeft){
115916
+ if( p->pRight->iDocid==pCsr->iPrevId ){
115917
+ fts3EvalZeroPoslist(p->pRight->pPhrase);
115918
+ }
115919
+ }
115920
+ if( p->iDocid==pCsr->iPrevId ){
115921
+ fts3EvalZeroPoslist(p->pPhrase);
115922
+ }
115923
+ }
115924
+
115925
+ break;
115926
+
115927
+ case FTSQUERY_OR: {
115928
+ int bHit1 = fts3EvalDeferredTest(pCsr, pExpr->pLeft, pRc);
115929
+ int bHit2 = fts3EvalDeferredTest(pCsr, pExpr->pRight, pRc);
115930
+ bHit = bHit1 || bHit2;
115931
+ break;
115932
+ }
115933
+
115934
+ case FTSQUERY_NOT:
115935
+ bHit = (
115936
+ fts3EvalDeferredTest(pCsr, pExpr->pLeft, pRc)
115937
+ && !fts3EvalDeferredTest(pCsr, pExpr->pRight, pRc)
115938
+ );
115939
+ break;
115940
+
115941
+ default: {
115942
+ if( pCsr->pDeferred
115943
+ && (pExpr->iDocid==pCsr->iPrevId || pExpr->bDeferred)
115944
+ ){
115945
+ Fts3Phrase *pPhrase = pExpr->pPhrase;
115946
+ assert( pExpr->bDeferred || pPhrase->doclist.bFreeList==0 );
115947
+ if( pExpr->bDeferred ){
115948
+ fts3EvalZeroPoslist(pPhrase);
115949
+ }
115950
+ *pRc = fts3EvalDeferredPhrase(pCsr, pPhrase);
115951
+ bHit = (pPhrase->doclist.pList!=0);
115952
+ pExpr->iDocid = pCsr->iPrevId;
115953
+ }else{
115954
+ bHit = (pExpr->bEof==0 && pExpr->iDocid==pCsr->iPrevId);
115955
+ }
115956
+ break;
115957
+ }
115958
+ }
115959
+ }
115960
+ return bHit;
115961
+}
115962
+
115963
+/*
115964
+** Return 1 if both of the following are true:
115965
+**
115966
+** 1. *pRc is SQLITE_OK when this function returns, and
115967
+**
115968
+** 2. After scanning the current FTS table row for the deferred tokens,
115969
+** it is determined that the row does not match the query.
115970
+**
115971
+** Or, if no error occurs and it seems the current row does match the FTS
115972
+** query, return 0.
115973
+*/
115974
+static int fts3EvalLoadDeferred(Fts3Cursor *pCsr, int *pRc){
115975
+ int rc = *pRc;
115976
+ int bMiss = 0;
115977
+ if( rc==SQLITE_OK ){
115978
+ if( pCsr->pDeferred ){
115979
+ rc = fts3CursorSeek(0, pCsr);
115980
+ if( rc==SQLITE_OK ){
115981
+ rc = sqlite3Fts3CacheDeferredDoclists(pCsr);
115982
+ }
115983
+ }
115984
+ bMiss = (0==fts3EvalDeferredTest(pCsr, pCsr->pExpr, &rc));
115985
+ sqlite3Fts3FreeDeferredDoclists(pCsr);
115986
+ *pRc = rc;
115987
+ }
115988
+ return (rc==SQLITE_OK && bMiss);
115989
+}
115990
+
115991
+/*
115992
+** Advance to the next document that matches the FTS expression in
115993
+** Fts3Cursor.pExpr.
115994
+*/
115995
+SQLITE_PRIVATE int sqlite3Fts3EvalNext(Fts3Cursor *pCsr){
115996
+ int rc = SQLITE_OK; /* Return Code */
115997
+ Fts3Expr *pExpr = pCsr->pExpr;
115998
+ assert( pCsr->isEof==0 );
115999
+ if( pExpr==0 ){
116000
+ pCsr->isEof = 1;
116001
+ }else{
116002
+ do {
116003
+ if( pCsr->isRequireSeek==0 ){
116004
+ sqlite3_reset(pCsr->pStmt);
116005
+ }
116006
+ assert( sqlite3_data_count(pCsr->pStmt)==0 );
116007
+ fts3EvalNext(pCsr, pExpr, &rc);
116008
+ pCsr->isEof = pExpr->bEof;
116009
+ pCsr->isRequireSeek = 1;
116010
+ pCsr->isMatchinfoNeeded = 1;
116011
+ pCsr->iPrevId = pExpr->iDocid;
116012
+ }while( pCsr->isEof==0 && fts3EvalLoadDeferred(pCsr, &rc) );
116013
+ }
116014
+ return rc;
116015
+}
116016
+
116017
+/*
116018
+** Restart interation for expression pExpr so that the next call to
116019
+** sqlite3Fts3EvalNext() visits the first row. Do not allow incremental
116020
+** loading or merging of phrase doclists for this iteration.
116021
+**
116022
+** If *pRc is other than SQLITE_OK when this function is called, it is
116023
+** a no-op. If an error occurs within this function, *pRc is set to an
116024
+** SQLite error code before returning.
116025
+*/
116026
+static void fts3EvalRestart(
116027
+ Fts3Cursor *pCsr,
116028
+ Fts3Expr *pExpr,
116029
+ int *pRc
116030
+){
116031
+ if( pExpr && *pRc==SQLITE_OK ){
116032
+ Fts3Phrase *pPhrase = pExpr->pPhrase;
116033
+
116034
+ if( pPhrase ){
116035
+ fts3EvalZeroPoslist(pPhrase);
116036
+ if( pPhrase->bIncr ){
116037
+ sqlite3Fts3EvalPhraseCleanup(pPhrase);
116038
+ memset(&pPhrase->doclist, 0, sizeof(Fts3Doclist));
116039
+ *pRc = sqlite3Fts3EvalStart(pCsr, pExpr, 0);
116040
+ }else{
116041
+ pPhrase->doclist.pNextDocid = 0;
116042
+ pPhrase->doclist.iDocid = 0;
116043
+ }
116044
+ }
116045
+
116046
+ pExpr->iDocid = 0;
116047
+ pExpr->bEof = 0;
116048
+ pExpr->bStart = 0;
116049
+
116050
+ fts3EvalRestart(pCsr, pExpr->pLeft, pRc);
116051
+ fts3EvalRestart(pCsr, pExpr->pRight, pRc);
116052
+ }
116053
+}
116054
+
116055
+/*
116056
+** After allocating the Fts3Expr.aMI[] array for each phrase in the
116057
+** expression rooted at pExpr, the cursor iterates through all rows matched
116058
+** by pExpr, calling this function for each row. This function increments
116059
+** the values in Fts3Expr.aMI[] according to the position-list currently
116060
+** found in Fts3Expr.pPhrase->doclist.pList for each of the phrase
116061
+** expression nodes.
116062
+*/
116063
+static void fts3EvalUpdateCounts(Fts3Expr *pExpr){
116064
+ if( pExpr ){
116065
+ Fts3Phrase *pPhrase = pExpr->pPhrase;
116066
+ if( pPhrase && pPhrase->doclist.pList ){
116067
+ int iCol = 0;
116068
+ char *p = pPhrase->doclist.pList;
116069
+
116070
+ assert( *p );
116071
+ while( 1 ){
116072
+ u8 c = 0;
116073
+ int iCnt = 0;
116074
+ while( 0xFE & (*p | c) ){
116075
+ if( (c&0x80)==0 ) iCnt++;
116076
+ c = *p++ & 0x80;
116077
+ }
116078
+
116079
+ /* aMI[iCol*3 + 1] = Number of occurrences
116080
+ ** aMI[iCol*3 + 2] = Number of rows containing at least one instance
116081
+ */
116082
+ pExpr->aMI[iCol*3 + 1] += iCnt;
116083
+ pExpr->aMI[iCol*3 + 2] += (iCnt>0);
116084
+ if( *p==0x00 ) break;
116085
+ p++;
116086
+ p += sqlite3Fts3GetVarint32(p, &iCol);
116087
+ }
116088
+ }
116089
+
116090
+ fts3EvalUpdateCounts(pExpr->pLeft);
116091
+ fts3EvalUpdateCounts(pExpr->pRight);
116092
+ }
116093
+}
116094
+
116095
+/*
116096
+** Expression pExpr must be of type FTSQUERY_PHRASE.
116097
+**
116098
+** If it is not already allocated and populated, this function allocates and
116099
+** populates the Fts3Expr.aMI[] array for expression pExpr. If pExpr is part
116100
+** of a NEAR expression, then it also allocates and populates the same array
116101
+** for all other phrases that are part of the NEAR expression.
116102
+**
116103
+** SQLITE_OK is returned if the aMI[] array is successfully allocated and
116104
+** populated. Otherwise, if an error occurs, an SQLite error code is returned.
116105
+*/
116106
+static int fts3EvalGatherStats(
116107
+ Fts3Cursor *pCsr, /* Cursor object */
116108
+ Fts3Expr *pExpr /* FTSQUERY_PHRASE expression */
116109
+){
116110
+ int rc = SQLITE_OK; /* Return code */
116111
+
116112
+ assert( pExpr->eType==FTSQUERY_PHRASE );
116113
+ if( pExpr->aMI==0 ){
116114
+ Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab;
116115
+ Fts3Expr *pRoot; /* Root of NEAR expression */
116116
+ Fts3Expr *p; /* Iterator used for several purposes */
116117
+
116118
+ sqlite3_int64 iPrevId = pCsr->iPrevId;
116119
+ sqlite3_int64 iDocid;
116120
+ u8 bEof;
116121
+
116122
+ /* Find the root of the NEAR expression */
116123
+ pRoot = pExpr;
116124
+ while( pRoot->pParent && pRoot->pParent->eType==FTSQUERY_NEAR ){
116125
+ pRoot = pRoot->pParent;
116126
+ }
116127
+ iDocid = pRoot->iDocid;
116128
+ bEof = pRoot->bEof;
116129
+ assert( pRoot->bStart );
116130
+
116131
+ /* Allocate space for the aMSI[] array of each FTSQUERY_PHRASE node */
116132
+ for(p=pRoot; p; p=p->pLeft){
116133
+ Fts3Expr *pE = (p->eType==FTSQUERY_PHRASE?p:p->pRight);
116134
+ assert( pE->aMI==0 );
116135
+ pE->aMI = (u32 *)sqlite3_malloc(pTab->nColumn * 3 * sizeof(u32));
116136
+ if( !pE->aMI ) return SQLITE_NOMEM;
116137
+ memset(pE->aMI, 0, pTab->nColumn * 3 * sizeof(u32));
116138
+ }
116139
+
116140
+ fts3EvalRestart(pCsr, pRoot, &rc);
116141
+
116142
+ while( pCsr->isEof==0 && rc==SQLITE_OK ){
116143
+
116144
+ do {
116145
+ /* Ensure the %_content statement is reset. */
116146
+ if( pCsr->isRequireSeek==0 ) sqlite3_reset(pCsr->pStmt);
116147
+ assert( sqlite3_data_count(pCsr->pStmt)==0 );
116148
+
116149
+ /* Advance to the next document */
116150
+ fts3EvalNext(pCsr, pRoot, &rc);
116151
+ pCsr->isEof = pRoot->bEof;
116152
+ pCsr->isRequireSeek = 1;
116153
+ pCsr->isMatchinfoNeeded = 1;
116154
+ pCsr->iPrevId = pRoot->iDocid;
116155
+ }while( pCsr->isEof==0
116156
+ && pRoot->eType==FTSQUERY_NEAR
116157
+ && fts3EvalLoadDeferred(pCsr, &rc)
116158
+ );
116159
+
116160
+ if( rc==SQLITE_OK && pCsr->isEof==0 ){
116161
+ fts3EvalUpdateCounts(pRoot);
116162
+ }
116163
+ }
116164
+
116165
+ pCsr->isEof = 0;
116166
+ pCsr->iPrevId = iPrevId;
116167
+
116168
+ if( bEof ){
116169
+ pRoot->bEof = bEof;
116170
+ }else{
116171
+ /* Caution: pRoot may iterate through docids in ascending or descending
116172
+ ** order. For this reason, even though it seems more defensive, the
116173
+ ** do loop can not be written:
116174
+ **
116175
+ ** do {...} while( pRoot->iDocid<iDocid && rc==SQLITE_OK );
116176
+ */
116177
+ fts3EvalRestart(pCsr, pRoot, &rc);
116178
+ do {
116179
+ fts3EvalNext(pCsr, pRoot, &rc);
116180
+ assert( pRoot->bEof==0 );
116181
+ }while( pRoot->iDocid!=iDocid && rc==SQLITE_OK );
116182
+ fts3EvalLoadDeferred(pCsr, &rc);
116183
+ }
116184
+ }
116185
+ return rc;
116186
+}
116187
+
116188
+/*
116189
+** This function is used by the matchinfo() module to query a phrase
116190
+** expression node for the following information:
116191
+**
116192
+** 1. The total number of occurrences of the phrase in each column of
116193
+** the FTS table (considering all rows), and
116194
+**
116195
+** 2. For each column, the number of rows in the table for which the
116196
+** column contains at least one instance of the phrase.
116197
+**
116198
+** If no error occurs, SQLITE_OK is returned and the values for each column
116199
+** written into the array aiOut as follows:
116200
+**
116201
+** aiOut[iCol*3 + 1] = Number of occurrences
116202
+** aiOut[iCol*3 + 2] = Number of rows containing at least one instance
116203
+**
116204
+** Caveats:
116205
+**
116206
+** * If a phrase consists entirely of deferred tokens, then all output
116207
+** values are set to the number of documents in the table. In other
116208
+** words we assume that very common tokens occur exactly once in each
116209
+** column of each row of the table.
116210
+**
116211
+** * If a phrase contains some deferred tokens (and some non-deferred
116212
+** tokens), count the potential occurrence identified by considering
116213
+** the non-deferred tokens instead of actual phrase occurrences.
116214
+**
116215
+** * If the phrase is part of a NEAR expression, then only phrase instances
116216
+** that meet the NEAR constraint are included in the counts.
116217
+*/
116218
+SQLITE_PRIVATE int sqlite3Fts3EvalPhraseStats(
116219
+ Fts3Cursor *pCsr, /* FTS cursor handle */
116220
+ Fts3Expr *pExpr, /* Phrase expression */
116221
+ u32 *aiOut /* Array to write results into (see above) */
116222
+){
116223
+ Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab;
116224
+ int rc = SQLITE_OK;
116225
+ int iCol;
116226
+
116227
+ if( pExpr->bDeferred && pExpr->pParent->eType!=FTSQUERY_NEAR ){
116228
+ assert( pCsr->nDoc>0 );
116229
+ for(iCol=0; iCol<pTab->nColumn; iCol++){
116230
+ aiOut[iCol*3 + 1] = pCsr->nDoc;
116231
+ aiOut[iCol*3 + 2] = pCsr->nDoc;
116232
+ }
116233
+ }else{
116234
+ rc = fts3EvalGatherStats(pCsr, pExpr);
116235
+ if( rc==SQLITE_OK ){
116236
+ assert( pExpr->aMI );
116237
+ for(iCol=0; iCol<pTab->nColumn; iCol++){
116238
+ aiOut[iCol*3 + 1] = pExpr->aMI[iCol*3 + 1];
116239
+ aiOut[iCol*3 + 2] = pExpr->aMI[iCol*3 + 2];
116240
+ }
116241
+ }
116242
+ }
116243
+
116244
+ return rc;
116245
+}
116246
+
116247
+/*
116248
+** The expression pExpr passed as the second argument to this function
116249
+** must be of type FTSQUERY_PHRASE.
116250
+**
116251
+** The returned value is either NULL or a pointer to a buffer containing
116252
+** a position-list indicating the occurrences of the phrase in column iCol
116253
+** of the current row.
116254
+**
116255
+** More specifically, the returned buffer contains 1 varint for each
116256
+** occurence of the phrase in the column, stored using the normal (delta+2)
116257
+** compression and is terminated by either an 0x01 or 0x00 byte. For example,
116258
+** if the requested column contains "a b X c d X X" and the position-list
116259
+** for 'X' is requested, the buffer returned may contain:
116260
+**
116261
+** 0x04 0x05 0x03 0x01 or 0x04 0x05 0x03 0x00
116262
+**
116263
+** This function works regardless of whether or not the phrase is deferred,
116264
+** incremental, or neither.
116265
+*/
116266
+SQLITE_PRIVATE char *sqlite3Fts3EvalPhrasePoslist(
116267
+ Fts3Cursor *pCsr, /* FTS3 cursor object */
116268
+ Fts3Expr *pExpr, /* Phrase to return doclist for */
116269
+ int iCol /* Column to return position list for */
116270
+){
116271
+ Fts3Phrase *pPhrase = pExpr->pPhrase;
116272
+ Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab;
116273
+ char *pIter = pPhrase->doclist.pList;
116274
+ int iThis;
116275
+
116276
+ assert( iCol>=0 && iCol<pTab->nColumn );
116277
+ if( !pIter
116278
+ || pExpr->bEof
116279
+ || pExpr->iDocid!=pCsr->iPrevId
116280
+ || (pPhrase->iColumn<pTab->nColumn && pPhrase->iColumn!=iCol)
116281
+ ){
116282
+ return 0;
116283
+ }
116284
+
116285
+ assert( pPhrase->doclist.nList>0 );
116286
+ if( *pIter==0x01 ){
116287
+ pIter++;
116288
+ pIter += sqlite3Fts3GetVarint32(pIter, &iThis);
116289
+ }else{
116290
+ iThis = 0;
116291
+ }
116292
+ while( iThis<iCol ){
116293
+ fts3ColumnlistCopy(0, &pIter);
116294
+ if( *pIter==0x00 ) return 0;
116295
+ pIter++;
116296
+ pIter += sqlite3Fts3GetVarint32(pIter, &iThis);
116297
+ }
116298
+
116299
+ return ((iCol==iThis)?pIter:0);
116300
+}
116301
+
116302
+/*
116303
+** Free all components of the Fts3Phrase structure that were allocated by
116304
+** the eval module. Specifically, this means to free:
116305
+**
116306
+** * the contents of pPhrase->doclist, and
116307
+** * any Fts3MultiSegReader objects held by phrase tokens.
116308
+*/
116309
+SQLITE_PRIVATE void sqlite3Fts3EvalPhraseCleanup(Fts3Phrase *pPhrase){
116310
+ if( pPhrase ){
116311
+ int i;
116312
+ sqlite3_free(pPhrase->doclist.aAll);
116313
+ fts3EvalZeroPoslist(pPhrase);
116314
+ memset(&pPhrase->doclist, 0, sizeof(Fts3Doclist));
116315
+ for(i=0; i<pPhrase->nToken; i++){
116316
+ fts3SegReaderCursorFree(pPhrase->aToken[i].pSegcsr);
116317
+ pPhrase->aToken[i].pSegcsr = 0;
116318
+ }
116319
+ }
116320
+}
116321
+
115622116322
#endif
115623116323
115624116324
/************** End of fts3.c ************************************************/
115625116325
/************** Begin file fts3_aux.c ****************************************/
115626116326
/*
@@ -115648,11 +116348,11 @@
115648116348
Fts3Table *pFts3Tab;
115649116349
};
115650116350
115651116351
struct Fts3auxCursor {
115652116352
sqlite3_vtab_cursor base; /* Base class used by SQLite core */
115653
- Fts3SegReaderCursor csr; /* Must be right after "base" */
116353
+ Fts3MultiSegReader csr; /* Must be right after "base" */
115654116354
Fts3SegFilter filter;
115655116355
char *zStop;
115656116356
int nStop; /* Byte-length of string zStop */
115657116357
int isEof; /* True if cursor is at EOF */
115658116358
sqlite3_int64 iRowid; /* Current rowid */
@@ -115716,10 +116416,11 @@
115716116416
115717116417
p->pFts3Tab = (Fts3Table *)&p[1];
115718116418
p->pFts3Tab->zDb = (char *)&p->pFts3Tab[1];
115719116419
p->pFts3Tab->zName = &p->pFts3Tab->zDb[nDb+1];
115720116420
p->pFts3Tab->db = db;
116421
+ p->pFts3Tab->nIndex = 1;
115721116422
115722116423
memcpy((char *)p->pFts3Tab->zDb, zDb, nDb);
115723116424
memcpy((char *)p->pFts3Tab->zName, zFts3, nFts3);
115724116425
sqlite3Fts3Dequote((char *)p->pFts3Tab->zName);
115725116426
@@ -115996,11 +116697,11 @@
115996116697
pCsr->zStop = sqlite3_mprintf("%s", sqlite3_value_text(apVal[iIdx]));
115997116698
pCsr->nStop = sqlite3_value_bytes(apVal[iIdx]);
115998116699
if( pCsr->zStop==0 ) return SQLITE_NOMEM;
115999116700
}
116000116701
116001
- rc = sqlite3Fts3SegReaderCursor(pFts3, FTS3_SEGCURSOR_ALL,
116702
+ rc = sqlite3Fts3SegReaderCursor(pFts3, 0, FTS3_SEGCURSOR_ALL,
116002116703
pCsr->filter.zTerm, pCsr->filter.nTerm, 0, isScan, &pCsr->csr
116003116704
);
116004116705
if( rc==SQLITE_OK ){
116005116706
rc = sqlite3Fts3SegReaderStart(pFts3, &pCsr->csr, &pCsr->filter);
116006116707
}
@@ -116175,16 +116876,25 @@
116175116876
** Default span for NEAR operators.
116176116877
*/
116177116878
#define SQLITE_FTS3_DEFAULT_NEAR_PARAM 10
116178116879
116179116880
116881
+/*
116882
+** isNot:
116883
+** This variable is used by function getNextNode(). When getNextNode() is
116884
+** called, it sets ParseContext.isNot to true if the 'next node' is a
116885
+** FTSQUERY_PHRASE with a unary "-" attached to it. i.e. "mysql" in the
116886
+** FTS3 query "sqlite -mysql". Otherwise, ParseContext.isNot is set to
116887
+** zero.
116888
+*/
116180116889
typedef struct ParseContext ParseContext;
116181116890
struct ParseContext {
116182116891
sqlite3_tokenizer *pTokenizer; /* Tokenizer module */
116183116892
const char **azCol; /* Array of column names for fts3 table */
116184116893
int nCol; /* Number of entries in azCol[] */
116185116894
int iDefaultCol; /* Default column to query */
116895
+ int isNot; /* True if getNextNode() sees a unary - */
116186116896
sqlite3_context *pCtx; /* Write error message here */
116187116897
int nNest; /* Number of nested brackets */
116188116898
};
116189116899
116190116900
/*
@@ -116266,11 +116976,11 @@
116266116976
if( iEnd<n && z[iEnd]=='*' ){
116267116977
pRet->pPhrase->aToken[0].isPrefix = 1;
116268116978
iEnd++;
116269116979
}
116270116980
if( !sqlite3_fts3_enable_parentheses && iStart>0 && z[iStart-1]=='-' ){
116271
- pRet->pPhrase->isNot = 1;
116981
+ pParse->isNot = 1;
116272116982
}
116273116983
}
116274116984
nConsumed = iEnd;
116275116985
}
116276116986
@@ -116318,71 +117028,86 @@
116318117028
Fts3Expr *p = 0;
116319117029
sqlite3_tokenizer_cursor *pCursor = 0;
116320117030
char *zTemp = 0;
116321117031
int nTemp = 0;
116322117032
117033
+ const int nSpace = sizeof(Fts3Expr) + sizeof(Fts3Phrase);
117034
+ int nToken = 0;
117035
+
117036
+ /* The final Fts3Expr data structure, including the Fts3Phrase,
117037
+ ** Fts3PhraseToken structures token buffers are all stored as a single
117038
+ ** allocation so that the expression can be freed with a single call to
117039
+ ** sqlite3_free(). Setting this up requires a two pass approach.
117040
+ **
117041
+ ** The first pass, in the block below, uses a tokenizer cursor to iterate
117042
+ ** through the tokens in the expression. This pass uses fts3ReallocOrFree()
117043
+ ** to assemble data in two dynamic buffers:
117044
+ **
117045
+ ** Buffer p: Points to the Fts3Expr structure, followed by the Fts3Phrase
117046
+ ** structure, followed by the array of Fts3PhraseToken
117047
+ ** structures. This pass only populates the Fts3PhraseToken array.
117048
+ **
117049
+ ** Buffer zTemp: Contains copies of all tokens.
117050
+ **
117051
+ ** The second pass, in the block that begins "if( rc==SQLITE_DONE )" below,
117052
+ ** appends buffer zTemp to buffer p, and fills in the Fts3Expr and Fts3Phrase
117053
+ ** structures.
117054
+ */
116323117055
rc = pModule->xOpen(pTokenizer, zInput, nInput, &pCursor);
116324117056
if( rc==SQLITE_OK ){
116325117057
int ii;
116326117058
pCursor->pTokenizer = pTokenizer;
116327117059
for(ii=0; rc==SQLITE_OK; ii++){
116328
- const char *zToken;
116329
- int nToken, iBegin, iEnd, iPos;
116330
- rc = pModule->xNext(pCursor, &zToken, &nToken, &iBegin, &iEnd, &iPos);
117060
+ const char *zByte;
117061
+ int nByte, iBegin, iEnd, iPos;
117062
+ rc = pModule->xNext(pCursor, &zByte, &nByte, &iBegin, &iEnd, &iPos);
116331117063
if( rc==SQLITE_OK ){
116332
- int nByte = sizeof(Fts3Expr) + sizeof(Fts3Phrase);
116333
- p = fts3ReallocOrFree(p, nByte+ii*sizeof(Fts3PhraseToken));
116334
- zTemp = fts3ReallocOrFree(zTemp, nTemp + nToken);
116335
- if( !p || !zTemp ){
116336
- goto no_mem;
116337
- }
116338
- if( ii==0 ){
116339
- memset(p, 0, nByte);
116340
- p->pPhrase = (Fts3Phrase *)&p[1];
116341
- }
116342
- p->pPhrase = (Fts3Phrase *)&p[1];
116343
- memset(&p->pPhrase->aToken[ii], 0, sizeof(Fts3PhraseToken));
116344
- p->pPhrase->nToken = ii+1;
116345
- p->pPhrase->aToken[ii].n = nToken;
116346
- memcpy(&zTemp[nTemp], zToken, nToken);
116347
- nTemp += nToken;
116348
- if( iEnd<nInput && zInput[iEnd]=='*' ){
116349
- p->pPhrase->aToken[ii].isPrefix = 1;
116350
- }else{
116351
- p->pPhrase->aToken[ii].isPrefix = 0;
116352
- }
117064
+ Fts3PhraseToken *pToken;
117065
+
117066
+ p = fts3ReallocOrFree(p, nSpace + ii*sizeof(Fts3PhraseToken));
117067
+ if( !p ) goto no_mem;
117068
+
117069
+ zTemp = fts3ReallocOrFree(zTemp, nTemp + nByte);
117070
+ if( !zTemp ) goto no_mem;
117071
+
117072
+ assert( nToken==ii );
117073
+ pToken = &((Fts3Phrase *)(&p[1]))->aToken[ii];
117074
+ memset(pToken, 0, sizeof(Fts3PhraseToken));
117075
+
117076
+ memcpy(&zTemp[nTemp], zByte, nByte);
117077
+ nTemp += nByte;
117078
+
117079
+ pToken->n = nByte;
117080
+ pToken->isPrefix = (iEnd<nInput && zInput[iEnd]=='*');
117081
+ nToken = ii+1;
116353117082
}
116354117083
}
116355117084
116356117085
pModule->xClose(pCursor);
116357117086
pCursor = 0;
116358117087
}
116359117088
116360117089
if( rc==SQLITE_DONE ){
116361117090
int jj;
116362
- char *zNew = NULL;
116363
- int nNew = 0;
116364
- int nByte = sizeof(Fts3Expr) + sizeof(Fts3Phrase);
116365
- nByte += (p?(p->pPhrase->nToken-1):0) * sizeof(Fts3PhraseToken);
116366
- p = fts3ReallocOrFree(p, nByte + nTemp);
116367
- if( !p ){
116368
- goto no_mem;
116369
- }
116370
- if( zTemp ){
116371
- zNew = &(((char *)p)[nByte]);
116372
- memcpy(zNew, zTemp, nTemp);
116373
- }else{
116374
- memset(p, 0, nByte+nTemp);
116375
- }
117091
+ char *zBuf = 0;
117092
+
117093
+ p = fts3ReallocOrFree(p, nSpace + nToken*sizeof(Fts3PhraseToken) + nTemp);
117094
+ if( !p ) goto no_mem;
117095
+ memset(p, 0, (char *)&(((Fts3Phrase *)&p[1])->aToken[0])-(char *)p);
117096
+ p->eType = FTSQUERY_PHRASE;
116376117097
p->pPhrase = (Fts3Phrase *)&p[1];
117098
+ p->pPhrase->iColumn = pParse->iDefaultCol;
117099
+ p->pPhrase->nToken = nToken;
117100
+
117101
+ zBuf = (char *)&p->pPhrase->aToken[nToken];
117102
+ memcpy(zBuf, zTemp, nTemp);
117103
+ sqlite3_free(zTemp);
117104
+
116377117105
for(jj=0; jj<p->pPhrase->nToken; jj++){
116378
- p->pPhrase->aToken[jj].z = &zNew[nNew];
116379
- nNew += p->pPhrase->aToken[jj].n;
117106
+ p->pPhrase->aToken[jj].z = zBuf;
117107
+ zBuf += p->pPhrase->aToken[jj].n;
116380117108
}
116381
- sqlite3_free(zTemp);
116382
- p->eType = FTSQUERY_PHRASE;
116383
- p->pPhrase->iColumn = pParse->iDefaultCol;
116384117109
rc = SQLITE_OK;
116385117110
}
116386117111
116387117112
*ppExpr = p;
116388117113
return rc;
@@ -116434,10 +117159,12 @@
116434117159
int rc;
116435117160
Fts3Expr *pRet = 0;
116436117161
116437117162
const char *zInput = z;
116438117163
int nInput = n;
117164
+
117165
+ pParse->isNot = 0;
116439117166
116440117167
/* Skip over any whitespace before checking for a keyword, an open or
116441117168
** close bracket, or a quoted string.
116442117169
*/
116443117170
while( nInput>0 && fts3isspace(*zInput) ){
@@ -116653,11 +117380,11 @@
116653117380
rc = getNextNode(pParse, zIn, nIn, &p, &nByte);
116654117381
if( rc==SQLITE_OK ){
116655117382
int isPhrase;
116656117383
116657117384
if( !sqlite3_fts3_enable_parentheses
116658
- && p->eType==FTSQUERY_PHRASE && p->pPhrase->isNot
117385
+ && p->eType==FTSQUERY_PHRASE && pParse->isNot
116659117386
){
116660117387
/* Create an implicit NOT operator. */
116661117388
Fts3Expr *pNot = fts3MallocZero(sizeof(Fts3Expr));
116662117389
if( !pNot ){
116663117390
sqlite3Fts3ExprFree(p);
@@ -116671,11 +117398,10 @@
116671117398
}
116672117399
pNotBranch = pNot;
116673117400
p = pPrev;
116674117401
}else{
116675117402
int eType = p->eType;
116676
- assert( eType!=FTSQUERY_PHRASE || !p->pPhrase->isNot );
116677117403
isPhrase = (eType==FTSQUERY_PHRASE || p->pLeft);
116678117404
116679117405
/* The isRequirePhrase variable is set to true if a phrase or
116680117406
** an expression contained in parenthesis is required. If a
116681117407
** binary operator (AND, OR, NOT or NEAR) is encounted when
@@ -116834,13 +117560,15 @@
116834117560
/*
116835117561
** Free a parsed fts3 query expression allocated by sqlite3Fts3ExprParse().
116836117562
*/
116837117563
SQLITE_PRIVATE void sqlite3Fts3ExprFree(Fts3Expr *p){
116838117564
if( p ){
117565
+ assert( p->eType==FTSQUERY_PHRASE || p->pPhrase==0 );
116839117566
sqlite3Fts3ExprFree(p->pLeft);
116840117567
sqlite3Fts3ExprFree(p->pRight);
116841
- sqlite3_free(p->aDoclist);
117568
+ sqlite3Fts3EvalPhraseCleanup(p->pPhrase);
117569
+ sqlite3_free(p->aMI);
116842117570
sqlite3_free(p);
116843117571
}
116844117572
}
116845117573
116846117574
/****************************************************************************
@@ -116893,11 +117621,11 @@
116893117621
switch( pExpr->eType ){
116894117622
case FTSQUERY_PHRASE: {
116895117623
Fts3Phrase *pPhrase = pExpr->pPhrase;
116896117624
int i;
116897117625
zBuf = sqlite3_mprintf(
116898
- "%zPHRASE %d %d", zBuf, pPhrase->iColumn, pPhrase->isNot);
117626
+ "%zPHRASE %d 0", zBuf, pPhrase->iColumn);
116899117627
for(i=0; zBuf && i<pPhrase->nToken; i++){
116900117628
zBuf = sqlite3_mprintf("%z %.*s%s", zBuf,
116901117629
pPhrase->aToken[i].n, pPhrase->aToken[i].z,
116902117630
(pPhrase->aToken[i].isPrefix?"+":"")
116903117631
);
@@ -118811,18 +119539,44 @@
118811119539
** it is always safe to read up to two varints from it without risking an
118812119540
** overread, even if the node data is corrupted.
118813119541
*/
118814119542
#define FTS3_NODE_PADDING (FTS3_VARINT_MAX*2)
118815119543
119544
+/*
119545
+** Under certain circumstances, b-tree nodes (doclists) can be loaded into
119546
+** memory incrementally instead of all at once. This can be a big performance
119547
+** win (reduced IO and CPU) if SQLite stops calling the virtual table xNext()
119548
+** method before retrieving all query results (as may happen, for example,
119549
+** if a query has a LIMIT clause).
119550
+**
119551
+** Incremental loading is used for b-tree nodes FTS3_NODE_CHUNK_THRESHOLD
119552
+** bytes and larger. Nodes are loaded in chunks of FTS3_NODE_CHUNKSIZE bytes.
119553
+** The code is written so that the hard lower-limit for each of these values
119554
+** is 1. Clearly such small values would be inefficient, but can be useful
119555
+** for testing purposes.
119556
+**
119557
+** If this module is built with SQLITE_TEST defined, these constants may
119558
+** be overridden at runtime for testing purposes. File fts3_test.c contains
119559
+** a Tcl interface to read and write the values.
119560
+*/
119561
+#ifdef SQLITE_TEST
119562
+int test_fts3_node_chunksize = (4*1024);
119563
+int test_fts3_node_chunk_threshold = (4*1024)*4;
119564
+# define FTS3_NODE_CHUNKSIZE test_fts3_node_chunksize
119565
+# define FTS3_NODE_CHUNK_THRESHOLD test_fts3_node_chunk_threshold
119566
+#else
119567
+# define FTS3_NODE_CHUNKSIZE (4*1024)
119568
+# define FTS3_NODE_CHUNK_THRESHOLD (FTS3_NODE_CHUNKSIZE*4)
119569
+#endif
119570
+
118816119571
typedef struct PendingList PendingList;
118817119572
typedef struct SegmentNode SegmentNode;
118818119573
typedef struct SegmentWriter SegmentWriter;
118819119574
118820119575
/*
118821
-** Data structure used while accumulating terms in the pending-terms hash
118822
-** table. The hash table entry maps from term (a string) to a malloc'd
118823
-** instance of this structure.
119576
+** An instance of the following data structure is used to build doclists
119577
+** incrementally. See function fts3PendingListAppend() for details.
118824119578
*/
118825119579
struct PendingList {
118826119580
int nData;
118827119581
char *aData;
118828119582
int nSpace;
@@ -118849,11 +119603,10 @@
118849119603
** of type Fts3SegReader* are also used by code in fts3.c to iterate through
118850119604
** terms when querying the full-text index. See functions:
118851119605
**
118852119606
** sqlite3Fts3SegReaderNew()
118853119607
** sqlite3Fts3SegReaderFree()
118854
-** sqlite3Fts3SegReaderCost()
118855119608
** sqlite3Fts3SegReaderIterate()
118856119609
**
118857119610
** Methods used to manipulate Fts3SegReader structures:
118858119611
**
118859119612
** fts3SegReaderNext()
@@ -118868,10 +119621,13 @@
118868119621
sqlite3_int64 iEndBlock; /* Rowid of final block in segment (or 0) */
118869119622
sqlite3_int64 iCurrentBlock; /* Current leaf block (or 0) */
118870119623
118871119624
char *aNode; /* Pointer to node data (or NULL) */
118872119625
int nNode; /* Size of buffer at aNode (or 0) */
119626
+ int nPopulate; /* If >0, bytes of buffer aNode[] loaded */
119627
+ sqlite3_blob *pBlob; /* If not NULL, blob handle to read node */
119628
+
118873119629
Fts3HashElem **ppNextElem;
118874119630
118875119631
/* Variables set by fts3SegReaderNext(). These may be read directly
118876119632
** by the caller. They are valid from the time SegmentReaderNew() returns
118877119633
** until SegmentReaderNext() returns something other than SQLITE_OK
@@ -118881,12 +119637,15 @@
118881119637
char *zTerm; /* Pointer to current term */
118882119638
int nTermAlloc; /* Allocated size of zTerm buffer */
118883119639
char *aDoclist; /* Pointer to doclist of current entry */
118884119640
int nDoclist; /* Size of doclist in current entry */
118885119641
118886
- /* The following variables are used to iterate through the current doclist */
119642
+ /* The following variables are used by fts3SegReaderNextDocid() to iterate
119643
+ ** through the current doclist (aDoclist/nDoclist).
119644
+ */
118887119645
char *pOffsetList;
119646
+ int nOffsetList; /* For descending pending seg-readers only */
118888119647
sqlite3_int64 iDocid;
118889119648
};
118890119649
118891119650
#define fts3SegReaderIsPending(p) ((p)->ppNextElem!=0)
118892119651
#define fts3SegReaderIsRootOnly(p) ((p)->aNode==(char *)&(p)[1])
@@ -118920,10 +119679,18 @@
118920119679
** within the fts3SegWriterXXX() family of functions described above.
118921119680
**
118922119681
** fts3NodeAddTerm()
118923119682
** fts3NodeWrite()
118924119683
** fts3NodeFree()
119684
+**
119685
+** When a b+tree is written to the database (either as a result of a merge
119686
+** or the pending-terms table being flushed), leaves are written into the
119687
+** database file as soon as they are completely populated. The interior of
119688
+** the tree is assembled in memory and written out only once all leaves have
119689
+** been populated and stored. This is Ok, as the b+-tree fanout is usually
119690
+** very large, meaning that the interior of the tree consumes relatively
119691
+** little memory.
118925119692
*/
118926119693
struct SegmentNode {
118927119694
SegmentNode *pParent; /* Parent node (or NULL for root node) */
118928119695
SegmentNode *pRight; /* Pointer to right-sibling */
118929119696
SegmentNode *pLeftmost; /* Pointer to left-most node of this depth */
@@ -118950,21 +119717,26 @@
118950119717
#define SQL_NEXT_SEGMENT_INDEX 8
118951119718
#define SQL_INSERT_SEGMENTS 9
118952119719
#define SQL_NEXT_SEGMENTS_ID 10
118953119720
#define SQL_INSERT_SEGDIR 11
118954119721
#define SQL_SELECT_LEVEL 12
118955
-#define SQL_SELECT_ALL_LEVEL 13
119722
+#define SQL_SELECT_LEVEL_RANGE 13
118956119723
#define SQL_SELECT_LEVEL_COUNT 14
118957
-#define SQL_SELECT_SEGDIR_COUNT_MAX 15
118958
-#define SQL_DELETE_SEGDIR_BY_LEVEL 16
119724
+#define SQL_SELECT_SEGDIR_MAX_LEVEL 15
119725
+#define SQL_DELETE_SEGDIR_LEVEL 16
118959119726
#define SQL_DELETE_SEGMENTS_RANGE 17
118960119727
#define SQL_CONTENT_INSERT 18
118961119728
#define SQL_DELETE_DOCSIZE 19
118962119729
#define SQL_REPLACE_DOCSIZE 20
118963119730
#define SQL_SELECT_DOCSIZE 21
118964119731
#define SQL_SELECT_DOCTOTAL 22
118965119732
#define SQL_REPLACE_DOCTOTAL 23
119733
+
119734
+#define SQL_SELECT_ALL_PREFIX_LEVEL 24
119735
+#define SQL_DELETE_ALL_TERMS_SEGDIR 25
119736
+
119737
+#define SQL_DELETE_SEGDIR_RANGE 26
118966119738
118967119739
/*
118968119740
** This function is used to obtain an SQLite prepared statement handle
118969119741
** for the statement identified by the second argument. If successful,
118970119742
** *pp is set to the requested statement handle and SQLITE_OK returned.
@@ -118997,23 +119769,29 @@
118997119769
118998119770
/* Return segments in order from oldest to newest.*/
118999119771
/* 12 */ "SELECT idx, start_block, leaves_end_block, end_block, root "
119000119772
"FROM %Q.'%q_segdir' WHERE level = ? ORDER BY idx ASC",
119001119773
/* 13 */ "SELECT idx, start_block, leaves_end_block, end_block, root "
119002
- "FROM %Q.'%q_segdir' ORDER BY level DESC, idx ASC",
119774
+ "FROM %Q.'%q_segdir' WHERE level BETWEEN ? AND ?"
119775
+ "ORDER BY level DESC, idx ASC",
119003119776
119004119777
/* 14 */ "SELECT count(*) FROM %Q.'%q_segdir' WHERE level = ?",
119005
-/* 15 */ "SELECT count(*), max(level) FROM %Q.'%q_segdir'",
119778
+/* 15 */ "SELECT max(level) FROM %Q.'%q_segdir' WHERE level BETWEEN ? AND ?",
119006119779
119007119780
/* 16 */ "DELETE FROM %Q.'%q_segdir' WHERE level = ?",
119008119781
/* 17 */ "DELETE FROM %Q.'%q_segments' WHERE blockid BETWEEN ? AND ?",
119009119782
/* 18 */ "INSERT INTO %Q.'%q_content' VALUES(%s)",
119010119783
/* 19 */ "DELETE FROM %Q.'%q_docsize' WHERE docid = ?",
119011119784
/* 20 */ "REPLACE INTO %Q.'%q_docsize' VALUES(?,?)",
119012119785
/* 21 */ "SELECT size FROM %Q.'%q_docsize' WHERE docid=?",
119013119786
/* 22 */ "SELECT value FROM %Q.'%q_stat' WHERE id=0",
119014119787
/* 23 */ "REPLACE INTO %Q.'%q_stat' VALUES(0,?)",
119788
+/* 24 */ "",
119789
+/* 25 */ "",
119790
+
119791
+/* 26 */ "DELETE FROM %Q.'%q_segdir' WHERE level BETWEEN ? AND ?",
119792
+
119015119793
};
119016119794
int rc = SQLITE_OK;
119017119795
sqlite3_stmt *pStmt;
119018119796
119019119797
assert( SizeofArray(azSql)==SizeofArray(p->aStmt) );
@@ -119165,18 +119943,36 @@
119165119943
** 1: start_block
119166119944
** 2: leaves_end_block
119167119945
** 3: end_block
119168119946
** 4: root
119169119947
*/
119170
-SQLITE_PRIVATE int sqlite3Fts3AllSegdirs(Fts3Table *p, int iLevel, sqlite3_stmt **ppStmt){
119948
+SQLITE_PRIVATE int sqlite3Fts3AllSegdirs(
119949
+ Fts3Table *p, /* FTS3 table */
119950
+ int iIndex, /* Index for p->aIndex[] */
119951
+ int iLevel, /* Level to select */
119952
+ sqlite3_stmt **ppStmt /* OUT: Compiled statement */
119953
+){
119171119954
int rc;
119172119955
sqlite3_stmt *pStmt = 0;
119956
+
119957
+ assert( iLevel==FTS3_SEGCURSOR_ALL || iLevel>=0 );
119958
+ assert( iLevel<FTS3_SEGDIR_MAXLEVEL );
119959
+ assert( iIndex>=0 && iIndex<p->nIndex );
119960
+
119173119961
if( iLevel<0 ){
119174
- rc = fts3SqlStmt(p, SQL_SELECT_ALL_LEVEL, &pStmt, 0);
119962
+ /* "SELECT * FROM %_segdir WHERE level BETWEEN ? AND ? ORDER BY ..." */
119963
+ rc = fts3SqlStmt(p, SQL_SELECT_LEVEL_RANGE, &pStmt, 0);
119964
+ if( rc==SQLITE_OK ){
119965
+ sqlite3_bind_int(pStmt, 1, iIndex*FTS3_SEGDIR_MAXLEVEL);
119966
+ sqlite3_bind_int(pStmt, 2, (iIndex+1)*FTS3_SEGDIR_MAXLEVEL-1);
119967
+ }
119175119968
}else{
119969
+ /* "SELECT * FROM %_segdir WHERE level = ? ORDER BY ..." */
119176119970
rc = fts3SqlStmt(p, SQL_SELECT_LEVEL, &pStmt, 0);
119177
- if( rc==SQLITE_OK ) sqlite3_bind_int(pStmt, 1, iLevel);
119971
+ if( rc==SQLITE_OK ){
119972
+ sqlite3_bind_int(pStmt, 1, iLevel+iIndex*FTS3_SEGDIR_MAXLEVEL);
119973
+ }
119178119974
}
119179119975
*ppStmt = pStmt;
119180119976
return rc;
119181119977
}
119182119978
@@ -119286,10 +120082,51 @@
119286120082
*pp = p;
119287120083
return 1;
119288120084
}
119289120085
return 0;
119290120086
}
120087
+
120088
+/*
120089
+** Free a PendingList object allocated by fts3PendingListAppend().
120090
+*/
120091
+static void fts3PendingListDelete(PendingList *pList){
120092
+ sqlite3_free(pList);
120093
+}
120094
+
120095
+/*
120096
+** Add an entry to one of the pending-terms hash tables.
120097
+*/
120098
+static int fts3PendingTermsAddOne(
120099
+ Fts3Table *p,
120100
+ int iCol,
120101
+ int iPos,
120102
+ Fts3Hash *pHash, /* Pending terms hash table to add entry to */
120103
+ const char *zToken,
120104
+ int nToken
120105
+){
120106
+ PendingList *pList;
120107
+ int rc = SQLITE_OK;
120108
+
120109
+ pList = (PendingList *)fts3HashFind(pHash, zToken, nToken);
120110
+ if( pList ){
120111
+ p->nPendingData -= (pList->nData + nToken + sizeof(Fts3HashElem));
120112
+ }
120113
+ if( fts3PendingListAppend(&pList, p->iPrevDocid, iCol, iPos, &rc) ){
120114
+ if( pList==fts3HashInsert(pHash, zToken, nToken, pList) ){
120115
+ /* Malloc failed while inserting the new entry. This can only
120116
+ ** happen if there was no previous entry for this token.
120117
+ */
120118
+ assert( 0==fts3HashFind(pHash, zToken, nToken) );
120119
+ sqlite3_free(pList);
120120
+ rc = SQLITE_NOMEM;
120121
+ }
120122
+ }
120123
+ if( rc==SQLITE_OK ){
120124
+ p->nPendingData += (pList->nData + nToken + sizeof(Fts3HashElem));
120125
+ }
120126
+ return rc;
120127
+}
119291120128
119292120129
/*
119293120130
** Tokenize the nul-terminated string zText and add all tokens to the
119294120131
** pending-terms hash-table. The docid used is that currently stored in
119295120132
** p->iPrevDocid, and the column is specified by argument iCol.
@@ -119335,12 +120172,11 @@
119335120172
119336120173
xNext = pModule->xNext;
119337120174
while( SQLITE_OK==rc
119338120175
&& SQLITE_OK==(rc = xNext(pCsr, &zToken, &nToken, &iStart, &iEnd, &iPos))
119339120176
){
119340
- PendingList *pList;
119341
-
120177
+ int i;
119342120178
if( iPos>=nWord ) nWord = iPos+1;
119343120179
119344120180
/* Positions cannot be negative; we use -1 as a terminator internally.
119345120181
** Tokens must have a non-zero length.
119346120182
*/
@@ -119347,26 +120183,23 @@
119347120183
if( iPos<0 || !zToken || nToken<=0 ){
119348120184
rc = SQLITE_ERROR;
119349120185
break;
119350120186
}
119351120187
119352
- pList = (PendingList *)fts3HashFind(&p->pendingTerms, zToken, nToken);
119353
- if( pList ){
119354
- p->nPendingData -= (pList->nData + nToken + sizeof(Fts3HashElem));
119355
- }
119356
- if( fts3PendingListAppend(&pList, p->iPrevDocid, iCol, iPos, &rc) ){
119357
- if( pList==fts3HashInsert(&p->pendingTerms, zToken, nToken, pList) ){
119358
- /* Malloc failed while inserting the new entry. This can only
119359
- ** happen if there was no previous entry for this token.
119360
- */
119361
- assert( 0==fts3HashFind(&p->pendingTerms, zToken, nToken) );
119362
- sqlite3_free(pList);
119363
- rc = SQLITE_NOMEM;
119364
- }
119365
- }
119366
- if( rc==SQLITE_OK ){
119367
- p->nPendingData += (pList->nData + nToken + sizeof(Fts3HashElem));
120188
+ /* Add the term to the terms index */
120189
+ rc = fts3PendingTermsAddOne(
120190
+ p, iCol, iPos, &p->aIndex[0].hPending, zToken, nToken
120191
+ );
120192
+
120193
+ /* Add the term to each of the prefix indexes that it is not too
120194
+ ** short for. */
120195
+ for(i=1; rc==SQLITE_OK && i<p->nIndex; i++){
120196
+ struct Fts3Index *pIndex = &p->aIndex[i];
120197
+ if( nToken<pIndex->nPrefix ) continue;
120198
+ rc = fts3PendingTermsAddOne(
120199
+ p, iCol, iPos, &pIndex->hPending, zToken, pIndex->nPrefix
120200
+ );
119368120201
}
119369120202
}
119370120203
119371120204
pModule->xClose(pCsr);
119372120205
*pnWord = nWord;
@@ -119392,18 +120225,23 @@
119392120225
p->iPrevDocid = iDocid;
119393120226
return SQLITE_OK;
119394120227
}
119395120228
119396120229
/*
119397
-** Discard the contents of the pending-terms hash table.
120230
+** Discard the contents of the pending-terms hash tables.
119398120231
*/
119399120232
SQLITE_PRIVATE void sqlite3Fts3PendingTermsClear(Fts3Table *p){
119400
- Fts3HashElem *pElem;
119401
- for(pElem=fts3HashFirst(&p->pendingTerms); pElem; pElem=fts3HashNext(pElem)){
119402
- sqlite3_free(fts3HashData(pElem));
120233
+ int i;
120234
+ for(i=0; i<p->nIndex; i++){
120235
+ Fts3HashElem *pElem;
120236
+ Fts3Hash *pHash = &p->aIndex[i].hPending;
120237
+ for(pElem=fts3HashFirst(pHash); pElem; pElem=fts3HashNext(pElem)){
120238
+ PendingList *pList = (PendingList *)fts3HashData(pElem);
120239
+ fts3PendingListDelete(pList);
120240
+ }
120241
+ fts3HashClear(pHash);
119403120242
}
119404
- fts3HashClear(&p->pendingTerms);
119405120243
p->nPendingData = 0;
119406120244
}
119407120245
119408120246
/*
119409120247
** This function is called by the xUpdate() method as part of an INSERT
@@ -119555,11 +120393,11 @@
119555120393
119556120394
/*
119557120395
** Forward declaration to account for the circular dependency between
119558120396
** functions fts3SegmentMerge() and fts3AllocateSegdirIdx().
119559120397
*/
119560
-static int fts3SegmentMerge(Fts3Table *, int);
120398
+static int fts3SegmentMerge(Fts3Table *, int, int);
119561120399
119562120400
/*
119563120401
** This function allocates a new level iLevel index in the segdir table.
119564120402
** Usually, indexes are allocated within a level sequentially starting
119565120403
** with 0, so the allocated index is one greater than the value returned
@@ -119572,19 +120410,24 @@
119572120410
** allocated index is 0.
119573120411
**
119574120412
** If successful, *piIdx is set to the allocated index slot and SQLITE_OK
119575120413
** returned. Otherwise, an SQLite error code is returned.
119576120414
*/
119577
-static int fts3AllocateSegdirIdx(Fts3Table *p, int iLevel, int *piIdx){
120415
+static int fts3AllocateSegdirIdx(
120416
+ Fts3Table *p,
120417
+ int iIndex, /* Index for p->aIndex */
120418
+ int iLevel,
120419
+ int *piIdx
120420
+){
119578120421
int rc; /* Return Code */
119579120422
sqlite3_stmt *pNextIdx; /* Query for next idx at level iLevel */
119580120423
int iNext = 0; /* Result of query pNextIdx */
119581120424
119582120425
/* Set variable iNext to the next available segdir index at level iLevel. */
119583120426
rc = fts3SqlStmt(p, SQL_NEXT_SEGMENT_INDEX, &pNextIdx, 0);
119584120427
if( rc==SQLITE_OK ){
119585
- sqlite3_bind_int(pNextIdx, 1, iLevel);
120428
+ sqlite3_bind_int(pNextIdx, 1, iIndex*FTS3_SEGDIR_MAXLEVEL + iLevel);
119586120429
if( SQLITE_ROW==sqlite3_step(pNextIdx) ){
119587120430
iNext = sqlite3_column_int(pNextIdx, 0);
119588120431
}
119589120432
rc = sqlite3_reset(pNextIdx);
119590120433
}
@@ -119594,11 +120437,11 @@
119594120437
** full, merge all segments in level iLevel into a single iLevel+1
119595120438
** segment and allocate (newly freed) index 0 at level iLevel. Otherwise,
119596120439
** if iNext is less than FTS3_MERGE_COUNT, allocate index iNext.
119597120440
*/
119598120441
if( iNext>=FTS3_MERGE_COUNT ){
119599
- rc = fts3SegmentMerge(p, iLevel);
120442
+ rc = fts3SegmentMerge(p, iIndex, iLevel);
119600120443
*piIdx = 0;
119601120444
}else{
119602120445
*piIdx = iNext;
119603120446
}
119604120447
}
@@ -119635,11 +120478,12 @@
119635120478
*/
119636120479
SQLITE_PRIVATE int sqlite3Fts3ReadBlock(
119637120480
Fts3Table *p, /* FTS3 table handle */
119638120481
sqlite3_int64 iBlockid, /* Access the row with blockid=$iBlockid */
119639120482
char **paBlob, /* OUT: Blob data in malloc'd buffer */
119640
- int *pnBlob /* OUT: Size of blob data */
120483
+ int *pnBlob, /* OUT: Size of blob data */
120484
+ int *pnLoad /* OUT: Bytes actually loaded */
119641120485
){
119642120486
int rc; /* Return code */
119643120487
119644120488
/* pnBlob must be non-NULL. paBlob may be NULL or non-NULL. */
119645120489
assert( pnBlob);
@@ -119656,25 +120500,29 @@
119656120500
);
119657120501
}
119658120502
119659120503
if( rc==SQLITE_OK ){
119660120504
int nByte = sqlite3_blob_bytes(p->pSegments);
120505
+ *pnBlob = nByte;
119661120506
if( paBlob ){
119662120507
char *aByte = sqlite3_malloc(nByte + FTS3_NODE_PADDING);
119663120508
if( !aByte ){
119664120509
rc = SQLITE_NOMEM;
119665120510
}else{
120511
+ if( pnLoad && nByte>(FTS3_NODE_CHUNK_THRESHOLD) ){
120512
+ nByte = FTS3_NODE_CHUNKSIZE;
120513
+ *pnLoad = nByte;
120514
+ }
119666120515
rc = sqlite3_blob_read(p->pSegments, aByte, nByte, 0);
119667120516
memset(&aByte[nByte], 0, FTS3_NODE_PADDING);
119668120517
if( rc!=SQLITE_OK ){
119669120518
sqlite3_free(aByte);
119670120519
aByte = 0;
119671120520
}
119672120521
}
119673120522
*paBlob = aByte;
119674120523
}
119675
- *pnBlob = nByte;
119676120524
}
119677120525
119678120526
return rc;
119679120527
}
119680120528
@@ -119684,17 +120532,59 @@
119684120532
*/
119685120533
SQLITE_PRIVATE void sqlite3Fts3SegmentsClose(Fts3Table *p){
119686120534
sqlite3_blob_close(p->pSegments);
119687120535
p->pSegments = 0;
119688120536
}
120537
+
120538
+static int fts3SegReaderIncrRead(Fts3SegReader *pReader){
120539
+ int nRead; /* Number of bytes to read */
120540
+ int rc; /* Return code */
120541
+
120542
+ nRead = MIN(pReader->nNode - pReader->nPopulate, FTS3_NODE_CHUNKSIZE);
120543
+ rc = sqlite3_blob_read(
120544
+ pReader->pBlob,
120545
+ &pReader->aNode[pReader->nPopulate],
120546
+ nRead,
120547
+ pReader->nPopulate
120548
+ );
120549
+
120550
+ if( rc==SQLITE_OK ){
120551
+ pReader->nPopulate += nRead;
120552
+ memset(&pReader->aNode[pReader->nPopulate], 0, FTS3_NODE_PADDING);
120553
+ if( pReader->nPopulate==pReader->nNode ){
120554
+ sqlite3_blob_close(pReader->pBlob);
120555
+ pReader->pBlob = 0;
120556
+ pReader->nPopulate = 0;
120557
+ }
120558
+ }
120559
+ return rc;
120560
+}
120561
+
120562
+static int fts3SegReaderRequire(Fts3SegReader *pReader, char *pFrom, int nByte){
120563
+ int rc = SQLITE_OK;
120564
+ assert( !pReader->pBlob
120565
+ || (pFrom>=pReader->aNode && pFrom<&pReader->aNode[pReader->nNode])
120566
+ );
120567
+ while( pReader->pBlob && rc==SQLITE_OK
120568
+ && (pFrom - pReader->aNode + nByte)>pReader->nPopulate
120569
+ ){
120570
+ rc = fts3SegReaderIncrRead(pReader);
120571
+ }
120572
+ return rc;
120573
+}
119689120574
119690120575
/*
119691120576
** Move the iterator passed as the first argument to the next term in the
119692120577
** segment. If successful, SQLITE_OK is returned. If there is no next term,
119693120578
** SQLITE_DONE. Otherwise, an SQLite error code.
119694120579
*/
119695
-static int fts3SegReaderNext(Fts3Table *p, Fts3SegReader *pReader){
120580
+static int fts3SegReaderNext(
120581
+ Fts3Table *p,
120582
+ Fts3SegReader *pReader,
120583
+ int bIncr
120584
+){
120585
+ int rc; /* Return code of various sub-routines */
119696120586
char *pNext; /* Cursor variable */
119697120587
int nPrefix; /* Number of bytes in term prefix */
119698120588
int nSuffix; /* Number of bytes in term suffix */
119699120589
119700120590
if( !pReader->aDoclist ){
@@ -119702,11 +120592,10 @@
119702120592
}else{
119703120593
pNext = &pReader->aDoclist[pReader->nDoclist];
119704120594
}
119705120595
119706120596
if( !pNext || pNext>=&pReader->aNode[pReader->nNode] ){
119707
- int rc; /* Return code from Fts3ReadBlock() */
119708120597
119709120598
if( fts3SegReaderIsPending(pReader) ){
119710120599
Fts3HashElem *pElem = *(pReader->ppNextElem);
119711120600
if( pElem==0 ){
119712120601
pReader->aNode = 0;
@@ -119722,10 +120611,12 @@
119722120611
return SQLITE_OK;
119723120612
}
119724120613
119725120614
if( !fts3SegReaderIsRootOnly(pReader) ){
119726120615
sqlite3_free(pReader->aNode);
120616
+ sqlite3_blob_close(pReader->pBlob);
120617
+ pReader->pBlob = 0;
119727120618
}
119728120619
pReader->aNode = 0;
119729120620
119730120621
/* If iCurrentBlock>=iLeafEndBlock, this is an EOF condition. All leaf
119731120622
** blocks have already been traversed. */
@@ -119733,19 +120624,29 @@
119733120624
if( pReader->iCurrentBlock>=pReader->iLeafEndBlock ){
119734120625
return SQLITE_OK;
119735120626
}
119736120627
119737120628
rc = sqlite3Fts3ReadBlock(
119738
- p, ++pReader->iCurrentBlock, &pReader->aNode, &pReader->nNode
120629
+ p, ++pReader->iCurrentBlock, &pReader->aNode, &pReader->nNode,
120630
+ (bIncr ? &pReader->nPopulate : 0)
119739120631
);
119740120632
if( rc!=SQLITE_OK ) return rc;
120633
+ assert( pReader->pBlob==0 );
120634
+ if( bIncr && pReader->nPopulate<pReader->nNode ){
120635
+ pReader->pBlob = p->pSegments;
120636
+ p->pSegments = 0;
120637
+ }
119741120638
pNext = pReader->aNode;
119742120639
}
120640
+
120641
+ assert( !fts3SegReaderIsPending(pReader) );
120642
+
120643
+ rc = fts3SegReaderRequire(pReader, pNext, FTS3_VARINT_MAX*2);
120644
+ if( rc!=SQLITE_OK ) return rc;
119743120645
119744120646
/* Because of the FTS3_NODE_PADDING bytes of padding, the following is
119745
- ** safe (no risk of overread) even if the node data is corrupted.
119746
- */
120647
+ ** safe (no risk of overread) even if the node data is corrupted. */
119747120648
pNext += sqlite3Fts3GetVarint32(pNext, &nPrefix);
119748120649
pNext += sqlite3Fts3GetVarint32(pNext, &nSuffix);
119749120650
if( nPrefix<0 || nSuffix<=0
119750120651
|| &pNext[nSuffix]>&pReader->aNode[pReader->nNode]
119751120652
){
@@ -119759,10 +120660,14 @@
119759120660
return SQLITE_NOMEM;
119760120661
}
119761120662
pReader->zTerm = zNew;
119762120663
pReader->nTermAlloc = nNew;
119763120664
}
120665
+
120666
+ rc = fts3SegReaderRequire(pReader, pNext, nSuffix+FTS3_VARINT_MAX);
120667
+ if( rc!=SQLITE_OK ) return rc;
120668
+
119764120669
memcpy(&pReader->zTerm[nPrefix], pNext, nSuffix);
119765120670
pReader->nTerm = nPrefix+nSuffix;
119766120671
pNext += nSuffix;
119767120672
pNext += sqlite3Fts3GetVarint32(pNext, &pReader->nDoclist);
119768120673
pReader->aDoclist = pNext;
@@ -119771,11 +120676,11 @@
119771120676
/* Check that the doclist does not appear to extend past the end of the
119772120677
** b-tree node. And that the final byte of the doclist is 0x00. If either
119773120678
** of these statements is untrue, then the data structure is corrupt.
119774120679
*/
119775120680
if( &pReader->aDoclist[pReader->nDoclist]>&pReader->aNode[pReader->nNode]
119776
- || pReader->aDoclist[pReader->nDoclist-1]
120681
+ || (pReader->nPopulate==0 && pReader->aDoclist[pReader->nDoclist-1])
119777120682
){
119778120683
return SQLITE_CORRUPT_VTAB;
119779120684
}
119780120685
return SQLITE_OK;
119781120686
}
@@ -119782,16 +120687,30 @@
119782120687
119783120688
/*
119784120689
** Set the SegReader to point to the first docid in the doclist associated
119785120690
** with the current term.
119786120691
*/
119787
-static void fts3SegReaderFirstDocid(Fts3SegReader *pReader){
119788
- int n;
120692
+static int fts3SegReaderFirstDocid(Fts3Table *pTab, Fts3SegReader *pReader){
120693
+ int rc = SQLITE_OK;
119789120694
assert( pReader->aDoclist );
119790120695
assert( !pReader->pOffsetList );
119791
- n = sqlite3Fts3GetVarint(pReader->aDoclist, &pReader->iDocid);
119792
- pReader->pOffsetList = &pReader->aDoclist[n];
120696
+ if( pTab->bDescIdx && fts3SegReaderIsPending(pReader) ){
120697
+ u8 bEof = 0;
120698
+ pReader->iDocid = 0;
120699
+ pReader->nOffsetList = 0;
120700
+ sqlite3Fts3DoclistPrev(0,
120701
+ pReader->aDoclist, pReader->nDoclist, &pReader->pOffsetList,
120702
+ &pReader->iDocid, &pReader->nOffsetList, &bEof
120703
+ );
120704
+ }else{
120705
+ rc = fts3SegReaderRequire(pReader, pReader->aDoclist, FTS3_VARINT_MAX);
120706
+ if( rc==SQLITE_OK ){
120707
+ int n = sqlite3Fts3GetVarint(pReader->aDoclist, &pReader->iDocid);
120708
+ pReader->pOffsetList = &pReader->aDoclist[n];
120709
+ }
120710
+ }
120711
+ return rc;
119793120712
}
119794120713
119795120714
/*
119796120715
** Advance the SegReader to point to the next docid in the doclist
119797120716
** associated with the current term.
@@ -119800,132 +120719,126 @@
119800120719
** *ppOffsetList is set to point to the first column-offset list
119801120720
** in the doclist entry (i.e. immediately past the docid varint).
119802120721
** *pnOffsetList is set to the length of the set of column-offset
119803120722
** lists, not including the nul-terminator byte. For example:
119804120723
*/
119805
-static void fts3SegReaderNextDocid(
119806
- Fts3SegReader *pReader,
119807
- char **ppOffsetList,
119808
- int *pnOffsetList
120724
+static int fts3SegReaderNextDocid(
120725
+ Fts3Table *pTab,
120726
+ Fts3SegReader *pReader, /* Reader to advance to next docid */
120727
+ char **ppOffsetList, /* OUT: Pointer to current position-list */
120728
+ int *pnOffsetList /* OUT: Length of *ppOffsetList in bytes */
119809120729
){
120730
+ int rc = SQLITE_OK;
119810120731
char *p = pReader->pOffsetList;
119811120732
char c = 0;
119812120733
119813
- /* Pointer p currently points at the first byte of an offset list. The
119814
- ** following two lines advance it to point one byte past the end of
119815
- ** the same offset list.
119816
- */
119817
- while( *p | c ) c = *p++ & 0x80;
119818
- p++;
119819
-
119820
- /* If required, populate the output variables with a pointer to and the
119821
- ** size of the previous offset-list.
119822
- */
119823
- if( ppOffsetList ){
119824
- *ppOffsetList = pReader->pOffsetList;
119825
- *pnOffsetList = (int)(p - pReader->pOffsetList - 1);
119826
- }
119827
-
119828
- /* If there are no more entries in the doclist, set pOffsetList to
119829
- ** NULL. Otherwise, set Fts3SegReader.iDocid to the next docid and
119830
- ** Fts3SegReader.pOffsetList to point to the next offset list before
119831
- ** returning.
119832
- */
119833
- if( p>=&pReader->aDoclist[pReader->nDoclist] ){
119834
- pReader->pOffsetList = 0;
120734
+ assert( p );
120735
+
120736
+ if( pTab->bDescIdx && fts3SegReaderIsPending(pReader) ){
120737
+ /* A pending-terms seg-reader for an FTS4 table that uses order=desc.
120738
+ ** Pending-terms doclists are always built up in ascending order, so
120739
+ ** we have to iterate through them backwards here. */
120740
+ u8 bEof = 0;
120741
+ if( ppOffsetList ){
120742
+ *ppOffsetList = pReader->pOffsetList;
120743
+ *pnOffsetList = pReader->nOffsetList - 1;
120744
+ }
120745
+ sqlite3Fts3DoclistPrev(0,
120746
+ pReader->aDoclist, pReader->nDoclist, &p, &pReader->iDocid,
120747
+ &pReader->nOffsetList, &bEof
120748
+ );
120749
+ if( bEof ){
120750
+ pReader->pOffsetList = 0;
120751
+ }else{
120752
+ pReader->pOffsetList = p;
120753
+ }
119835120754
}else{
119836
- sqlite3_int64 iDelta;
119837
- pReader->pOffsetList = p + sqlite3Fts3GetVarint(p, &iDelta);
119838
- pReader->iDocid += iDelta;
119839
- }
119840
-}
119841
-
119842
-/*
119843
-** This function is called to estimate the amount of data that will be
119844
-** loaded from the disk If SegReaderIterate() is called on this seg-reader,
119845
-** in units of average document size.
119846
-**
119847
-** This can be used as follows: If the caller has a small doclist that
119848
-** contains references to N documents, and is considering merging it with
119849
-** a large doclist (size X "average documents"), it may opt not to load
119850
-** the large doclist if X>N.
119851
-*/
119852
-SQLITE_PRIVATE int sqlite3Fts3SegReaderCost(
119853
- Fts3Cursor *pCsr, /* FTS3 cursor handle */
119854
- Fts3SegReader *pReader, /* Segment-reader handle */
119855
- int *pnCost /* IN/OUT: Number of bytes read */
120755
+
120756
+ /* Pointer p currently points at the first byte of an offset list. The
120757
+ ** following block advances it to point one byte past the end of
120758
+ ** the same offset list. */
120759
+ while( 1 ){
120760
+
120761
+ /* The following line of code (and the "p++" below the while() loop) is
120762
+ ** normally all that is required to move pointer p to the desired
120763
+ ** position. The exception is if this node is being loaded from disk
120764
+ ** incrementally and pointer "p" now points to the first byte passed
120765
+ ** the populated part of pReader->aNode[].
120766
+ */
120767
+ while( *p | c ) c = *p++ & 0x80;
120768
+ assert( *p==0 );
120769
+
120770
+ if( pReader->pBlob==0 || p<&pReader->aNode[pReader->nPopulate] ) break;
120771
+ rc = fts3SegReaderIncrRead(pReader);
120772
+ if( rc!=SQLITE_OK ) return rc;
120773
+ }
120774
+ p++;
120775
+
120776
+ /* If required, populate the output variables with a pointer to and the
120777
+ ** size of the previous offset-list.
120778
+ */
120779
+ if( ppOffsetList ){
120780
+ *ppOffsetList = pReader->pOffsetList;
120781
+ *pnOffsetList = (int)(p - pReader->pOffsetList - 1);
120782
+ }
120783
+
120784
+ /* If there are no more entries in the doclist, set pOffsetList to
120785
+ ** NULL. Otherwise, set Fts3SegReader.iDocid to the next docid and
120786
+ ** Fts3SegReader.pOffsetList to point to the next offset list before
120787
+ ** returning.
120788
+ */
120789
+ if( p>=&pReader->aDoclist[pReader->nDoclist] ){
120790
+ pReader->pOffsetList = 0;
120791
+ }else{
120792
+ rc = fts3SegReaderRequire(pReader, p, FTS3_VARINT_MAX);
120793
+ if( rc==SQLITE_OK ){
120794
+ sqlite3_int64 iDelta;
120795
+ pReader->pOffsetList = p + sqlite3Fts3GetVarint(p, &iDelta);
120796
+ if( pTab->bDescIdx ){
120797
+ pReader->iDocid -= iDelta;
120798
+ }else{
120799
+ pReader->iDocid += iDelta;
120800
+ }
120801
+ }
120802
+ }
120803
+ }
120804
+
120805
+ return SQLITE_OK;
120806
+}
120807
+
120808
+
120809
+SQLITE_PRIVATE int sqlite3Fts3MsrOvfl(
120810
+ Fts3Cursor *pCsr,
120811
+ Fts3MultiSegReader *pMsr,
120812
+ int *pnOvfl
119856120813
){
119857120814
Fts3Table *p = (Fts3Table*)pCsr->base.pVtab;
119858
- int rc = SQLITE_OK; /* Return code */
119859
- int nCost = 0; /* Cost in bytes to return */
119860
- int pgsz = p->nPgsz; /* Database page size */
119861
-
119862
- /* If this seg-reader is reading the pending-terms table, or if all data
119863
- ** for the segment is stored on the root page of the b-tree, then the cost
119864
- ** is zero. In this case all required data is already in main memory.
119865
- */
119866
- if( p->bHasStat
119867
- && !fts3SegReaderIsPending(pReader)
119868
- && !fts3SegReaderIsRootOnly(pReader)
119869
- ){
119870
- int nBlob = 0;
119871
- sqlite3_int64 iBlock;
119872
-
119873
- if( pCsr->nRowAvg==0 ){
119874
- /* The average document size, which is required to calculate the cost
119875
- ** of each doclist, has not yet been determined. Read the required
119876
- ** data from the %_stat table to calculate it.
119877
- **
119878
- ** Entry 0 of the %_stat table is a blob containing (nCol+1) FTS3
119879
- ** varints, where nCol is the number of columns in the FTS3 table.
119880
- ** The first varint is the number of documents currently stored in
119881
- ** the table. The following nCol varints contain the total amount of
119882
- ** data stored in all rows of each column of the table, from left
119883
- ** to right.
119884
- */
119885
- sqlite3_stmt *pStmt;
119886
- sqlite3_int64 nDoc = 0;
119887
- sqlite3_int64 nByte = 0;
119888
- const char *pEnd;
119889
- const char *a;
119890
-
119891
- rc = sqlite3Fts3SelectDoctotal(p, &pStmt);
119892
- if( rc!=SQLITE_OK ) return rc;
119893
- a = sqlite3_column_blob(pStmt, 0);
119894
- assert( a );
119895
-
119896
- pEnd = &a[sqlite3_column_bytes(pStmt, 0)];
119897
- a += sqlite3Fts3GetVarint(a, &nDoc);
119898
- while( a<pEnd ){
119899
- a += sqlite3Fts3GetVarint(a, &nByte);
119900
- }
119901
- if( nDoc==0 || nByte==0 ){
119902
- sqlite3_reset(pStmt);
119903
- return SQLITE_CORRUPT_VTAB;
119904
- }
119905
-
119906
- pCsr->nRowAvg = (int)(((nByte / nDoc) + pgsz) / pgsz);
119907
- assert( pCsr->nRowAvg>0 );
119908
- rc = sqlite3_reset(pStmt);
119909
- if( rc!=SQLITE_OK ) return rc;
119910
- }
119911
-
119912
- /* Assume that a blob flows over onto overflow pages if it is larger
119913
- ** than (pgsz-35) bytes in size (the file-format documentation
119914
- ** confirms this).
119915
- */
119916
- for(iBlock=pReader->iStartBlock; iBlock<=pReader->iLeafEndBlock; iBlock++){
119917
- rc = sqlite3Fts3ReadBlock(p, iBlock, 0, &nBlob);
119918
- if( rc!=SQLITE_OK ) break;
119919
- if( (nBlob+35)>pgsz ){
119920
- int nOvfl = (nBlob + 34)/pgsz;
119921
- nCost += ((nOvfl + pCsr->nRowAvg - 1)/pCsr->nRowAvg);
119922
- }
119923
- }
119924
- }
119925
-
119926
- *pnCost += nCost;
120815
+ int nOvfl = 0;
120816
+ int ii;
120817
+ int rc = SQLITE_OK;
120818
+ int pgsz = p->nPgsz;
120819
+
120820
+ assert( p->bHasStat );
120821
+ assert( pgsz>0 );
120822
+
120823
+ for(ii=0; rc==SQLITE_OK && ii<pMsr->nSegment; ii++){
120824
+ Fts3SegReader *pReader = pMsr->apSegment[ii];
120825
+ if( !fts3SegReaderIsPending(pReader)
120826
+ && !fts3SegReaderIsRootOnly(pReader)
120827
+ ){
120828
+ int jj;
120829
+ for(jj=pReader->iStartBlock; jj<=pReader->iLeafEndBlock; jj++){
120830
+ int nBlob;
120831
+ rc = sqlite3Fts3ReadBlock(p, jj, 0, &nBlob, 0);
120832
+ if( rc!=SQLITE_OK ) break;
120833
+ if( (nBlob+35)>pgsz ){
120834
+ nOvfl += (nBlob + 34)/pgsz;
120835
+ }
120836
+ }
120837
+ }
120838
+ }
120839
+ *pnOvfl = nOvfl;
119927120840
return rc;
119928120841
}
119929120842
119930120843
/*
119931120844
** Free all allocations associated with the iterator passed as the
@@ -119934,10 +120847,11 @@
119934120847
SQLITE_PRIVATE void sqlite3Fts3SegReaderFree(Fts3SegReader *pReader){
119935120848
if( pReader && !fts3SegReaderIsPending(pReader) ){
119936120849
sqlite3_free(pReader->zTerm);
119937120850
if( !fts3SegReaderIsRootOnly(pReader) ){
119938120851
sqlite3_free(pReader->aNode);
120852
+ sqlite3_blob_close(pReader->pBlob);
119939120853
}
119940120854
}
119941120855
sqlite3_free(pReader);
119942120856
}
119943120857
@@ -120010,28 +120924,46 @@
120010120924
}
120011120925
120012120926
/*
120013120927
** This function is used to allocate an Fts3SegReader that iterates through
120014120928
** a subset of the terms stored in the Fts3Table.pendingTerms array.
120929
+**
120930
+** If the isPrefixIter parameter is zero, then the returned SegReader iterates
120931
+** through each term in the pending-terms table. Or, if isPrefixIter is
120932
+** non-zero, it iterates through each term and its prefixes. For example, if
120933
+** the pending terms hash table contains the terms "sqlite", "mysql" and
120934
+** "firebird", then the iterator visits the following 'terms' (in the order
120935
+** shown):
120936
+**
120937
+** f fi fir fire fireb firebi firebir firebird
120938
+** m my mys mysq mysql
120939
+** s sq sql sqli sqlit sqlite
120940
+**
120941
+** Whereas if isPrefixIter is zero, the terms visited are:
120942
+**
120943
+** firebird mysql sqlite
120015120944
*/
120016120945
SQLITE_PRIVATE int sqlite3Fts3SegReaderPending(
120017120946
Fts3Table *p, /* Virtual table handle */
120947
+ int iIndex, /* Index for p->aIndex */
120018120948
const char *zTerm, /* Term to search for */
120019120949
int nTerm, /* Size of buffer zTerm */
120020
- int isPrefix, /* True for a term-prefix query */
120950
+ int bPrefix, /* True for a prefix iterator */
120021120951
Fts3SegReader **ppReader /* OUT: SegReader for pending-terms */
120022120952
){
120023120953
Fts3SegReader *pReader = 0; /* Fts3SegReader object to return */
120024120954
Fts3HashElem **aElem = 0; /* Array of term hash entries to scan */
120025120955
int nElem = 0; /* Size of array at aElem */
120026120956
int rc = SQLITE_OK; /* Return Code */
120957
+ Fts3Hash *pHash;
120027120958
120028
- if( isPrefix ){
120959
+ pHash = &p->aIndex[iIndex].hPending;
120960
+ if( bPrefix ){
120029120961
int nAlloc = 0; /* Size of allocated array at aElem */
120030120962
Fts3HashElem *pE = 0; /* Iterator variable */
120031120963
120032
- for(pE=fts3HashFirst(&p->pendingTerms); pE; pE=fts3HashNext(pE)){
120964
+ for(pE=fts3HashFirst(pHash); pE; pE=fts3HashNext(pE)){
120033120965
char *zKey = (char *)fts3HashKey(pE);
120034120966
int nKey = fts3HashKeysize(pE);
120035120967
if( nTerm==0 || (nKey>=nTerm && 0==memcmp(zKey, zTerm, nTerm)) ){
120036120968
if( nElem==nAlloc ){
120037120969
Fts3HashElem **aElem2;
@@ -120044,10 +120976,11 @@
120044120976
nElem = 0;
120045120977
break;
120046120978
}
120047120979
aElem = aElem2;
120048120980
}
120981
+
120049120982
aElem[nElem++] = pE;
120050120983
}
120051120984
}
120052120985
120053120986
/* If more than one term matches the prefix, sort the Fts3HashElem
@@ -120057,11 +120990,13 @@
120057120990
if( nElem>1 ){
120058120991
qsort(aElem, nElem, sizeof(Fts3HashElem *), fts3CompareElemByTerm);
120059120992
}
120060120993
120061120994
}else{
120062
- Fts3HashElem *pE = fts3HashFindElem(&p->pendingTerms, zTerm, nTerm);
120995
+ /* The query is a simple term lookup that matches at most one term in
120996
+ ** the index. All that is required is a straight hash-lookup. */
120997
+ Fts3HashElem *pE = fts3HashFindElem(pHash, zTerm, nTerm);
120063120998
if( pE ){
120064120999
aElem = &pE;
120065121000
nElem = 1;
120066121001
}
120067121002
}
@@ -120077,11 +121012,11 @@
120077121012
pReader->ppNextElem = (Fts3HashElem **)&pReader[1];
120078121013
memcpy(pReader->ppNextElem, aElem, nElem*sizeof(Fts3HashElem *));
120079121014
}
120080121015
}
120081121016
120082
- if( isPrefix ){
121017
+ if( bPrefix ){
120083121018
sqlite3_free(aElem);
120084121019
}
120085121020
*ppReader = pReader;
120086121021
return rc;
120087121022
}
@@ -120137,10 +121072,22 @@
120137121072
if( pLhs->iDocid==pRhs->iDocid ){
120138121073
rc = pRhs->iIdx - pLhs->iIdx;
120139121074
}else{
120140121075
rc = (pLhs->iDocid > pRhs->iDocid) ? 1 : -1;
120141121076
}
121077
+ }
121078
+ assert( pLhs->aNode && pRhs->aNode );
121079
+ return rc;
121080
+}
121081
+static int fts3SegReaderDoclistCmpRev(Fts3SegReader *pLhs, Fts3SegReader *pRhs){
121082
+ int rc = (pLhs->pOffsetList==0)-(pRhs->pOffsetList==0);
121083
+ if( rc==0 ){
121084
+ if( pLhs->iDocid==pRhs->iDocid ){
121085
+ rc = pRhs->iIdx - pLhs->iIdx;
121086
+ }else{
121087
+ rc = (pLhs->iDocid < pRhs->iDocid) ? 1 : -1;
121088
+ }
120142121089
}
120143121090
assert( pLhs->aNode && pRhs->aNode );
120144121091
return rc;
120145121092
}
120146121093
@@ -120689,25 +121636,34 @@
120689121636
}
120690121637
return rc;
120691121638
}
120692121639
120693121640
/*
120694
-** Set *pnSegment to the total number of segments in the database. Set
120695
-** *pnMax to the largest segment level in the database (segment levels
120696
-** are stored in the 'level' column of the %_segdir table).
121641
+** Set *pnMax to the largest segment level in the database for the index
121642
+** iIndex.
121643
+**
121644
+** Segment levels are stored in the 'level' column of the %_segdir table.
120697121645
**
120698121646
** Return SQLITE_OK if successful, or an SQLite error code if not.
120699121647
*/
120700
-static int fts3SegmentCountMax(Fts3Table *p, int *pnSegment, int *pnMax){
121648
+static int fts3SegmentMaxLevel(Fts3Table *p, int iIndex, int *pnMax){
120701121649
sqlite3_stmt *pStmt;
120702121650
int rc;
121651
+ assert( iIndex>=0 && iIndex<p->nIndex );
120703121652
120704
- rc = fts3SqlStmt(p, SQL_SELECT_SEGDIR_COUNT_MAX, &pStmt, 0);
121653
+ /* Set pStmt to the compiled version of:
121654
+ **
121655
+ ** SELECT max(level) FROM %Q.'%q_segdir' WHERE level BETWEEN ? AND ?
121656
+ **
121657
+ ** (1024 is actually the value of macro FTS3_SEGDIR_PREFIXLEVEL_STR).
121658
+ */
121659
+ rc = fts3SqlStmt(p, SQL_SELECT_SEGDIR_MAX_LEVEL, &pStmt, 0);
120705121660
if( rc!=SQLITE_OK ) return rc;
121661
+ sqlite3_bind_int(pStmt, 1, iIndex*FTS3_SEGDIR_MAXLEVEL);
121662
+ sqlite3_bind_int(pStmt, 2, (iIndex+1)*FTS3_SEGDIR_MAXLEVEL - 1);
120706121663
if( SQLITE_ROW==sqlite3_step(pStmt) ){
120707
- *pnSegment = sqlite3_column_int(pStmt, 0);
120708
- *pnMax = sqlite3_column_int(pStmt, 1);
121664
+ *pnMax = sqlite3_column_int(pStmt, 0);
120709121665
}
120710121666
return sqlite3_reset(pStmt);
120711121667
}
120712121668
120713121669
/*
@@ -120724,10 +121680,11 @@
120724121680
**
120725121681
** SQLITE_OK is returned if successful, otherwise an SQLite error code.
120726121682
*/
120727121683
static int fts3DeleteSegdir(
120728121684
Fts3Table *p, /* Virtual table handle */
121685
+ int iIndex, /* Index for p->aIndex */
120729121686
int iLevel, /* Level of %_segdir entries to delete */
120730121687
Fts3SegReader **apSegment, /* Array of SegReader objects */
120731121688
int nReader /* Size of array apSegment */
120732121689
){
120733121690
int rc; /* Return Code */
@@ -120746,23 +121703,28 @@
120746121703
}
120747121704
if( rc!=SQLITE_OK ){
120748121705
return rc;
120749121706
}
120750121707
121708
+ assert( iLevel>=0 || iLevel==FTS3_SEGCURSOR_ALL );
120751121709
if( iLevel==FTS3_SEGCURSOR_ALL ){
120752
- fts3SqlExec(&rc, p, SQL_DELETE_ALL_SEGDIR, 0);
120753
- }else if( iLevel==FTS3_SEGCURSOR_PENDING ){
120754
- sqlite3Fts3PendingTermsClear(p);
121710
+ rc = fts3SqlStmt(p, SQL_DELETE_SEGDIR_RANGE, &pDelete, 0);
121711
+ if( rc==SQLITE_OK ){
121712
+ sqlite3_bind_int(pDelete, 1, iIndex*FTS3_SEGDIR_MAXLEVEL);
121713
+ sqlite3_bind_int(pDelete, 2, (iIndex+1) * FTS3_SEGDIR_MAXLEVEL - 1);
121714
+ }
120755121715
}else{
120756
- assert( iLevel>=0 );
120757
- rc = fts3SqlStmt(p, SQL_DELETE_SEGDIR_BY_LEVEL, &pDelete, 0);
121716
+ rc = fts3SqlStmt(p, SQL_DELETE_SEGDIR_LEVEL, &pDelete, 0);
120758121717
if( rc==SQLITE_OK ){
120759
- sqlite3_bind_int(pDelete, 1, iLevel);
120760
- sqlite3_step(pDelete);
120761
- rc = sqlite3_reset(pDelete);
121718
+ sqlite3_bind_int(pDelete, 1, iIndex*FTS3_SEGDIR_MAXLEVEL + iLevel);
120762121719
}
120763121720
}
121721
+
121722
+ if( rc==SQLITE_OK ){
121723
+ sqlite3_step(pDelete);
121724
+ rc = sqlite3_reset(pDelete);
121725
+ }
120764121726
120765121727
return rc;
120766121728
}
120767121729
120768121730
/*
@@ -120805,14 +121767,124 @@
120805121767
}
120806121768
120807121769
*ppList = pList;
120808121770
*pnList = nList;
120809121771
}
121772
+
121773
+SQLITE_PRIVATE int sqlite3Fts3MsrIncrStart(
121774
+ Fts3Table *p, /* Virtual table handle */
121775
+ Fts3MultiSegReader *pCsr, /* Cursor object */
121776
+ int iCol, /* Column to match on. */
121777
+ const char *zTerm, /* Term to iterate through a doclist for */
121778
+ int nTerm /* Number of bytes in zTerm */
121779
+){
121780
+ int i;
121781
+ int nSegment = pCsr->nSegment;
121782
+ int (*xCmp)(Fts3SegReader *, Fts3SegReader *) = (
121783
+ p->bDescIdx ? fts3SegReaderDoclistCmpRev : fts3SegReaderDoclistCmp
121784
+ );
121785
+
121786
+ assert( pCsr->pFilter==0 );
121787
+ assert( zTerm && nTerm>0 );
121788
+
121789
+ /* Advance each segment iterator until it points to the term zTerm/nTerm. */
121790
+ for(i=0; i<nSegment; i++){
121791
+ Fts3SegReader *pSeg = pCsr->apSegment[i];
121792
+ do {
121793
+ int rc = fts3SegReaderNext(p, pSeg, 1);
121794
+ if( rc!=SQLITE_OK ) return rc;
121795
+ }while( fts3SegReaderTermCmp(pSeg, zTerm, nTerm)<0 );
121796
+ }
121797
+ fts3SegReaderSort(pCsr->apSegment, nSegment, nSegment, fts3SegReaderCmp);
121798
+
121799
+ /* Determine how many of the segments actually point to zTerm/nTerm. */
121800
+ for(i=0; i<nSegment; i++){
121801
+ Fts3SegReader *pSeg = pCsr->apSegment[i];
121802
+ if( !pSeg->aNode || fts3SegReaderTermCmp(pSeg, zTerm, nTerm) ){
121803
+ break;
121804
+ }
121805
+ }
121806
+ pCsr->nAdvance = i;
121807
+
121808
+ /* Advance each of the segments to point to the first docid. */
121809
+ for(i=0; i<pCsr->nAdvance; i++){
121810
+ int rc = fts3SegReaderFirstDocid(p, pCsr->apSegment[i]);
121811
+ if( rc!=SQLITE_OK ) return rc;
121812
+ }
121813
+ fts3SegReaderSort(pCsr->apSegment, i, i, xCmp);
121814
+
121815
+ assert( iCol<0 || iCol<p->nColumn );
121816
+ pCsr->iColFilter = iCol;
121817
+
121818
+ return SQLITE_OK;
121819
+}
121820
+
121821
+SQLITE_PRIVATE int sqlite3Fts3MsrIncrNext(
121822
+ Fts3Table *p, /* Virtual table handle */
121823
+ Fts3MultiSegReader *pMsr, /* Multi-segment-reader handle */
121824
+ sqlite3_int64 *piDocid, /* OUT: Docid value */
121825
+ char **paPoslist, /* OUT: Pointer to position list */
121826
+ int *pnPoslist /* OUT: Size of position list in bytes */
121827
+){
121828
+ int nMerge = pMsr->nAdvance;
121829
+ Fts3SegReader **apSegment = pMsr->apSegment;
121830
+ int (*xCmp)(Fts3SegReader *, Fts3SegReader *) = (
121831
+ p->bDescIdx ? fts3SegReaderDoclistCmpRev : fts3SegReaderDoclistCmp
121832
+ );
121833
+
121834
+ if( nMerge==0 ){
121835
+ *paPoslist = 0;
121836
+ return SQLITE_OK;
121837
+ }
121838
+
121839
+ while( 1 ){
121840
+ Fts3SegReader *pSeg;
121841
+ pSeg = pMsr->apSegment[0];
121842
+
121843
+ if( pSeg->pOffsetList==0 ){
121844
+ *paPoslist = 0;
121845
+ break;
121846
+ }else{
121847
+ int rc;
121848
+ char *pList;
121849
+ int nList;
121850
+ int j;
121851
+ sqlite3_int64 iDocid = apSegment[0]->iDocid;
121852
+
121853
+ rc = fts3SegReaderNextDocid(p, apSegment[0], &pList, &nList);
121854
+ j = 1;
121855
+ while( rc==SQLITE_OK
121856
+ && j<nMerge
121857
+ && apSegment[j]->pOffsetList
121858
+ && apSegment[j]->iDocid==iDocid
121859
+ ){
121860
+ rc = fts3SegReaderNextDocid(p, apSegment[j], 0, 0);
121861
+ j++;
121862
+ }
121863
+ if( rc!=SQLITE_OK ) return rc;
121864
+ fts3SegReaderSort(pMsr->apSegment, nMerge, j, xCmp);
121865
+
121866
+ if( pMsr->iColFilter>=0 ){
121867
+ fts3ColumnFilter(pMsr->iColFilter, &pList, &nList);
121868
+ }
121869
+
121870
+ if( nList>0 ){
121871
+ *piDocid = iDocid;
121872
+ *paPoslist = pList;
121873
+ *pnPoslist = nList;
121874
+ break;
121875
+ }
121876
+ }
121877
+
121878
+ }
121879
+
121880
+ return SQLITE_OK;
121881
+}
120810121882
120811121883
SQLITE_PRIVATE int sqlite3Fts3SegReaderStart(
120812121884
Fts3Table *p, /* Virtual table handle */
120813
- Fts3SegReaderCursor *pCsr, /* Cursor object */
121885
+ Fts3MultiSegReader *pCsr, /* Cursor object */
120814121886
Fts3SegFilter *pFilter /* Restrictions on range of iteration */
120815121887
){
120816121888
int i;
120817121889
120818121890
/* Initialize the cursor object */
@@ -120827,11 +121899,11 @@
120827121899
for(i=0; i<pCsr->nSegment; i++){
120828121900
int nTerm = pFilter->nTerm;
120829121901
const char *zTerm = pFilter->zTerm;
120830121902
Fts3SegReader *pSeg = pCsr->apSegment[i];
120831121903
do {
120832
- int rc = fts3SegReaderNext(p, pSeg);
121904
+ int rc = fts3SegReaderNext(p, pSeg, 0);
120833121905
if( rc!=SQLITE_OK ) return rc;
120834121906
}while( zTerm && fts3SegReaderTermCmp(pSeg, zTerm, nTerm)<0 );
120835121907
}
120836121908
fts3SegReaderSort(
120837121909
pCsr->apSegment, pCsr->nSegment, pCsr->nSegment, fts3SegReaderCmp);
@@ -120839,11 +121911,11 @@
120839121911
return SQLITE_OK;
120840121912
}
120841121913
120842121914
SQLITE_PRIVATE int sqlite3Fts3SegReaderStep(
120843121915
Fts3Table *p, /* Virtual table handle */
120844
- Fts3SegReaderCursor *pCsr /* Cursor object */
121916
+ Fts3MultiSegReader *pCsr /* Cursor object */
120845121917
){
120846121918
int rc = SQLITE_OK;
120847121919
120848121920
int isIgnoreEmpty = (pCsr->pFilter->flags & FTS3_SEGMENT_IGNORE_EMPTY);
120849121921
int isRequirePos = (pCsr->pFilter->flags & FTS3_SEGMENT_REQUIRE_POS);
@@ -120852,10 +121924,13 @@
120852121924
int isScan = (pCsr->pFilter->flags & FTS3_SEGMENT_SCAN);
120853121925
120854121926
Fts3SegReader **apSegment = pCsr->apSegment;
120855121927
int nSegment = pCsr->nSegment;
120856121928
Fts3SegFilter *pFilter = pCsr->pFilter;
121929
+ int (*xCmp)(Fts3SegReader *, Fts3SegReader *) = (
121930
+ p->bDescIdx ? fts3SegReaderDoclistCmpRev : fts3SegReaderDoclistCmp
121931
+ );
120857121932
120858121933
if( pCsr->nSegment==0 ) return SQLITE_OK;
120859121934
120860121935
do {
120861121936
int nMerge;
@@ -120863,11 +121938,11 @@
120863121938
120864121939
/* Advance the first pCsr->nAdvance entries in the apSegment[] array
120865121940
** forward. Then sort the list in order of current term again.
120866121941
*/
120867121942
for(i=0; i<pCsr->nAdvance; i++){
120868
- rc = fts3SegReaderNext(p, apSegment[i]);
121943
+ rc = fts3SegReaderNext(p, apSegment[i], 0);
120869121944
if( rc!=SQLITE_OK ) return rc;
120870121945
}
120871121946
fts3SegReaderSort(apSegment, nSegment, pCsr->nAdvance, fts3SegReaderCmp);
120872121947
pCsr->nAdvance = 0;
120873121948
@@ -120902,11 +121977,14 @@
120902121977
){
120903121978
nMerge++;
120904121979
}
120905121980
120906121981
assert( isIgnoreEmpty || (isRequirePos && !isColFilter) );
120907
- if( nMerge==1 && !isIgnoreEmpty ){
121982
+ if( nMerge==1
121983
+ && !isIgnoreEmpty
121984
+ && (p->bDescIdx==0 || fts3SegReaderIsPending(apSegment[0])==0)
121985
+ ){
120908121986
pCsr->aDoclist = apSegment[0]->aDoclist;
120909121987
pCsr->nDoclist = apSegment[0]->nDoclist;
120910121988
rc = SQLITE_ROW;
120911121989
}else{
120912121990
int nDoclist = 0; /* Size of doclist */
@@ -120915,56 +121993,66 @@
120915121993
/* The current term of the first nMerge entries in the array
120916121994
** of Fts3SegReader objects is the same. The doclists must be merged
120917121995
** and a single term returned with the merged doclist.
120918121996
*/
120919121997
for(i=0; i<nMerge; i++){
120920
- fts3SegReaderFirstDocid(apSegment[i]);
121998
+ fts3SegReaderFirstDocid(p, apSegment[i]);
120921121999
}
120922
- fts3SegReaderSort(apSegment, nMerge, nMerge, fts3SegReaderDoclistCmp);
122000
+ fts3SegReaderSort(apSegment, nMerge, nMerge, xCmp);
120923122001
while( apSegment[0]->pOffsetList ){
120924122002
int j; /* Number of segments that share a docid */
120925122003
char *pList;
120926122004
int nList;
120927122005
int nByte;
120928122006
sqlite3_int64 iDocid = apSegment[0]->iDocid;
120929
- fts3SegReaderNextDocid(apSegment[0], &pList, &nList);
122007
+ fts3SegReaderNextDocid(p, apSegment[0], &pList, &nList);
120930122008
j = 1;
120931122009
while( j<nMerge
120932122010
&& apSegment[j]->pOffsetList
120933122011
&& apSegment[j]->iDocid==iDocid
120934122012
){
120935
- fts3SegReaderNextDocid(apSegment[j], 0, 0);
122013
+ fts3SegReaderNextDocid(p, apSegment[j], 0, 0);
120936122014
j++;
120937122015
}
120938122016
120939122017
if( isColFilter ){
120940122018
fts3ColumnFilter(pFilter->iCol, &pList, &nList);
120941122019
}
120942122020
120943122021
if( !isIgnoreEmpty || nList>0 ){
120944
- nByte = sqlite3Fts3VarintLen(iDocid-iPrev) + (isRequirePos?nList+1:0);
122022
+
122023
+ /* Calculate the 'docid' delta value to write into the merged
122024
+ ** doclist. */
122025
+ sqlite3_int64 iDelta;
122026
+ if( p->bDescIdx && nDoclist>0 ){
122027
+ iDelta = iPrev - iDocid;
122028
+ }else{
122029
+ iDelta = iDocid - iPrev;
122030
+ }
122031
+ assert( iDelta>0 || (nDoclist==0 && iDelta==iDocid) );
122032
+ assert( nDoclist>0 || iDelta==iDocid );
122033
+
122034
+ nByte = sqlite3Fts3VarintLen(iDelta) + (isRequirePos?nList+1:0);
120945122035
if( nDoclist+nByte>pCsr->nBuffer ){
120946122036
char *aNew;
120947122037
pCsr->nBuffer = (nDoclist+nByte)*2;
120948122038
aNew = sqlite3_realloc(pCsr->aBuffer, pCsr->nBuffer);
120949122039
if( !aNew ){
120950122040
return SQLITE_NOMEM;
120951122041
}
120952122042
pCsr->aBuffer = aNew;
120953122043
}
120954
- nDoclist += sqlite3Fts3PutVarint(
120955
- &pCsr->aBuffer[nDoclist], iDocid-iPrev
120956
- );
122044
+ nDoclist += sqlite3Fts3PutVarint(&pCsr->aBuffer[nDoclist], iDelta);
120957122045
iPrev = iDocid;
120958122046
if( isRequirePos ){
120959122047
memcpy(&pCsr->aBuffer[nDoclist], pList, nList);
120960122048
nDoclist += nList;
120961122049
pCsr->aBuffer[nDoclist++] = '\0';
120962122050
}
120963122051
}
120964122052
120965
- fts3SegReaderSort(apSegment, nMerge, j, fts3SegReaderDoclistCmp);
122053
+ fts3SegReaderSort(apSegment, nMerge, j, xCmp);
120966122054
}
120967122055
if( nDoclist>0 ){
120968122056
pCsr->aDoclist = pCsr->aBuffer;
120969122057
pCsr->nDoclist = nDoclist;
120970122058
rc = SQLITE_ROW;
@@ -120973,13 +122061,14 @@
120973122061
pCsr->nAdvance = nMerge;
120974122062
}while( rc==SQLITE_OK );
120975122063
120976122064
return rc;
120977122065
}
122066
+
120978122067
120979122068
SQLITE_PRIVATE void sqlite3Fts3SegReaderFinish(
120980
- Fts3SegReaderCursor *pCsr /* Cursor object */
122069
+ Fts3MultiSegReader *pCsr /* Cursor object */
120981122070
){
120982122071
if( pCsr ){
120983122072
int i;
120984122073
for(i=0; i<pCsr->nSegment; i++){
120985122074
sqlite3Fts3SegReaderFree(pCsr->apSegment[i]);
@@ -121002,47 +122091,60 @@
121002122091
** If this function is called with iLevel<0, but there is only one
121003122092
** segment in the database, SQLITE_DONE is returned immediately.
121004122093
** Otherwise, if successful, SQLITE_OK is returned. If an error occurs,
121005122094
** an SQLite error code is returned.
121006122095
*/
121007
-static int fts3SegmentMerge(Fts3Table *p, int iLevel){
122096
+static int fts3SegmentMerge(Fts3Table *p, int iIndex, int iLevel){
121008122097
int rc; /* Return code */
121009122098
int iIdx = 0; /* Index of new segment */
121010
- int iNewLevel = 0; /* Level to create new segment at */
122099
+ int iNewLevel = 0; /* Level/index to create new segment at */
121011122100
SegmentWriter *pWriter = 0; /* Used to write the new, merged, segment */
121012122101
Fts3SegFilter filter; /* Segment term filter condition */
121013
- Fts3SegReaderCursor csr; /* Cursor to iterate through level(s) */
122102
+ Fts3MultiSegReader csr; /* Cursor to iterate through level(s) */
122103
+ int bIgnoreEmpty = 0; /* True to ignore empty segments */
121014122104
121015
- rc = sqlite3Fts3SegReaderCursor(p, iLevel, 0, 0, 1, 0, &csr);
122105
+ assert( iLevel==FTS3_SEGCURSOR_ALL
122106
+ || iLevel==FTS3_SEGCURSOR_PENDING
122107
+ || iLevel>=0
122108
+ );
122109
+ assert( iLevel<FTS3_SEGDIR_MAXLEVEL );
122110
+ assert( iIndex>=0 && iIndex<p->nIndex );
122111
+
122112
+ rc = sqlite3Fts3SegReaderCursor(p, iIndex, iLevel, 0, 0, 1, 0, &csr);
121016122113
if( rc!=SQLITE_OK || csr.nSegment==0 ) goto finished;
121017122114
121018122115
if( iLevel==FTS3_SEGCURSOR_ALL ){
121019122116
/* This call is to merge all segments in the database to a single
121020122117
** segment. The level of the new segment is equal to the the numerically
121021
- ** greatest segment level currently present in the database. The index
121022
- ** of the new segment is always 0. */
121023
- int nDummy; /* TODO: Remove this */
122118
+ ** greatest segment level currently present in the database for this
122119
+ ** index. The idx of the new segment is always 0. */
121024122120
if( csr.nSegment==1 ){
121025122121
rc = SQLITE_DONE;
121026122122
goto finished;
121027122123
}
121028
- rc = fts3SegmentCountMax(p, &nDummy, &iNewLevel);
122124
+ rc = fts3SegmentMaxLevel(p, iIndex, &iNewLevel);
122125
+ bIgnoreEmpty = 1;
122126
+
122127
+ }else if( iLevel==FTS3_SEGCURSOR_PENDING ){
122128
+ iNewLevel = iIndex * FTS3_SEGDIR_MAXLEVEL;
122129
+ rc = fts3AllocateSegdirIdx(p, iIndex, 0, &iIdx);
121029122130
}else{
121030
- /* This call is to merge all segments at level iLevel. Find the next
122131
+ /* This call is to merge all segments at level iLevel. find the next
121031122132
** available segment index at level iLevel+1. The call to
121032122133
** fts3AllocateSegdirIdx() will merge the segments at level iLevel+1 to
121033122134
** a single iLevel+2 segment if necessary. */
121034
- iNewLevel = iLevel+1;
121035
- rc = fts3AllocateSegdirIdx(p, iNewLevel, &iIdx);
122135
+ rc = fts3AllocateSegdirIdx(p, iIndex, iLevel+1, &iIdx);
122136
+ iNewLevel = iIndex * FTS3_SEGDIR_MAXLEVEL + iLevel+1;
121036122137
}
121037122138
if( rc!=SQLITE_OK ) goto finished;
121038122139
assert( csr.nSegment>0 );
121039
- assert( iNewLevel>=0 );
122140
+ assert( iNewLevel>=(iIndex*FTS3_SEGDIR_MAXLEVEL) );
122141
+ assert( iNewLevel<((iIndex+1)*FTS3_SEGDIR_MAXLEVEL) );
121040122142
121041122143
memset(&filter, 0, sizeof(Fts3SegFilter));
121042122144
filter.flags = FTS3_SEGMENT_REQUIRE_POS;
121043
- filter.flags |= (iLevel==FTS3_SEGCURSOR_ALL ? FTS3_SEGMENT_IGNORE_EMPTY : 0);
122145
+ filter.flags |= (bIgnoreEmpty ? FTS3_SEGMENT_IGNORE_EMPTY : 0);
121044122146
121045122147
rc = sqlite3Fts3SegReaderStart(p, &csr, &filter);
121046122148
while( SQLITE_OK==rc ){
121047122149
rc = sqlite3Fts3SegReaderStep(p, &csr);
121048122150
if( rc!=SQLITE_ROW ) break;
@@ -121050,12 +122152,14 @@
121050122152
csr.zTerm, csr.nTerm, csr.aDoclist, csr.nDoclist);
121051122153
}
121052122154
if( rc!=SQLITE_OK ) goto finished;
121053122155
assert( pWriter );
121054122156
121055
- rc = fts3DeleteSegdir(p, iLevel, csr.apSegment, csr.nSegment);
121056
- if( rc!=SQLITE_OK ) goto finished;
122157
+ if( iLevel!=FTS3_SEGCURSOR_PENDING ){
122158
+ rc = fts3DeleteSegdir(p, iIndex, iLevel, csr.apSegment, csr.nSegment);
122159
+ if( rc!=SQLITE_OK ) goto finished;
122160
+ }
121057122161
rc = fts3SegWriterFlush(p, pWriter, iNewLevel, iIdx);
121058122162
121059122163
finished:
121060122164
fts3SegWriterFree(pWriter);
121061122165
sqlite3Fts3SegReaderFinish(&csr);
@@ -121062,14 +122166,21 @@
121062122166
return rc;
121063122167
}
121064122168
121065122169
121066122170
/*
121067
-** Flush the contents of pendingTerms to a level 0 segment.
122171
+** Flush the contents of pendingTerms to level 0 segments.
121068122172
*/
121069122173
SQLITE_PRIVATE int sqlite3Fts3PendingTermsFlush(Fts3Table *p){
121070
- return fts3SegmentMerge(p, FTS3_SEGCURSOR_PENDING);
122174
+ int rc = SQLITE_OK;
122175
+ int i;
122176
+ for(i=0; rc==SQLITE_OK && i<p->nIndex; i++){
122177
+ rc = fts3SegmentMerge(p, i, FTS3_SEGCURSOR_PENDING);
122178
+ if( rc==SQLITE_DONE ) rc = SQLITE_OK;
122179
+ }
122180
+ sqlite3Fts3PendingTermsClear(p);
122181
+ return rc;
121071122182
}
121072122183
121073122184
/*
121074122185
** Encode N integers as varints into a blob.
121075122186
*/
@@ -121215,10 +122326,27 @@
121215122326
sqlite3_bind_blob(pStmt, 1, pBlob, nBlob, SQLITE_STATIC);
121216122327
sqlite3_step(pStmt);
121217122328
*pRC = sqlite3_reset(pStmt);
121218122329
sqlite3_free(a);
121219122330
}
122331
+
122332
+static int fts3DoOptimize(Fts3Table *p, int bReturnDone){
122333
+ int i;
122334
+ int bSeenDone = 0;
122335
+ int rc = SQLITE_OK;
122336
+ for(i=0; rc==SQLITE_OK && i<p->nIndex; i++){
122337
+ rc = fts3SegmentMerge(p, i, FTS3_SEGCURSOR_ALL);
122338
+ if( rc==SQLITE_DONE ){
122339
+ bSeenDone = 1;
122340
+ rc = SQLITE_OK;
122341
+ }
122342
+ }
122343
+ sqlite3Fts3SegmentsClose(p);
122344
+ sqlite3Fts3PendingTermsClear(p);
122345
+
122346
+ return (rc==SQLITE_OK && bReturnDone && bSeenDone) ? SQLITE_DONE : rc;
122347
+}
121220122348
121221122349
/*
121222122350
** Handle a 'special' INSERT of the form:
121223122351
**
121224122352
** "INSERT INTO tbl(tbl) VALUES(<expr>)"
@@ -121232,16 +122360,11 @@
121232122360
int nVal = sqlite3_value_bytes(pVal);
121233122361
121234122362
if( !zVal ){
121235122363
return SQLITE_NOMEM;
121236122364
}else if( nVal==8 && 0==sqlite3_strnicmp(zVal, "optimize", 8) ){
121237
- rc = fts3SegmentMerge(p, FTS3_SEGCURSOR_ALL);
121238
- if( rc==SQLITE_DONE ){
121239
- rc = SQLITE_OK;
121240
- }else{
121241
- sqlite3Fts3PendingTermsClear(p);
121242
- }
122365
+ rc = fts3DoOptimize(p, 0);
121243122366
#ifdef SQLITE_TEST
121244122367
}else if( nVal>9 && 0==sqlite3_strnicmp(zVal, "nodesize=", 9) ){
121245122368
p->nNodeSize = atoi(&zVal[9]);
121246122369
rc = SQLITE_OK;
121247122370
}else if( nVal>11 && 0==sqlite3_strnicmp(zVal, "maxpending=", 9) ){
@@ -121250,61 +122373,23 @@
121250122373
#endif
121251122374
}else{
121252122375
rc = SQLITE_ERROR;
121253122376
}
121254122377
121255
- sqlite3Fts3SegmentsClose(p);
121256122378
return rc;
121257122379
}
121258122380
121259
-/*
121260
-** Return the deferred doclist associated with deferred token pDeferred.
121261
-** This function assumes that sqlite3Fts3CacheDeferredDoclists() has already
121262
-** been called to allocate and populate the doclist.
121263
-*/
121264
-SQLITE_PRIVATE char *sqlite3Fts3DeferredDoclist(Fts3DeferredToken *pDeferred, int *pnByte){
121265
- if( pDeferred->pList ){
121266
- *pnByte = pDeferred->pList->nData;
121267
- return pDeferred->pList->aData;
121268
- }
121269
- *pnByte = 0;
121270
- return 0;
121271
-}
121272
-
121273
-/*
121274
-** Helper fucntion for FreeDeferredDoclists(). This function removes all
121275
-** references to deferred doclists from within the tree of Fts3Expr
121276
-** structures headed by
121277
-*/
121278
-static void fts3DeferredDoclistClear(Fts3Expr *pExpr){
121279
- if( pExpr ){
121280
- fts3DeferredDoclistClear(pExpr->pLeft);
121281
- fts3DeferredDoclistClear(pExpr->pRight);
121282
- if( pExpr->isLoaded ){
121283
- sqlite3_free(pExpr->aDoclist);
121284
- pExpr->isLoaded = 0;
121285
- pExpr->aDoclist = 0;
121286
- pExpr->nDoclist = 0;
121287
- pExpr->pCurrent = 0;
121288
- pExpr->iCurrent = 0;
121289
- }
121290
- }
121291
-}
121292
-
121293122381
/*
121294122382
** Delete all cached deferred doclists. Deferred doclists are cached
121295122383
** (allocated) by the sqlite3Fts3CacheDeferredDoclists() function.
121296122384
*/
121297122385
SQLITE_PRIVATE void sqlite3Fts3FreeDeferredDoclists(Fts3Cursor *pCsr){
121298122386
Fts3DeferredToken *pDef;
121299122387
for(pDef=pCsr->pDeferred; pDef; pDef=pDef->pNext){
121300
- sqlite3_free(pDef->pList);
122388
+ fts3PendingListDelete(pDef->pList);
121301122389
pDef->pList = 0;
121302122390
}
121303
- if( pCsr->pDeferred ){
121304
- fts3DeferredDoclistClear(pCsr->pExpr);
121305
- }
121306122391
}
121307122392
121308122393
/*
121309122394
** Free all entries in the pCsr->pDeffered list. Entries are added to
121310122395
** this list using sqlite3Fts3DeferToken().
@@ -121312,11 +122397,11 @@
121312122397
SQLITE_PRIVATE void sqlite3Fts3FreeDeferredTokens(Fts3Cursor *pCsr){
121313122398
Fts3DeferredToken *pDef;
121314122399
Fts3DeferredToken *pNext;
121315122400
for(pDef=pCsr->pDeferred; pDef; pDef=pNext){
121316122401
pNext = pDef->pNext;
121317
- sqlite3_free(pDef->pList);
122402
+ fts3PendingListDelete(pDef->pList);
121318122403
sqlite3_free(pDef);
121319122404
}
121320122405
pCsr->pDeferred = 0;
121321122406
}
121322122407
@@ -121376,10 +122461,37 @@
121376122461
}
121377122462
}
121378122463
121379122464
return rc;
121380122465
}
122466
+
122467
+SQLITE_PRIVATE int sqlite3Fts3DeferredTokenList(
122468
+ Fts3DeferredToken *p,
122469
+ char **ppData,
122470
+ int *pnData
122471
+){
122472
+ char *pRet;
122473
+ int nSkip;
122474
+ sqlite3_int64 dummy;
122475
+
122476
+ *ppData = 0;
122477
+ *pnData = 0;
122478
+
122479
+ if( p->pList==0 ){
122480
+ return SQLITE_OK;
122481
+ }
122482
+
122483
+ pRet = (char *)sqlite3_malloc(p->pList->nData);
122484
+ if( !pRet ) return SQLITE_NOMEM;
122485
+
122486
+ nSkip = sqlite3Fts3GetVarint(p->pList->aData, &dummy);
122487
+ *pnData = p->pList->nData - nSkip;
122488
+ *ppData = pRet;
122489
+
122490
+ memcpy(pRet, &p->pList->aData[nSkip], *pnData);
122491
+ return SQLITE_OK;
122492
+}
121381122493
121382122494
/*
121383122495
** Add an entry for token pToken to the pCsr->pDeferred list.
121384122496
*/
121385122497
SQLITE_PRIVATE int sqlite3Fts3DeferToken(
@@ -121451,11 +122563,11 @@
121451122563
){
121452122564
Fts3Table *p = (Fts3Table *)pVtab;
121453122565
int rc = SQLITE_OK; /* Return Code */
121454122566
int isRemove = 0; /* True for an UPDATE or DELETE */
121455122567
sqlite3_int64 iRemove = 0; /* Rowid removed by UPDATE or DELETE */
121456
- u32 *aSzIns; /* Sizes of inserted documents */
122568
+ u32 *aSzIns = 0; /* Sizes of inserted documents */
121457122569
u32 *aSzDel; /* Sizes of deleted documents */
121458122570
int nChng = 0; /* Net change in number of documents */
121459122571
int bInsertDone = 0;
121460122572
121461122573
assert( p->pSegments==0 );
@@ -121466,16 +122578,20 @@
121466122578
*/
121467122579
if( nArg>1
121468122580
&& sqlite3_value_type(apVal[0])==SQLITE_NULL
121469122581
&& sqlite3_value_type(apVal[p->nColumn+2])!=SQLITE_NULL
121470122582
){
121471
- return fts3SpecialInsert(p, apVal[p->nColumn+2]);
122583
+ rc = fts3SpecialInsert(p, apVal[p->nColumn+2]);
122584
+ goto update_out;
121472122585
}
121473122586
121474122587
/* Allocate space to hold the change in document sizes */
121475122588
aSzIns = sqlite3_malloc( sizeof(aSzIns[0])*(p->nColumn+1)*2 );
121476
- if( aSzIns==0 ) return SQLITE_NOMEM;
122589
+ if( aSzIns==0 ){
122590
+ rc = SQLITE_NOMEM;
122591
+ goto update_out;
122592
+ }
121477122593
aSzDel = &aSzIns[p->nColumn+1];
121478122594
memset(aSzIns, 0, sizeof(aSzIns[0])*(p->nColumn+1)*2);
121479122595
121480122596
/* If this is an INSERT operation, or an UPDATE that modifies the rowid
121481122597
** value, then this operation requires constraint handling.
@@ -121521,12 +122637,11 @@
121521122637
bInsertDone = 1;
121522122638
}
121523122639
}
121524122640
}
121525122641
if( rc!=SQLITE_OK ){
121526
- sqlite3_free(aSzIns);
121527
- return rc;
122642
+ goto update_out;
121528122643
}
121529122644
121530122645
/* If this is a DELETE or UPDATE operation, remove the old record. */
121531122646
if( sqlite3_value_type(apVal[0])!=SQLITE_NULL ){
121532122647
assert( sqlite3_value_type(apVal[0])==SQLITE_INTEGER );
@@ -121555,10 +122670,11 @@
121555122670
121556122671
if( p->bHasStat ){
121557122672
fts3UpdateDocTotals(&rc, p, aSzIns, aSzDel, nChng);
121558122673
}
121559122674
122675
+ update_out:
121560122676
sqlite3_free(aSzIns);
121561122677
sqlite3Fts3SegmentsClose(p);
121562122678
return rc;
121563122679
}
121564122680
@@ -121569,16 +122685,14 @@
121569122685
*/
121570122686
SQLITE_PRIVATE int sqlite3Fts3Optimize(Fts3Table *p){
121571122687
int rc;
121572122688
rc = sqlite3_exec(p->db, "SAVEPOINT fts3", 0, 0, 0);
121573122689
if( rc==SQLITE_OK ){
121574
- rc = fts3SegmentMerge(p, FTS3_SEGCURSOR_ALL);
121575
- if( rc==SQLITE_OK ){
121576
- rc = sqlite3_exec(p->db, "RELEASE fts3", 0, 0, 0);
121577
- if( rc==SQLITE_OK ){
121578
- sqlite3Fts3PendingTermsClear(p);
121579
- }
122690
+ rc = fts3DoOptimize(p, 1);
122691
+ if( rc==SQLITE_OK || rc==SQLITE_DONE ){
122692
+ int rc2 = sqlite3_exec(p->db, "RELEASE fts3", 0, 0, 0);
122693
+ if( rc2!=SQLITE_OK ) rc = rc2;
121580122694
}else{
121581122695
sqlite3_exec(p->db, "ROLLBACK TO fts3", 0, 0, 0);
121582122696
sqlite3_exec(p->db, "RELEASE fts3", 0, 0, 0);
121583122697
}
121584122698
}
@@ -121763,76 +122877,24 @@
121763122877
){
121764122878
int iPhrase = 0; /* Variable used as the phrase counter */
121765122879
return fts3ExprIterate2(pExpr, &iPhrase, x, pCtx);
121766122880
}
121767122881
121768
-/*
121769
-** The argument to this function is always a phrase node. Its doclist
121770
-** (Fts3Expr.aDoclist[]) and the doclists associated with all phrase nodes
121771
-** to the left of this one in the query tree have already been loaded.
121772
-**
121773
-** If this phrase node is part of a series of phrase nodes joined by
121774
-** NEAR operators (and is not the left-most of said series), then elements are
121775
-** removed from the phrases doclist consistent with the NEAR restriction. If
121776
-** required, elements may be removed from the doclists of phrases to the
121777
-** left of this one that are part of the same series of NEAR operator
121778
-** connected phrases.
121779
-**
121780
-** If an OOM error occurs, SQLITE_NOMEM is returned. Otherwise, SQLITE_OK.
121781
-*/
121782
-static int fts3ExprNearTrim(Fts3Expr *pExpr){
121783
- int rc = SQLITE_OK;
121784
- Fts3Expr *pParent = pExpr->pParent;
121785
-
121786
- assert( pExpr->eType==FTSQUERY_PHRASE );
121787
- while( rc==SQLITE_OK
121788
- && pParent
121789
- && pParent->eType==FTSQUERY_NEAR
121790
- && pParent->pRight==pExpr
121791
- ){
121792
- /* This expression (pExpr) is the right-hand-side of a NEAR operator.
121793
- ** Find the expression to the left of the same operator.
121794
- */
121795
- int nNear = pParent->nNear;
121796
- Fts3Expr *pLeft = pParent->pLeft;
121797
-
121798
- if( pLeft->eType!=FTSQUERY_PHRASE ){
121799
- assert( pLeft->eType==FTSQUERY_NEAR );
121800
- assert( pLeft->pRight->eType==FTSQUERY_PHRASE );