| | @@ -16,11 +16,11 @@ |
| 16 | 16 | ** if you want a wrapper to interface SQLite with your choice of programming |
| 17 | 17 | ** language. The code for the "sqlite3" command-line shell is also in a |
| 18 | 18 | ** separate file. This file contains only code for the core SQLite library. |
| 19 | 19 | ** |
| 20 | 20 | ** The content in this amalgamation comes from Fossil check-in |
| 21 | | -** 7d95ed60f0a17ea13b4bc19c2ab2ec9052f. |
| 21 | +** a0b9aecbaca9a8e784fd2bcb50f78cbdcf4. |
| 22 | 22 | */ |
| 23 | 23 | #define SQLITE_CORE 1 |
| 24 | 24 | #define SQLITE_AMALGAMATION 1 |
| 25 | 25 | #ifndef SQLITE_PRIVATE |
| 26 | 26 | # define SQLITE_PRIVATE static |
| | @@ -459,11 +459,11 @@ |
| 459 | 459 | ** [sqlite3_libversion_number()], [sqlite3_sourceid()], |
| 460 | 460 | ** [sqlite_version()] and [sqlite_source_id()]. |
| 461 | 461 | */ |
| 462 | 462 | #define SQLITE_VERSION "3.43.0" |
| 463 | 463 | #define SQLITE_VERSION_NUMBER 3043000 |
| 464 | | -#define SQLITE_SOURCE_ID "2023-07-08 17:42:24 07d95ed60f0a17ea13b4bc19c2ab2ec9052fedd27c9e1e57a1ec6e3a6470e5b7" |
| 464 | +#define SQLITE_SOURCE_ID "2023-08-02 16:06:02 ea0b9aecbaca9a8e784fd2bcb50f78cbdcf4c5cfb45a7700bb222e4cc104c644" |
| 465 | 465 | |
| 466 | 466 | /* |
| 467 | 467 | ** CAPI3REF: Run-Time Library Version Numbers |
| 468 | 468 | ** KEYWORDS: sqlite3_version sqlite3_sourceid |
| 469 | 469 | ** |
| | @@ -4732,10 +4732,45 @@ |
| 4732 | 4732 | ** ^The sqlite3_stmt_isexplain(S) interface returns 0 if S is |
| 4733 | 4733 | ** an ordinary statement or a NULL pointer. |
| 4734 | 4734 | */ |
| 4735 | 4735 | SQLITE_API int sqlite3_stmt_isexplain(sqlite3_stmt *pStmt); |
| 4736 | 4736 | |
| 4737 | +/* |
| 4738 | +** CAPI3REF: Change The EXPLAIN Setting For A Prepared Statement |
| 4739 | +** METHOD: sqlite3_stmt |
| 4740 | +** |
| 4741 | +** The sqlite3_stmt_explain(S,E) interface changes the EXPLAIN |
| 4742 | +** setting for [prepared statement] S. If E is zero, then S becomes |
| 4743 | +** a normal prepared statement. If E is 1, then S behaves as if |
| 4744 | +** its SQL text began with "[EXPLAIN]". If E is 2, then S behaves as if |
| 4745 | +** its SQL text began with "[EXPLAIN QUERY PLAN]". |
| 4746 | +** |
| 4747 | +** Calling sqlite3_stmt_explain(S,E) might cause S to be reprepared. |
| 4748 | +** SQLite tries to avoid a reprepare, but a reprepare might be necessary |
| 4749 | +** on the first transition into EXPLAIN or EXPLAIN QUERY PLAN mode. |
| 4750 | +** |
| 4751 | +** Because of the potential need to reprepare, a call to |
| 4752 | +** sqlite3_stmt_explain(S,E) will fail with SQLITE_ERROR if S cannot be |
| 4753 | +** reprepared because it was created using [sqlite3_prepare()] instead of |
| 4754 | +** the newer [sqlite3_prepare_v2()] or [sqlite3_prepare_v3()] interfaces and |
| 4755 | +** hence has no saved SQL text with which to reprepare. |
| 4756 | +** |
| 4757 | +** Changing the explain setting for a prepared statement does not change |
| 4758 | +** the original SQL text for the statement. Hence, if the SQL text originally |
| 4759 | +** began with EXPLAIN or EXPLAIN QUERY PLAN, but sqlite3_stmt_explain(S,0) |
| 4760 | +** is called to convert the statement into an ordinary statement, the EXPLAIN |
| 4761 | +** or EXPLAIN QUERY PLAN keywords will still appear in the sqlite3_sql(S) |
| 4762 | +** output, even though the statement now acts like a normal SQL statement. |
| 4763 | +** |
| 4764 | +** This routine returns SQLITE_OK if the explain mode is successfully |
| 4765 | +** changed, or an error code if the explain mode could not be changed. |
| 4766 | +** The explain mode cannot be changed while a statement is active. |
| 4767 | +** Hence, it is good practice to call [sqlite3_reset(S)] |
| 4768 | +** immediately prior to calling sqlite3_stmt_explain(S,E). |
| 4769 | +*/ |
| 4770 | +SQLITE_API int sqlite3_stmt_explain(sqlite3_stmt *pStmt, int eMode); |
| 4771 | + |
| 4737 | 4772 | /* |
| 4738 | 4773 | ** CAPI3REF: Determine If A Prepared Statement Has Been Reset |
| 4739 | 4774 | ** METHOD: sqlite3_stmt |
| 4740 | 4775 | ** |
| 4741 | 4776 | ** ^The sqlite3_stmt_busy(S) interface returns true (non-zero) if the |
| | @@ -14590,12 +14625,35 @@ |
| 14590 | 14625 | ** |
| 14591 | 14626 | ** In other words, S is a buffer and E is a pointer to the first byte after |
| 14592 | 14627 | ** the end of buffer S. This macro returns true if P points to something |
| 14593 | 14628 | ** contained within the buffer S. |
| 14594 | 14629 | */ |
| 14595 | | -#define SQLITE_WITHIN(P,S,E) (((uptr)(P)>=(uptr)(S))&&((uptr)(P)<(uptr)(E))) |
| 14630 | +#define SQLITE_WITHIN(P,S,E) (((uptr)(P)>=(uptr)(S))&&((uptr)(P)<(uptr)(E))) |
| 14596 | 14631 | |
| 14632 | +/* |
| 14633 | +** P is one byte past the end of a large buffer. Return true if a span of bytes |
| 14634 | +** between S..E crosses the end of that buffer. In other words, return true |
| 14635 | +** if the sub-buffer S..E-1 overflows the buffer show last byte is P-1. |
| 14636 | +** |
| 14637 | +** S is the start of the span. E is one byte past the end of end of span. |
| 14638 | +** |
| 14639 | +** P |
| 14640 | +** |-----------------| FALSE |
| 14641 | +** |-------| |
| 14642 | +** S E |
| 14643 | +** |
| 14644 | +** P |
| 14645 | +** |-----------------| |
| 14646 | +** |-------| TRUE |
| 14647 | +** S E |
| 14648 | +** |
| 14649 | +** P |
| 14650 | +** |-----------------| |
| 14651 | +** |-------| FALSE |
| 14652 | +** S E |
| 14653 | +*/ |
| 14654 | +#define SQLITE_OVERFLOW(P,S,E) (((uptr)(S)<(uptr)(P))&&((uptr)(E)>(uptr)(P))) |
| 14597 | 14655 | |
| 14598 | 14656 | /* |
| 14599 | 14657 | ** Macros to determine whether the machine is big or little endian, |
| 14600 | 14658 | ** and whether or not that determination is run-time or compile-time. |
| 14601 | 14659 | ** |
| | @@ -14959,10 +15017,11 @@ |
| 14959 | 15017 | typedef struct OnOrUsing OnOrUsing; |
| 14960 | 15018 | typedef struct Parse Parse; |
| 14961 | 15019 | typedef struct ParseCleanup ParseCleanup; |
| 14962 | 15020 | typedef struct PreUpdate PreUpdate; |
| 14963 | 15021 | typedef struct PrintfArguments PrintfArguments; |
| 15022 | +typedef struct RCStr RCStr; |
| 14964 | 15023 | typedef struct RenameToken RenameToken; |
| 14965 | 15024 | typedef struct Returning Returning; |
| 14966 | 15025 | typedef struct RowSet RowSet; |
| 14967 | 15026 | typedef struct Savepoint Savepoint; |
| 14968 | 15027 | typedef struct Select Select; |
| | @@ -15925,13 +15984,11 @@ |
| 15925 | 15984 | SQLITE_PRIVATE int sqlite3BtreeEof(BtCursor*); |
| 15926 | 15985 | SQLITE_PRIVATE int sqlite3BtreePrevious(BtCursor*, int flags); |
| 15927 | 15986 | SQLITE_PRIVATE i64 sqlite3BtreeIntegerKey(BtCursor*); |
| 15928 | 15987 | SQLITE_PRIVATE void sqlite3BtreeCursorPin(BtCursor*); |
| 15929 | 15988 | SQLITE_PRIVATE void sqlite3BtreeCursorUnpin(BtCursor*); |
| 15930 | | -#ifdef SQLITE_ENABLE_OFFSET_SQL_FUNC |
| 15931 | 15989 | SQLITE_PRIVATE i64 sqlite3BtreeOffset(BtCursor*); |
| 15932 | | -#endif |
| 15933 | 15990 | SQLITE_PRIVATE int sqlite3BtreePayload(BtCursor*, u32 offset, u32 amt, void*); |
| 15934 | 15991 | SQLITE_PRIVATE const void *sqlite3BtreePayloadFetch(BtCursor*, u32 *pAmt); |
| 15935 | 15992 | SQLITE_PRIVATE u32 sqlite3BtreePayloadSize(BtCursor*); |
| 15936 | 15993 | SQLITE_PRIVATE sqlite3_int64 sqlite3BtreeMaxRecordSize(BtCursor*); |
| 15937 | 15994 | |
| | @@ -17469,10 +17526,11 @@ |
| 17469 | 17526 | #define SQLITE_FlttnUnionAll 0x00800000 /* Disable the UNION ALL flattener */ |
| 17470 | 17527 | /* TH3 expects this value ^^^^^^^^^^ See flatten04.test */ |
| 17471 | 17528 | #define SQLITE_IndexedExpr 0x01000000 /* Pull exprs from index when able */ |
| 17472 | 17529 | #define SQLITE_Coroutines 0x02000000 /* Co-routines for subqueries */ |
| 17473 | 17530 | #define SQLITE_NullUnusedCols 0x04000000 /* NULL unused columns in subqueries */ |
| 17531 | +#define SQLITE_OnePass 0x08000000 /* Single-pass DELETE and UPDATE */ |
| 17474 | 17532 | #define SQLITE_AllOpts 0xffffffff /* All optimizations */ |
| 17475 | 17533 | |
| 17476 | 17534 | /* |
| 17477 | 17535 | ** Macros for testing whether or not optimizations are enabled or disabled. |
| 17478 | 17536 | */ |
| | @@ -19385,11 +19443,11 @@ |
| 19385 | 19443 | ParseCleanup *pCleanup; /* List of cleanup operations to run after parse */ |
| 19386 | 19444 | union { |
| 19387 | 19445 | int addrCrTab; /* Address of OP_CreateBtree on CREATE TABLE */ |
| 19388 | 19446 | Returning *pReturning; /* The RETURNING clause */ |
| 19389 | 19447 | } u1; |
| 19390 | | - u32 nQueryLoop; /* Est number of iterations of a query (10*log2(N)) */ |
| 19448 | + LogEst nQueryLoop; /* Est number of iterations of a query (10*log2(N)) */ |
| 19391 | 19449 | u32 oldmask; /* Mask of old.* columns referenced */ |
| 19392 | 19450 | u32 newmask; /* Mask of new.* columns referenced */ |
| 19393 | 19451 | #ifndef SQLITE_OMIT_PROGRESS_CALLBACK |
| 19394 | 19452 | u32 nProgressSteps; /* xProgress steps taken during sqlite3_prepare() */ |
| 19395 | 19453 | #endif |
| | @@ -19657,10 +19715,29 @@ |
| 19657 | 19715 | #define SQLITE_PRINTF_SQLFUNC 0x02 /* SQL function arguments to VXPrintf */ |
| 19658 | 19716 | #define SQLITE_PRINTF_MALLOCED 0x04 /* True if xText is allocated space */ |
| 19659 | 19717 | |
| 19660 | 19718 | #define isMalloced(X) (((X)->printfFlags & SQLITE_PRINTF_MALLOCED)!=0) |
| 19661 | 19719 | |
| 19720 | +/* |
| 19721 | +** The following object is the header for an "RCStr" or "reference-counted |
| 19722 | +** string". An RCStr is passed around and used like any other char* |
| 19723 | +** that has been dynamically allocated. The important interface |
| 19724 | +** differences: |
| 19725 | +** |
| 19726 | +** 1. RCStr strings are reference counted. They are deallocated |
| 19727 | +** when the reference count reaches zero. |
| 19728 | +** |
| 19729 | +** 2. Use sqlite3RCStrUnref() to free an RCStr string rather than |
| 19730 | +** sqlite3_free() |
| 19731 | +** |
| 19732 | +** 3. Make a (read-only) copy of a read-only RCStr string using |
| 19733 | +** sqlite3RCStrRef(). |
| 19734 | +*/ |
| 19735 | +struct RCStr { |
| 19736 | + u64 nRCRef; /* Number of references */ |
| 19737 | + /* Total structure size should be a multiple of 8 bytes for alignment */ |
| 19738 | +}; |
| 19662 | 19739 | |
| 19663 | 19740 | /* |
| 19664 | 19741 | ** A pointer to this structure is used to communicate information |
| 19665 | 19742 | ** from sqlite3Init and OP_ParseSchema into the sqlite3InitCallback. |
| 19666 | 19743 | */ |
| | @@ -20776,10 +20853,11 @@ |
| 20776 | 20853 | # define sqlite3FileSuffix3(X,Y) |
| 20777 | 20854 | #endif |
| 20778 | 20855 | SQLITE_PRIVATE u8 sqlite3GetBoolean(const char *z,u8); |
| 20779 | 20856 | |
| 20780 | 20857 | SQLITE_PRIVATE const void *sqlite3ValueText(sqlite3_value*, u8); |
| 20858 | +SQLITE_PRIVATE int sqlite3ValueIsOfClass(const sqlite3_value*, void(*)(void*)); |
| 20781 | 20859 | SQLITE_PRIVATE int sqlite3ValueBytes(sqlite3_value*, u8); |
| 20782 | 20860 | SQLITE_PRIVATE void sqlite3ValueSetStr(sqlite3_value*, int, const void *,u8, |
| 20783 | 20861 | void(*)(void*)); |
| 20784 | 20862 | SQLITE_PRIVATE void sqlite3ValueSetNull(sqlite3_value*); |
| 20785 | 20863 | SQLITE_PRIVATE void sqlite3ValueFree(sqlite3_value*); |
| | @@ -20883,10 +20961,15 @@ |
| 20883 | 20961 | SQLITE_PRIVATE void *sqlite3OomFault(sqlite3*); |
| 20884 | 20962 | SQLITE_PRIVATE void sqlite3OomClear(sqlite3*); |
| 20885 | 20963 | SQLITE_PRIVATE int sqlite3ApiExit(sqlite3 *db, int); |
| 20886 | 20964 | SQLITE_PRIVATE int sqlite3OpenTempDatabase(Parse *); |
| 20887 | 20965 | |
| 20966 | +SQLITE_PRIVATE char *sqlite3RCStrRef(char*); |
| 20967 | +SQLITE_PRIVATE void sqlite3RCStrUnref(char*); |
| 20968 | +SQLITE_PRIVATE char *sqlite3RCStrNew(u64); |
| 20969 | +SQLITE_PRIVATE char *sqlite3RCStrResize(char*,u64); |
| 20970 | + |
| 20888 | 20971 | SQLITE_PRIVATE void sqlite3StrAccumInit(StrAccum*, sqlite3*, char*, int, int); |
| 20889 | 20972 | SQLITE_PRIVATE int sqlite3StrAccumEnlarge(StrAccum*, i64); |
| 20890 | 20973 | SQLITE_PRIVATE char *sqlite3StrAccumFinish(StrAccum*); |
| 20891 | 20974 | SQLITE_PRIVATE void sqlite3StrAccumSetError(StrAccum*, u8); |
| 20892 | 20975 | SQLITE_PRIVATE void sqlite3ResultStrAccum(sqlite3_context*,StrAccum*); |
| | @@ -22623,10 +22706,13 @@ |
| 22623 | 22706 | typedef struct VdbeSorter VdbeSorter; |
| 22624 | 22707 | |
| 22625 | 22708 | /* Elements of the linked list at Vdbe.pAuxData */ |
| 22626 | 22709 | typedef struct AuxData AuxData; |
| 22627 | 22710 | |
| 22711 | +/* A cache of large TEXT or BLOB values in a VdbeCursor */ |
| 22712 | +typedef struct VdbeTxtBlbCache VdbeTxtBlbCache; |
| 22713 | + |
| 22628 | 22714 | /* Types of VDBE cursors */ |
| 22629 | 22715 | #define CURTYPE_BTREE 0 |
| 22630 | 22716 | #define CURTYPE_SORTER 1 |
| 22631 | 22717 | #define CURTYPE_VTAB 2 |
| 22632 | 22718 | #define CURTYPE_PSEUDO 3 |
| | @@ -22654,10 +22740,11 @@ |
| 22654 | 22740 | #endif |
| 22655 | 22741 | Bool isEphemeral:1; /* True for an ephemeral table */ |
| 22656 | 22742 | Bool useRandomRowid:1; /* Generate new record numbers semi-randomly */ |
| 22657 | 22743 | Bool isOrdered:1; /* True if the table is not BTREE_UNORDERED */ |
| 22658 | 22744 | Bool noReuse:1; /* OpenEphemeral may not reuse this cursor */ |
| 22745 | + Bool colCache:1; /* pCache pointer is initialized and non-NULL */ |
| 22659 | 22746 | u16 seekHit; /* See the OP_SeekHit and OP_IfNoHope opcodes */ |
| 22660 | 22747 | union { /* pBtx for isEphermeral. pAltMap otherwise */ |
| 22661 | 22748 | Btree *pBtx; /* Separate file holding temporary table */ |
| 22662 | 22749 | u32 *aAltMap; /* Mapping from table to index column numbers */ |
| 22663 | 22750 | } ub; |
| | @@ -22694,10 +22781,11 @@ |
| 22694 | 22781 | u32 payloadSize; /* Total number of bytes in the record */ |
| 22695 | 22782 | u32 szRow; /* Byte available in aRow */ |
| 22696 | 22783 | #ifdef SQLITE_ENABLE_COLUMN_USED_MASK |
| 22697 | 22784 | u64 maskUsed; /* Mask of columns used by this cursor */ |
| 22698 | 22785 | #endif |
| 22786 | + VdbeTxtBlbCache *pCache; /* Cache of large TEXT or BLOB values */ |
| 22699 | 22787 | |
| 22700 | 22788 | /* 2*nField extra array elements allocated for aType[], beyond the one |
| 22701 | 22789 | ** static element declared in the structure. nField total array slots for |
| 22702 | 22790 | ** aType[] and nField+1 array slots for aOffset[] */ |
| 22703 | 22791 | u32 aType[1]; /* Type values record decode. MUST BE LAST */ |
| | @@ -22706,15 +22794,28 @@ |
| 22706 | 22794 | /* Return true if P is a null-only cursor |
| 22707 | 22795 | */ |
| 22708 | 22796 | #define IsNullCursor(P) \ |
| 22709 | 22797 | ((P)->eCurType==CURTYPE_PSEUDO && (P)->nullRow && (P)->seekResult==0) |
| 22710 | 22798 | |
| 22711 | | - |
| 22712 | 22799 | /* |
| 22713 | 22800 | ** A value for VdbeCursor.cacheStatus that means the cache is always invalid. |
| 22714 | 22801 | */ |
| 22715 | 22802 | #define CACHE_STALE 0 |
| 22803 | + |
| 22804 | +/* |
| 22805 | +** Large TEXT or BLOB values can be slow to load, so we want to avoid |
| 22806 | +** loading them more than once. For that reason, large TEXT and BLOB values |
| 22807 | +** can be stored in a cache defined by this object, and attached to the |
| 22808 | +** VdbeCursor using the pCache field. |
| 22809 | +*/ |
| 22810 | +struct VdbeTxtBlbCache { |
| 22811 | + char *pCValue; /* A RCStr buffer to hold the value */ |
| 22812 | + i64 iOffset; /* File offset of the row being cached */ |
| 22813 | + int iCol; /* Column for which the cache is valid */ |
| 22814 | + u32 cacheStatus; /* Vdbe.cacheCtr value */ |
| 22815 | + u32 colCacheCtr; /* Column cache counter */ |
| 22816 | +}; |
| 22716 | 22817 | |
| 22717 | 22818 | /* |
| 22718 | 22819 | ** When a sub-program is executed (OP_Program), a structure of this type |
| 22719 | 22820 | ** is allocated to store the current value of the program counter, as |
| 22720 | 22821 | ** well as the current memory cell array and various other frame specific |
| | @@ -23032,20 +23133,22 @@ |
| 23032 | 23133 | #ifdef SQLITE_DEBUG |
| 23033 | 23134 | int rcApp; /* errcode set by sqlite3_result_error_code() */ |
| 23034 | 23135 | u32 nWrite; /* Number of write operations that have occurred */ |
| 23035 | 23136 | #endif |
| 23036 | 23137 | u16 nResColumn; /* Number of columns in one row of the result set */ |
| 23138 | + u16 nResAlloc; /* Column slots allocated to aColName[] */ |
| 23037 | 23139 | u8 errorAction; /* Recovery action to do in case of an error */ |
| 23038 | 23140 | u8 minWriteFileFormat; /* Minimum file format for writable database files */ |
| 23039 | 23141 | u8 prepFlags; /* SQLITE_PREPARE_* flags */ |
| 23040 | 23142 | u8 eVdbeState; /* On of the VDBE_*_STATE values */ |
| 23041 | 23143 | bft expired:2; /* 1: recompile VM immediately 2: when convenient */ |
| 23042 | | - bft explain:2; /* True if EXPLAIN present on SQL command */ |
| 23144 | + bft explain:2; /* 0: normal, 1: EXPLAIN, 2: EXPLAIN QUERY PLAN */ |
| 23043 | 23145 | bft changeCntOn:1; /* True to update the change-counter */ |
| 23044 | 23146 | bft usesStmtJournal:1; /* True if uses a statement journal */ |
| 23045 | 23147 | bft readOnly:1; /* True for statements that do not write */ |
| 23046 | 23148 | bft bIsReader:1; /* True for statements that read */ |
| 23149 | + bft haveEqpOps:1; /* Bytecode supports EXPLAIN QUERY PLAN */ |
| 23047 | 23150 | yDbMask btreeMask; /* Bitmask of db->aDb[] entries referenced */ |
| 23048 | 23151 | yDbMask lockMask; /* Subset of btreeMask that requires a lock */ |
| 23049 | 23152 | u32 aCounter[9]; /* Counters used by sqlite3_stmt_status() */ |
| 23050 | 23153 | char *zSql; /* Text of the SQL statement that generated this */ |
| 23051 | 23154 | #ifdef SQLITE_ENABLE_NORMALIZE |
| | @@ -23178,10 +23281,11 @@ |
| 23178 | 23281 | #endif |
| 23179 | 23282 | #ifdef SQLITE_DEBUG |
| 23180 | 23283 | SQLITE_PRIVATE int sqlite3VdbeMemIsRowSet(const Mem*); |
| 23181 | 23284 | #endif |
| 23182 | 23285 | SQLITE_PRIVATE int sqlite3VdbeMemSetRowSet(Mem*); |
| 23286 | +SQLITE_PRIVATE void sqlite3VdbeMemZeroTerminateIfAble(Mem*); |
| 23183 | 23287 | SQLITE_PRIVATE int sqlite3VdbeMemMakeWriteable(Mem*); |
| 23184 | 23288 | SQLITE_PRIVATE int sqlite3VdbeMemStringify(Mem*, u8, u8); |
| 23185 | 23289 | SQLITE_PRIVATE int sqlite3IntFloatCompare(i64,double); |
| 23186 | 23290 | SQLITE_PRIVATE i64 sqlite3VdbeIntValue(const Mem*); |
| 23187 | 23291 | SQLITE_PRIVATE int sqlite3VdbeMemIntegerify(Mem*); |
| | @@ -31698,10 +31802,79 @@ |
| 31698 | 31802 | va_start(ap,zFormat); |
| 31699 | 31803 | sqlite3_str_vappendf(p, zFormat, ap); |
| 31700 | 31804 | va_end(ap); |
| 31701 | 31805 | } |
| 31702 | 31806 | |
| 31807 | + |
| 31808 | +/***************************************************************************** |
| 31809 | +** Reference counted string storage |
| 31810 | +*****************************************************************************/ |
| 31811 | + |
| 31812 | +/* |
| 31813 | +** Increase the reference count of the string by one. |
| 31814 | +** |
| 31815 | +** The input parameter is returned. |
| 31816 | +*/ |
| 31817 | +SQLITE_PRIVATE char *sqlite3RCStrRef(char *z){ |
| 31818 | + RCStr *p = (RCStr*)z; |
| 31819 | + assert( p!=0 ); |
| 31820 | + p--; |
| 31821 | + p->nRCRef++; |
| 31822 | + return z; |
| 31823 | +} |
| 31824 | + |
| 31825 | +/* |
| 31826 | +** Decrease the reference count by one. Free the string when the |
| 31827 | +** reference count reaches zero. |
| 31828 | +*/ |
| 31829 | +SQLITE_PRIVATE void sqlite3RCStrUnref(char *z){ |
| 31830 | + RCStr *p = (RCStr*)z; |
| 31831 | + assert( p!=0 ); |
| 31832 | + p--; |
| 31833 | + assert( p->nRCRef>0 ); |
| 31834 | + if( p->nRCRef>=2 ){ |
| 31835 | + p->nRCRef--; |
| 31836 | + }else{ |
| 31837 | + sqlite3_free(p); |
| 31838 | + } |
| 31839 | +} |
| 31840 | + |
| 31841 | +/* |
| 31842 | +** Create a new string that is capable of holding N bytes of text, not counting |
| 31843 | +** the zero byte at the end. The string is uninitialized. |
| 31844 | +** |
| 31845 | +** The reference count is initially 1. Call sqlite3RCStrUnref() to free the |
| 31846 | +** newly allocated string. |
| 31847 | +** |
| 31848 | +** This routine returns 0 on an OOM. |
| 31849 | +*/ |
| 31850 | +SQLITE_PRIVATE char *sqlite3RCStrNew(u64 N){ |
| 31851 | + RCStr *p = sqlite3_malloc64( N + sizeof(*p) + 1 ); |
| 31852 | + if( p==0 ) return 0; |
| 31853 | + p->nRCRef = 1; |
| 31854 | + return (char*)&p[1]; |
| 31855 | +} |
| 31856 | + |
| 31857 | +/* |
| 31858 | +** Change the size of the string so that it is able to hold N bytes. |
| 31859 | +** The string might be reallocated, so return the new allocation. |
| 31860 | +*/ |
| 31861 | +SQLITE_PRIVATE char *sqlite3RCStrResize(char *z, u64 N){ |
| 31862 | + RCStr *p = (RCStr*)z; |
| 31863 | + RCStr *pNew; |
| 31864 | + assert( p!=0 ); |
| 31865 | + p--; |
| 31866 | + assert( p->nRCRef==1 ); |
| 31867 | + pNew = sqlite3_realloc64(p, N+sizeof(RCStr)+1); |
| 31868 | + if( pNew==0 ){ |
| 31869 | + sqlite3_free(p); |
| 31870 | + return 0; |
| 31871 | + }else{ |
| 31872 | + return (char*)&pNew[1]; |
| 31873 | + } |
| 31874 | +} |
| 31875 | + |
| 31703 | 31876 | /************** End of printf.c **********************************************/ |
| 31704 | 31877 | /************** Begin file treeview.c ****************************************/ |
| 31705 | 31878 | /* |
| 31706 | 31879 | ** 2015-06-08 |
| 31707 | 31880 | ** |
| | @@ -34572,11 +34745,16 @@ |
| 34572 | 34745 | }else{ |
| 34573 | 34746 | while( e<=-100 ){ e+=100; r *= 1.0e-100L; } |
| 34574 | 34747 | while( e<=-10 ){ e+=10; r *= 1.0e-10L; } |
| 34575 | 34748 | while( e<=-1 ){ e+=1; r *= 1.0e-01L; } |
| 34576 | 34749 | } |
| 34577 | | - *pResult = r; |
| 34750 | + assert( r>=0.0 ); |
| 34751 | + if( r>+1.7976931348623157081452742373e+308L ){ |
| 34752 | + *pResult = +INFINITY; |
| 34753 | + }else{ |
| 34754 | + *pResult = (double)r; |
| 34755 | + } |
| 34578 | 34756 | }else{ |
| 34579 | 34757 | double rr[2]; |
| 34580 | 34758 | u64 s2; |
| 34581 | 34759 | rr[0] = (double)s; |
| 34582 | 34760 | s2 = (u64)rr[0]; |
| | @@ -34827,11 +35005,13 @@ |
| 34827 | 35005 | if( z[k]!=0 ) return 1; |
| 34828 | 35006 | return 0; |
| 34829 | 35007 | }else |
| 34830 | 35008 | #endif /* SQLITE_OMIT_HEX_INTEGER */ |
| 34831 | 35009 | { |
| 34832 | | - return sqlite3Atoi64(z, pOut, sqlite3Strlen30(z), SQLITE_UTF8); |
| 35010 | + int n = (int)(0x3fffffff&strspn(z,"+- \n\t0123456789")); |
| 35011 | + if( z[n] ) n++; |
| 35012 | + return sqlite3Atoi64(z, pOut, n, SQLITE_UTF8); |
| 34833 | 35013 | } |
| 34834 | 35014 | } |
| 34835 | 35015 | |
| 34836 | 35016 | /* |
| 34837 | 35017 | ** If zNum represents an integer that will fit in 32-bits, then set |
| | @@ -65559,27 +65739,19 @@ |
| 65559 | 65739 | /* Allocate space for the WalIterator object. */ |
| 65560 | 65740 | nSegment = walFramePage(iLast) + 1; |
| 65561 | 65741 | nByte = sizeof(WalIterator) |
| 65562 | 65742 | + (nSegment-1)*sizeof(struct WalSegment) |
| 65563 | 65743 | + iLast*sizeof(ht_slot); |
| 65564 | | - p = (WalIterator *)sqlite3_malloc64(nByte); |
| 65744 | + p = (WalIterator *)sqlite3_malloc64(nByte |
| 65745 | + + sizeof(ht_slot) * (iLast>HASHTABLE_NPAGE?HASHTABLE_NPAGE:iLast) |
| 65746 | + ); |
| 65565 | 65747 | if( !p ){ |
| 65566 | 65748 | return SQLITE_NOMEM_BKPT; |
| 65567 | 65749 | } |
| 65568 | 65750 | memset(p, 0, nByte); |
| 65569 | 65751 | p->nSegment = nSegment; |
| 65570 | | - |
| 65571 | | - /* Allocate temporary space used by the merge-sort routine. This block |
| 65572 | | - ** of memory will be freed before this function returns. |
| 65573 | | - */ |
| 65574 | | - aTmp = (ht_slot *)sqlite3_malloc64( |
| 65575 | | - sizeof(ht_slot) * (iLast>HASHTABLE_NPAGE?HASHTABLE_NPAGE:iLast) |
| 65576 | | - ); |
| 65577 | | - if( !aTmp ){ |
| 65578 | | - rc = SQLITE_NOMEM_BKPT; |
| 65579 | | - } |
| 65580 | | - |
| 65752 | + aTmp = (ht_slot*)&(((u8*)p)[nByte]); |
| 65581 | 65753 | for(i=walFramePage(nBackfill+1); rc==SQLITE_OK && i<nSegment; i++){ |
| 65582 | 65754 | WalHashLoc sLoc; |
| 65583 | 65755 | |
| 65584 | 65756 | rc = walHashGet(pWal, i, &sLoc); |
| 65585 | 65757 | if( rc==SQLITE_OK ){ |
| | @@ -65603,12 +65775,10 @@ |
| 65603 | 65775 | p->aSegment[i].nEntry = nEntry; |
| 65604 | 65776 | p->aSegment[i].aIndex = aIndex; |
| 65605 | 65777 | p->aSegment[i].aPgno = (u32 *)sLoc.aPgno; |
| 65606 | 65778 | } |
| 65607 | 65779 | } |
| 65608 | | - sqlite3_free(aTmp); |
| 65609 | | - |
| 65610 | 65780 | if( rc!=SQLITE_OK ){ |
| 65611 | 65781 | walIteratorFree(p); |
| 65612 | 65782 | p = 0; |
| 65613 | 65783 | } |
| 65614 | 65784 | *pp = p; |
| | @@ -68121,11 +68291,11 @@ |
| 68121 | 68291 | ** 0x00 becomes 0x00000000 |
| 68122 | 68292 | ** 0x7f becomes 0x0000007f |
| 68123 | 68293 | ** 0x81 0x00 becomes 0x00000080 |
| 68124 | 68294 | ** 0x82 0x00 becomes 0x00000100 |
| 68125 | 68295 | ** 0x80 0x7f becomes 0x0000007f |
| 68126 | | -** 0x8a 0x91 0xd1 0xac 0x78 becomes 0x12345678 |
| 68296 | +** 0x81 0x91 0xd1 0xac 0x78 becomes 0x12345678 |
| 68127 | 68297 | ** 0x81 0x81 0x81 0x81 0x01 becomes 0x10204081 |
| 68128 | 68298 | ** |
| 68129 | 68299 | ** Variable length integers are used for rowids and to hold the number of |
| 68130 | 68300 | ** bytes of key and data in a btree cell. |
| 68131 | 68301 | ** |
| | @@ -70505,11 +70675,11 @@ |
| 70505 | 70675 | if( *pRC ) return; |
| 70506 | 70676 | assert( pCell!=0 ); |
| 70507 | 70677 | pPage->xParseCell(pPage, pCell, &info); |
| 70508 | 70678 | if( info.nLocal<info.nPayload ){ |
| 70509 | 70679 | Pgno ovfl; |
| 70510 | | - if( SQLITE_WITHIN(pSrc->aDataEnd, pCell, pCell+info.nLocal) ){ |
| 70680 | + if( SQLITE_OVERFLOW(pSrc->aDataEnd, pCell, pCell+info.nLocal) ){ |
| 70511 | 70681 | testcase( pSrc!=pPage ); |
| 70512 | 70682 | *pRC = SQLITE_CORRUPT_BKPT; |
| 70513 | 70683 | return; |
| 70514 | 70684 | } |
| 70515 | 70685 | ovfl = get4byte(&pCell[info.nSize-4]); |
| | @@ -73797,11 +73967,10 @@ |
| 73797 | 73967 | SQLITE_PRIVATE void sqlite3BtreeCursorUnpin(BtCursor *pCur){ |
| 73798 | 73968 | assert( (pCur->curFlags & BTCF_Pinned)!=0 ); |
| 73799 | 73969 | pCur->curFlags &= ~BTCF_Pinned; |
| 73800 | 73970 | } |
| 73801 | 73971 | |
| 73802 | | -#ifdef SQLITE_ENABLE_OFFSET_SQL_FUNC |
| 73803 | 73972 | /* |
| 73804 | 73973 | ** Return the offset into the database file for the start of the |
| 73805 | 73974 | ** payload to which the cursor is pointing. |
| 73806 | 73975 | */ |
| 73807 | 73976 | SQLITE_PRIVATE i64 sqlite3BtreeOffset(BtCursor *pCur){ |
| | @@ -73809,11 +73978,10 @@ |
| 73809 | 73978 | assert( pCur->eState==CURSOR_VALID ); |
| 73810 | 73979 | getCellInfo(pCur); |
| 73811 | 73980 | return (i64)pCur->pBt->pageSize*((i64)pCur->pPage->pgno - 1) + |
| 73812 | 73981 | (i64)(pCur->info.pPayload - pCur->pPage->aData); |
| 73813 | 73982 | } |
| 73814 | | -#endif /* SQLITE_ENABLE_OFFSET_SQL_FUNC */ |
| 73815 | 73983 | |
| 73816 | 73984 | /* |
| 73817 | 73985 | ** Return the number of bytes of payload for the entry that pCur is |
| 73818 | 73986 | ** currently pointing to. For table btrees, this will be the amount |
| 73819 | 73987 | ** of data. For index btrees, this will be the size of the key. |
| | @@ -77666,11 +77834,11 @@ |
| 77666 | 77834 | iOvflSpace += sz; |
| 77667 | 77835 | assert( sz<=pBt->maxLocal+23 ); |
| 77668 | 77836 | assert( iOvflSpace <= (int)pBt->pageSize ); |
| 77669 | 77837 | for(k=0; ALWAYS(k<NB*2) && b.ixNx[k]<=j; k++){} |
| 77670 | 77838 | pSrcEnd = b.apEnd[k]; |
| 77671 | | - if( SQLITE_WITHIN(pSrcEnd, pCell, pCell+sz) ){ |
| 77839 | + if( SQLITE_OVERFLOW(pSrcEnd, pCell, pCell+sz) ){ |
| 77672 | 77840 | rc = SQLITE_CORRUPT_BKPT; |
| 77673 | 77841 | goto balance_cleanup; |
| 77674 | 77842 | } |
| 77675 | 77843 | rc = insertCell(pParent, nxDiv+i, pCell, sz, pTemp, pNew->pgno); |
| 77676 | 77844 | if( rc!=SQLITE_OK ) goto balance_cleanup; |
| | @@ -81439,10 +81607,44 @@ |
| 81439 | 81607 | assert( (pMem->flags & MEM_Dyn)==0 ); |
| 81440 | 81608 | pMem->z = pMem->zMalloc; |
| 81441 | 81609 | pMem->flags &= (MEM_Null|MEM_Int|MEM_Real|MEM_IntReal); |
| 81442 | 81610 | return SQLITE_OK; |
| 81443 | 81611 | } |
| 81612 | + |
| 81613 | +/* |
| 81614 | +** If pMem is already a string, detect if it is a zero-terminated |
| 81615 | +** string, or make it into one if possible, and mark it as such. |
| 81616 | +** |
| 81617 | +** This is an optimization. Correct operation continues even if |
| 81618 | +** this routine is a no-op. |
| 81619 | +*/ |
| 81620 | +SQLITE_PRIVATE void sqlite3VdbeMemZeroTerminateIfAble(Mem *pMem){ |
| 81621 | + if( (pMem->flags & (MEM_Str|MEM_Term|MEM_Ephem|MEM_Static))!=MEM_Str ){ |
| 81622 | + /* pMem must be a string, and it cannot be an ephemeral or static string */ |
| 81623 | + return; |
| 81624 | + } |
| 81625 | + if( pMem->enc!=SQLITE_UTF8 ) return; |
| 81626 | + if( NEVER(pMem->z==0) ) return; |
| 81627 | + if( pMem->flags & MEM_Dyn ){ |
| 81628 | + if( pMem->xDel==sqlite3_free |
| 81629 | + && sqlite3_msize(pMem->z) >= (u64)(pMem->n+1) |
| 81630 | + ){ |
| 81631 | + pMem->z[pMem->n] = 0; |
| 81632 | + pMem->flags |= MEM_Term; |
| 81633 | + return; |
| 81634 | + } |
| 81635 | + if( pMem->xDel==(void(*)(void*))sqlite3RCStrUnref ){ |
| 81636 | + /* Blindly assume that all RCStr objects are zero-terminated */ |
| 81637 | + pMem->flags |= MEM_Term; |
| 81638 | + return; |
| 81639 | + } |
| 81640 | + }else if( pMem->szMalloc >= pMem->n+1 ){ |
| 81641 | + pMem->z[pMem->n] = 0; |
| 81642 | + pMem->flags |= MEM_Term; |
| 81643 | + return; |
| 81644 | + } |
| 81645 | +} |
| 81444 | 81646 | |
| 81445 | 81647 | /* |
| 81446 | 81648 | ** It is already known that pMem contains an unterminated string. |
| 81447 | 81649 | ** Add the zero terminator. |
| 81448 | 81650 | ** |
| | @@ -81929,18 +82131,21 @@ |
| 81929 | 82131 | case SQLITE_AFF_REAL: { |
| 81930 | 82132 | sqlite3VdbeMemRealify(pMem); |
| 81931 | 82133 | break; |
| 81932 | 82134 | } |
| 81933 | 82135 | default: { |
| 82136 | + int rc; |
| 81934 | 82137 | assert( aff==SQLITE_AFF_TEXT ); |
| 81935 | 82138 | assert( MEM_Str==(MEM_Blob>>3) ); |
| 81936 | 82139 | pMem->flags |= (pMem->flags&MEM_Blob)>>3; |
| 81937 | 82140 | sqlite3ValueApplyAffinity(pMem, SQLITE_AFF_TEXT, encoding); |
| 81938 | 82141 | assert( pMem->flags & MEM_Str || pMem->db->mallocFailed ); |
| 81939 | 82142 | pMem->flags &= ~(MEM_Int|MEM_Real|MEM_IntReal|MEM_Blob|MEM_Zero); |
| 81940 | 82143 | if( encoding!=SQLITE_UTF8 ) pMem->n &= ~1; |
| 81941 | | - return sqlite3VdbeChangeEncoding(pMem, encoding); |
| 82144 | + rc = sqlite3VdbeChangeEncoding(pMem, encoding); |
| 82145 | + if( rc ) return rc; |
| 82146 | + sqlite3VdbeMemZeroTerminateIfAble(pMem); |
| 81942 | 82147 | } |
| 81943 | 82148 | } |
| 81944 | 82149 | return SQLITE_OK; |
| 81945 | 82150 | } |
| 81946 | 82151 | |
| | @@ -82459,10 +82664,28 @@ |
| 82459 | 82664 | if( pVal->flags&MEM_Null ){ |
| 82460 | 82665 | return 0; |
| 82461 | 82666 | } |
| 82462 | 82667 | return valueToText(pVal, enc); |
| 82463 | 82668 | } |
| 82669 | + |
| 82670 | +/* Return true if sqlit3_value object pVal is a string or blob value |
| 82671 | +** that uses the destructor specified in the second argument. |
| 82672 | +** |
| 82673 | +** TODO: Maybe someday promote this interface into a published API so |
| 82674 | +** that third-party extensions can get access to it? |
| 82675 | +*/ |
| 82676 | +SQLITE_PRIVATE int sqlite3ValueIsOfClass(const sqlite3_value *pVal, void(*xFree)(void*)){ |
| 82677 | + if( ALWAYS(pVal!=0) |
| 82678 | + && ALWAYS((pVal->flags & (MEM_Str|MEM_Blob))!=0) |
| 82679 | + && (pVal->flags & MEM_Dyn)!=0 |
| 82680 | + && pVal->xDel==xFree |
| 82681 | + ){ |
| 82682 | + return 1; |
| 82683 | + }else{ |
| 82684 | + return 0; |
| 82685 | + } |
| 82686 | +} |
| 82464 | 82687 | |
| 82465 | 82688 | /* |
| 82466 | 82689 | ** Create a new sqlite3_value object. |
| 82467 | 82690 | */ |
| 82468 | 82691 | SQLITE_PRIVATE sqlite3_value *sqlite3ValueNew(sqlite3 *db){ |
| | @@ -84596,11 +84819,10 @@ |
| 84596 | 84819 | if( bUndefine ) sqlite3VdbeChangeP5(pParse->pVdbe, 1); |
| 84597 | 84820 | } |
| 84598 | 84821 | } |
| 84599 | 84822 | #endif /* SQLITE_DEBUG */ |
| 84600 | 84823 | |
| 84601 | | - |
| 84602 | 84824 | /* |
| 84603 | 84825 | ** Change the value of the P4 operand for a specific instruction. |
| 84604 | 84826 | ** This routine is useful when a large program is loaded from a |
| 84605 | 84827 | ** static array using sqlite3VdbeAddOpList but we want to make a |
| 84606 | 84828 | ** few minor changes to the program. |
| | @@ -85517,11 +85739,11 @@ |
| 85517 | 85739 | if( p->explain==2 ){ |
| 85518 | 85740 | sqlite3VdbeMemSetInt64(pMem, pOp->p1); |
| 85519 | 85741 | sqlite3VdbeMemSetInt64(pMem+1, pOp->p2); |
| 85520 | 85742 | sqlite3VdbeMemSetInt64(pMem+2, pOp->p3); |
| 85521 | 85743 | sqlite3VdbeMemSetStr(pMem+3, zP4, -1, SQLITE_UTF8, sqlite3_free); |
| 85522 | | - p->nResColumn = 4; |
| 85744 | + assert( p->nResColumn==4 ); |
| 85523 | 85745 | }else{ |
| 85524 | 85746 | sqlite3VdbeMemSetInt64(pMem+0, i); |
| 85525 | 85747 | sqlite3VdbeMemSetStr(pMem+1, (char*)sqlite3OpcodeName(pOp->opcode), |
| 85526 | 85748 | -1, SQLITE_UTF8, SQLITE_STATIC); |
| 85527 | 85749 | sqlite3VdbeMemSetInt64(pMem+2, pOp->p1); |
| | @@ -85536,11 +85758,11 @@ |
| 85536 | 85758 | } |
| 85537 | 85759 | #else |
| 85538 | 85760 | sqlite3VdbeMemSetNull(pMem+7); |
| 85539 | 85761 | #endif |
| 85540 | 85762 | sqlite3VdbeMemSetStr(pMem+5, zP4, -1, SQLITE_UTF8, sqlite3_free); |
| 85541 | | - p->nResColumn = 8; |
| 85763 | + assert( p->nResColumn==8 ); |
| 85542 | 85764 | } |
| 85543 | 85765 | p->pResultRow = pMem; |
| 85544 | 85766 | if( db->mallocFailed ){ |
| 85545 | 85767 | p->rc = SQLITE_NOMEM; |
| 85546 | 85768 | rc = SQLITE_ERROR; |
| | @@ -85750,30 +85972,13 @@ |
| 85750 | 85972 | assert( EIGHT_BYTE_ALIGNMENT(&x.pSpace[x.nFree]) ); |
| 85751 | 85973 | |
| 85752 | 85974 | resolveP2Values(p, &nArg); |
| 85753 | 85975 | p->usesStmtJournal = (u8)(pParse->isMultiWrite && pParse->mayAbort); |
| 85754 | 85976 | if( pParse->explain ){ |
| 85755 | | - static const char * const azColName[] = { |
| 85756 | | - "addr", "opcode", "p1", "p2", "p3", "p4", "p5", "comment", |
| 85757 | | - "id", "parent", "notused", "detail" |
| 85758 | | - }; |
| 85759 | | - int iFirst, mx, i; |
| 85760 | 85977 | if( nMem<10 ) nMem = 10; |
| 85761 | 85978 | p->explain = pParse->explain; |
| 85762 | | - if( pParse->explain==2 ){ |
| 85763 | | - sqlite3VdbeSetNumCols(p, 4); |
| 85764 | | - iFirst = 8; |
| 85765 | | - mx = 12; |
| 85766 | | - }else{ |
| 85767 | | - sqlite3VdbeSetNumCols(p, 8); |
| 85768 | | - iFirst = 0; |
| 85769 | | - mx = 8; |
| 85770 | | - } |
| 85771 | | - for(i=iFirst; i<mx; i++){ |
| 85772 | | - sqlite3VdbeSetColName(p, i-iFirst, COLNAME_NAME, |
| 85773 | | - azColName[i], SQLITE_STATIC); |
| 85774 | | - } |
| 85979 | + p->nResColumn = 12 - 4*p->explain; |
| 85775 | 85980 | } |
| 85776 | 85981 | p->expired = 0; |
| 85777 | 85982 | |
| 85778 | 85983 | /* Memory for registers, parameters, cursor, etc, is allocated in one or two |
| 85779 | 85984 | ** passes. On the first pass, we try to reuse unused memory at the |
| | @@ -85820,12 +86025,28 @@ |
| 85820 | 86025 | ** Close a VDBE cursor and release all the resources that cursor |
| 85821 | 86026 | ** happens to hold. |
| 85822 | 86027 | */ |
| 85823 | 86028 | SQLITE_PRIVATE void sqlite3VdbeFreeCursor(Vdbe *p, VdbeCursor *pCx){ |
| 85824 | 86029 | if( pCx ) sqlite3VdbeFreeCursorNN(p,pCx); |
| 86030 | +} |
| 86031 | +static SQLITE_NOINLINE void freeCursorWithCache(Vdbe *p, VdbeCursor *pCx){ |
| 86032 | + VdbeTxtBlbCache *pCache = pCx->pCache; |
| 86033 | + assert( pCx->colCache ); |
| 86034 | + pCx->colCache = 0; |
| 86035 | + pCx->pCache = 0; |
| 86036 | + if( pCache->pCValue ){ |
| 86037 | + sqlite3RCStrUnref(pCache->pCValue); |
| 86038 | + pCache->pCValue = 0; |
| 86039 | + } |
| 86040 | + sqlite3DbFree(p->db, pCache); |
| 86041 | + sqlite3VdbeFreeCursorNN(p, pCx); |
| 85825 | 86042 | } |
| 85826 | 86043 | SQLITE_PRIVATE void sqlite3VdbeFreeCursorNN(Vdbe *p, VdbeCursor *pCx){ |
| 86044 | + if( pCx->colCache ){ |
| 86045 | + freeCursorWithCache(p, pCx); |
| 86046 | + return; |
| 86047 | + } |
| 85827 | 86048 | switch( pCx->eCurType ){ |
| 85828 | 86049 | case CURTYPE_SORTER: { |
| 85829 | 86050 | sqlite3VdbeSorterClose(p->db, pCx); |
| 85830 | 86051 | break; |
| 85831 | 86052 | } |
| | @@ -85922,16 +86143,16 @@ |
| 85922 | 86143 | */ |
| 85923 | 86144 | SQLITE_PRIVATE void sqlite3VdbeSetNumCols(Vdbe *p, int nResColumn){ |
| 85924 | 86145 | int n; |
| 85925 | 86146 | sqlite3 *db = p->db; |
| 85926 | 86147 | |
| 85927 | | - if( p->nResColumn ){ |
| 85928 | | - releaseMemArray(p->aColName, p->nResColumn*COLNAME_N); |
| 86148 | + if( p->nResAlloc ){ |
| 86149 | + releaseMemArray(p->aColName, p->nResAlloc*COLNAME_N); |
| 85929 | 86150 | sqlite3DbFree(db, p->aColName); |
| 85930 | 86151 | } |
| 85931 | 86152 | n = nResColumn*COLNAME_N; |
| 85932 | | - p->nResColumn = (u16)nResColumn; |
| 86153 | + p->nResColumn = p->nResAlloc = (u16)nResColumn; |
| 85933 | 86154 | p->aColName = (Mem*)sqlite3DbMallocRawNN(db, sizeof(Mem)*n ); |
| 85934 | 86155 | if( p->aColName==0 ) return; |
| 85935 | 86156 | initMemArray(p->aColName, n, db, MEM_Null); |
| 85936 | 86157 | } |
| 85937 | 86158 | |
| | @@ -85952,18 +86173,18 @@ |
| 85952 | 86173 | const char *zName, /* Pointer to buffer containing name */ |
| 85953 | 86174 | void (*xDel)(void*) /* Memory management strategy for zName */ |
| 85954 | 86175 | ){ |
| 85955 | 86176 | int rc; |
| 85956 | 86177 | Mem *pColName; |
| 85957 | | - assert( idx<p->nResColumn ); |
| 86178 | + assert( idx<p->nResAlloc ); |
| 85958 | 86179 | assert( var<COLNAME_N ); |
| 85959 | 86180 | if( p->db->mallocFailed ){ |
| 85960 | 86181 | assert( !zName || xDel!=SQLITE_DYNAMIC ); |
| 85961 | 86182 | return SQLITE_NOMEM_BKPT; |
| 85962 | 86183 | } |
| 85963 | 86184 | assert( p->aColName!=0 ); |
| 85964 | | - pColName = &(p->aColName[idx+var*p->nResColumn]); |
| 86185 | + pColName = &(p->aColName[idx+var*p->nResAlloc]); |
| 85965 | 86186 | rc = sqlite3VdbeMemSetStr(pColName, zName, -1, SQLITE_UTF8, xDel); |
| 85966 | 86187 | assert( rc!=0 || !zName || (pColName->flags&MEM_Term)!=0 ); |
| 85967 | 86188 | return rc; |
| 85968 | 86189 | } |
| 85969 | 86190 | |
| | @@ -86783,11 +87004,11 @@ |
| 86783 | 87004 | static void sqlite3VdbeClearObject(sqlite3 *db, Vdbe *p){ |
| 86784 | 87005 | SubProgram *pSub, *pNext; |
| 86785 | 87006 | assert( db!=0 ); |
| 86786 | 87007 | assert( p->db==0 || p->db==db ); |
| 86787 | 87008 | if( p->aColName ){ |
| 86788 | | - releaseMemArray(p->aColName, p->nResColumn*COLNAME_N); |
| 87009 | + releaseMemArray(p->aColName, p->nResAlloc*COLNAME_N); |
| 86789 | 87010 | sqlite3DbNNFreeNN(db, p->aColName); |
| 86790 | 87011 | } |
| 86791 | 87012 | for(pSub=p->pProgram; pSub; pSub=pNext){ |
| 86792 | 87013 | pNext = pSub->pNext; |
| 86793 | 87014 | vdbeFreeOpArray(db, pSub->aOp, pSub->nOp); |
| | @@ -89092,10 +89313,11 @@ |
| 89092 | 89313 | } |
| 89093 | 89314 | if( n>0x7fffffff ){ |
| 89094 | 89315 | (void)invokeValueDestructor(z, xDel, pCtx); |
| 89095 | 89316 | }else{ |
| 89096 | 89317 | setResultStrOrError(pCtx, z, (int)n, enc, xDel); |
| 89318 | + sqlite3VdbeMemZeroTerminateIfAble(pCtx->pOut); |
| 89097 | 89319 | } |
| 89098 | 89320 | } |
| 89099 | 89321 | #ifndef SQLITE_OMIT_UTF16 |
| 89100 | 89322 | SQLITE_API void sqlite3_result_text16( |
| 89101 | 89323 | sqlite3_context *pCtx, |
| | @@ -89704,11 +89926,12 @@ |
| 89704 | 89926 | /* |
| 89705 | 89927 | ** Return the number of columns in the result set for the statement pStmt. |
| 89706 | 89928 | */ |
| 89707 | 89929 | SQLITE_API int sqlite3_column_count(sqlite3_stmt *pStmt){ |
| 89708 | 89930 | Vdbe *pVm = (Vdbe *)pStmt; |
| 89709 | | - return pVm ? pVm->nResColumn : 0; |
| 89931 | + if( pVm==0 ) return 0; |
| 89932 | + return pVm->nResColumn; |
| 89710 | 89933 | } |
| 89711 | 89934 | |
| 89712 | 89935 | /* |
| 89713 | 89936 | ** Return the number of values available from the current row of the |
| 89714 | 89937 | ** currently executing statement pStmt. |
| | @@ -89877,10 +90100,36 @@ |
| 89877 | 90100 | int iType = sqlite3_value_type( columnMem(pStmt,i) ); |
| 89878 | 90101 | columnMallocFailure(pStmt); |
| 89879 | 90102 | return iType; |
| 89880 | 90103 | } |
| 89881 | 90104 | |
| 90105 | +/* |
| 90106 | +** Column names appropriate for EXPLAIN or EXPLAIN QUERY PLAN. |
| 90107 | +*/ |
| 90108 | +static const char * const azExplainColNames8[] = { |
| 90109 | + "addr", "opcode", "p1", "p2", "p3", "p4", "p5", "comment", /* EXPLAIN */ |
| 90110 | + "id", "parent", "notused", "detail" /* EQP */ |
| 90111 | +}; |
| 90112 | +static const u16 azExplainColNames16data[] = { |
| 90113 | + /* 0 */ 'a', 'd', 'd', 'r', 0, |
| 90114 | + /* 5 */ 'o', 'p', 'c', 'o', 'd', 'e', 0, |
| 90115 | + /* 12 */ 'p', '1', 0, |
| 90116 | + /* 15 */ 'p', '2', 0, |
| 90117 | + /* 18 */ 'p', '3', 0, |
| 90118 | + /* 21 */ 'p', '4', 0, |
| 90119 | + /* 24 */ 'p', '5', 0, |
| 90120 | + /* 27 */ 'c', 'o', 'm', 'm', 'e', 'n', 't', 0, |
| 90121 | + /* 35 */ 'i', 'd', 0, |
| 90122 | + /* 38 */ 'p', 'a', 'r', 'e', 'n', 't', 0, |
| 90123 | + /* 45 */ 'n', 'o', 't', 'u', 's', 'e', 'd', 0, |
| 90124 | + /* 53 */ 'd', 'e', 't', 'a', 'i', 'l', 0 |
| 90125 | +}; |
| 90126 | +static const u8 iExplainColNames16[] = { |
| 90127 | + 0, 5, 12, 15, 18, 21, 24, 27, |
| 90128 | + 35, 38, 45, 53 |
| 90129 | +}; |
| 90130 | + |
| 89882 | 90131 | /* |
| 89883 | 90132 | ** Convert the N-th element of pStmt->pColName[] into a string using |
| 89884 | 90133 | ** xFunc() then return that string. If N is out of range, return 0. |
| 89885 | 90134 | ** |
| 89886 | 90135 | ** There are up to 5 names for each column. useType determines which |
| | @@ -89909,19 +90158,33 @@ |
| 89909 | 90158 | if( pStmt==0 ){ |
| 89910 | 90159 | (void)SQLITE_MISUSE_BKPT; |
| 89911 | 90160 | return 0; |
| 89912 | 90161 | } |
| 89913 | 90162 | #endif |
| 90163 | + if( N<0 ) return 0; |
| 89914 | 90164 | ret = 0; |
| 89915 | 90165 | p = (Vdbe *)pStmt; |
| 89916 | 90166 | db = p->db; |
| 89917 | 90167 | assert( db!=0 ); |
| 89918 | | - n = sqlite3_column_count(pStmt); |
| 89919 | | - if( N<n && N>=0 ){ |
| 90168 | + sqlite3_mutex_enter(db->mutex); |
| 90169 | + |
| 90170 | + if( p->explain ){ |
| 90171 | + if( useType>0 ) goto columnName_end; |
| 90172 | + n = p->explain==1 ? 8 : 4; |
| 90173 | + if( N>=n ) goto columnName_end; |
| 90174 | + if( useUtf16 ){ |
| 90175 | + int i = iExplainColNames16[N + 8*p->explain - 8]; |
| 90176 | + ret = (void*)&azExplainColNames16data[i]; |
| 90177 | + }else{ |
| 90178 | + ret = (void*)azExplainColNames8[N + 8*p->explain - 8]; |
| 90179 | + } |
| 90180 | + goto columnName_end; |
| 90181 | + } |
| 90182 | + n = p->nResColumn; |
| 90183 | + if( N<n ){ |
| 89920 | 90184 | u8 prior_mallocFailed = db->mallocFailed; |
| 89921 | 90185 | N += useType*n; |
| 89922 | | - sqlite3_mutex_enter(db->mutex); |
| 89923 | 90186 | #ifndef SQLITE_OMIT_UTF16 |
| 89924 | 90187 | if( useUtf16 ){ |
| 89925 | 90188 | ret = sqlite3_value_text16((sqlite3_value*)&p->aColName[N]); |
| 89926 | 90189 | }else |
| 89927 | 90190 | #endif |
| | @@ -89934,12 +90197,13 @@ |
| 89934 | 90197 | assert( db->mallocFailed==0 || db->mallocFailed==1 ); |
| 89935 | 90198 | if( db->mallocFailed > prior_mallocFailed ){ |
| 89936 | 90199 | sqlite3OomClear(db); |
| 89937 | 90200 | ret = 0; |
| 89938 | 90201 | } |
| 89939 | | - sqlite3_mutex_leave(db->mutex); |
| 89940 | 90202 | } |
| 90203 | +columnName_end: |
| 90204 | + sqlite3_mutex_leave(db->mutex); |
| 89941 | 90205 | return ret; |
| 89942 | 90206 | } |
| 89943 | 90207 | |
| 89944 | 90208 | /* |
| 89945 | 90209 | ** Return the name of the Nth column of the result set returned by SQL |
| | @@ -90391,10 +90655,43 @@ |
| 90391 | 90655 | ** statement is an EXPLAIN QUERY PLAN |
| 90392 | 90656 | */ |
| 90393 | 90657 | SQLITE_API int sqlite3_stmt_isexplain(sqlite3_stmt *pStmt){ |
| 90394 | 90658 | return pStmt ? ((Vdbe*)pStmt)->explain : 0; |
| 90395 | 90659 | } |
| 90660 | + |
| 90661 | +/* |
| 90662 | +** Set the explain mode for a statement. |
| 90663 | +*/ |
| 90664 | +SQLITE_API int sqlite3_stmt_explain(sqlite3_stmt *pStmt, int eMode){ |
| 90665 | + Vdbe *v = (Vdbe*)pStmt; |
| 90666 | + int rc; |
| 90667 | + sqlite3_mutex_enter(v->db->mutex); |
| 90668 | + if( v->explain==eMode ){ |
| 90669 | + rc = SQLITE_OK; |
| 90670 | + }else if( eMode<0 || eMode>2 ){ |
| 90671 | + rc = SQLITE_ERROR; |
| 90672 | + }else if( (v->prepFlags & SQLITE_PREPARE_SAVESQL)==0 ){ |
| 90673 | + rc = SQLITE_ERROR; |
| 90674 | + }else if( v->eVdbeState!=VDBE_READY_STATE ){ |
| 90675 | + rc = SQLITE_BUSY; |
| 90676 | + }else if( v->nMem>=10 && (eMode!=2 || v->haveEqpOps) ){ |
| 90677 | + /* No reprepare necessary */ |
| 90678 | + v->explain = eMode; |
| 90679 | + rc = SQLITE_OK; |
| 90680 | + }else{ |
| 90681 | + v->explain = eMode; |
| 90682 | + rc = sqlite3Reprepare(v); |
| 90683 | + v->haveEqpOps = eMode==2; |
| 90684 | + } |
| 90685 | + if( v->explain ){ |
| 90686 | + v->nResColumn = 12 - 4*v->explain; |
| 90687 | + }else{ |
| 90688 | + v->nResColumn = v->nResAlloc; |
| 90689 | + } |
| 90690 | + sqlite3_mutex_leave(v->db->mutex); |
| 90691 | + return rc; |
| 90692 | +} |
| 90396 | 90693 | |
| 90397 | 90694 | /* |
| 90398 | 90695 | ** Return true if the prepared statement is in need of being reset. |
| 90399 | 90696 | */ |
| 90400 | 90697 | SQLITE_API int sqlite3_stmt_busy(sqlite3_stmt *pStmt){ |
| | @@ -91631,10 +91928,13 @@ |
| 91631 | 91928 | for(j=0; j<25 && j<pMem->n; j++){ |
| 91632 | 91929 | c = pMem->z[j]; |
| 91633 | 91930 | sqlite3_str_appendchar(pStr, 1, (c>=0x20&&c<=0x7f) ? c : '.'); |
| 91634 | 91931 | } |
| 91635 | 91932 | sqlite3_str_appendf(pStr, "]%s", encnames[pMem->enc]); |
| 91933 | + if( f & MEM_Term ){ |
| 91934 | + sqlite3_str_appendf(pStr, "(0-term)"); |
| 91935 | + } |
| 91636 | 91936 | } |
| 91637 | 91937 | } |
| 91638 | 91938 | #endif |
| 91639 | 91939 | |
| 91640 | 91940 | #ifdef SQLITE_DEBUG |
| | @@ -91766,10 +92066,97 @@ |
| 91766 | 92066 | h += 4093 + (p->flags & (MEM_Str|MEM_Blob)); |
| 91767 | 92067 | } |
| 91768 | 92068 | } |
| 91769 | 92069 | return h; |
| 91770 | 92070 | } |
| 92071 | + |
| 92072 | + |
| 92073 | +/* |
| 92074 | +** For OP_Column, factor out the case where content is loaded from |
| 92075 | +** overflow pages, so that the code to implement this case is separate |
| 92076 | +** the common case where all content fits on the page. Factoring out |
| 92077 | +** the code reduces register pressure and helps the common case |
| 92078 | +** to run faster. |
| 92079 | +*/ |
| 92080 | +static SQLITE_NOINLINE int vdbeColumnFromOverflow( |
| 92081 | + VdbeCursor *pC, /* The BTree cursor from which we are reading */ |
| 92082 | + int iCol, /* The column to read */ |
| 92083 | + int t, /* The serial-type code for the column value */ |
| 92084 | + i64 iOffset, /* Offset to the start of the content value */ |
| 92085 | + u32 cacheStatus, /* Current Vdbe.cacheCtr value */ |
| 92086 | + u32 colCacheCtr, /* Current value of the column cache counter */ |
| 92087 | + Mem *pDest /* Store the value into this register. */ |
| 92088 | +){ |
| 92089 | + int rc; |
| 92090 | + sqlite3 *db = pDest->db; |
| 92091 | + int encoding = pDest->enc; |
| 92092 | + int len = sqlite3VdbeSerialTypeLen(t); |
| 92093 | + assert( pC->eCurType==CURTYPE_BTREE ); |
| 92094 | + if( len>db->aLimit[SQLITE_LIMIT_LENGTH] ) return SQLITE_TOOBIG; |
| 92095 | + if( len > 4000 && pC->pKeyInfo==0 ){ |
| 92096 | + /* Cache large column values that are on overflow pages using |
| 92097 | + ** an RCStr (reference counted string) so that if they are reloaded, |
| 92098 | + ** that do not have to be copied a second time. The overhead of |
| 92099 | + ** creating and managing the cache is such that this is only |
| 92100 | + ** profitable for larger TEXT and BLOB values. |
| 92101 | + ** |
| 92102 | + ** Only do this on table-btrees so that writes to index-btrees do not |
| 92103 | + ** need to clear the cache. This buys performance in the common case |
| 92104 | + ** in exchange for generality. |
| 92105 | + */ |
| 92106 | + VdbeTxtBlbCache *pCache; |
| 92107 | + char *pBuf; |
| 92108 | + if( pC->colCache==0 ){ |
| 92109 | + pC->pCache = sqlite3DbMallocZero(db, sizeof(VdbeTxtBlbCache) ); |
| 92110 | + if( pC->pCache==0 ) return SQLITE_NOMEM; |
| 92111 | + pC->colCache = 1; |
| 92112 | + } |
| 92113 | + pCache = pC->pCache; |
| 92114 | + if( pCache->pCValue==0 |
| 92115 | + || pCache->iCol!=iCol |
| 92116 | + || pCache->cacheStatus!=cacheStatus |
| 92117 | + || pCache->colCacheCtr!=colCacheCtr |
| 92118 | + || pCache->iOffset!=sqlite3BtreeOffset(pC->uc.pCursor) |
| 92119 | + ){ |
| 92120 | + if( pCache->pCValue ) sqlite3RCStrUnref(pCache->pCValue); |
| 92121 | + pBuf = pCache->pCValue = sqlite3RCStrNew( len+3 ); |
| 92122 | + if( pBuf==0 ) return SQLITE_NOMEM; |
| 92123 | + rc = sqlite3BtreePayload(pC->uc.pCursor, iOffset, len, pBuf); |
| 92124 | + if( rc ) return rc; |
| 92125 | + pBuf[len] = 0; |
| 92126 | + pBuf[len+1] = 0; |
| 92127 | + pBuf[len+2] = 0; |
| 92128 | + pCache->iCol = iCol; |
| 92129 | + pCache->cacheStatus = cacheStatus; |
| 92130 | + pCache->colCacheCtr = colCacheCtr; |
| 92131 | + pCache->iOffset = sqlite3BtreeOffset(pC->uc.pCursor); |
| 92132 | + }else{ |
| 92133 | + pBuf = pCache->pCValue; |
| 92134 | + } |
| 92135 | + assert( t>=12 ); |
| 92136 | + sqlite3RCStrRef(pBuf); |
| 92137 | + if( t&1 ){ |
| 92138 | + rc = sqlite3VdbeMemSetStr(pDest, pBuf, len, encoding, |
| 92139 | + (void(*)(void*))sqlite3RCStrUnref); |
| 92140 | + pDest->flags |= MEM_Term; |
| 92141 | + }else{ |
| 92142 | + rc = sqlite3VdbeMemSetStr(pDest, pBuf, len, 0, |
| 92143 | + (void(*)(void*))sqlite3RCStrUnref); |
| 92144 | + } |
| 92145 | + }else{ |
| 92146 | + rc = sqlite3VdbeMemFromBtree(pC->uc.pCursor, iOffset, len, pDest); |
| 92147 | + if( rc ) return rc; |
| 92148 | + sqlite3VdbeSerialGet((const u8*)pDest->z, t, pDest); |
| 92149 | + if( (t&1)!=0 && encoding==SQLITE_UTF8 ){ |
| 92150 | + pDest->z[len] = 0; |
| 92151 | + pDest->flags |= MEM_Term; |
| 92152 | + } |
| 92153 | + } |
| 92154 | + pDest->flags &= ~MEM_Ephem; |
| 92155 | + return rc; |
| 92156 | +} |
| 92157 | + |
| 91771 | 92158 | |
| 91772 | 92159 | /* |
| 91773 | 92160 | ** Return the symbolic name for the data type of a pMem |
| 91774 | 92161 | */ |
| 91775 | 92162 | static const char *vdbeMemTypeName(Mem *pMem){ |
| | @@ -91809,10 +92196,11 @@ |
| 91809 | 92196 | Mem *aMem = p->aMem; /* Copy of p->aMem */ |
| 91810 | 92197 | Mem *pIn1 = 0; /* 1st input operand */ |
| 91811 | 92198 | Mem *pIn2 = 0; /* 2nd input operand */ |
| 91812 | 92199 | Mem *pIn3 = 0; /* 3rd input operand */ |
| 91813 | 92200 | Mem *pOut = 0; /* Output operand */ |
| 92201 | + u32 colCacheCtr = 0; /* Column cache counter */ |
| 91814 | 92202 | #if defined(SQLITE_ENABLE_STMT_SCANSTATUS) || defined(VDBE_PROFILE) |
| 91815 | 92203 | u64 *pnCycle = 0; |
| 91816 | 92204 | int bStmtScanStatus = IS_STMT_SCANSTATUS(db)!=0; |
| 91817 | 92205 | #endif |
| 91818 | 92206 | /*** INSERT STACK UNION HERE ***/ |
| | @@ -94133,10 +94521,11 @@ |
| 94133 | 94521 | pDest->flags = aFlag[t&1]; |
| 94134 | 94522 | } |
| 94135 | 94523 | }else{ |
| 94136 | 94524 | u8 p5; |
| 94137 | 94525 | pDest->enc = encoding; |
| 94526 | + assert( pDest->db==db ); |
| 94138 | 94527 | /* This branch happens only when content is on overflow pages */ |
| 94139 | 94528 | if( ((p5 = (pOp->p5 & OPFLAG_BYTELENARG))!=0 |
| 94140 | 94529 | && (p5==OPFLAG_TYPEOFARG |
| 94141 | 94530 | || (t>=12 && ((t&1)==0 || p5==OPFLAG_BYTELENARG)) |
| 94142 | 94531 | ) |
| | @@ -94156,15 +94545,17 @@ |
| 94156 | 94545 | ** as that array is 256 bytes long (plenty for VdbeMemPrettyPrint()) |
| 94157 | 94546 | ** and it begins with a bunch of zeros. |
| 94158 | 94547 | */ |
| 94159 | 94548 | sqlite3VdbeSerialGet((u8*)sqlite3CtypeMap, t, pDest); |
| 94160 | 94549 | }else{ |
| 94161 | | - if( len>db->aLimit[SQLITE_LIMIT_LENGTH] ) goto too_big; |
| 94162 | | - rc = sqlite3VdbeMemFromBtree(pC->uc.pCursor, aOffset[p2], len, pDest); |
| 94163 | | - if( rc!=SQLITE_OK ) goto abort_due_to_error; |
| 94164 | | - sqlite3VdbeSerialGet((const u8*)pDest->z, t, pDest); |
| 94165 | | - pDest->flags &= ~MEM_Ephem; |
| 94550 | + rc = vdbeColumnFromOverflow(pC, p2, t, aOffset[p2], |
| 94551 | + p->cacheCtr, colCacheCtr, pDest); |
| 94552 | + if( rc ){ |
| 94553 | + if( rc==SQLITE_NOMEM ) goto no_mem; |
| 94554 | + if( rc==SQLITE_TOOBIG ) goto too_big; |
| 94555 | + goto abort_due_to_error; |
| 94556 | + } |
| 94166 | 94557 | } |
| 94167 | 94558 | } |
| 94168 | 94559 | |
| 94169 | 94560 | op_column_out: |
| 94170 | 94561 | UPDATE_MAX_BLOBSIZE(pDest); |
| | @@ -96693,10 +97084,11 @@ |
| 96693 | 97084 | (pOp->p5 & (OPFLAG_APPEND|OPFLAG_SAVEPOSITION|OPFLAG_PREFORMAT)), |
| 96694 | 97085 | seekResult |
| 96695 | 97086 | ); |
| 96696 | 97087 | pC->deferredMoveto = 0; |
| 96697 | 97088 | pC->cacheStatus = CACHE_STALE; |
| 97089 | + colCacheCtr++; |
| 96698 | 97090 | |
| 96699 | 97091 | /* Invoke the update-hook if required. */ |
| 96700 | 97092 | if( rc ) goto abort_due_to_error; |
| 96701 | 97093 | if( pTab ){ |
| 96702 | 97094 | assert( db->xUpdateCallback!=0 ); |
| | @@ -96853,10 +97245,11 @@ |
| 96853 | 97245 | } |
| 96854 | 97246 | #endif |
| 96855 | 97247 | |
| 96856 | 97248 | rc = sqlite3BtreeDelete(pC->uc.pCursor, pOp->p5); |
| 96857 | 97249 | pC->cacheStatus = CACHE_STALE; |
| 97250 | + colCacheCtr++; |
| 96858 | 97251 | pC->seekResult = 0; |
| 96859 | 97252 | if( rc ) goto abort_due_to_error; |
| 96860 | 97253 | |
| 96861 | 97254 | /* Invoke the update-hook if required. */ |
| 96862 | 97255 | if( opflags & OPFLAG_NCHANGE ){ |
| | @@ -103347,10 +103740,12 @@ |
| 103347 | 103740 | "p3 INT," |
| 103348 | 103741 | "p4 TEXT," |
| 103349 | 103742 | "p5 INT," |
| 103350 | 103743 | "comment TEXT," |
| 103351 | 103744 | "subprog TEXT," |
| 103745 | + "nexec INT," |
| 103746 | + "ncycle INT," |
| 103352 | 103747 | "stmt HIDDEN" |
| 103353 | 103748 | ");", |
| 103354 | 103749 | |
| 103355 | 103750 | /* Tables_used() schema */ |
| 103356 | 103751 | "CREATE TABLE x(" |
| | @@ -103509,11 +103904,11 @@ |
| 103509 | 103904 | pCur->zType = "index"; |
| 103510 | 103905 | } |
| 103511 | 103906 | } |
| 103512 | 103907 | } |
| 103513 | 103908 | } |
| 103514 | | - i += 10; |
| 103909 | + i += 20; |
| 103515 | 103910 | } |
| 103516 | 103911 | } |
| 103517 | 103912 | switch( i ){ |
| 103518 | 103913 | case 0: /* addr */ |
| 103519 | 103914 | sqlite3_result_int(ctx, pCur->iAddr); |
| | @@ -103559,20 +103954,35 @@ |
| 103559 | 103954 | }else{ |
| 103560 | 103955 | sqlite3_result_text(ctx, "(FK)", 4, SQLITE_STATIC); |
| 103561 | 103956 | } |
| 103562 | 103957 | break; |
| 103563 | 103958 | } |
| 103564 | | - case 10: /* tables_used.type */ |
| 103959 | + |
| 103960 | +#ifdef SQLITE_ENABLE_STMT_SCANSTATUS |
| 103961 | + case 9: /* nexec */ |
| 103962 | + sqlite3_result_int(ctx, pOp->nExec); |
| 103963 | + break; |
| 103964 | + case 10: /* ncycle */ |
| 103965 | + sqlite3_result_int(ctx, pOp->nCycle); |
| 103966 | + break; |
| 103967 | +#else |
| 103968 | + case 9: /* nexec */ |
| 103969 | + case 10: /* ncycle */ |
| 103970 | + sqlite3_result_int(ctx, 0); |
| 103971 | + break; |
| 103972 | +#endif |
| 103973 | + |
| 103974 | + case 20: /* tables_used.type */ |
| 103565 | 103975 | sqlite3_result_text(ctx, pCur->zType, -1, SQLITE_STATIC); |
| 103566 | 103976 | break; |
| 103567 | | - case 11: /* tables_used.schema */ |
| 103977 | + case 21: /* tables_used.schema */ |
| 103568 | 103978 | sqlite3_result_text(ctx, pCur->zSchema, -1, SQLITE_STATIC); |
| 103569 | 103979 | break; |
| 103570 | | - case 12: /* tables_used.name */ |
| 103980 | + case 22: /* tables_used.name */ |
| 103571 | 103981 | sqlite3_result_text(ctx, pCur->zName, -1, SQLITE_STATIC); |
| 103572 | 103982 | break; |
| 103573 | | - case 13: /* tables_used.wr */ |
| 103983 | + case 23: /* tables_used.wr */ |
| 103574 | 103984 | sqlite3_result_int(ctx, pOp->opcode==OP_OpenWrite); |
| 103575 | 103985 | break; |
| 103576 | 103986 | } |
| 103577 | 103987 | return SQLITE_OK; |
| 103578 | 103988 | } |
| | @@ -103642,11 +104052,11 @@ |
| 103642 | 104052 | ){ |
| 103643 | 104053 | int i; |
| 103644 | 104054 | int rc = SQLITE_CONSTRAINT; |
| 103645 | 104055 | struct sqlite3_index_constraint *p; |
| 103646 | 104056 | bytecodevtab *pVTab = (bytecodevtab*)tab; |
| 103647 | | - int iBaseCol = pVTab->bTablesUsed ? 4 : 8; |
| 104057 | + int iBaseCol = pVTab->bTablesUsed ? 4 : 10; |
| 103648 | 104058 | pIdxInfo->estimatedCost = (double)100; |
| 103649 | 104059 | pIdxInfo->estimatedRows = 100; |
| 103650 | 104060 | pIdxInfo->idxNum = 0; |
| 103651 | 104061 | for(i=0, p=pIdxInfo->aConstraint; i<pIdxInfo->nConstraint; i++, p++){ |
| 103652 | 104062 | if( p->usable==0 ) continue; |
| | @@ -111269,12 +111679,11 @@ |
| 111269 | 111679 | testcase( (pDef->funcFlags & OPFLAG_BYTELENARG)==OPFLAG_BYTELENARG); |
| 111270 | 111680 | pFarg->a[0].pExpr->op2 = pDef->funcFlags & OPFLAG_BYTELENARG; |
| 111271 | 111681 | } |
| 111272 | 111682 | } |
| 111273 | 111683 | |
| 111274 | | - sqlite3ExprCodeExprList(pParse, pFarg, r1, 0, |
| 111275 | | - SQLITE_ECEL_DUP|SQLITE_ECEL_FACTOR); |
| 111684 | + sqlite3ExprCodeExprList(pParse, pFarg, r1, 0, SQLITE_ECEL_FACTOR); |
| 111276 | 111685 | }else{ |
| 111277 | 111686 | r1 = 0; |
| 111278 | 111687 | } |
| 111279 | 111688 | #ifndef SQLITE_OMIT_VIRTUALTABLE |
| 111280 | 111689 | /* Possibly overload the function if the first argument is |
| | @@ -125336,11 +125745,12 @@ |
| 125336 | 125745 | */ |
| 125337 | 125746 | pWInfo = sqlite3WhereBegin(pParse, pTabList, pWhere, 0, 0,0,wcf,iTabCur+1); |
| 125338 | 125747 | if( pWInfo==0 ) goto delete_from_cleanup; |
| 125339 | 125748 | eOnePass = sqlite3WhereOkOnePass(pWInfo, aiCurOnePass); |
| 125340 | 125749 | assert( IsVirtual(pTab)==0 || eOnePass!=ONEPASS_MULTI ); |
| 125341 | | - assert( IsVirtual(pTab) || bComplex || eOnePass!=ONEPASS_OFF ); |
| 125750 | + assert( IsVirtual(pTab) || bComplex || eOnePass!=ONEPASS_OFF |
| 125751 | + || OptimizationDisabled(db, SQLITE_OnePass) ); |
| 125342 | 125752 | if( eOnePass!=ONEPASS_SINGLE ) sqlite3MultiWrite(pParse); |
| 125343 | 125753 | if( sqlite3WhereUsesDeferredSeek(pWInfo) ){ |
| 125344 | 125754 | sqlite3VdbeAddOp1(v, OP_FinishSeek, iTabCur); |
| 125345 | 125755 | } |
| 125346 | 125756 | |
| | @@ -127068,10 +127478,11 @@ |
| 127068 | 127478 | *zOut++ = 0x80 + (u8)((c>>12) & 0x3F); |
| 127069 | 127479 | *zOut++ = 0x80 + (u8)((c>>6) & 0x3F); |
| 127070 | 127480 | *zOut++ = 0x80 + (u8)(c & 0x3F); |
| 127071 | 127481 | } \ |
| 127072 | 127482 | } |
| 127483 | + *zOut = 0; |
| 127073 | 127484 | sqlite3_result_text64(context, (char*)z, zOut-z, sqlite3_free, SQLITE_UTF8); |
| 127074 | 127485 | } |
| 127075 | 127486 | |
| 127076 | 127487 | /* |
| 127077 | 127488 | ** The hex() function. Interpret the argument as a blob. Return |
| | @@ -127605,15 +128016,14 @@ |
| 127605 | 128016 | p->iSum = x; |
| 127606 | 128017 | }else{ |
| 127607 | 128018 | p->ovrfl = 1; |
| 127608 | 128019 | kahanBabuskaNeumaierInit(p, p->iSum); |
| 127609 | 128020 | p->approx = 1; |
| 127610 | | - kahanBabuskaNeumaierStep(p, sqlite3_value_double(argv[0])); |
| 128021 | + kahanBabuskaNeumaierStepInt64(p, sqlite3_value_int64(argv[0])); |
| 127611 | 128022 | } |
| 127612 | 128023 | } |
| 127613 | 128024 | }else{ |
| 127614 | | - p->approx = 1; |
| 127615 | 128025 | if( type==SQLITE_INTEGER ){ |
| 127616 | 128026 | kahanBabuskaNeumaierStepInt64(p, sqlite3_value_int64(argv[0])); |
| 127617 | 128027 | }else{ |
| 127618 | 128028 | p->ovrfl = 0; |
| 127619 | 128029 | kahanBabuskaNeumaierStep(p, sqlite3_value_double(argv[0])); |
| | @@ -139192,11 +139602,16 @@ |
| 139192 | 139602 | memset(PARSE_HDR(&sParse), 0, PARSE_HDR_SZ); |
| 139193 | 139603 | memset(PARSE_TAIL(&sParse), 0, PARSE_TAIL_SZ); |
| 139194 | 139604 | sParse.pOuterParse = db->pParse; |
| 139195 | 139605 | db->pParse = &sParse; |
| 139196 | 139606 | sParse.db = db; |
| 139197 | | - sParse.pReprepare = pReprepare; |
| 139607 | + if( pReprepare ){ |
| 139608 | + sParse.pReprepare = pReprepare; |
| 139609 | + sParse.explain = sqlite3_stmt_isexplain((sqlite3_stmt*)pReprepare); |
| 139610 | + }else{ |
| 139611 | + assert( sParse.pReprepare==0 ); |
| 139612 | + } |
| 139198 | 139613 | assert( ppStmt && *ppStmt==0 ); |
| 139199 | 139614 | if( db->mallocFailed ){ |
| 139200 | 139615 | sqlite3ErrorMsg(&sParse, "out of memory"); |
| 139201 | 139616 | db->errCode = rc = SQLITE_NOMEM; |
| 139202 | 139617 | goto end_prepare; |
| | @@ -141671,17 +142086,10 @@ |
| 141671 | 142086 | ExprList *pEList; |
| 141672 | 142087 | sqlite3 *db = pParse->db; |
| 141673 | 142088 | int fullName; /* TABLE.COLUMN if no AS clause and is a direct table ref */ |
| 141674 | 142089 | int srcName; /* COLUMN or TABLE.COLUMN if no AS clause and is direct */ |
| 141675 | 142090 | |
| 141676 | | -#ifndef SQLITE_OMIT_EXPLAIN |
| 141677 | | - /* If this is an EXPLAIN, skip this step */ |
| 141678 | | - if( pParse->explain ){ |
| 141679 | | - return; |
| 141680 | | - } |
| 141681 | | -#endif |
| 141682 | | - |
| 141683 | 142091 | if( pParse->colNamesSet ) return; |
| 141684 | 142092 | /* Column names are determined by the left-most term of a compound select */ |
| 141685 | 142093 | while( pSelect->pPrior ) pSelect = pSelect->pPrior; |
| 141686 | 142094 | TREETRACE(0x80,pParse,pSelect,("generating column names\n")); |
| 141687 | 142095 | pTabList = pSelect->pSrc; |
| | @@ -143869,11 +144277,12 @@ |
| 143869 | 144277 | ** is the first element of the parent query. Two subcases: |
| 143870 | 144278 | ** (27a) the subquery is not a compound query. |
| 143871 | 144279 | ** (27b) the subquery is a compound query and the RIGHT JOIN occurs |
| 143872 | 144280 | ** in any arm of the compound query. (See also (17g).) |
| 143873 | 144281 | ** |
| 143874 | | -** (28) The subquery is not a MATERIALIZED CTE. |
| 144282 | +** (28) The subquery is not a MATERIALIZED CTE. (This is handled |
| 144283 | +** in the caller before ever reaching this routine.) |
| 143875 | 144284 | ** |
| 143876 | 144285 | ** |
| 143877 | 144286 | ** In this routine, the "p" parameter is a pointer to the outer query. |
| 143878 | 144287 | ** The subquery is p->pSrc->a[iFrom]. isAgg is true if the outer query |
| 143879 | 144288 | ** uses aggregates. |
| | @@ -143979,13 +144388,13 @@ |
| 143979 | 144388 | |
| 143980 | 144389 | assert( pSubSrc->nSrc>0 ); /* True by restriction (7) */ |
| 143981 | 144390 | if( iFrom>0 && (pSubSrc->a[0].fg.jointype & JT_LTORJ)!=0 ){ |
| 143982 | 144391 | return 0; /* Restriction (27a) */ |
| 143983 | 144392 | } |
| 143984 | | - if( pSubitem->fg.isCte && pSubitem->u2.pCteUse->eM10d==M10d_Yes ){ |
| 143985 | | - return 0; /* (28) */ |
| 143986 | | - } |
| 144393 | + |
| 144394 | + /* Condition (28) is blocked by the caller */ |
| 144395 | + assert( !pSubitem->fg.isCte || pSubitem->u2.pCteUse->eM10d!=M10d_Yes ); |
| 143987 | 144396 | |
| 143988 | 144397 | /* Restriction (17): If the sub-query is a compound SELECT, then it must |
| 143989 | 144398 | ** use only the UNION ALL operator. And none of the simple select queries |
| 143990 | 144399 | ** that make up the compound SELECT are allowed to be aggregate or distinct |
| 143991 | 144400 | ** queries. |
| | @@ -146861,10 +147270,18 @@ |
| 146861 | 147270 | if( pTab->nCol!=pSub->pEList->nExpr ){ |
| 146862 | 147271 | sqlite3ErrorMsg(pParse, "expected %d columns for '%s' but got %d", |
| 146863 | 147272 | pTab->nCol, pTab->zName, pSub->pEList->nExpr); |
| 146864 | 147273 | goto select_end; |
| 146865 | 147274 | } |
| 147275 | + |
| 147276 | + /* Do not attempt the usual optimizations (flattening and ORDER BY |
| 147277 | + ** elimination) on a MATERIALIZED common table expression because |
| 147278 | + ** a MATERIALIZED common table expression is an optimization fence. |
| 147279 | + */ |
| 147280 | + if( pItem->fg.isCte && pItem->u2.pCteUse->eM10d==M10d_Yes ){ |
| 147281 | + continue; |
| 147282 | + } |
| 146866 | 147283 | |
| 146867 | 147284 | /* Do not try to flatten an aggregate subquery. |
| 146868 | 147285 | ** |
| 146869 | 147286 | ** Flattening an aggregate subquery is only possible if the outer query |
| 146870 | 147287 | ** is not a join. But if the outer query is not a join, then the subquery |
| | @@ -146891,10 +147308,12 @@ |
| 146891 | 147308 | ** (5) The ORDER BY isn't going to accomplish anything because |
| 146892 | 147309 | ** one of: |
| 146893 | 147310 | ** (a) The outer query has a different ORDER BY clause |
| 146894 | 147311 | ** (b) The subquery is part of a join |
| 146895 | 147312 | ** See forum post 062d576715d277c8 |
| 147313 | + ** |
| 147314 | + ** Also retain the ORDER BY if the OmitOrderBy optimization is disabled. |
| 146896 | 147315 | */ |
| 146897 | 147316 | if( pSub->pOrderBy!=0 |
| 146898 | 147317 | && (p->pOrderBy!=0 || pTabList->nSrc>1) /* Condition (5) */ |
| 146899 | 147318 | && pSub->pLimit==0 /* Condition (1) */ |
| 146900 | 147319 | && (pSub->selFlags & SF_OrderByReqd)==0 /* Condition (2) */ |
| | @@ -150394,11 +150813,11 @@ |
| 150394 | 150813 | if( !pParse->nested |
| 150395 | 150814 | && !pTrigger |
| 150396 | 150815 | && !hasFK |
| 150397 | 150816 | && !chngKey |
| 150398 | 150817 | && !bReplace |
| 150399 | | - && (sNC.ncFlags & NC_Subquery)==0 |
| 150818 | + && (pWhere==0 || !ExprHasProperty(pWhere, EP_Subquery)) |
| 150400 | 150819 | ){ |
| 150401 | 150820 | flags |= WHERE_ONEPASS_MULTIROW; |
| 150402 | 150821 | } |
| 150403 | 150822 | pWInfo = sqlite3WhereBegin(pParse, pTabList, pWhere,0,0,0,flags,iIdxCur); |
| 150404 | 150823 | if( pWInfo==0 ) goto update_cleanup; |
| | @@ -160954,11 +161373,11 @@ |
| 160954 | 161373 | return 0; /* Discard pTemplate */ |
| 160955 | 161374 | } |
| 160956 | 161375 | |
| 160957 | 161376 | /* If pTemplate is always better than p, then cause p to be overwritten |
| 160958 | 161377 | ** with pTemplate. pTemplate is better than p if: |
| 160959 | | - ** (1) pTemplate has no more dependences than p, and |
| 161378 | + ** (1) pTemplate has no more dependencies than p, and |
| 160960 | 161379 | ** (2) pTemplate has an equal or lower cost than p. |
| 160961 | 161380 | */ |
| 160962 | 161381 | if( (p->prereq & pTemplate->prereq)==pTemplate->prereq /* (1) */ |
| 160963 | 161382 | && p->rRun>=pTemplate->rRun /* (2a) */ |
| 160964 | 161383 | && p->nOut>=pTemplate->nOut /* (2b) */ |
| | @@ -163422,11 +163841,12 @@ |
| 163422 | 163841 | /* TUNING: For simple queries, only the best path is tracked. |
| 163423 | 163842 | ** For 2-way joins, the 5 best paths are followed. |
| 163424 | 163843 | ** For joins of 3 or more tables, track the 10 best paths */ |
| 163425 | 163844 | mxChoice = (nLoop<=1) ? 1 : (nLoop==2 ? 5 : 10); |
| 163426 | 163845 | assert( nLoop<=pWInfo->pTabList->nSrc ); |
| 163427 | | - WHERETRACE(0x002, ("---- begin solver. (nRowEst=%d)\n", nRowEst)); |
| 163846 | + WHERETRACE(0x002, ("---- begin solver. (nRowEst=%d, nQueryLoop=%d)\n", |
| 163847 | + nRowEst, pParse->nQueryLoop)); |
| 163428 | 163848 | |
| 163429 | 163849 | /* If nRowEst is zero and there is an ORDER BY clause, ignore it. In this |
| 163430 | 163850 | ** case the purpose of this call is to estimate the number of rows returned |
| 163431 | 163851 | ** by the overall query. Once this estimate has been obtained, the caller |
| 163432 | 163852 | ** will invoke this function a second time, passing the estimate as the |
| | @@ -163541,13 +163961,14 @@ |
| 163541 | 163961 | rUnsorted -= 2; /* TUNING: Slight bias in favor of no-sort plans */ |
| 163542 | 163962 | } |
| 163543 | 163963 | |
| 163544 | 163964 | /* TUNING: A full-scan of a VIEW or subquery in the outer loop |
| 163545 | 163965 | ** is not so bad. */ |
| 163546 | | - if( iLoop==0 && (pWLoop->wsFlags & WHERE_VIEWSCAN)!=0 ){ |
| 163966 | + if( iLoop==0 && (pWLoop->wsFlags & WHERE_VIEWSCAN)!=0 && nLoop>1 ){ |
| 163547 | 163967 | rCost += -10; |
| 163548 | 163968 | nOut += -30; |
| 163969 | + WHERETRACE(0x80,("VIEWSCAN cost reduction for %c\n",pWLoop->cId)); |
| 163549 | 163970 | } |
| 163550 | 163971 | |
| 163551 | 163972 | /* Check to see if pWLoop should be added to the set of |
| 163552 | 163973 | ** mxChoice best-so-far paths. |
| 163553 | 163974 | ** |
| | @@ -164174,10 +164595,32 @@ |
| 164174 | 164595 | if( p->pIENext==0 ){ |
| 164175 | 164596 | sqlite3ParserAddCleanup(pParse, whereIndexedExprCleanup, pParse); |
| 164176 | 164597 | } |
| 164177 | 164598 | } |
| 164178 | 164599 | } |
| 164600 | + |
| 164601 | +/* |
| 164602 | +** Set the reverse-scan order mask to one for all tables in the query |
| 164603 | +** with the exception of MATERIALIZED common table expressions that have |
| 164604 | +** their own internal ORDER BY clauses. |
| 164605 | +** |
| 164606 | +** This implements the PRAGMA reverse_unordered_selects=ON setting. |
| 164607 | +** (Also SQLITE_DBCONFIG_REVERSE_SCANORDER). |
| 164608 | +*/ |
| 164609 | +static SQLITE_NOINLINE void whereReverseScanOrder(WhereInfo *pWInfo){ |
| 164610 | + int ii; |
| 164611 | + for(ii=0; ii<pWInfo->pTabList->nSrc; ii++){ |
| 164612 | + SrcItem *pItem = &pWInfo->pTabList->a[ii]; |
| 164613 | + if( !pItem->fg.isCte |
| 164614 | + || pItem->u2.pCteUse->eM10d!=M10d_Yes |
| 164615 | + || NEVER(pItem->pSelect==0) |
| 164616 | + || pItem->pSelect->pOrderBy==0 |
| 164617 | + ){ |
| 164618 | + pWInfo->revMask |= MASKBIT(ii); |
| 164619 | + } |
| 164620 | + } |
| 164621 | +} |
| 164179 | 164622 | |
| 164180 | 164623 | /* |
| 164181 | 164624 | ** Generate the beginning of the loop used for WHERE clause processing. |
| 164182 | 164625 | ** The return value is a pointer to an opaque structure that contains |
| 164183 | 164626 | ** information needed to terminate the loop. Later, the calling routine |
| | @@ -164539,12 +164982,13 @@ |
| 164539 | 164982 | if( pWInfo->pOrderBy ){ |
| 164540 | 164983 | wherePathSolver(pWInfo, pWInfo->nRowOut+1); |
| 164541 | 164984 | if( db->mallocFailed ) goto whereBeginError; |
| 164542 | 164985 | } |
| 164543 | 164986 | } |
| 164987 | + assert( pWInfo->pTabList!=0 ); |
| 164544 | 164988 | if( pWInfo->pOrderBy==0 && (db->flags & SQLITE_ReverseOrder)!=0 ){ |
| 164545 | | - pWInfo->revMask = ALLBITS; |
| 164989 | + whereReverseScanOrder(pWInfo); |
| 164546 | 164990 | } |
| 164547 | 164991 | if( pParse->nErr ){ |
| 164548 | 164992 | goto whereBeginError; |
| 164549 | 164993 | } |
| 164550 | 164994 | assert( db->mallocFailed==0 ); |
| | @@ -164640,10 +165084,11 @@ |
| 164640 | 165084 | assert( !(wsFlags & WHERE_VIRTUALTABLE) || IsVirtual(pTabList->a[0].pTab) ); |
| 164641 | 165085 | if( bOnerow || ( |
| 164642 | 165086 | 0!=(wctrlFlags & WHERE_ONEPASS_MULTIROW) |
| 164643 | 165087 | && !IsVirtual(pTabList->a[0].pTab) |
| 164644 | 165088 | && (0==(wsFlags & WHERE_MULTI_OR) || (wctrlFlags & WHERE_DUPLICATES_OK)) |
| 165089 | + && OptimizationEnabled(db, SQLITE_OnePass) |
| 164645 | 165090 | )){ |
| 164646 | 165091 | pWInfo->eOnePass = bOnerow ? ONEPASS_SINGLE : ONEPASS_MULTI; |
| 164647 | 165092 | if( HasRowid(pTabList->a[0].pTab) && (wsFlags & WHERE_IDX_ONLY) ){ |
| 164648 | 165093 | if( wctrlFlags & WHERE_ONEPASS_MULTIROW ){ |
| 164649 | 165094 | bFordelete = OPFLAG_FORDELETE; |
| | @@ -171937,14 +172382,14 @@ |
| 171937 | 172382 | ** break; |
| 171938 | 172383 | */ |
| 171939 | 172384 | /********** Begin reduce actions **********************************************/ |
| 171940 | 172385 | YYMINORTYPE yylhsminor; |
| 171941 | 172386 | case 0: /* explain ::= EXPLAIN */ |
| 171942 | | -{ pParse->explain = 1; } |
| 172387 | +{ if( pParse->pReprepare==0 ) pParse->explain = 1; } |
| 171943 | 172388 | break; |
| 171944 | 172389 | case 1: /* explain ::= EXPLAIN QUERY PLAN */ |
| 171945 | | -{ pParse->explain = 2; } |
| 172390 | +{ if( pParse->pReprepare==0 ) pParse->explain = 2; } |
| 171946 | 172391 | break; |
| 171947 | 172392 | case 2: /* cmdx ::= cmd */ |
| 171948 | 172393 | { sqlite3FinishCoding(pParse); } |
| 171949 | 172394 | break; |
| 171950 | 172395 | case 3: /* cmd ::= BEGIN transtype trans_opt */ |
| | @@ -200394,28 +200839,54 @@ |
| 200394 | 200839 | ** Growing our own isspace() routine this way is twice as fast as |
| 200395 | 200840 | ** the library isspace() function, resulting in a 7% overall performance |
| 200396 | 200841 | ** increase for the parser. (Ubuntu14.10 gcc 4.8.4 x64 with -Os). |
| 200397 | 200842 | */ |
| 200398 | 200843 | static const char jsonIsSpace[] = { |
| 200399 | | - 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 1, 0, 0, |
| 200400 | | - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, |
| 200401 | | - 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, |
| 200402 | | - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, |
| 200403 | | - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, |
| 200404 | | - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, |
| 200405 | | - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, |
| 200406 | | - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, |
| 200407 | | - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, |
| 200408 | | - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, |
| 200409 | | - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, |
| 200410 | | - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, |
| 200411 | | - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, |
| 200412 | | - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, |
| 200413 | | - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, |
| 200414 | | - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, |
| 200844 | + 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 1, 0, 0, |
| 200845 | + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, |
| 200846 | + 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, |
| 200847 | + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, |
| 200848 | + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, |
| 200849 | + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, |
| 200850 | + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, |
| 200851 | + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, |
| 200852 | + |
| 200853 | + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, |
| 200854 | + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, |
| 200855 | + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, |
| 200856 | + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, |
| 200857 | + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, |
| 200858 | + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, |
| 200859 | + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, |
| 200860 | + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, |
| 200415 | 200861 | }; |
| 200416 | 200862 | #define fast_isspace(x) (jsonIsSpace[(unsigned char)x]) |
| 200863 | + |
| 200864 | +/* |
| 200865 | +** Characters that are special to JSON. Control charaters, |
| 200866 | +** '"' and '\\'. |
| 200867 | +*/ |
| 200868 | +static const char jsonIsOk[256] = { |
| 200869 | + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, |
| 200870 | + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, |
| 200871 | + 1, 1, 0, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1, |
| 200872 | + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, |
| 200873 | + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, |
| 200874 | + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, |
| 200875 | + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, |
| 200876 | + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, |
| 200877 | + |
| 200878 | + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, |
| 200879 | + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, |
| 200880 | + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, |
| 200881 | + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, |
| 200882 | + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, |
| 200883 | + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, |
| 200884 | + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, |
| 200885 | + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1 |
| 200886 | +}; |
| 200887 | + |
| 200417 | 200888 | |
| 200418 | 200889 | #if !defined(SQLITE_DEBUG) && !defined(SQLITE_COVERAGE_TEST) |
| 200419 | 200890 | # define VVA(X) |
| 200420 | 200891 | #else |
| 200421 | 200892 | # define VVA(X) X |
| | @@ -200423,10 +200894,11 @@ |
| 200423 | 200894 | |
| 200424 | 200895 | /* Objects */ |
| 200425 | 200896 | typedef struct JsonString JsonString; |
| 200426 | 200897 | typedef struct JsonNode JsonNode; |
| 200427 | 200898 | typedef struct JsonParse JsonParse; |
| 200899 | +typedef struct JsonCleanup JsonCleanup; |
| 200428 | 200900 | |
| 200429 | 200901 | /* An instance of this object represents a JSON string |
| 200430 | 200902 | ** under construction. Really, this is a generic string accumulator |
| 200431 | 200903 | ** that can be and is used to create strings other than JSON. |
| 200432 | 200904 | */ |
| | @@ -200437,75 +200909,120 @@ |
| 200437 | 200909 | u64 nUsed; /* Bytes of zBuf[] currently used */ |
| 200438 | 200910 | u8 bStatic; /* True if zBuf is static space */ |
| 200439 | 200911 | u8 bErr; /* True if an error has been encountered */ |
| 200440 | 200912 | char zSpace[100]; /* Initial static space */ |
| 200441 | 200913 | }; |
| 200914 | + |
| 200915 | +/* A deferred cleanup task. A list of JsonCleanup objects might be |
| 200916 | +** run when the JsonParse object is destroyed. |
| 200917 | +*/ |
| 200918 | +struct JsonCleanup { |
| 200919 | + JsonCleanup *pJCNext; /* Next in a list */ |
| 200920 | + void (*xOp)(void*); /* Routine to run */ |
| 200921 | + void *pArg; /* Argument to xOp() */ |
| 200922 | +}; |
| 200442 | 200923 | |
| 200443 | 200924 | /* JSON type values |
| 200444 | 200925 | */ |
| 200445 | | -#define JSON_NULL 0 |
| 200446 | | -#define JSON_TRUE 1 |
| 200447 | | -#define JSON_FALSE 2 |
| 200448 | | -#define JSON_INT 3 |
| 200449 | | -#define JSON_REAL 4 |
| 200450 | | -#define JSON_STRING 5 |
| 200451 | | -#define JSON_ARRAY 6 |
| 200452 | | -#define JSON_OBJECT 7 |
| 200926 | +#define JSON_SUBST 0 /* Special edit node. Uses u.iPrev */ |
| 200927 | +#define JSON_NULL 1 |
| 200928 | +#define JSON_TRUE 2 |
| 200929 | +#define JSON_FALSE 3 |
| 200930 | +#define JSON_INT 4 |
| 200931 | +#define JSON_REAL 5 |
| 200932 | +#define JSON_STRING 6 |
| 200933 | +#define JSON_ARRAY 7 |
| 200934 | +#define JSON_OBJECT 8 |
| 200453 | 200935 | |
| 200454 | 200936 | /* The "subtype" set for JSON values */ |
| 200455 | 200937 | #define JSON_SUBTYPE 74 /* Ascii for "J" */ |
| 200456 | 200938 | |
| 200457 | 200939 | /* |
| 200458 | 200940 | ** Names of the various JSON types: |
| 200459 | 200941 | */ |
| 200460 | 200942 | static const char * const jsonType[] = { |
| 200943 | + "subst", |
| 200461 | 200944 | "null", "true", "false", "integer", "real", "text", "array", "object" |
| 200462 | 200945 | }; |
| 200463 | 200946 | |
| 200464 | 200947 | /* Bit values for the JsonNode.jnFlag field |
| 200465 | 200948 | */ |
| 200466 | | -#define JNODE_RAW 0x01 /* Content is raw, not JSON encoded */ |
| 200467 | | -#define JNODE_ESCAPE 0x02 /* Content is text with \ escapes */ |
| 200468 | | -#define JNODE_REMOVE 0x04 /* Do not output */ |
| 200469 | | -#define JNODE_REPLACE 0x08 /* Replace with JsonNode.u.iReplace */ |
| 200470 | | -#define JNODE_PATCH 0x10 /* Patch with JsonNode.u.pPatch */ |
| 200471 | | -#define JNODE_APPEND 0x20 /* More ARRAY/OBJECT entries at u.iAppend */ |
| 200472 | | -#define JNODE_LABEL 0x40 /* Is a label of an object */ |
| 200473 | | -#define JNODE_JSON5 0x80 /* Node contains JSON5 enhancements */ |
| 200949 | +#define JNODE_RAW 0x01 /* Content is raw, not JSON encoded */ |
| 200950 | +#define JNODE_ESCAPE 0x02 /* Content is text with \ escapes */ |
| 200951 | +#define JNODE_REMOVE 0x04 /* Do not output */ |
| 200952 | +#define JNODE_REPLACE 0x08 /* Target of a JSON_SUBST node */ |
| 200953 | +#define JNODE_APPEND 0x10 /* More ARRAY/OBJECT entries at u.iAppend */ |
| 200954 | +#define JNODE_LABEL 0x20 /* Is a label of an object */ |
| 200955 | +#define JNODE_JSON5 0x40 /* Node contains JSON5 enhancements */ |
| 200474 | 200956 | |
| 200475 | 200957 | |
| 200476 | | -/* A single node of parsed JSON |
| 200958 | +/* A single node of parsed JSON. An array of these nodes describes |
| 200959 | +** a parse of JSON + edits. |
| 200960 | +** |
| 200961 | +** Use the json_parse() SQL function (available when compiled with |
| 200962 | +** -DSQLITE_DEBUG) to see a dump of complete JsonParse objects, including |
| 200963 | +** a complete listing and decoding of the array of JsonNodes. |
| 200477 | 200964 | */ |
| 200478 | 200965 | struct JsonNode { |
| 200479 | 200966 | u8 eType; /* One of the JSON_ type values */ |
| 200480 | 200967 | u8 jnFlags; /* JNODE flags */ |
| 200481 | 200968 | u8 eU; /* Which union element to use */ |
| 200482 | | - u32 n; /* Bytes of content, or number of sub-nodes */ |
| 200969 | + u32 n; /* Bytes of content for INT, REAL or STRING |
| 200970 | + ** Number of sub-nodes for ARRAY and OBJECT |
| 200971 | + ** Node that SUBST applies to */ |
| 200483 | 200972 | union { |
| 200484 | 200973 | const char *zJContent; /* 1: Content for INT, REAL, and STRING */ |
| 200485 | 200974 | u32 iAppend; /* 2: More terms for ARRAY and OBJECT */ |
| 200486 | 200975 | u32 iKey; /* 3: Key for ARRAY objects in json_tree() */ |
| 200487 | | - u32 iReplace; /* 4: Replacement content for JNODE_REPLACE */ |
| 200488 | | - JsonNode *pPatch; /* 5: Node chain of patch for JNODE_PATCH */ |
| 200976 | + u32 iPrev; /* 4: Previous SUBST node, or 0 */ |
| 200489 | 200977 | } u; |
| 200490 | 200978 | }; |
| 200491 | 200979 | |
| 200492 | | -/* A completely parsed JSON string |
| 200980 | + |
| 200981 | +/* A parsed and possibly edited JSON string. Lifecycle: |
| 200982 | +** |
| 200983 | +** 1. JSON comes in and is parsed into an array aNode[]. The original |
| 200984 | +** JSON text is stored in zJson. |
| 200985 | +** |
| 200986 | +** 2. Zero or more changes are made (via json_remove() or json_replace() |
| 200987 | +** or similar) to the aNode[] array. |
| 200988 | +** |
| 200989 | +** 3. A new, edited and mimified JSON string is generated from aNode |
| 200990 | +** and stored in zAlt. The JsonParse object always owns zAlt. |
| 200991 | +** |
| 200992 | +** Step 1 always happens. Step 2 and 3 may or may not happen, depending |
| 200993 | +** on the operation. |
| 200994 | +** |
| 200995 | +** aNode[].u.zJContent entries typically point into zJson. Hence zJson |
| 200996 | +** must remain valid for the lifespan of the parse. For edits, |
| 200997 | +** aNode[].u.zJContent might point to malloced space other than zJson. |
| 200998 | +** Entries in pClup are responsible for freeing that extra malloced space. |
| 200999 | +** |
| 201000 | +** When walking the parse tree in aNode[], edits are ignored if useMod is |
| 201001 | +** false. |
| 200493 | 201002 | */ |
| 200494 | 201003 | struct JsonParse { |
| 200495 | 201004 | u32 nNode; /* Number of slots of aNode[] used */ |
| 200496 | 201005 | u32 nAlloc; /* Number of slots of aNode[] allocated */ |
| 200497 | 201006 | JsonNode *aNode; /* Array of nodes containing the parse */ |
| 200498 | | - const char *zJson; /* Original JSON string */ |
| 201007 | + char *zJson; /* Original JSON string (before edits) */ |
| 201008 | + char *zAlt; /* Revised and/or mimified JSON */ |
| 200499 | 201009 | u32 *aUp; /* Index of parent of each node */ |
| 201010 | + JsonCleanup *pClup;/* Cleanup operations prior to freeing this object */ |
| 200500 | 201011 | u16 iDepth; /* Nesting depth */ |
| 200501 | 201012 | u8 nErr; /* Number of errors seen */ |
| 200502 | 201013 | u8 oom; /* Set to true if out of memory */ |
| 201014 | + u8 bJsonIsRCStr; /* True if zJson is an RCStr */ |
| 200503 | 201015 | u8 hasNonstd; /* True if input uses non-standard features like JSON5 */ |
| 201016 | + u8 useMod; /* Actually use the edits contain inside aNode */ |
| 201017 | + u8 hasMod; /* aNode contains edits from the original zJson */ |
| 201018 | + u32 nJPRef; /* Number of references to this object */ |
| 200504 | 201019 | int nJson; /* Length of the zJson string in bytes */ |
| 201020 | + int nAlt; /* Length of alternative JSON string zAlt, in bytes */ |
| 200505 | 201021 | u32 iErr; /* Error location in zJson[] */ |
| 200506 | | - u32 iHold; /* Replace cache line with the lowest iHold value */ |
| 201022 | + u32 iSubst; /* Last JSON_SUBST entry in aNode[] */ |
| 201023 | + u32 iHold; /* Age of this entry in the cache for LRU replacement */ |
| 200507 | 201024 | }; |
| 200508 | 201025 | |
| 200509 | 201026 | /* |
| 200510 | 201027 | ** Maximum nesting depth of JSON for this implementation. |
| 200511 | 201028 | ** |
| | @@ -200534,19 +201051,17 @@ |
| 200534 | 201051 | p->pCtx = pCtx; |
| 200535 | 201052 | p->bErr = 0; |
| 200536 | 201053 | jsonZero(p); |
| 200537 | 201054 | } |
| 200538 | 201055 | |
| 200539 | | - |
| 200540 | 201056 | /* Free all allocated memory and reset the JsonString object back to its |
| 200541 | 201057 | ** initial state. |
| 200542 | 201058 | */ |
| 200543 | 201059 | static void jsonReset(JsonString *p){ |
| 200544 | | - if( !p->bStatic ) sqlite3_free(p->zBuf); |
| 201060 | + if( !p->bStatic ) sqlite3RCStrUnref(p->zBuf); |
| 200545 | 201061 | jsonZero(p); |
| 200546 | 201062 | } |
| 200547 | | - |
| 200548 | 201063 | |
| 200549 | 201064 | /* Report an out-of-memory (OOM) condition |
| 200550 | 201065 | */ |
| 200551 | 201066 | static void jsonOom(JsonString *p){ |
| 200552 | 201067 | p->bErr = 1; |
| | @@ -200560,38 +201075,61 @@ |
| 200560 | 201075 | static int jsonGrow(JsonString *p, u32 N){ |
| 200561 | 201076 | u64 nTotal = N<p->nAlloc ? p->nAlloc*2 : p->nAlloc+N+10; |
| 200562 | 201077 | char *zNew; |
| 200563 | 201078 | if( p->bStatic ){ |
| 200564 | 201079 | if( p->bErr ) return 1; |
| 200565 | | - zNew = sqlite3_malloc64(nTotal); |
| 201080 | + zNew = sqlite3RCStrNew(nTotal); |
| 200566 | 201081 | if( zNew==0 ){ |
| 200567 | 201082 | jsonOom(p); |
| 200568 | 201083 | return SQLITE_NOMEM; |
| 200569 | 201084 | } |
| 200570 | 201085 | memcpy(zNew, p->zBuf, (size_t)p->nUsed); |
| 200571 | 201086 | p->zBuf = zNew; |
| 200572 | 201087 | p->bStatic = 0; |
| 200573 | 201088 | }else{ |
| 200574 | | - zNew = sqlite3_realloc64(p->zBuf, nTotal); |
| 200575 | | - if( zNew==0 ){ |
| 200576 | | - jsonOom(p); |
| 201089 | + p->zBuf = sqlite3RCStrResize(p->zBuf, nTotal); |
| 201090 | + if( p->zBuf==0 ){ |
| 201091 | + p->bErr = 1; |
| 201092 | + jsonZero(p); |
| 200577 | 201093 | return SQLITE_NOMEM; |
| 200578 | 201094 | } |
| 200579 | | - p->zBuf = zNew; |
| 200580 | 201095 | } |
| 200581 | 201096 | p->nAlloc = nTotal; |
| 200582 | 201097 | return SQLITE_OK; |
| 200583 | 201098 | } |
| 200584 | 201099 | |
| 200585 | 201100 | /* Append N bytes from zIn onto the end of the JsonString string. |
| 200586 | 201101 | */ |
| 200587 | | -static void jsonAppendRaw(JsonString *p, const char *zIn, u32 N){ |
| 200588 | | - if( N==0 ) return; |
| 200589 | | - if( (N+p->nUsed >= p->nAlloc) && jsonGrow(p,N)!=0 ) return; |
| 201102 | +static SQLITE_NOINLINE void jsonAppendExpand( |
| 201103 | + JsonString *p, |
| 201104 | + const char *zIn, |
| 201105 | + u32 N |
| 201106 | +){ |
| 201107 | + assert( N>0 ); |
| 201108 | + if( jsonGrow(p,N) ) return; |
| 200590 | 201109 | memcpy(p->zBuf+p->nUsed, zIn, N); |
| 200591 | 201110 | p->nUsed += N; |
| 200592 | 201111 | } |
| 201112 | +static void jsonAppendRaw(JsonString *p, const char *zIn, u32 N){ |
| 201113 | + if( N==0 ) return; |
| 201114 | + if( N+p->nUsed >= p->nAlloc ){ |
| 201115 | + jsonAppendExpand(p,zIn,N); |
| 201116 | + }else{ |
| 201117 | + memcpy(p->zBuf+p->nUsed, zIn, N); |
| 201118 | + p->nUsed += N; |
| 201119 | + } |
| 201120 | +} |
| 201121 | +static void jsonAppendRawNZ(JsonString *p, const char *zIn, u32 N){ |
| 201122 | + assert( N>0 ); |
| 201123 | + if( N+p->nUsed >= p->nAlloc ){ |
| 201124 | + jsonAppendExpand(p,zIn,N); |
| 201125 | + }else{ |
| 201126 | + memcpy(p->zBuf+p->nUsed, zIn, N); |
| 201127 | + p->nUsed += N; |
| 201128 | + } |
| 201129 | +} |
| 201130 | + |
| 200593 | 201131 | |
| 200594 | 201132 | /* Append formatted text (not to exceed N bytes) to the JsonString. |
| 200595 | 201133 | */ |
| 200596 | 201134 | static void jsonPrintf(int N, JsonString *p, const char *zFormat, ...){ |
| 200597 | 201135 | va_list ap; |
| | @@ -200602,23 +201140,49 @@ |
| 200602 | 201140 | p->nUsed += (int)strlen(p->zBuf+p->nUsed); |
| 200603 | 201141 | } |
| 200604 | 201142 | |
| 200605 | 201143 | /* Append a single character |
| 200606 | 201144 | */ |
| 200607 | | -static void jsonAppendChar(JsonString *p, char c){ |
| 200608 | | - if( p->nUsed>=p->nAlloc && jsonGrow(p,1)!=0 ) return; |
| 201145 | +static SQLITE_NOINLINE void jsonAppendCharExpand(JsonString *p, char c){ |
| 201146 | + if( jsonGrow(p,1) ) return; |
| 200609 | 201147 | p->zBuf[p->nUsed++] = c; |
| 200610 | 201148 | } |
| 201149 | +static void jsonAppendChar(JsonString *p, char c){ |
| 201150 | + if( p->nUsed>=p->nAlloc ){ |
| 201151 | + jsonAppendCharExpand(p,c); |
| 201152 | + }else{ |
| 201153 | + p->zBuf[p->nUsed++] = c; |
| 201154 | + } |
| 201155 | +} |
| 201156 | + |
| 201157 | +/* Try to force the string to be a zero-terminated RCStr string. |
| 201158 | +** |
| 201159 | +** Return true on success. Return false if an OOM prevents this |
| 201160 | +** from happening. |
| 201161 | +*/ |
| 201162 | +static int jsonForceRCStr(JsonString *p){ |
| 201163 | + jsonAppendChar(p, 0); |
| 201164 | + if( p->bErr ) return 0; |
| 201165 | + p->nUsed--; |
| 201166 | + if( p->bStatic==0 ) return 1; |
| 201167 | + p->nAlloc = 0; |
| 201168 | + p->nUsed++; |
| 201169 | + jsonGrow(p, p->nUsed); |
| 201170 | + p->nUsed--; |
| 201171 | + return p->bStatic==0; |
| 201172 | +} |
| 201173 | + |
| 200611 | 201174 | |
| 200612 | 201175 | /* Append a comma separator to the output buffer, if the previous |
| 200613 | 201176 | ** character is not '[' or '{'. |
| 200614 | 201177 | */ |
| 200615 | 201178 | static void jsonAppendSeparator(JsonString *p){ |
| 200616 | 201179 | char c; |
| 200617 | 201180 | if( p->nUsed==0 ) return; |
| 200618 | 201181 | c = p->zBuf[p->nUsed-1]; |
| 200619 | | - if( c!='[' && c!='{' ) jsonAppendChar(p, ','); |
| 201182 | + if( c=='[' || c=='{' ) return; |
| 201183 | + jsonAppendChar(p, ','); |
| 200620 | 201184 | } |
| 200621 | 201185 | |
| 200622 | 201186 | /* Append the N-byte string in zIn to the end of the JsonString string |
| 200623 | 201187 | ** under construction. Enclose the string in "..." and escape |
| 200624 | 201188 | ** any double-quotes or backslash characters contained within the |
| | @@ -200628,15 +201192,20 @@ |
| 200628 | 201192 | u32 i; |
| 200629 | 201193 | if( zIn==0 || ((N+p->nUsed+2 >= p->nAlloc) && jsonGrow(p,N+2)!=0) ) return; |
| 200630 | 201194 | p->zBuf[p->nUsed++] = '"'; |
| 200631 | 201195 | for(i=0; i<N; i++){ |
| 200632 | 201196 | unsigned char c = ((unsigned const char*)zIn)[i]; |
| 200633 | | - if( c=='"' || c=='\\' ){ |
| 201197 | + if( jsonIsOk[c] ){ |
| 201198 | + p->zBuf[p->nUsed++] = c; |
| 201199 | + }else if( c=='"' || c=='\\' ){ |
| 200634 | 201200 | json_simple_escape: |
| 200635 | 201201 | if( (p->nUsed+N+3-i > p->nAlloc) && jsonGrow(p,N+3-i)!=0 ) return; |
| 200636 | 201202 | p->zBuf[p->nUsed++] = '\\'; |
| 200637 | | - }else if( c<=0x1f ){ |
| 201203 | + p->zBuf[p->nUsed++] = c; |
| 201204 | + }else if( c=='\'' ){ |
| 201205 | + p->zBuf[p->nUsed++] = c; |
| 201206 | + }else{ |
| 200638 | 201207 | static const char aSpecial[] = { |
| 200639 | 201208 | 0, 0, 0, 0, 0, 0, 0, 0, 'b', 't', 'n', 0, 'f', 'r', 0, 0, |
| 200640 | 201209 | 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 |
| 200641 | 201210 | }; |
| 200642 | 201211 | assert( sizeof(aSpecial)==32 ); |
| | @@ -200643,23 +201212,23 @@ |
| 200643 | 201212 | assert( aSpecial['\b']=='b' ); |
| 200644 | 201213 | assert( aSpecial['\f']=='f' ); |
| 200645 | 201214 | assert( aSpecial['\n']=='n' ); |
| 200646 | 201215 | assert( aSpecial['\r']=='r' ); |
| 200647 | 201216 | assert( aSpecial['\t']=='t' ); |
| 201217 | + assert( c>=0 && c<sizeof(aSpecial) ); |
| 200648 | 201218 | if( aSpecial[c] ){ |
| 200649 | 201219 | c = aSpecial[c]; |
| 200650 | 201220 | goto json_simple_escape; |
| 200651 | 201221 | } |
| 200652 | 201222 | if( (p->nUsed+N+7+i > p->nAlloc) && jsonGrow(p,N+7-i)!=0 ) return; |
| 200653 | 201223 | p->zBuf[p->nUsed++] = '\\'; |
| 200654 | 201224 | p->zBuf[p->nUsed++] = 'u'; |
| 200655 | 201225 | p->zBuf[p->nUsed++] = '0'; |
| 200656 | 201226 | p->zBuf[p->nUsed++] = '0'; |
| 200657 | | - p->zBuf[p->nUsed++] = '0' + (c>>4); |
| 200658 | | - c = "0123456789abcdef"[c&0xf]; |
| 201227 | + p->zBuf[p->nUsed++] = "0123456789abcdef"[c>>4]; |
| 201228 | + p->zBuf[p->nUsed++] = "0123456789abcdef"[c&0xf]; |
| 200659 | 201229 | } |
| 200660 | | - p->zBuf[p->nUsed++] = c; |
| 200661 | 201230 | } |
| 200662 | 201231 | p->zBuf[p->nUsed++] = '"'; |
| 200663 | 201232 | assert( p->nUsed<p->nAlloc ); |
| 200664 | 201233 | } |
| 200665 | 201234 | |
| | @@ -200674,11 +201243,11 @@ |
| 200674 | 201243 | zIn++; |
| 200675 | 201244 | N -= 2; |
| 200676 | 201245 | while( N>0 ){ |
| 200677 | 201246 | for(i=0; i<N && zIn[i]!='\\'; i++){} |
| 200678 | 201247 | if( i>0 ){ |
| 200679 | | - jsonAppendRaw(p, zIn, i); |
| 201248 | + jsonAppendRawNZ(p, zIn, i); |
| 200680 | 201249 | zIn += i; |
| 200681 | 201250 | N -= i; |
| 200682 | 201251 | if( N==0 ) break; |
| 200683 | 201252 | } |
| 200684 | 201253 | assert( zIn[0]=='\\' ); |
| | @@ -200685,20 +201254,20 @@ |
| 200685 | 201254 | switch( (u8)zIn[1] ){ |
| 200686 | 201255 | case '\'': |
| 200687 | 201256 | jsonAppendChar(p, '\''); |
| 200688 | 201257 | break; |
| 200689 | 201258 | case 'v': |
| 200690 | | - jsonAppendRaw(p, "\\u0009", 6); |
| 201259 | + jsonAppendRawNZ(p, "\\u0009", 6); |
| 200691 | 201260 | break; |
| 200692 | 201261 | case 'x': |
| 200693 | | - jsonAppendRaw(p, "\\u00", 4); |
| 200694 | | - jsonAppendRaw(p, &zIn[2], 2); |
| 201262 | + jsonAppendRawNZ(p, "\\u00", 4); |
| 201263 | + jsonAppendRawNZ(p, &zIn[2], 2); |
| 200695 | 201264 | zIn += 2; |
| 200696 | 201265 | N -= 2; |
| 200697 | 201266 | break; |
| 200698 | 201267 | case '0': |
| 200699 | | - jsonAppendRaw(p, "\\u0000", 6); |
| 201268 | + jsonAppendRawNZ(p, "\\u0000", 6); |
| 200700 | 201269 | break; |
| 200701 | 201270 | case '\r': |
| 200702 | 201271 | if( zIn[2]=='\n' ){ |
| 200703 | 201272 | zIn++; |
| 200704 | 201273 | N--; |
| | @@ -200712,11 +201281,11 @@ |
| 200712 | 201281 | assert( 0xa8==(u8)zIn[3] || 0xa9==(u8)zIn[3] ); |
| 200713 | 201282 | zIn += 2; |
| 200714 | 201283 | N -= 2; |
| 200715 | 201284 | break; |
| 200716 | 201285 | default: |
| 200717 | | - jsonAppendRaw(p, zIn, 2); |
| 201286 | + jsonAppendRawNZ(p, zIn, 2); |
| 200718 | 201287 | break; |
| 200719 | 201288 | } |
| 200720 | 201289 | zIn += 2; |
| 200721 | 201290 | N -= 2; |
| 200722 | 201291 | } |
| | @@ -200742,15 +201311,16 @@ |
| 200742 | 201311 | int rc = sqlite3DecOrHexToI64(zIn, &i); |
| 200743 | 201312 | if( rc<=1 ){ |
| 200744 | 201313 | jsonPrintf(100,p,"%lld",i); |
| 200745 | 201314 | }else{ |
| 200746 | 201315 | assert( rc==2 ); |
| 200747 | | - jsonAppendRaw(p, "9.0e999", 7); |
| 201316 | + jsonAppendRawNZ(p, "9.0e999", 7); |
| 200748 | 201317 | } |
| 200749 | 201318 | return; |
| 200750 | 201319 | } |
| 200751 | | - jsonAppendRaw(p, zIn, N); |
| 201320 | + assert( N>0 ); |
| 201321 | + jsonAppendRawNZ(p, zIn, N); |
| 200752 | 201322 | } |
| 200753 | 201323 | |
| 200754 | 201324 | /* |
| 200755 | 201325 | ** The zIn[0..N] string is a JSON5 real literal. Append to p a translation |
| 200756 | 201326 | ** of the string literal that standard JSON and that omits all JSON5 |
| | @@ -200778,11 +201348,11 @@ |
| 200778 | 201348 | jsonAppendChar(p, '0'); |
| 200779 | 201349 | break; |
| 200780 | 201350 | } |
| 200781 | 201351 | } |
| 200782 | 201352 | if( N>0 ){ |
| 200783 | | - jsonAppendRaw(p, zIn, N); |
| 201353 | + jsonAppendRawNZ(p, zIn, N); |
| 200784 | 201354 | } |
| 200785 | 201355 | } |
| 200786 | 201356 | |
| 200787 | 201357 | |
| 200788 | 201358 | |
| | @@ -200794,11 +201364,11 @@ |
| 200794 | 201364 | JsonString *p, /* Append to this JSON string */ |
| 200795 | 201365 | sqlite3_value *pValue /* Value to append */ |
| 200796 | 201366 | ){ |
| 200797 | 201367 | switch( sqlite3_value_type(pValue) ){ |
| 200798 | 201368 | case SQLITE_NULL: { |
| 200799 | | - jsonAppendRaw(p, "null", 4); |
| 201369 | + jsonAppendRawNZ(p, "null", 4); |
| 200800 | 201370 | break; |
| 200801 | 201371 | } |
| 200802 | 201372 | case SQLITE_FLOAT: { |
| 200803 | 201373 | jsonPrintf(100, p, "%!0.15g", sqlite3_value_double(pValue)); |
| 200804 | 201374 | break; |
| | @@ -200830,19 +201400,29 @@ |
| 200830 | 201400 | } |
| 200831 | 201401 | } |
| 200832 | 201402 | |
| 200833 | 201403 | |
| 200834 | 201404 | /* Make the JSON in p the result of the SQL function. |
| 201405 | +** |
| 201406 | +** The JSON string is reset. |
| 200835 | 201407 | */ |
| 200836 | 201408 | static void jsonResult(JsonString *p){ |
| 200837 | 201409 | if( p->bErr==0 ){ |
| 200838 | | - sqlite3_result_text64(p->pCtx, p->zBuf, p->nUsed, |
| 200839 | | - p->bStatic ? SQLITE_TRANSIENT : sqlite3_free, |
| 200840 | | - SQLITE_UTF8); |
| 200841 | | - jsonZero(p); |
| 201410 | + if( p->bStatic ){ |
| 201411 | + sqlite3_result_text64(p->pCtx, p->zBuf, p->nUsed, |
| 201412 | + SQLITE_TRANSIENT, SQLITE_UTF8); |
| 201413 | + }else if( jsonForceRCStr(p) ){ |
| 201414 | + sqlite3RCStrRef(p->zBuf); |
| 201415 | + sqlite3_result_text64(p->pCtx, p->zBuf, p->nUsed, |
| 201416 | + (void(*)(void*))sqlite3RCStrUnref, |
| 201417 | + SQLITE_UTF8); |
| 201418 | + } |
| 200842 | 201419 | } |
| 200843 | | - assert( p->bStatic ); |
| 201420 | + if( p->bErr==1 ){ |
| 201421 | + sqlite3_result_error_nomem(p->pCtx); |
| 201422 | + } |
| 201423 | + jsonReset(p); |
| 200844 | 201424 | } |
| 200845 | 201425 | |
| 200846 | 201426 | /************************************************************************** |
| 200847 | 201427 | ** Utility routines for dealing with JsonNode and JsonParse objects |
| 200848 | 201428 | **************************************************************************/ |
| | @@ -200863,58 +201443,117 @@ |
| 200863 | 201443 | /* |
| 200864 | 201444 | ** Reclaim all memory allocated by a JsonParse object. But do not |
| 200865 | 201445 | ** delete the JsonParse object itself. |
| 200866 | 201446 | */ |
| 200867 | 201447 | static void jsonParseReset(JsonParse *pParse){ |
| 200868 | | - sqlite3_free(pParse->aNode); |
| 200869 | | - pParse->aNode = 0; |
| 201448 | + while( pParse->pClup ){ |
| 201449 | + JsonCleanup *pTask = pParse->pClup; |
| 201450 | + pParse->pClup = pTask->pJCNext; |
| 201451 | + pTask->xOp(pTask->pArg); |
| 201452 | + sqlite3_free(pTask); |
| 201453 | + } |
| 201454 | + assert( pParse->nJPRef<=1 ); |
| 201455 | + if( pParse->aNode ){ |
| 201456 | + sqlite3_free(pParse->aNode); |
| 201457 | + pParse->aNode = 0; |
| 201458 | + } |
| 200870 | 201459 | pParse->nNode = 0; |
| 200871 | 201460 | pParse->nAlloc = 0; |
| 200872 | | - sqlite3_free(pParse->aUp); |
| 200873 | | - pParse->aUp = 0; |
| 201461 | + if( pParse->aUp ){ |
| 201462 | + sqlite3_free(pParse->aUp); |
| 201463 | + pParse->aUp = 0; |
| 201464 | + } |
| 201465 | + if( pParse->bJsonIsRCStr ){ |
| 201466 | + sqlite3RCStrUnref(pParse->zJson); |
| 201467 | + pParse->zJson = 0; |
| 201468 | + pParse->bJsonIsRCStr = 0; |
| 201469 | + } |
| 201470 | + if( pParse->zAlt ){ |
| 201471 | + sqlite3RCStrUnref(pParse->zAlt); |
| 201472 | + pParse->zAlt = 0; |
| 201473 | + } |
| 200874 | 201474 | } |
| 200875 | 201475 | |
| 200876 | 201476 | /* |
| 200877 | 201477 | ** Free a JsonParse object that was obtained from sqlite3_malloc(). |
| 201478 | +** |
| 201479 | +** Note that destroying JsonParse might call sqlite3RCStrUnref() to |
| 201480 | +** destroy the zJson value. The RCStr object might recursively invoke |
| 201481 | +** JsonParse to destroy this pParse object again. Take care to ensure |
| 201482 | +** that this recursive destructor sequence terminates harmlessly. |
| 200878 | 201483 | */ |
| 200879 | 201484 | static void jsonParseFree(JsonParse *pParse){ |
| 200880 | | - jsonParseReset(pParse); |
| 200881 | | - sqlite3_free(pParse); |
| 201485 | + if( pParse->nJPRef>1 ){ |
| 201486 | + pParse->nJPRef--; |
| 201487 | + }else{ |
| 201488 | + jsonParseReset(pParse); |
| 201489 | + sqlite3_free(pParse); |
| 201490 | + } |
| 201491 | +} |
| 201492 | + |
| 201493 | +/* |
| 201494 | +** Add a cleanup task to the JsonParse object. |
| 201495 | +** |
| 201496 | +** If an OOM occurs, the cleanup operation happens immediately |
| 201497 | +** and this function returns SQLITE_NOMEM. |
| 201498 | +*/ |
| 201499 | +static int jsonParseAddCleanup( |
| 201500 | + JsonParse *pParse, /* Add the cleanup task to this parser */ |
| 201501 | + void(*xOp)(void*), /* The cleanup task */ |
| 201502 | + void *pArg /* Argument to the cleanup */ |
| 201503 | +){ |
| 201504 | + JsonCleanup *pTask = sqlite3_malloc64( sizeof(*pTask) ); |
| 201505 | + if( pTask==0 ){ |
| 201506 | + pParse->oom = 1; |
| 201507 | + xOp(pArg); |
| 201508 | + return SQLITE_ERROR; |
| 201509 | + } |
| 201510 | + pTask->pJCNext = pParse->pClup; |
| 201511 | + pParse->pClup = pTask; |
| 201512 | + pTask->xOp = xOp; |
| 201513 | + pTask->pArg = pArg; |
| 201514 | + return SQLITE_OK; |
| 200882 | 201515 | } |
| 200883 | 201516 | |
| 200884 | 201517 | /* |
| 200885 | 201518 | ** Convert the JsonNode pNode into a pure JSON string and |
| 200886 | 201519 | ** append to pOut. Subsubstructure is also included. Return |
| 200887 | 201520 | ** the number of JsonNode objects that are encoded. |
| 200888 | 201521 | */ |
| 200889 | 201522 | static void jsonRenderNode( |
| 201523 | + JsonParse *pParse, /* the complete parse of the JSON */ |
| 200890 | 201524 | JsonNode *pNode, /* The node to render */ |
| 200891 | | - JsonString *pOut, /* Write JSON here */ |
| 200892 | | - sqlite3_value **aReplace /* Replacement values */ |
| 201525 | + JsonString *pOut /* Write JSON here */ |
| 200893 | 201526 | ){ |
| 200894 | 201527 | assert( pNode!=0 ); |
| 200895 | | - if( pNode->jnFlags & (JNODE_REPLACE|JNODE_PATCH) ){ |
| 200896 | | - if( (pNode->jnFlags & JNODE_REPLACE)!=0 && ALWAYS(aReplace!=0) ){ |
| 200897 | | - assert( pNode->eU==4 ); |
| 200898 | | - jsonAppendValue(pOut, aReplace[pNode->u.iReplace]); |
| 200899 | | - return; |
| 200900 | | - } |
| 200901 | | - assert( pNode->eU==5 ); |
| 200902 | | - pNode = pNode->u.pPatch; |
| 201528 | + while( (pNode->jnFlags & JNODE_REPLACE)!=0 && pParse->useMod ){ |
| 201529 | + u32 idx = (u32)(pNode - pParse->aNode); |
| 201530 | + u32 i = pParse->iSubst; |
| 201531 | + while( 1 /*exit-by-break*/ ){ |
| 201532 | + assert( i<pParse->nNode ); |
| 201533 | + assert( pParse->aNode[i].eType==JSON_SUBST ); |
| 201534 | + assert( pParse->aNode[i].eU==4 ); |
| 201535 | + assert( pParse->aNode[i].u.iPrev<i ); |
| 201536 | + if( pParse->aNode[i].n==idx ){ |
| 201537 | + pNode = &pParse->aNode[i+1]; |
| 201538 | + break; |
| 201539 | + } |
| 201540 | + i = pParse->aNode[i].u.iPrev; |
| 201541 | + } |
| 200903 | 201542 | } |
| 200904 | 201543 | switch( pNode->eType ){ |
| 200905 | 201544 | default: { |
| 200906 | 201545 | assert( pNode->eType==JSON_NULL ); |
| 200907 | | - jsonAppendRaw(pOut, "null", 4); |
| 201546 | + jsonAppendRawNZ(pOut, "null", 4); |
| 200908 | 201547 | break; |
| 200909 | 201548 | } |
| 200910 | 201549 | case JSON_TRUE: { |
| 200911 | | - jsonAppendRaw(pOut, "true", 4); |
| 201550 | + jsonAppendRawNZ(pOut, "true", 4); |
| 200912 | 201551 | break; |
| 200913 | 201552 | } |
| 200914 | 201553 | case JSON_FALSE: { |
| 200915 | | - jsonAppendRaw(pOut, "false", 5); |
| 201554 | + jsonAppendRawNZ(pOut, "false", 5); |
| 200916 | 201555 | break; |
| 200917 | 201556 | } |
| 200918 | 201557 | case JSON_STRING: { |
| 200919 | 201558 | assert( pNode->eU==1 ); |
| 200920 | 201559 | if( pNode->jnFlags & JNODE_RAW ){ |
| | @@ -200926,46 +201565,50 @@ |
| 200926 | 201565 | jsonAppendString(pOut, pNode->u.zJContent, pNode->n); |
| 200927 | 201566 | } |
| 200928 | 201567 | }else if( pNode->jnFlags & JNODE_JSON5 ){ |
| 200929 | 201568 | jsonAppendNormalizedString(pOut, pNode->u.zJContent, pNode->n); |
| 200930 | 201569 | }else{ |
| 200931 | | - jsonAppendRaw(pOut, pNode->u.zJContent, pNode->n); |
| 201570 | + assert( pNode->n>0 ); |
| 201571 | + jsonAppendRawNZ(pOut, pNode->u.zJContent, pNode->n); |
| 200932 | 201572 | } |
| 200933 | 201573 | break; |
| 200934 | 201574 | } |
| 200935 | 201575 | case JSON_REAL: { |
| 200936 | 201576 | assert( pNode->eU==1 ); |
| 200937 | 201577 | if( pNode->jnFlags & JNODE_JSON5 ){ |
| 200938 | 201578 | jsonAppendNormalizedReal(pOut, pNode->u.zJContent, pNode->n); |
| 200939 | 201579 | }else{ |
| 200940 | | - jsonAppendRaw(pOut, pNode->u.zJContent, pNode->n); |
| 201580 | + assert( pNode->n>0 ); |
| 201581 | + jsonAppendRawNZ(pOut, pNode->u.zJContent, pNode->n); |
| 200941 | 201582 | } |
| 200942 | 201583 | break; |
| 200943 | 201584 | } |
| 200944 | 201585 | case JSON_INT: { |
| 200945 | 201586 | assert( pNode->eU==1 ); |
| 200946 | 201587 | if( pNode->jnFlags & JNODE_JSON5 ){ |
| 200947 | 201588 | jsonAppendNormalizedInt(pOut, pNode->u.zJContent, pNode->n); |
| 200948 | 201589 | }else{ |
| 200949 | | - jsonAppendRaw(pOut, pNode->u.zJContent, pNode->n); |
| 201590 | + assert( pNode->n>0 ); |
| 201591 | + jsonAppendRawNZ(pOut, pNode->u.zJContent, pNode->n); |
| 200950 | 201592 | } |
| 200951 | 201593 | break; |
| 200952 | 201594 | } |
| 200953 | 201595 | case JSON_ARRAY: { |
| 200954 | 201596 | u32 j = 1; |
| 200955 | 201597 | jsonAppendChar(pOut, '['); |
| 200956 | 201598 | for(;;){ |
| 200957 | 201599 | while( j<=pNode->n ){ |
| 200958 | | - if( (pNode[j].jnFlags & JNODE_REMOVE)==0 ){ |
| 201600 | + if( (pNode[j].jnFlags & JNODE_REMOVE)==0 || pParse->useMod==0 ){ |
| 200959 | 201601 | jsonAppendSeparator(pOut); |
| 200960 | | - jsonRenderNode(&pNode[j], pOut, aReplace); |
| 201602 | + jsonRenderNode(pParse, &pNode[j], pOut); |
| 200961 | 201603 | } |
| 200962 | 201604 | j += jsonNodeSize(&pNode[j]); |
| 200963 | 201605 | } |
| 200964 | 201606 | if( (pNode->jnFlags & JNODE_APPEND)==0 ) break; |
| 201607 | + if( pParse->useMod==0 ) break; |
| 200965 | 201608 | assert( pNode->eU==2 ); |
| 200966 | | - pNode = &pNode[pNode->u.iAppend]; |
| 201609 | + pNode = &pParse->aNode[pNode->u.iAppend]; |
| 200967 | 201610 | j = 1; |
| 200968 | 201611 | } |
| 200969 | 201612 | jsonAppendChar(pOut, ']'); |
| 200970 | 201613 | break; |
| 200971 | 201614 | } |
| | @@ -200972,21 +201615,22 @@ |
| 200972 | 201615 | case JSON_OBJECT: { |
| 200973 | 201616 | u32 j = 1; |
| 200974 | 201617 | jsonAppendChar(pOut, '{'); |
| 200975 | 201618 | for(;;){ |
| 200976 | 201619 | while( j<=pNode->n ){ |
| 200977 | | - if( (pNode[j+1].jnFlags & JNODE_REMOVE)==0 ){ |
| 201620 | + if( (pNode[j+1].jnFlags & JNODE_REMOVE)==0 || pParse->useMod==0 ){ |
| 200978 | 201621 | jsonAppendSeparator(pOut); |
| 200979 | | - jsonRenderNode(&pNode[j], pOut, aReplace); |
| 201622 | + jsonRenderNode(pParse, &pNode[j], pOut); |
| 200980 | 201623 | jsonAppendChar(pOut, ':'); |
| 200981 | | - jsonRenderNode(&pNode[j+1], pOut, aReplace); |
| 201624 | + jsonRenderNode(pParse, &pNode[j+1], pOut); |
| 200982 | 201625 | } |
| 200983 | 201626 | j += 1 + jsonNodeSize(&pNode[j+1]); |
| 200984 | 201627 | } |
| 200985 | 201628 | if( (pNode->jnFlags & JNODE_APPEND)==0 ) break; |
| 201629 | + if( pParse->useMod==0 ) break; |
| 200986 | 201630 | assert( pNode->eU==2 ); |
| 200987 | | - pNode = &pNode[pNode->u.iAppend]; |
| 201631 | + pNode = &pParse->aNode[pNode->u.iAppend]; |
| 200988 | 201632 | j = 1; |
| 200989 | 201633 | } |
| 200990 | 201634 | jsonAppendChar(pOut, '}'); |
| 200991 | 201635 | break; |
| 200992 | 201636 | } |
| | @@ -200995,19 +201639,30 @@ |
| 200995 | 201639 | |
| 200996 | 201640 | /* |
| 200997 | 201641 | ** Return a JsonNode and all its descendants as a JSON string. |
| 200998 | 201642 | */ |
| 200999 | 201643 | static void jsonReturnJson( |
| 201644 | + JsonParse *pParse, /* The complete JSON */ |
| 201000 | 201645 | JsonNode *pNode, /* Node to return */ |
| 201001 | 201646 | sqlite3_context *pCtx, /* Return value for this function */ |
| 201002 | | - sqlite3_value **aReplace /* Array of replacement values */ |
| 201647 | + int bGenerateAlt /* Also store the rendered text in zAlt */ |
| 201003 | 201648 | ){ |
| 201004 | 201649 | JsonString s; |
| 201005 | | - jsonInit(&s, pCtx); |
| 201006 | | - jsonRenderNode(pNode, &s, aReplace); |
| 201007 | | - jsonResult(&s); |
| 201008 | | - sqlite3_result_subtype(pCtx, JSON_SUBTYPE); |
| 201650 | + if( pParse->oom ){ |
| 201651 | + sqlite3_result_error_nomem(pCtx); |
| 201652 | + return; |
| 201653 | + } |
| 201654 | + if( pParse->nErr==0 ){ |
| 201655 | + jsonInit(&s, pCtx); |
| 201656 | + jsonRenderNode(pParse, pNode, &s); |
| 201657 | + if( bGenerateAlt && pParse->zAlt==0 && jsonForceRCStr(&s) ){ |
| 201658 | + pParse->zAlt = sqlite3RCStrRef(s.zBuf); |
| 201659 | + pParse->nAlt = s.nUsed; |
| 201660 | + } |
| 201661 | + jsonResult(&s); |
| 201662 | + sqlite3_result_subtype(pCtx, JSON_SUBTYPE); |
| 201663 | + } |
| 201009 | 201664 | } |
| 201010 | 201665 | |
| 201011 | 201666 | /* |
| 201012 | 201667 | ** Translate a single byte of Hex into an integer. |
| 201013 | 201668 | ** This routine only works if h really is a valid hexadecimal |
| | @@ -201041,13 +201696,13 @@ |
| 201041 | 201696 | |
| 201042 | 201697 | /* |
| 201043 | 201698 | ** Make the JsonNode the return value of the function. |
| 201044 | 201699 | */ |
| 201045 | 201700 | static void jsonReturn( |
| 201701 | + JsonParse *pParse, /* Complete JSON parse tree */ |
| 201046 | 201702 | JsonNode *pNode, /* Node to return */ |
| 201047 | | - sqlite3_context *pCtx, /* Return value for this function */ |
| 201048 | | - sqlite3_value **aReplace /* Array of replacement values */ |
| 201703 | + sqlite3_context *pCtx /* Return value for this function */ |
| 201049 | 201704 | ){ |
| 201050 | 201705 | switch( pNode->eType ){ |
| 201051 | 201706 | default: { |
| 201052 | 201707 | assert( pNode->eType==JSON_NULL ); |
| 201053 | 201708 | sqlite3_result_null(pCtx); |
| | @@ -201064,11 +201719,10 @@ |
| 201064 | 201719 | case JSON_INT: { |
| 201065 | 201720 | sqlite3_int64 i = 0; |
| 201066 | 201721 | int rc; |
| 201067 | 201722 | int bNeg = 0; |
| 201068 | 201723 | const char *z; |
| 201069 | | - |
| 201070 | 201724 | |
| 201071 | 201725 | assert( pNode->eU==1 ); |
| 201072 | 201726 | z = pNode->u.zJContent; |
| 201073 | 201727 | if( z[0]=='-' ){ z++; bNeg = 1; } |
| 201074 | 201728 | else if( z[0]=='+' ){ z++; } |
| | @@ -201190,11 +201844,11 @@ |
| 201190 | 201844 | } |
| 201191 | 201845 | break; |
| 201192 | 201846 | } |
| 201193 | 201847 | case JSON_ARRAY: |
| 201194 | 201848 | case JSON_OBJECT: { |
| 201195 | | - jsonReturnJson(pNode, pCtx, aReplace); |
| 201849 | + jsonReturnJson(pParse, pNode, pCtx, 0); |
| 201196 | 201850 | break; |
| 201197 | 201851 | } |
| 201198 | 201852 | } |
| 201199 | 201853 | } |
| 201200 | 201854 | |
| | @@ -201212,10 +201866,16 @@ |
| 201212 | 201866 | #else |
| 201213 | 201867 | # define JSON_NOINLINE |
| 201214 | 201868 | #endif |
| 201215 | 201869 | |
| 201216 | 201870 | |
| 201871 | +/* |
| 201872 | +** Add a single node to pParse->aNode after first expanding the |
| 201873 | +** size of the aNode array. Return the index of the new node. |
| 201874 | +** |
| 201875 | +** If an OOM error occurs, set pParse->oom and return -1. |
| 201876 | +*/ |
| 201217 | 201877 | static JSON_NOINLINE int jsonParseAddNodeExpand( |
| 201218 | 201878 | JsonParse *pParse, /* Append the node to this object */ |
| 201219 | 201879 | u32 eType, /* Node type */ |
| 201220 | 201880 | u32 n, /* Content size or sub-node count */ |
| 201221 | 201881 | const char *zContent /* Content */ |
| | @@ -201228,11 +201888,11 @@ |
| 201228 | 201888 | pNew = sqlite3_realloc64(pParse->aNode, sizeof(JsonNode)*nNew); |
| 201229 | 201889 | if( pNew==0 ){ |
| 201230 | 201890 | pParse->oom = 1; |
| 201231 | 201891 | return -1; |
| 201232 | 201892 | } |
| 201233 | | - pParse->nAlloc = nNew; |
| 201893 | + pParse->nAlloc = sqlite3_msize(pNew)/sizeof(JsonNode); |
| 201234 | 201894 | pParse->aNode = pNew; |
| 201235 | 201895 | assert( pParse->nNode<pParse->nAlloc ); |
| 201236 | 201896 | return jsonParseAddNode(pParse, eType, n, zContent); |
| 201237 | 201897 | } |
| 201238 | 201898 | |
| | @@ -201246,11 +201906,12 @@ |
| 201246 | 201906 | u32 eType, /* Node type */ |
| 201247 | 201907 | u32 n, /* Content size or sub-node count */ |
| 201248 | 201908 | const char *zContent /* Content */ |
| 201249 | 201909 | ){ |
| 201250 | 201910 | JsonNode *p; |
| 201251 | | - if( pParse->aNode==0 || pParse->nNode>=pParse->nAlloc ){ |
| 201911 | + assert( pParse->aNode!=0 || pParse->nNode>=pParse->nAlloc ); |
| 201912 | + if( pParse->nNode>=pParse->nAlloc ){ |
| 201252 | 201913 | return jsonParseAddNodeExpand(pParse, eType, n, zContent); |
| 201253 | 201914 | } |
| 201254 | 201915 | p = &pParse->aNode[pParse->nNode]; |
| 201255 | 201916 | p->eType = (u8)(eType & 0xff); |
| 201256 | 201917 | p->jnFlags = (u8)(eType >> 8); |
| | @@ -201257,10 +201918,54 @@ |
| 201257 | 201918 | VVA( p->eU = zContent ? 1 : 0 ); |
| 201258 | 201919 | p->n = n; |
| 201259 | 201920 | p->u.zJContent = zContent; |
| 201260 | 201921 | return pParse->nNode++; |
| 201261 | 201922 | } |
| 201923 | + |
| 201924 | +/* |
| 201925 | +** Add an array of new nodes to the current pParse->aNode array. |
| 201926 | +** Return the index of the first node added. |
| 201927 | +** |
| 201928 | +** If an OOM error occurs, set pParse->oom. |
| 201929 | +*/ |
| 201930 | +static void jsonParseAddNodeArray( |
| 201931 | + JsonParse *pParse, /* Append the node to this object */ |
| 201932 | + JsonNode *aNode, /* Array of nodes to add */ |
| 201933 | + u32 nNode /* Number of elements in aNew */ |
| 201934 | +){ |
| 201935 | + if( pParse->nNode + nNode > pParse->nAlloc ){ |
| 201936 | + u32 nNew = pParse->nNode + nNode; |
| 201937 | + JsonNode *aNew = sqlite3_realloc64(pParse->aNode, nNew*sizeof(JsonNode)); |
| 201938 | + if( aNew==0 ){ |
| 201939 | + pParse->oom = 1; |
| 201940 | + return; |
| 201941 | + } |
| 201942 | + pParse->nAlloc = sqlite3_msize(aNew)/sizeof(JsonNode); |
| 201943 | + pParse->aNode = aNew; |
| 201944 | + } |
| 201945 | + memcpy(&pParse->aNode[pParse->nNode], aNode, nNode*sizeof(JsonNode)); |
| 201946 | + pParse->nNode += nNode; |
| 201947 | +} |
| 201948 | + |
| 201949 | +/* |
| 201950 | +** Add a new JSON_SUBST node. The node immediately following |
| 201951 | +** this new node will be the substitute content for iNode. |
| 201952 | +*/ |
| 201953 | +static int jsonParseAddSubstNode( |
| 201954 | + JsonParse *pParse, /* Add the JSON_SUBST here */ |
| 201955 | + u32 iNode /* References this node */ |
| 201956 | +){ |
| 201957 | + int idx = jsonParseAddNode(pParse, JSON_SUBST, iNode, 0); |
| 201958 | + if( pParse->oom ) return -1; |
| 201959 | + pParse->aNode[iNode].jnFlags |= JNODE_REPLACE; |
| 201960 | + pParse->aNode[idx].eU = 4; |
| 201961 | + pParse->aNode[idx].u.iPrev = pParse->iSubst; |
| 201962 | + pParse->iSubst = idx; |
| 201963 | + pParse->hasMod = 1; |
| 201964 | + pParse->useMod = 1; |
| 201965 | + return idx; |
| 201966 | +} |
| 201262 | 201967 | |
| 201263 | 201968 | /* |
| 201264 | 201969 | ** Return true if z[] begins with 2 (or more) hexadecimal digits |
| 201265 | 201970 | */ |
| 201266 | 201971 | static int jsonIs2Hex(const char *z){ |
| | @@ -201424,11 +202129,11 @@ |
| 201424 | 202129 | ** Parse a single JSON value which begins at pParse->zJson[i]. Return the |
| 201425 | 202130 | ** index of the first character past the end of the value parsed. |
| 201426 | 202131 | ** |
| 201427 | 202132 | ** Special return values: |
| 201428 | 202133 | ** |
| 201429 | | -** 0 End if input |
| 202134 | +** 0 End of input |
| 201430 | 202135 | ** -1 Syntax error |
| 201431 | 202136 | ** -2 '}' seen |
| 201432 | 202137 | ** -3 ']' seen |
| 201433 | 202138 | ** -4 ',' seen |
| 201434 | 202139 | ** -5 ':' seen |
| | @@ -201599,19 +202304,16 @@ |
| 201599 | 202304 | case '"': |
| 201600 | 202305 | /* Parse string */ |
| 201601 | 202306 | jnFlags = 0; |
| 201602 | 202307 | parse_string: |
| 201603 | 202308 | cDelim = z[i]; |
| 201604 | | - j = i+1; |
| 201605 | | - for(;;){ |
| 202309 | + for(j=i+1; 1; j++){ |
| 202310 | + if( jsonIsOk[(unsigned char)z[j]] ) continue; |
| 201606 | 202311 | c = z[j]; |
| 201607 | | - if( (c & ~0x1f)==0 ){ |
| 201608 | | - /* Control characters are not allowed in strings */ |
| 201609 | | - pParse->iErr = j; |
| 201610 | | - return -1; |
| 201611 | | - } |
| 201612 | | - if( c=='\\' ){ |
| 202312 | + if( c==cDelim ){ |
| 202313 | + break; |
| 202314 | + }else if( c=='\\' ){ |
| 201613 | 202315 | c = z[++j]; |
| 201614 | 202316 | if( c=='"' || c=='\\' || c=='/' || c=='b' || c=='f' |
| 201615 | 202317 | || c=='n' || c=='r' || c=='t' |
| 201616 | 202318 | || (c=='u' && jsonIs4Hex(&z[j+1])) ){ |
| 201617 | 202319 | jnFlags |= JNODE_ESCAPE; |
| | @@ -201627,14 +202329,15 @@ |
| 201627 | 202329 | pParse->hasNonstd = 1; |
| 201628 | 202330 | }else{ |
| 201629 | 202331 | pParse->iErr = j; |
| 201630 | 202332 | return -1; |
| 201631 | 202333 | } |
| 201632 | | - }else if( c==cDelim ){ |
| 201633 | | - break; |
| 202334 | + }else if( c<=0x1f ){ |
| 202335 | + /* Control characters are not allowed in strings */ |
| 202336 | + pParse->iErr = j; |
| 202337 | + return -1; |
| 201634 | 202338 | } |
| 201635 | | - j++; |
| 201636 | 202339 | } |
| 201637 | 202340 | jsonParseAddNode(pParse, JSON_STRING | (jnFlags<<8), j+1-i, &z[i]); |
| 201638 | 202341 | return j+1; |
| 201639 | 202342 | } |
| 201640 | 202343 | case 't': { |
| | @@ -201866,24 +202569,22 @@ |
| 201866 | 202569 | } /* End switch(z[i]) */ |
| 201867 | 202570 | } |
| 201868 | 202571 | |
| 201869 | 202572 | /* |
| 201870 | 202573 | ** Parse a complete JSON string. Return 0 on success or non-zero if there |
| 201871 | | -** are any errors. If an error occurs, free all memory associated with |
| 201872 | | -** pParse. |
| 202574 | +** are any errors. If an error occurs, free all memory held by pParse, |
| 202575 | +** but not pParse itself. |
| 201873 | 202576 | ** |
| 201874 | | -** pParse is uninitialized when this routine is called. |
| 202577 | +** pParse must be initialized to an empty parse object prior to calling |
| 202578 | +** this routine. |
| 201875 | 202579 | */ |
| 201876 | 202580 | static int jsonParse( |
| 201877 | 202581 | JsonParse *pParse, /* Initialize and fill this JsonParse object */ |
| 201878 | | - sqlite3_context *pCtx, /* Report errors here */ |
| 201879 | | - const char *zJson /* Input JSON text to be parsed */ |
| 202582 | + sqlite3_context *pCtx /* Report errors here */ |
| 201880 | 202583 | ){ |
| 201881 | 202584 | int i; |
| 201882 | | - memset(pParse, 0, sizeof(*pParse)); |
| 201883 | | - if( zJson==0 ) return 1; |
| 201884 | | - pParse->zJson = zJson; |
| 202585 | + const char *zJson = pParse->zJson; |
| 201885 | 202586 | i = jsonParseValue(pParse, 0); |
| 201886 | 202587 | if( pParse->oom ) i = -1; |
| 201887 | 202588 | if( i>0 ){ |
| 201888 | 202589 | assert( pParse->iDepth==0 ); |
| 201889 | 202590 | while( fast_isspace(zJson[i]) ) i++; |
| | @@ -201907,10 +202608,11 @@ |
| 201907 | 202608 | jsonParseReset(pParse); |
| 201908 | 202609 | return 1; |
| 201909 | 202610 | } |
| 201910 | 202611 | return 0; |
| 201911 | 202612 | } |
| 202613 | + |
| 201912 | 202614 | |
| 201913 | 202615 | /* Mark node i of pParse as being a child of iParent. Call recursively |
| 201914 | 202616 | ** to fill in all the descendants of node i. |
| 201915 | 202617 | */ |
| 201916 | 202618 | static void jsonParseFillInParentage(JsonParse *pParse, u32 i, u32 iParent){ |
| | @@ -201957,51 +202659,77 @@ |
| 201957 | 202659 | */ |
| 201958 | 202660 | #define JSON_CACHE_ID (-429938) /* First cache entry */ |
| 201959 | 202661 | #define JSON_CACHE_SZ 4 /* Max number of cache entries */ |
| 201960 | 202662 | |
| 201961 | 202663 | /* |
| 201962 | | -** Obtain a complete parse of the JSON found in the first argument |
| 201963 | | -** of the argv array. Use the sqlite3_get_auxdata() cache for this |
| 201964 | | -** parse if it is available. If the cache is not available or if it |
| 201965 | | -** is no longer valid, parse the JSON again and return the new parse, |
| 201966 | | -** and also register the new parse so that it will be available for |
| 202664 | +** Obtain a complete parse of the JSON found in the pJson argument |
| 202665 | +** |
| 202666 | +** Use the sqlite3_get_auxdata() cache to find a preexisting parse |
| 202667 | +** if it is available. If the cache is not available or if it |
| 202668 | +** is no longer valid, parse the JSON again and return the new parse. |
| 202669 | +** Also register the new parse so that it will be available for |
| 201967 | 202670 | ** future sqlite3_get_auxdata() calls. |
| 201968 | 202671 | ** |
| 201969 | 202672 | ** If an error occurs and pErrCtx!=0 then report the error on pErrCtx |
| 201970 | 202673 | ** and return NULL. |
| 201971 | 202674 | ** |
| 201972 | | -** If an error occurs and pErrCtx==0 then return the Parse object with |
| 201973 | | -** JsonParse.nErr non-zero. If the caller invokes this routine with |
| 201974 | | -** pErrCtx==0 and it gets back a JsonParse with nErr!=0, then the caller |
| 201975 | | -** is responsible for invoking jsonParseFree() on the returned value. |
| 201976 | | -** But the caller may invoke jsonParseFree() *only* if pParse->nErr!=0. |
| 202675 | +** The returned pointer (if it is not NULL) is owned by the cache in |
| 202676 | +** most cases, not the caller. The caller does NOT need to invoke |
| 202677 | +** jsonParseFree(), in most cases. |
| 202678 | +** |
| 202679 | +** Except, if an error occurs and pErrCtx==0 then return the JsonParse |
| 202680 | +** object with JsonParse.nErr non-zero and the caller will own the JsonParse |
| 202681 | +** object. In that case, it will be the responsibility of the caller to |
| 202682 | +** invoke jsonParseFree(). To summarize: |
| 202683 | +** |
| 202684 | +** pErrCtx!=0 || p->nErr==0 ==> Return value p is owned by the |
| 202685 | +** cache. Call does not need to |
| 202686 | +** free it. |
| 202687 | +** |
| 202688 | +** pErrCtx==0 && p->nErr!=0 ==> Return value is owned by the caller |
| 202689 | +** and so the caller must free it. |
| 201977 | 202690 | */ |
| 201978 | 202691 | static JsonParse *jsonParseCached( |
| 201979 | | - sqlite3_context *pCtx, |
| 201980 | | - sqlite3_value **argv, |
| 201981 | | - sqlite3_context *pErrCtx |
| 202692 | + sqlite3_context *pCtx, /* Context to use for cache search */ |
| 202693 | + sqlite3_value *pJson, /* Function param containing JSON text */ |
| 202694 | + sqlite3_context *pErrCtx, /* Write parse errors here if not NULL */ |
| 202695 | + int bUnedited /* No prior edits allowed */ |
| 201982 | 202696 | ){ |
| 201983 | | - const char *zJson = (const char*)sqlite3_value_text(argv[0]); |
| 201984 | | - int nJson = sqlite3_value_bytes(argv[0]); |
| 202697 | + char *zJson = (char*)sqlite3_value_text(pJson); |
| 202698 | + int nJson = sqlite3_value_bytes(pJson); |
| 201985 | 202699 | JsonParse *p; |
| 201986 | 202700 | JsonParse *pMatch = 0; |
| 201987 | 202701 | int iKey; |
| 201988 | 202702 | int iMinKey = 0; |
| 201989 | 202703 | u32 iMinHold = 0xffffffff; |
| 201990 | 202704 | u32 iMaxHold = 0; |
| 202705 | + int bJsonRCStr; |
| 202706 | + |
| 201991 | 202707 | if( zJson==0 ) return 0; |
| 201992 | 202708 | for(iKey=0; iKey<JSON_CACHE_SZ; iKey++){ |
| 201993 | 202709 | p = (JsonParse*)sqlite3_get_auxdata(pCtx, JSON_CACHE_ID+iKey); |
| 201994 | 202710 | if( p==0 ){ |
| 201995 | 202711 | iMinKey = iKey; |
| 201996 | 202712 | break; |
| 201997 | 202713 | } |
| 201998 | 202714 | if( pMatch==0 |
| 201999 | 202715 | && p->nJson==nJson |
| 202000 | | - && memcmp(p->zJson,zJson,nJson)==0 |
| 202716 | + && (p->hasMod==0 || bUnedited==0) |
| 202717 | + && (p->zJson==zJson || memcmp(p->zJson,zJson,nJson)==0) |
| 202718 | + ){ |
| 202719 | + p->nErr = 0; |
| 202720 | + p->useMod = 0; |
| 202721 | + pMatch = p; |
| 202722 | + }else |
| 202723 | + if( pMatch==0 |
| 202724 | + && p->zAlt!=0 |
| 202725 | + && bUnedited==0 |
| 202726 | + && p->nAlt==nJson |
| 202727 | + && memcmp(p->zAlt, zJson, nJson)==0 |
| 202001 | 202728 | ){ |
| 202002 | 202729 | p->nErr = 0; |
| 202730 | + p->useMod = 1; |
| 202003 | 202731 | pMatch = p; |
| 202004 | 202732 | }else if( p->iHold<iMinHold ){ |
| 202005 | 202733 | iMinHold = p->iHold; |
| 202006 | 202734 | iMinKey = iKey; |
| 202007 | 202735 | } |
| | @@ -202008,32 +202736,48 @@ |
| 202008 | 202736 | if( p->iHold>iMaxHold ){ |
| 202009 | 202737 | iMaxHold = p->iHold; |
| 202010 | 202738 | } |
| 202011 | 202739 | } |
| 202012 | 202740 | if( pMatch ){ |
| 202741 | + /* The input JSON text was found in the cache. Use the preexisting |
| 202742 | + ** parse of this JSON */ |
| 202013 | 202743 | pMatch->nErr = 0; |
| 202014 | 202744 | pMatch->iHold = iMaxHold+1; |
| 202745 | + assert( pMatch->nJPRef>0 ); /* pMatch is owned by the cache */ |
| 202015 | 202746 | return pMatch; |
| 202016 | 202747 | } |
| 202017 | | - p = sqlite3_malloc64( sizeof(*p) + nJson + 1 ); |
| 202748 | + |
| 202749 | + /* The input JSON was not found anywhere in the cache. We will need |
| 202750 | + ** to parse it ourselves and generate a new JsonParse object. |
| 202751 | + */ |
| 202752 | + bJsonRCStr = sqlite3ValueIsOfClass(pJson,(void(*)(void*))sqlite3RCStrUnref); |
| 202753 | + p = sqlite3_malloc64( sizeof(*p) + (bJsonRCStr ? 0 : nJson+1) ); |
| 202018 | 202754 | if( p==0 ){ |
| 202019 | 202755 | sqlite3_result_error_nomem(pCtx); |
| 202020 | 202756 | return 0; |
| 202021 | 202757 | } |
| 202022 | 202758 | memset(p, 0, sizeof(*p)); |
| 202023 | | - p->zJson = (char*)&p[1]; |
| 202024 | | - memcpy((char*)p->zJson, zJson, nJson+1); |
| 202025 | | - if( jsonParse(p, pErrCtx, p->zJson) ){ |
| 202759 | + if( bJsonRCStr ){ |
| 202760 | + p->zJson = sqlite3RCStrRef(zJson); |
| 202761 | + p->bJsonIsRCStr = 1; |
| 202762 | + }else{ |
| 202763 | + p->zJson = (char*)&p[1]; |
| 202764 | + memcpy(p->zJson, zJson, nJson+1); |
| 202765 | + } |
| 202766 | + p->nJPRef = 1; |
| 202767 | + if( jsonParse(p, pErrCtx) ){ |
| 202026 | 202768 | if( pErrCtx==0 ){ |
| 202027 | 202769 | p->nErr = 1; |
| 202770 | + assert( p->nJPRef==1 ); /* Caller will own the new JsonParse object p */ |
| 202028 | 202771 | return p; |
| 202029 | 202772 | } |
| 202030 | | - sqlite3_free(p); |
| 202773 | + jsonParseFree(p); |
| 202031 | 202774 | return 0; |
| 202032 | 202775 | } |
| 202033 | 202776 | p->nJson = nJson; |
| 202034 | 202777 | p->iHold = iMaxHold+1; |
| 202778 | + /* Transfer ownership of the new JsonParse to the cache */ |
| 202035 | 202779 | sqlite3_set_auxdata(pCtx, JSON_CACHE_ID+iMinKey, p, |
| 202036 | 202780 | (void(*)(void*))jsonParseFree); |
| 202037 | 202781 | return (JsonParse*)sqlite3_get_auxdata(pCtx, JSON_CACHE_ID+iMinKey); |
| 202038 | 202782 | } |
| 202039 | 202783 | |
| | @@ -202080,13 +202824,30 @@ |
| 202080 | 202824 | int *pApnd, /* Append nodes to complete path if not NULL */ |
| 202081 | 202825 | const char **pzErr /* Make *pzErr point to any syntax error in zPath */ |
| 202082 | 202826 | ){ |
| 202083 | 202827 | u32 i, j, nKey; |
| 202084 | 202828 | const char *zKey; |
| 202085 | | - JsonNode *pRoot = &pParse->aNode[iRoot]; |
| 202829 | + JsonNode *pRoot; |
| 202830 | + if( pParse->oom ) return 0; |
| 202831 | + pRoot = &pParse->aNode[iRoot]; |
| 202832 | + while( (pRoot->jnFlags & JNODE_REPLACE)!=0 && pParse->useMod ){ |
| 202833 | + u32 idx = (u32)(pRoot - pParse->aNode); |
| 202834 | + i = pParse->iSubst; |
| 202835 | + while( 1 /*exit-by-break*/ ){ |
| 202836 | + assert( i<pParse->nNode ); |
| 202837 | + assert( pParse->aNode[i].eType==JSON_SUBST ); |
| 202838 | + assert( pParse->aNode[i].eU==4 ); |
| 202839 | + assert( pParse->aNode[i].u.iPrev<i ); |
| 202840 | + if( pParse->aNode[i].n==idx ){ |
| 202841 | + pRoot = &pParse->aNode[i+1]; |
| 202842 | + iRoot = i+1; |
| 202843 | + break; |
| 202844 | + } |
| 202845 | + i = pParse->aNode[i].u.iPrev; |
| 202846 | + } |
| 202847 | + } |
| 202086 | 202848 | if( zPath[0]==0 ) return pRoot; |
| 202087 | | - if( pRoot->jnFlags & JNODE_REPLACE ) return 0; |
| 202088 | 202849 | if( zPath[0]=='.' ){ |
| 202089 | 202850 | if( pRoot->eType!=JSON_OBJECT ) return 0; |
| 202090 | 202851 | zPath++; |
| 202091 | 202852 | if( zPath[0]=='"' ){ |
| 202092 | 202853 | zKey = zPath + 1; |
| | @@ -202116,27 +202877,29 @@ |
| 202116 | 202877 | } |
| 202117 | 202878 | j++; |
| 202118 | 202879 | j += jsonNodeSize(&pRoot[j]); |
| 202119 | 202880 | } |
| 202120 | 202881 | if( (pRoot->jnFlags & JNODE_APPEND)==0 ) break; |
| 202882 | + if( pParse->useMod==0 ) break; |
| 202121 | 202883 | assert( pRoot->eU==2 ); |
| 202122 | | - iRoot += pRoot->u.iAppend; |
| 202884 | + iRoot = pRoot->u.iAppend; |
| 202123 | 202885 | pRoot = &pParse->aNode[iRoot]; |
| 202124 | 202886 | j = 1; |
| 202125 | 202887 | } |
| 202126 | 202888 | if( pApnd ){ |
| 202127 | 202889 | u32 iStart, iLabel; |
| 202128 | 202890 | JsonNode *pNode; |
| 202891 | + assert( pParse->useMod ); |
| 202129 | 202892 | iStart = jsonParseAddNode(pParse, JSON_OBJECT, 2, 0); |
| 202130 | 202893 | iLabel = jsonParseAddNode(pParse, JSON_STRING, nKey, zKey); |
| 202131 | 202894 | zPath += i; |
| 202132 | 202895 | pNode = jsonLookupAppend(pParse, zPath, pApnd, pzErr); |
| 202133 | 202896 | if( pParse->oom ) return 0; |
| 202134 | 202897 | if( pNode ){ |
| 202135 | 202898 | pRoot = &pParse->aNode[iRoot]; |
| 202136 | 202899 | assert( pRoot->eU==0 ); |
| 202137 | | - pRoot->u.iAppend = iStart - iRoot; |
| 202900 | + pRoot->u.iAppend = iStart; |
| 202138 | 202901 | pRoot->jnFlags |= JNODE_APPEND; |
| 202139 | 202902 | VVA( pRoot->eU = 2 ); |
| 202140 | 202903 | pParse->aNode[iLabel].jnFlags |= JNODE_RAW; |
| 202141 | 202904 | } |
| 202142 | 202905 | return pNode; |
| | @@ -202153,16 +202916,17 @@ |
| 202153 | 202916 | JsonNode *pBase = pRoot; |
| 202154 | 202917 | int iBase = iRoot; |
| 202155 | 202918 | if( pRoot->eType!=JSON_ARRAY ) return 0; |
| 202156 | 202919 | for(;;){ |
| 202157 | 202920 | while( j<=pBase->n ){ |
| 202158 | | - if( (pBase[j].jnFlags & JNODE_REMOVE)==0 ) i++; |
| 202921 | + if( (pBase[j].jnFlags & JNODE_REMOVE)==0 || pParse->useMod==0 ) i++; |
| 202159 | 202922 | j += jsonNodeSize(&pBase[j]); |
| 202160 | 202923 | } |
| 202161 | 202924 | if( (pBase->jnFlags & JNODE_APPEND)==0 ) break; |
| 202925 | + if( pParse->useMod==0 ) break; |
| 202162 | 202926 | assert( pBase->eU==2 ); |
| 202163 | | - iBase += pBase->u.iAppend; |
| 202927 | + iBase = pBase->u.iAppend; |
| 202164 | 202928 | pBase = &pParse->aNode[iBase]; |
| 202165 | 202929 | j = 1; |
| 202166 | 202930 | } |
| 202167 | 202931 | j = 2; |
| 202168 | 202932 | if( zPath[2]=='-' && sqlite3Isdigit(zPath[3]) ){ |
| | @@ -202186,33 +202950,37 @@ |
| 202186 | 202950 | } |
| 202187 | 202951 | if( pRoot->eType!=JSON_ARRAY ) return 0; |
| 202188 | 202952 | zPath += j + 1; |
| 202189 | 202953 | j = 1; |
| 202190 | 202954 | for(;;){ |
| 202191 | | - while( j<=pRoot->n && (i>0 || (pRoot[j].jnFlags & JNODE_REMOVE)!=0) ){ |
| 202192 | | - if( (pRoot[j].jnFlags & JNODE_REMOVE)==0 ) i--; |
| 202955 | + while( j<=pRoot->n |
| 202956 | + && (i>0 || ((pRoot[j].jnFlags & JNODE_REMOVE)!=0 && pParse->useMod)) |
| 202957 | + ){ |
| 202958 | + if( (pRoot[j].jnFlags & JNODE_REMOVE)==0 || pParse->useMod==0 ) i--; |
| 202193 | 202959 | j += jsonNodeSize(&pRoot[j]); |
| 202194 | 202960 | } |
| 202195 | 202961 | if( (pRoot->jnFlags & JNODE_APPEND)==0 ) break; |
| 202962 | + if( pParse->useMod==0 ) break; |
| 202196 | 202963 | assert( pRoot->eU==2 ); |
| 202197 | | - iRoot += pRoot->u.iAppend; |
| 202964 | + iRoot = pRoot->u.iAppend; |
| 202198 | 202965 | pRoot = &pParse->aNode[iRoot]; |
| 202199 | 202966 | j = 1; |
| 202200 | 202967 | } |
| 202201 | 202968 | if( j<=pRoot->n ){ |
| 202202 | 202969 | return jsonLookupStep(pParse, iRoot+j, zPath, pApnd, pzErr); |
| 202203 | 202970 | } |
| 202204 | 202971 | if( i==0 && pApnd ){ |
| 202205 | 202972 | u32 iStart; |
| 202206 | 202973 | JsonNode *pNode; |
| 202974 | + assert( pParse->useMod ); |
| 202207 | 202975 | iStart = jsonParseAddNode(pParse, JSON_ARRAY, 1, 0); |
| 202208 | 202976 | pNode = jsonLookupAppend(pParse, zPath, pApnd, pzErr); |
| 202209 | 202977 | if( pParse->oom ) return 0; |
| 202210 | 202978 | if( pNode ){ |
| 202211 | 202979 | pRoot = &pParse->aNode[iRoot]; |
| 202212 | 202980 | assert( pRoot->eU==0 ); |
| 202213 | | - pRoot->u.iAppend = iStart - iRoot; |
| 202981 | + pRoot->u.iAppend = iStart; |
| 202214 | 202982 | pRoot->jnFlags |= JNODE_APPEND; |
| 202215 | 202983 | VVA( pRoot->eU = 2 ); |
| 202216 | 202984 | } |
| 202217 | 202985 | return pNode; |
| 202218 | 202986 | } |
| | @@ -202334,52 +203102,95 @@ |
| 202334 | 203102 | |
| 202335 | 203103 | |
| 202336 | 203104 | /**************************************************************************** |
| 202337 | 203105 | ** SQL functions used for testing and debugging |
| 202338 | 203106 | ****************************************************************************/ |
| 203107 | + |
| 203108 | +#if SQLITE_DEBUG |
| 203109 | +/* |
| 203110 | +** Print N node entries. |
| 203111 | +*/ |
| 203112 | +static void jsonDebugPrintNodeEntries( |
| 203113 | + JsonNode *aNode, /* First node entry to print */ |
| 203114 | + int N /* Number of node entries to print */ |
| 203115 | +){ |
| 203116 | + int i; |
| 203117 | + for(i=0; i<N; i++){ |
| 203118 | + const char *zType; |
| 203119 | + if( aNode[i].jnFlags & JNODE_LABEL ){ |
| 203120 | + zType = "label"; |
| 203121 | + }else{ |
| 203122 | + zType = jsonType[aNode[i].eType]; |
| 203123 | + } |
| 203124 | + printf("node %4u: %-7s n=%-5d", i, zType, aNode[i].n); |
| 203125 | + if( (aNode[i].jnFlags & ~JNODE_LABEL)!=0 ){ |
| 203126 | + u8 f = aNode[i].jnFlags; |
| 203127 | + if( f & JNODE_RAW ) printf(" RAW"); |
| 203128 | + if( f & JNODE_ESCAPE ) printf(" ESCAPE"); |
| 203129 | + if( f & JNODE_REMOVE ) printf(" REMOVE"); |
| 203130 | + if( f & JNODE_REPLACE ) printf(" REPLACE"); |
| 203131 | + if( f & JNODE_APPEND ) printf(" APPEND"); |
| 203132 | + if( f & JNODE_JSON5 ) printf(" JSON5"); |
| 203133 | + } |
| 203134 | + switch( aNode[i].eU ){ |
| 203135 | + case 1: printf(" zJContent=[%.*s]\n", |
| 203136 | + aNode[i].n, aNode[i].u.zJContent); break; |
| 203137 | + case 2: printf(" iAppend=%u\n", aNode[i].u.iAppend); break; |
| 203138 | + case 3: printf(" iKey=%u\n", aNode[i].u.iKey); break; |
| 203139 | + case 4: printf(" iPrev=%u\n", aNode[i].u.iPrev); break; |
| 203140 | + default: printf("\n"); |
| 203141 | + } |
| 203142 | + } |
| 203143 | +} |
| 203144 | +#endif /* SQLITE_DEBUG */ |
| 203145 | + |
| 203146 | + |
| 203147 | +#if 0 /* 1 for debugging. 0 normally. Requires -DSQLITE_DEBUG too */ |
| 203148 | +static void jsonDebugPrintParse(JsonParse *p){ |
| 203149 | + jsonDebugPrintNodeEntries(p->aNode, p->nNode); |
| 203150 | +} |
| 203151 | +static void jsonDebugPrintNode(JsonNode *pNode){ |
| 203152 | + jsonDebugPrintNodeEntries(pNode, jsonNodeSize(pNode)); |
| 203153 | +} |
| 203154 | +#else |
| 203155 | + /* The usual case */ |
| 203156 | +# define jsonDebugPrintNode(X) |
| 203157 | +# define jsonDebugPrintParse(X) |
| 203158 | +#endif |
| 202339 | 203159 | |
| 202340 | 203160 | #ifdef SQLITE_DEBUG |
| 202341 | 203161 | /* |
| 202342 | | -** The json_parse(JSON) function returns a string which describes |
| 202343 | | -** a parse of the JSON provided. Or it returns NULL if JSON is not |
| 202344 | | -** well-formed. |
| 203162 | +** SQL function: json_parse(JSON) |
| 203163 | +** |
| 203164 | +** Parse JSON using jsonParseCached(). Then print a dump of that |
| 203165 | +** parse on standard output. Return the mimified JSON result, just |
| 203166 | +** like the json() function. |
| 202345 | 203167 | */ |
| 202346 | 203168 | static void jsonParseFunc( |
| 202347 | 203169 | sqlite3_context *ctx, |
| 202348 | 203170 | int argc, |
| 202349 | 203171 | sqlite3_value **argv |
| 202350 | 203172 | ){ |
| 202351 | | - JsonString s; /* Output string - not real JSON */ |
| 202352 | | - JsonParse x; /* The parse */ |
| 202353 | | - u32 i; |
| 203173 | + JsonParse *p; /* The parse */ |
| 202354 | 203174 | |
| 202355 | 203175 | assert( argc==1 ); |
| 202356 | | - if( jsonParse(&x, ctx, (const char*)sqlite3_value_text(argv[0])) ) return; |
| 202357 | | - jsonParseFindParents(&x); |
| 202358 | | - jsonInit(&s, ctx); |
| 202359 | | - for(i=0; i<x.nNode; i++){ |
| 202360 | | - const char *zType; |
| 202361 | | - if( x.aNode[i].jnFlags & JNODE_LABEL ){ |
| 202362 | | - assert( x.aNode[i].eType==JSON_STRING ); |
| 202363 | | - zType = "label"; |
| 202364 | | - }else{ |
| 202365 | | - zType = jsonType[x.aNode[i].eType]; |
| 202366 | | - } |
| 202367 | | - jsonPrintf(100, &s,"node %3u: %7s n=%-4d up=%-4d", |
| 202368 | | - i, zType, x.aNode[i].n, x.aUp[i]); |
| 202369 | | - assert( x.aNode[i].eU==0 || x.aNode[i].eU==1 ); |
| 202370 | | - if( x.aNode[i].u.zJContent!=0 ){ |
| 202371 | | - assert( x.aNode[i].eU==1 ); |
| 202372 | | - jsonAppendRaw(&s, " ", 1); |
| 202373 | | - jsonAppendRaw(&s, x.aNode[i].u.zJContent, x.aNode[i].n); |
| 202374 | | - }else{ |
| 202375 | | - assert( x.aNode[i].eU==0 ); |
| 202376 | | - } |
| 202377 | | - jsonAppendRaw(&s, "\n", 1); |
| 202378 | | - } |
| 202379 | | - jsonParseReset(&x); |
| 202380 | | - jsonResult(&s); |
| 203176 | + p = jsonParseCached(ctx, argv[0], ctx, 0); |
| 203177 | + if( p==0 ) return; |
| 203178 | + printf("nNode = %u\n", p->nNode); |
| 203179 | + printf("nAlloc = %u\n", p->nAlloc); |
| 203180 | + printf("nJson = %d\n", p->nJson); |
| 203181 | + printf("nAlt = %d\n", p->nAlt); |
| 203182 | + printf("nErr = %u\n", p->nErr); |
| 203183 | + printf("oom = %u\n", p->oom); |
| 203184 | + printf("hasNonstd = %u\n", p->hasNonstd); |
| 203185 | + printf("useMod = %u\n", p->useMod); |
| 203186 | + printf("hasMod = %u\n", p->hasMod); |
| 203187 | + printf("nJPRef = %u\n", p->nJPRef); |
| 203188 | + printf("iSubst = %u\n", p->iSubst); |
| 203189 | + printf("iHold = %u\n", p->iHold); |
| 203190 | + jsonDebugPrintNodeEntries(p->aNode, p->nNode); |
| 203191 | + jsonReturnJson(p, p->aNode, ctx, 1); |
| 202381 | 203192 | } |
| 202382 | 203193 | |
| 202383 | 203194 | /* |
| 202384 | 203195 | ** The json_test1(JSON) function return true (1) if the input is JSON |
| 202385 | 203196 | ** text generated by another json function. It returns (0) if the input |
| | @@ -202459,11 +203270,11 @@ |
| 202459 | 203270 | JsonParse *p; /* The parse */ |
| 202460 | 203271 | sqlite3_int64 n = 0; |
| 202461 | 203272 | u32 i; |
| 202462 | 203273 | JsonNode *pNode; |
| 202463 | 203274 | |
| 202464 | | - p = jsonParseCached(ctx, argv, ctx); |
| 203275 | + p = jsonParseCached(ctx, argv[0], ctx, 0); |
| 202465 | 203276 | if( p==0 ) return; |
| 202466 | 203277 | assert( p->nNode ); |
| 202467 | 203278 | if( argc==2 ){ |
| 202468 | 203279 | const char *zPath = (const char*)sqlite3_value_text(argv[1]); |
| 202469 | 203280 | pNode = jsonLookup(p, zPath, 0, ctx); |
| | @@ -202472,13 +203283,18 @@ |
| 202472 | 203283 | } |
| 202473 | 203284 | if( pNode==0 ){ |
| 202474 | 203285 | return; |
| 202475 | 203286 | } |
| 202476 | 203287 | if( pNode->eType==JSON_ARRAY ){ |
| 202477 | | - assert( (pNode->jnFlags & JNODE_APPEND)==0 ); |
| 202478 | | - for(i=1; i<=pNode->n; n++){ |
| 202479 | | - i += jsonNodeSize(&pNode[i]); |
| 203288 | + while( 1 /*exit-by-break*/ ){ |
| 203289 | + for(i=1; i<=pNode->n; n++){ |
| 203290 | + i += jsonNodeSize(&pNode[i]); |
| 203291 | + } |
| 203292 | + if( (pNode->jnFlags & JNODE_APPEND)==0 ) break; |
| 203293 | + if( p->useMod==0 ) break; |
| 203294 | + assert( pNode->eU==2 ); |
| 203295 | + pNode = &p->aNode[pNode->u.iAppend]; |
| 202480 | 203296 | } |
| 202481 | 203297 | } |
| 202482 | 203298 | sqlite3_result_int64(ctx, n); |
| 202483 | 203299 | } |
| 202484 | 203300 | |
| | @@ -202521,11 +203337,11 @@ |
| 202521 | 203337 | const char *zPath; |
| 202522 | 203338 | int flags = SQLITE_PTR_TO_INT(sqlite3_user_data(ctx)); |
| 202523 | 203339 | JsonString jx; |
| 202524 | 203340 | |
| 202525 | 203341 | if( argc<2 ) return; |
| 202526 | | - p = jsonParseCached(ctx, argv, ctx); |
| 203342 | + p = jsonParseCached(ctx, argv[0], ctx, 0); |
| 202527 | 203343 | if( p==0 ) return; |
| 202528 | 203344 | if( argc==2 ){ |
| 202529 | 203345 | /* With a single PATH argument */ |
| 202530 | 203346 | zPath = (const char*)sqlite3_value_text(argv[1]); |
| 202531 | 203347 | if( zPath==0 ) return; |
| | @@ -202539,15 +203355,15 @@ |
| 202539 | 203355 | ** LABEL ==> $.LABEL // PG compatible |
| 202540 | 203356 | ** [NUMBER] ==> $[NUMBER] // Not PG. Purely for convenience |
| 202541 | 203357 | */ |
| 202542 | 203358 | jsonInit(&jx, ctx); |
| 202543 | 203359 | if( sqlite3Isdigit(zPath[0]) ){ |
| 202544 | | - jsonAppendRaw(&jx, "$[", 2); |
| 203360 | + jsonAppendRawNZ(&jx, "$[", 2); |
| 202545 | 203361 | jsonAppendRaw(&jx, zPath, (int)strlen(zPath)); |
| 202546 | | - jsonAppendRaw(&jx, "]", 2); |
| 203362 | + jsonAppendRawNZ(&jx, "]", 2); |
| 202547 | 203363 | }else{ |
| 202548 | | - jsonAppendRaw(&jx, "$.", 1 + (zPath[0]!='[')); |
| 203364 | + jsonAppendRawNZ(&jx, "$.", 1 + (zPath[0]!='[')); |
| 202549 | 203365 | jsonAppendRaw(&jx, zPath, (int)strlen(zPath)); |
| 202550 | 203366 | jsonAppendChar(&jx, 0); |
| 202551 | 203367 | } |
| 202552 | 203368 | pNode = jx.bErr ? 0 : jsonLookup(p, jx.zBuf, 0, ctx); |
| 202553 | 203369 | jsonReset(&jx); |
| | @@ -202554,19 +203370,19 @@ |
| 202554 | 203370 | }else{ |
| 202555 | 203371 | pNode = jsonLookup(p, zPath, 0, ctx); |
| 202556 | 203372 | } |
| 202557 | 203373 | if( pNode ){ |
| 202558 | 203374 | if( flags & JSON_JSON ){ |
| 202559 | | - jsonReturnJson(pNode, ctx, 0); |
| 203375 | + jsonReturnJson(p, pNode, ctx, 0); |
| 202560 | 203376 | }else{ |
| 202561 | | - jsonReturn(pNode, ctx, 0); |
| 203377 | + jsonReturn(p, pNode, ctx); |
| 202562 | 203378 | sqlite3_result_subtype(ctx, 0); |
| 202563 | 203379 | } |
| 202564 | 203380 | } |
| 202565 | 203381 | }else{ |
| 202566 | 203382 | pNode = jsonLookup(p, zPath, 0, ctx); |
| 202567 | | - if( p->nErr==0 && pNode ) jsonReturn(pNode, ctx, 0); |
| 203383 | + if( p->nErr==0 && pNode ) jsonReturn(p, pNode, ctx); |
| 202568 | 203384 | } |
| 202569 | 203385 | }else{ |
| 202570 | 203386 | /* Two or more PATH arguments results in a JSON array with each |
| 202571 | 203387 | ** element of the array being the value selected by one of the PATHs */ |
| 202572 | 203388 | int i; |
| | @@ -202576,13 +203392,13 @@ |
| 202576 | 203392 | zPath = (const char*)sqlite3_value_text(argv[i]); |
| 202577 | 203393 | pNode = jsonLookup(p, zPath, 0, ctx); |
| 202578 | 203394 | if( p->nErr ) break; |
| 202579 | 203395 | jsonAppendSeparator(&jx); |
| 202580 | 203396 | if( pNode ){ |
| 202581 | | - jsonRenderNode(pNode, &jx, 0); |
| 203397 | + jsonRenderNode(p, pNode, &jx); |
| 202582 | 203398 | }else{ |
| 202583 | | - jsonAppendRaw(&jx, "null", 4); |
| 203399 | + jsonAppendRawNZ(&jx, "null", 4); |
| 202584 | 203400 | } |
| 202585 | 203401 | } |
| 202586 | 203402 | if( i==argc ){ |
| 202587 | 203403 | jsonAppendChar(&jx, ']'); |
| 202588 | 203404 | jsonResult(&jx); |
| | @@ -202623,49 +203439,42 @@ |
| 202623 | 203439 | zKey = pPatch[i].u.zJContent; |
| 202624 | 203440 | for(j=1; j<pTarget->n; j += jsonNodeSize(&pTarget[j+1])+1 ){ |
| 202625 | 203441 | assert( pTarget[j].eType==JSON_STRING ); |
| 202626 | 203442 | assert( pTarget[j].jnFlags & JNODE_LABEL ); |
| 202627 | 203443 | if( jsonSameLabel(&pPatch[i], &pTarget[j]) ){ |
| 202628 | | - if( pTarget[j+1].jnFlags & (JNODE_REMOVE|JNODE_PATCH) ) break; |
| 203444 | + if( pTarget[j+1].jnFlags & (JNODE_REMOVE|JNODE_REPLACE) ) break; |
| 202629 | 203445 | if( pPatch[i+1].eType==JSON_NULL ){ |
| 202630 | 203446 | pTarget[j+1].jnFlags |= JNODE_REMOVE; |
| 202631 | 203447 | }else{ |
| 202632 | 203448 | JsonNode *pNew = jsonMergePatch(pParse, iTarget+j+1, &pPatch[i+1]); |
| 202633 | 203449 | if( pNew==0 ) return 0; |
| 203450 | + if( pNew!=&pParse->aNode[iTarget+j+1] ){ |
| 203451 | + jsonParseAddSubstNode(pParse, iTarget+j+1); |
| 203452 | + jsonParseAddNodeArray(pParse, pNew, jsonNodeSize(pNew)); |
| 203453 | + } |
| 202634 | 203454 | pTarget = &pParse->aNode[iTarget]; |
| 202635 | | - if( pNew!=&pTarget[j+1] ){ |
| 202636 | | - assert( pTarget[j+1].eU==0 |
| 202637 | | - || pTarget[j+1].eU==1 |
| 202638 | | - || pTarget[j+1].eU==2 ); |
| 202639 | | - testcase( pTarget[j+1].eU==1 ); |
| 202640 | | - testcase( pTarget[j+1].eU==2 ); |
| 202641 | | - VVA( pTarget[j+1].eU = 5 ); |
| 202642 | | - pTarget[j+1].u.pPatch = pNew; |
| 202643 | | - pTarget[j+1].jnFlags |= JNODE_PATCH; |
| 202644 | | - } |
| 202645 | 203455 | } |
| 202646 | 203456 | break; |
| 202647 | 203457 | } |
| 202648 | 203458 | } |
| 202649 | 203459 | if( j>=pTarget->n && pPatch[i+1].eType!=JSON_NULL ){ |
| 202650 | | - int iStart, iPatch; |
| 202651 | | - iStart = jsonParseAddNode(pParse, JSON_OBJECT, 2, 0); |
| 203460 | + int iStart; |
| 203461 | + JsonNode *pApnd; |
| 203462 | + u32 nApnd; |
| 203463 | + iStart = jsonParseAddNode(pParse, JSON_OBJECT, 0, 0); |
| 202652 | 203464 | jsonParseAddNode(pParse, JSON_STRING, nKey, zKey); |
| 202653 | | - iPatch = jsonParseAddNode(pParse, JSON_TRUE, 0, 0); |
| 203465 | + pApnd = &pPatch[i+1]; |
| 203466 | + if( pApnd->eType==JSON_OBJECT ) jsonRemoveAllNulls(pApnd); |
| 203467 | + nApnd = jsonNodeSize(pApnd); |
| 203468 | + jsonParseAddNodeArray(pParse, pApnd, jsonNodeSize(pApnd)); |
| 202654 | 203469 | if( pParse->oom ) return 0; |
| 202655 | | - jsonRemoveAllNulls(pPatch); |
| 202656 | | - pTarget = &pParse->aNode[iTarget]; |
| 202657 | | - assert( pParse->aNode[iRoot].eU==0 || pParse->aNode[iRoot].eU==2 ); |
| 202658 | | - testcase( pParse->aNode[iRoot].eU==2 ); |
| 203470 | + pParse->aNode[iStart].n = 1+nApnd; |
| 202659 | 203471 | pParse->aNode[iRoot].jnFlags |= JNODE_APPEND; |
| 203472 | + pParse->aNode[iRoot].u.iAppend = iStart; |
| 202660 | 203473 | VVA( pParse->aNode[iRoot].eU = 2 ); |
| 202661 | | - pParse->aNode[iRoot].u.iAppend = iStart - iRoot; |
| 202662 | 203474 | iRoot = iStart; |
| 202663 | | - assert( pParse->aNode[iPatch].eU==0 ); |
| 202664 | | - VVA( pParse->aNode[iPatch].eU = 5 ); |
| 202665 | | - pParse->aNode[iPatch].jnFlags |= JNODE_PATCH; |
| 202666 | | - pParse->aNode[iPatch].u.pPatch = &pPatch[i+1]; |
| 203475 | + pTarget = &pParse->aNode[iTarget]; |
| 202667 | 203476 | } |
| 202668 | 203477 | } |
| 202669 | 203478 | return pTarget; |
| 202670 | 203479 | } |
| 202671 | 203480 | |
| | @@ -202677,29 +203486,32 @@ |
| 202677 | 203486 | static void jsonPatchFunc( |
| 202678 | 203487 | sqlite3_context *ctx, |
| 202679 | 203488 | int argc, |
| 202680 | 203489 | sqlite3_value **argv |
| 202681 | 203490 | ){ |
| 202682 | | - JsonParse x; /* The JSON that is being patched */ |
| 202683 | | - JsonParse y; /* The patch */ |
| 203491 | + JsonParse *pX; /* The JSON that is being patched */ |
| 203492 | + JsonParse *pY; /* The patch */ |
| 202684 | 203493 | JsonNode *pResult; /* The result of the merge */ |
| 202685 | 203494 | |
| 202686 | 203495 | UNUSED_PARAMETER(argc); |
| 202687 | | - if( jsonParse(&x, ctx, (const char*)sqlite3_value_text(argv[0])) ) return; |
| 202688 | | - if( jsonParse(&y, ctx, (const char*)sqlite3_value_text(argv[1])) ){ |
| 202689 | | - jsonParseReset(&x); |
| 202690 | | - return; |
| 202691 | | - } |
| 202692 | | - pResult = jsonMergePatch(&x, 0, y.aNode); |
| 202693 | | - assert( pResult!=0 || x.oom ); |
| 202694 | | - if( pResult ){ |
| 202695 | | - jsonReturnJson(pResult, ctx, 0); |
| 203496 | + pX = jsonParseCached(ctx, argv[0], ctx, 1); |
| 203497 | + if( pX==0 ) return; |
| 203498 | + assert( pX->hasMod==0 ); |
| 203499 | + pX->hasMod = 1; |
| 203500 | + pY = jsonParseCached(ctx, argv[1], ctx, 1); |
| 203501 | + if( pY==0 ) return; |
| 203502 | + pX->useMod = 1; |
| 203503 | + pY->useMod = 1; |
| 203504 | + pResult = jsonMergePatch(pX, 0, pY->aNode); |
| 203505 | + assert( pResult!=0 || pX->oom ); |
| 203506 | + if( pResult && pX->oom==0 ){ |
| 203507 | + jsonDebugPrintParse(pX); |
| 203508 | + jsonDebugPrintNode(pResult); |
| 203509 | + jsonReturnJson(pX, pResult, ctx, 0); |
| 202696 | 203510 | }else{ |
| 202697 | 203511 | sqlite3_result_error_nomem(ctx); |
| 202698 | 203512 | } |
| 202699 | | - jsonParseReset(&x); |
| 202700 | | - jsonParseReset(&y); |
| 202701 | 203513 | } |
| 202702 | 203514 | |
| 202703 | 203515 | |
| 202704 | 203516 | /* |
| 202705 | 203517 | ** Implementation of the json_object(NAME,VALUE,...) function. Return a JSON |
| | @@ -202751,30 +203563,122 @@ |
| 202751 | 203563 | static void jsonRemoveFunc( |
| 202752 | 203564 | sqlite3_context *ctx, |
| 202753 | 203565 | int argc, |
| 202754 | 203566 | sqlite3_value **argv |
| 202755 | 203567 | ){ |
| 202756 | | - JsonParse x; /* The parse */ |
| 203568 | + JsonParse *pParse; /* The parse */ |
| 202757 | 203569 | JsonNode *pNode; |
| 202758 | 203570 | const char *zPath; |
| 202759 | 203571 | u32 i; |
| 202760 | 203572 | |
| 202761 | 203573 | if( argc<1 ) return; |
| 202762 | | - if( jsonParse(&x, ctx, (const char*)sqlite3_value_text(argv[0])) ) return; |
| 202763 | | - assert( x.nNode ); |
| 203574 | + pParse = jsonParseCached(ctx, argv[0], ctx, argc>1); |
| 203575 | + if( pParse==0 ) return; |
| 202764 | 203576 | for(i=1; i<(u32)argc; i++){ |
| 202765 | 203577 | zPath = (const char*)sqlite3_value_text(argv[i]); |
| 202766 | 203578 | if( zPath==0 ) goto remove_done; |
| 202767 | | - pNode = jsonLookup(&x, zPath, 0, ctx); |
| 202768 | | - if( x.nErr ) goto remove_done; |
| 202769 | | - if( pNode ) pNode->jnFlags |= JNODE_REMOVE; |
| 203579 | + pNode = jsonLookup(pParse, zPath, 0, ctx); |
| 203580 | + if( pParse->nErr ) goto remove_done; |
| 203581 | + if( pNode ){ |
| 203582 | + pNode->jnFlags |= JNODE_REMOVE; |
| 203583 | + pParse->hasMod = 1; |
| 203584 | + pParse->useMod = 1; |
| 203585 | + } |
| 202770 | 203586 | } |
| 202771 | | - if( (x.aNode[0].jnFlags & JNODE_REMOVE)==0 ){ |
| 202772 | | - jsonReturnJson(x.aNode, ctx, 0); |
| 203587 | + if( (pParse->aNode[0].jnFlags & JNODE_REMOVE)==0 ){ |
| 203588 | + jsonReturnJson(pParse, pParse->aNode, ctx, 1); |
| 202773 | 203589 | } |
| 202774 | 203590 | remove_done: |
| 202775 | | - jsonParseReset(&x); |
| 203591 | + jsonDebugPrintParse(p); |
| 203592 | +} |
| 203593 | + |
| 203594 | +/* |
| 203595 | +** Substitute the value at iNode with the pValue parameter. |
| 203596 | +*/ |
| 203597 | +static void jsonReplaceNode( |
| 203598 | + sqlite3_context *pCtx, |
| 203599 | + JsonParse *p, |
| 203600 | + int iNode, |
| 203601 | + sqlite3_value *pValue |
| 203602 | +){ |
| 203603 | + int idx = jsonParseAddSubstNode(p, iNode); |
| 203604 | + if( idx<=0 ){ |
| 203605 | + assert( p->oom ); |
| 203606 | + return; |
| 203607 | + } |
| 203608 | + switch( sqlite3_value_type(pValue) ){ |
| 203609 | + case SQLITE_NULL: { |
| 203610 | + jsonParseAddNode(p, JSON_NULL, 0, 0); |
| 203611 | + break; |
| 203612 | + } |
| 203613 | + case SQLITE_FLOAT: { |
| 203614 | + char *z = sqlite3_mprintf("%!0.15g", sqlite3_value_double(pValue)); |
| 203615 | + int n; |
| 203616 | + if( z==0 ){ |
| 203617 | + p->oom = 1; |
| 203618 | + break; |
| 203619 | + } |
| 203620 | + n = sqlite3Strlen30(z); |
| 203621 | + jsonParseAddNode(p, JSON_REAL, n, z); |
| 203622 | + jsonParseAddCleanup(p, sqlite3_free, z); |
| 203623 | + break; |
| 203624 | + } |
| 203625 | + case SQLITE_INTEGER: { |
| 203626 | + char *z = sqlite3_mprintf("%lld", sqlite3_value_int64(pValue)); |
| 203627 | + int n; |
| 203628 | + if( z==0 ){ |
| 203629 | + p->oom = 1; |
| 203630 | + break; |
| 203631 | + } |
| 203632 | + n = sqlite3Strlen30(z); |
| 203633 | + jsonParseAddNode(p, JSON_INT, n, z); |
| 203634 | + jsonParseAddCleanup(p, sqlite3_free, z); |
| 203635 | + |
| 203636 | + break; |
| 203637 | + } |
| 203638 | + case SQLITE_TEXT: { |
| 203639 | + const char *z = (const char*)sqlite3_value_text(pValue); |
| 203640 | + u32 n = (u32)sqlite3_value_bytes(pValue); |
| 203641 | + if( z==0 ){ |
| 203642 | + p->oom = 1; |
| 203643 | + break; |
| 203644 | + } |
| 203645 | + if( sqlite3_value_subtype(pValue)!=JSON_SUBTYPE ){ |
| 203646 | + char *zCopy = sqlite3DbStrDup(0, z); |
| 203647 | + int k; |
| 203648 | + if( zCopy ){ |
| 203649 | + jsonParseAddCleanup(p, sqlite3_free, zCopy); |
| 203650 | + }else{ |
| 203651 | + p->oom = 1; |
| 203652 | + sqlite3_result_error_nomem(pCtx); |
| 203653 | + } |
| 203654 | + k = jsonParseAddNode(p, JSON_STRING, n, zCopy); |
| 203655 | + assert( k>0 || p->oom ); |
| 203656 | + if( p->oom==0 ) p->aNode[k].jnFlags |= JNODE_RAW; |
| 203657 | + }else{ |
| 203658 | + JsonParse *pPatch = jsonParseCached(pCtx, pValue, pCtx, 1); |
| 203659 | + if( pPatch==0 ){ |
| 203660 | + p->oom = 1; |
| 203661 | + break; |
| 203662 | + } |
| 203663 | + jsonParseAddNodeArray(p, pPatch->aNode, pPatch->nNode); |
| 203664 | + /* The nodes copied out of pPatch and into p likely contain |
| 203665 | + ** u.zJContent pointers into pPatch->zJson. So preserve the |
| 203666 | + ** content of pPatch until p is destroyed. */ |
| 203667 | + assert( pPatch->nJPRef>=1 ); |
| 203668 | + pPatch->nJPRef++; |
| 203669 | + jsonParseAddCleanup(p, (void(*)(void*))jsonParseFree, pPatch); |
| 203670 | + } |
| 203671 | + break; |
| 203672 | + } |
| 203673 | + default: { |
| 203674 | + jsonParseAddNode(p, JSON_NULL, 0, 0); |
| 203675 | + sqlite3_result_error(pCtx, "JSON cannot hold BLOB values", -1); |
| 203676 | + p->nErr++; |
| 203677 | + break; |
| 203678 | + } |
| 203679 | + } |
| 202776 | 203680 | } |
| 202777 | 203681 | |
| 202778 | 203682 | /* |
| 202779 | 203683 | ** json_replace(JSON, PATH, VALUE, ...) |
| 202780 | 203684 | ** |
| | @@ -202784,42 +203688,34 @@ |
| 202784 | 203688 | static void jsonReplaceFunc( |
| 202785 | 203689 | sqlite3_context *ctx, |
| 202786 | 203690 | int argc, |
| 202787 | 203691 | sqlite3_value **argv |
| 202788 | 203692 | ){ |
| 202789 | | - JsonParse x; /* The parse */ |
| 203693 | + JsonParse *pParse; /* The parse */ |
| 202790 | 203694 | JsonNode *pNode; |
| 202791 | 203695 | const char *zPath; |
| 202792 | 203696 | u32 i; |
| 202793 | 203697 | |
| 202794 | 203698 | if( argc<1 ) return; |
| 202795 | 203699 | if( (argc&1)==0 ) { |
| 202796 | 203700 | jsonWrongNumArgs(ctx, "replace"); |
| 202797 | 203701 | return; |
| 202798 | 203702 | } |
| 202799 | | - if( jsonParse(&x, ctx, (const char*)sqlite3_value_text(argv[0])) ) return; |
| 202800 | | - assert( x.nNode ); |
| 203703 | + pParse = jsonParseCached(ctx, argv[0], ctx, argc>1); |
| 203704 | + if( pParse==0 ) return; |
| 202801 | 203705 | for(i=1; i<(u32)argc; i+=2){ |
| 202802 | 203706 | zPath = (const char*)sqlite3_value_text(argv[i]); |
| 202803 | | - pNode = jsonLookup(&x, zPath, 0, ctx); |
| 202804 | | - if( x.nErr ) goto replace_err; |
| 203707 | + pParse->useMod = 1; |
| 203708 | + pNode = jsonLookup(pParse, zPath, 0, ctx); |
| 203709 | + if( pParse->nErr ) goto replace_err; |
| 202805 | 203710 | if( pNode ){ |
| 202806 | | - assert( pNode->eU==0 || pNode->eU==1 || pNode->eU==4 ); |
| 202807 | | - testcase( pNode->eU!=0 && pNode->eU!=1 ); |
| 202808 | | - pNode->jnFlags |= (u8)JNODE_REPLACE; |
| 202809 | | - VVA( pNode->eU = 4 ); |
| 202810 | | - pNode->u.iReplace = i + 1; |
| 203711 | + jsonReplaceNode(ctx, pParse, (u32)(pNode - pParse->aNode), argv[i+1]); |
| 202811 | 203712 | } |
| 202812 | 203713 | } |
| 202813 | | - if( x.aNode[0].jnFlags & JNODE_REPLACE ){ |
| 202814 | | - assert( x.aNode[0].eU==4 ); |
| 202815 | | - sqlite3_result_value(ctx, argv[x.aNode[0].u.iReplace]); |
| 202816 | | - }else{ |
| 202817 | | - jsonReturnJson(x.aNode, ctx, argv); |
| 202818 | | - } |
| 203714 | + jsonReturnJson(pParse, pParse->aNode, ctx, 1); |
| 202819 | 203715 | replace_err: |
| 202820 | | - jsonParseReset(&x); |
| 203716 | + jsonDebugPrintParse(pParse); |
| 202821 | 203717 | } |
| 202822 | 203718 | |
| 202823 | 203719 | |
| 202824 | 203720 | /* |
| 202825 | 203721 | ** json_set(JSON, PATH, VALUE, ...) |
| | @@ -202836,11 +203732,11 @@ |
| 202836 | 203732 | static void jsonSetFunc( |
| 202837 | 203733 | sqlite3_context *ctx, |
| 202838 | 203734 | int argc, |
| 202839 | 203735 | sqlite3_value **argv |
| 202840 | 203736 | ){ |
| 202841 | | - JsonParse x; /* The parse */ |
| 203737 | + JsonParse *pParse; /* The parse */ |
| 202842 | 203738 | JsonNode *pNode; |
| 202843 | 203739 | const char *zPath; |
| 202844 | 203740 | u32 i; |
| 202845 | 203741 | int bApnd; |
| 202846 | 203742 | int bIsSet = sqlite3_user_data(ctx)!=0; |
| | @@ -202848,37 +203744,31 @@ |
| 202848 | 203744 | if( argc<1 ) return; |
| 202849 | 203745 | if( (argc&1)==0 ) { |
| 202850 | 203746 | jsonWrongNumArgs(ctx, bIsSet ? "set" : "insert"); |
| 202851 | 203747 | return; |
| 202852 | 203748 | } |
| 202853 | | - if( jsonParse(&x, ctx, (const char*)sqlite3_value_text(argv[0])) ) return; |
| 202854 | | - assert( x.nNode ); |
| 203749 | + pParse = jsonParseCached(ctx, argv[0], ctx, argc>1); |
| 203750 | + if( pParse==0 ) return; |
| 202855 | 203751 | for(i=1; i<(u32)argc; i+=2){ |
| 202856 | 203752 | zPath = (const char*)sqlite3_value_text(argv[i]); |
| 202857 | 203753 | bApnd = 0; |
| 202858 | | - pNode = jsonLookup(&x, zPath, &bApnd, ctx); |
| 202859 | | - if( x.oom ){ |
| 203754 | + pParse->useMod = 1; |
| 203755 | + pNode = jsonLookup(pParse, zPath, &bApnd, ctx); |
| 203756 | + if( pParse->oom ){ |
| 202860 | 203757 | sqlite3_result_error_nomem(ctx); |
| 202861 | 203758 | goto jsonSetDone; |
| 202862 | | - }else if( x.nErr ){ |
| 203759 | + }else if( pParse->nErr ){ |
| 202863 | 203760 | goto jsonSetDone; |
| 202864 | 203761 | }else if( pNode && (bApnd || bIsSet) ){ |
| 202865 | | - testcase( pNode->eU!=0 && pNode->eU!=1 ); |
| 202866 | | - assert( pNode->eU!=3 && pNode->eU!=5 ); |
| 202867 | | - VVA( pNode->eU = 4 ); |
| 202868 | | - pNode->jnFlags |= (u8)JNODE_REPLACE; |
| 202869 | | - pNode->u.iReplace = i + 1; |
| 203762 | + jsonReplaceNode(ctx, pParse, (u32)(pNode - pParse->aNode), argv[i+1]); |
| 202870 | 203763 | } |
| 202871 | 203764 | } |
| 202872 | | - if( x.aNode[0].jnFlags & JNODE_REPLACE ){ |
| 202873 | | - assert( x.aNode[0].eU==4 ); |
| 202874 | | - sqlite3_result_value(ctx, argv[x.aNode[0].u.iReplace]); |
| 202875 | | - }else{ |
| 202876 | | - jsonReturnJson(x.aNode, ctx, argv); |
| 202877 | | - } |
| 203765 | + jsonDebugPrintParse(pParse); |
| 203766 | + jsonReturnJson(pParse, pParse->aNode, ctx, 1); |
| 203767 | + |
| 202878 | 203768 | jsonSetDone: |
| 202879 | | - jsonParseReset(&x); |
| 203769 | + /* no cleanup required */; |
| 202880 | 203770 | } |
| 202881 | 203771 | |
| 202882 | 203772 | /* |
| 202883 | 203773 | ** json_type(JSON) |
| 202884 | 203774 | ** json_type(JSON, PATH) |
| | @@ -202893,11 +203783,11 @@ |
| 202893 | 203783 | ){ |
| 202894 | 203784 | JsonParse *p; /* The parse */ |
| 202895 | 203785 | const char *zPath; |
| 202896 | 203786 | JsonNode *pNode; |
| 202897 | 203787 | |
| 202898 | | - p = jsonParseCached(ctx, argv, ctx); |
| 203788 | + p = jsonParseCached(ctx, argv[0], ctx, 0); |
| 202899 | 203789 | if( p==0 ) return; |
| 202900 | 203790 | if( argc==2 ){ |
| 202901 | 203791 | zPath = (const char*)sqlite3_value_text(argv[1]); |
| 202902 | 203792 | pNode = jsonLookup(p, zPath, 0, ctx); |
| 202903 | 203793 | }else{ |
| | @@ -202920,16 +203810,16 @@ |
| 202920 | 203810 | sqlite3_value **argv |
| 202921 | 203811 | ){ |
| 202922 | 203812 | JsonParse *p; /* The parse */ |
| 202923 | 203813 | UNUSED_PARAMETER(argc); |
| 202924 | 203814 | if( sqlite3_value_type(argv[0])==SQLITE_NULL ) return; |
| 202925 | | - p = jsonParseCached(ctx, argv, 0); |
| 203815 | + p = jsonParseCached(ctx, argv[0], 0, 0); |
| 202926 | 203816 | if( p==0 || p->oom ){ |
| 202927 | 203817 | sqlite3_result_error_nomem(ctx); |
| 202928 | 203818 | sqlite3_free(p); |
| 202929 | 203819 | }else{ |
| 202930 | | - sqlite3_result_int(ctx, p->nErr==0 && p->hasNonstd==0); |
| 203820 | + sqlite3_result_int(ctx, p->nErr==0 && (p->hasNonstd==0 || p->useMod)); |
| 202931 | 203821 | if( p->nErr ) jsonParseFree(p); |
| 202932 | 203822 | } |
| 202933 | 203823 | } |
| 202934 | 203824 | |
| 202935 | 203825 | /* |
| | @@ -202966,20 +203856,20 @@ |
| 202966 | 203856 | sqlite3_value **argv |
| 202967 | 203857 | ){ |
| 202968 | 203858 | JsonParse *p; /* The parse */ |
| 202969 | 203859 | UNUSED_PARAMETER(argc); |
| 202970 | 203860 | if( sqlite3_value_type(argv[0])==SQLITE_NULL ) return; |
| 202971 | | - p = jsonParseCached(ctx, argv, 0); |
| 203861 | + p = jsonParseCached(ctx, argv[0], 0, 0); |
| 202972 | 203862 | if( p==0 || p->oom ){ |
| 202973 | 203863 | sqlite3_result_error_nomem(ctx); |
| 202974 | 203864 | sqlite3_free(p); |
| 202975 | 203865 | }else if( p->nErr==0 ){ |
| 202976 | 203866 | sqlite3_result_int(ctx, 0); |
| 202977 | 203867 | }else{ |
| 202978 | 203868 | int n = 1; |
| 202979 | 203869 | u32 i; |
| 202980 | | - const char *z = p->zJson; |
| 203870 | + const char *z = (const char*)sqlite3_value_text(argv[0]); |
| 202981 | 203871 | for(i=0; i<p->iErr && ALWAYS(z[i]); i++){ |
| 202982 | 203872 | if( (z[i]&0xc0)!=0x80 ) n++; |
| 202983 | 203873 | } |
| 202984 | 203874 | sqlite3_result_int(ctx, n); |
| 202985 | 203875 | jsonParseFree(p); |
| | @@ -203023,11 +203913,12 @@ |
| 203023 | 203913 | if( pStr->bErr ){ |
| 203024 | 203914 | if( pStr->bErr==1 ) sqlite3_result_error_nomem(ctx); |
| 203025 | 203915 | assert( pStr->bStatic ); |
| 203026 | 203916 | }else if( isFinal ){ |
| 203027 | 203917 | sqlite3_result_text(ctx, pStr->zBuf, (int)pStr->nUsed, |
| 203028 | | - pStr->bStatic ? SQLITE_TRANSIENT : sqlite3_free); |
| 203918 | + pStr->bStatic ? SQLITE_TRANSIENT : |
| 203919 | + (void(*)(void*))sqlite3RCStrUnref); |
| 203029 | 203920 | pStr->bStatic = 1; |
| 203030 | 203921 | }else{ |
| 203031 | 203922 | sqlite3_result_text(ctx, pStr->zBuf, (int)pStr->nUsed, SQLITE_TRANSIENT); |
| 203032 | 203923 | pStr->nUsed--; |
| 203033 | 203924 | } |
| | @@ -203131,11 +204022,12 @@ |
| 203131 | 204022 | if( pStr->bErr ){ |
| 203132 | 204023 | if( pStr->bErr==1 ) sqlite3_result_error_nomem(ctx); |
| 203133 | 204024 | assert( pStr->bStatic ); |
| 203134 | 204025 | }else if( isFinal ){ |
| 203135 | 204026 | sqlite3_result_text(ctx, pStr->zBuf, (int)pStr->nUsed, |
| 203136 | | - pStr->bStatic ? SQLITE_TRANSIENT : sqlite3_free); |
| 204027 | + pStr->bStatic ? SQLITE_TRANSIENT : |
| 204028 | + (void(*)(void*))sqlite3RCStrUnref); |
| 203137 | 204029 | pStr->bStatic = 1; |
| 203138 | 204030 | }else{ |
| 203139 | 204031 | sqlite3_result_text(ctx, pStr->zBuf, (int)pStr->nUsed, SQLITE_TRANSIENT); |
| 203140 | 204032 | pStr->nUsed--; |
| 203141 | 204033 | } |
| | @@ -203242,11 +204134,10 @@ |
| 203242 | 204134 | } |
| 203243 | 204135 | |
| 203244 | 204136 | /* Reset a JsonEachCursor back to its original state. Free any memory |
| 203245 | 204137 | ** held. */ |
| 203246 | 204138 | static void jsonEachCursorReset(JsonEachCursor *p){ |
| 203247 | | - sqlite3_free(p->zJson); |
| 203248 | 204139 | sqlite3_free(p->zRoot); |
| 203249 | 204140 | jsonParseReset(&p->sParse); |
| 203250 | 204141 | p->iRowid = 0; |
| 203251 | 204142 | p->i = 0; |
| 203252 | 204143 | p->iEnd = 0; |
| | @@ -203380,11 +204271,11 @@ |
| 203380 | 204271 | JsonNode *pThis = &p->sParse.aNode[p->i]; |
| 203381 | 204272 | switch( i ){ |
| 203382 | 204273 | case JEACH_KEY: { |
| 203383 | 204274 | if( p->i==0 ) break; |
| 203384 | 204275 | if( p->eType==JSON_OBJECT ){ |
| 203385 | | - jsonReturn(pThis, ctx, 0); |
| 204276 | + jsonReturn(&p->sParse, pThis, ctx); |
| 203386 | 204277 | }else if( p->eType==JSON_ARRAY ){ |
| 203387 | 204278 | u32 iKey; |
| 203388 | 204279 | if( p->bRecursive ){ |
| 203389 | 204280 | if( p->iRowid==0 ) break; |
| 203390 | 204281 | assert( p->sParse.aNode[p->sParse.aUp[p->i]].eU==3 ); |
| | @@ -203396,11 +204287,11 @@ |
| 203396 | 204287 | } |
| 203397 | 204288 | break; |
| 203398 | 204289 | } |
| 203399 | 204290 | case JEACH_VALUE: { |
| 203400 | 204291 | if( pThis->jnFlags & JNODE_LABEL ) pThis++; |
| 203401 | | - jsonReturn(pThis, ctx, 0); |
| 204292 | + jsonReturn(&p->sParse, pThis, ctx); |
| 203402 | 204293 | break; |
| 203403 | 204294 | } |
| 203404 | 204295 | case JEACH_TYPE: { |
| 203405 | 204296 | if( pThis->jnFlags & JNODE_LABEL ) pThis++; |
| 203406 | 204297 | sqlite3_result_text(ctx, jsonType[pThis->eType], -1, SQLITE_STATIC); |
| | @@ -203407,11 +204298,11 @@ |
| 203407 | 204298 | break; |
| 203408 | 204299 | } |
| 203409 | 204300 | case JEACH_ATOM: { |
| 203410 | 204301 | if( pThis->jnFlags & JNODE_LABEL ) pThis++; |
| 203411 | 204302 | if( pThis->eType>=JSON_ARRAY ) break; |
| 203412 | | - jsonReturn(pThis, ctx, 0); |
| 204303 | + jsonReturn(&p->sParse, pThis, ctx); |
| 203413 | 204304 | break; |
| 203414 | 204305 | } |
| 203415 | 204306 | case JEACH_ID: { |
| 203416 | 204307 | sqlite3_result_int64(ctx, |
| 203417 | 204308 | (sqlite3_int64)p->i + ((pThis->jnFlags & JNODE_LABEL)!=0)); |
| | @@ -203562,15 +204453,23 @@ |
| 203562 | 204453 | UNUSED_PARAMETER(argc); |
| 203563 | 204454 | jsonEachCursorReset(p); |
| 203564 | 204455 | if( idxNum==0 ) return SQLITE_OK; |
| 203565 | 204456 | z = (const char*)sqlite3_value_text(argv[0]); |
| 203566 | 204457 | if( z==0 ) return SQLITE_OK; |
| 203567 | | - n = sqlite3_value_bytes(argv[0]); |
| 203568 | | - p->zJson = sqlite3_malloc64( n+1 ); |
| 203569 | | - if( p->zJson==0 ) return SQLITE_NOMEM; |
| 203570 | | - memcpy(p->zJson, z, (size_t)n+1); |
| 203571 | | - if( jsonParse(&p->sParse, 0, p->zJson) ){ |
| 204458 | + memset(&p->sParse, 0, sizeof(p->sParse)); |
| 204459 | + p->sParse.nJPRef = 1; |
| 204460 | + if( sqlite3ValueIsOfClass(argv[0], (void(*)(void*))sqlite3RCStrUnref) ){ |
| 204461 | + p->sParse.zJson = sqlite3RCStrRef((char*)z); |
| 204462 | + }else{ |
| 204463 | + n = sqlite3_value_bytes(argv[0]); |
| 204464 | + p->sParse.zJson = sqlite3RCStrNew( n+1 ); |
| 204465 | + if( p->sParse.zJson==0 ) return SQLITE_NOMEM; |
| 204466 | + memcpy(p->sParse.zJson, z, (size_t)n+1); |
| 204467 | + } |
| 204468 | + p->sParse.bJsonIsRCStr = 1; |
| 204469 | + p->zJson = p->sParse.zJson; |
| 204470 | + if( jsonParse(&p->sParse, 0) ){ |
| 203572 | 204471 | int rc = SQLITE_NOMEM; |
| 203573 | 204472 | if( p->sParse.oom==0 ){ |
| 203574 | 204473 | sqlite3_free(cur->pVtab->zErrMsg); |
| 203575 | 204474 | cur->pVtab->zErrMsg = sqlite3_mprintf("malformed JSON"); |
| 203576 | 204475 | if( cur->pVtab->zErrMsg ) rc = SQLITE_ERROR; |
| | @@ -219306,10 +220205,11 @@ |
| 219306 | 220205 | }else{ |
| 219307 | 220206 | /* assert( db->pPreUpdate->pUnpacked ); */ |
| 219308 | 220207 | rc = pSession->hook.xOld(pSession->hook.pCtx, iCol, &pVal); |
| 219309 | 220208 | } |
| 219310 | 220209 | assert( rc==SQLITE_OK ); |
| 220210 | + (void)rc; /* Suppress warning about unused variable */ |
| 219311 | 220211 | if( sqlite3_value_type(pVal)!=eType ) return 0; |
| 219312 | 220212 | |
| 219313 | 220213 | /* A SessionChange object never has a NULL value in a PK column */ |
| 219314 | 220214 | assert( eType==SQLITE_INTEGER || eType==SQLITE_FLOAT |
| 219315 | 220215 | || eType==SQLITE_BLOB || eType==SQLITE_TEXT |
| | @@ -225188,10 +226088,14 @@ |
| 225188 | 226088 | ** nAutomerge: |
| 225189 | 226089 | ** The minimum number of segments that an auto-merge operation should |
| 225190 | 226090 | ** attempt to merge together. A value of 1 sets the object to use the |
| 225191 | 226091 | ** compile time default. Zero disables auto-merge altogether. |
| 225192 | 226092 | ** |
| 226093 | +** bContentlessDelete: |
| 226094 | +** True if the contentless_delete option was present in the CREATE |
| 226095 | +** VIRTUAL TABLE statement. |
| 226096 | +** |
| 225193 | 226097 | ** zContent: |
| 225194 | 226098 | ** |
| 225195 | 226099 | ** zContentRowid: |
| 225196 | 226100 | ** The value of the content_rowid= option, if one was specified. Or |
| 225197 | 226101 | ** the string "rowid" otherwise. This text is not quoted - if it is |
| | @@ -225222,10 +226126,11 @@ |
| 225222 | 226126 | char **azCol; /* Column names */ |
| 225223 | 226127 | u8 *abUnindexed; /* True for unindexed columns */ |
| 225224 | 226128 | int nPrefix; /* Number of prefix indexes */ |
| 225225 | 226129 | int *aPrefix; /* Sizes in bytes of nPrefix prefix indexes */ |
| 225226 | 226130 | int eContent; /* An FTS5_CONTENT value */ |
| 226131 | + int bContentlessDelete; /* "contentless_delete=" option (dflt==0) */ |
| 225227 | 226132 | char *zContent; /* content table */ |
| 225228 | 226133 | char *zContentRowid; /* "content_rowid=" option value */ |
| 225229 | 226134 | int bColumnsize; /* "columnsize=" option value (dflt==1) */ |
| 225230 | 226135 | int eDetail; /* FTS5_DETAIL_XXX value */ |
| 225231 | 226136 | char *zContentExprlist; |
| | @@ -225243,10 +226148,11 @@ |
| 225243 | 226148 | int nUsermerge; /* 'usermerge' setting */ |
| 225244 | 226149 | int nHashSize; /* Bytes of memory for in-memory hash */ |
| 225245 | 226150 | char *zRank; /* Name of rank function */ |
| 225246 | 226151 | char *zRankArgs; /* Arguments to rank function */ |
| 225247 | 226152 | int bSecureDelete; /* 'secure-delete' */ |
| 226153 | + int nDeleteMerge; /* 'deletemerge' */ |
| 225248 | 226154 | |
| 225249 | 226155 | /* If non-NULL, points to sqlite3_vtab.base.zErrmsg. Often NULL. */ |
| 225250 | 226156 | char **pzErrmsg; |
| 225251 | 226157 | |
| 225252 | 226158 | #ifdef SQLITE_DEBUG |
| | @@ -225565,10 +226471,13 @@ |
| 225565 | 226471 | static int sqlite3Fts5IndexMerge(Fts5Index *p, int nMerge); |
| 225566 | 226472 | static int sqlite3Fts5IndexReset(Fts5Index *p); |
| 225567 | 226473 | |
| 225568 | 226474 | static int sqlite3Fts5IndexLoadConfig(Fts5Index *p); |
| 225569 | 226475 | |
| 226476 | +static int sqlite3Fts5IndexGetOrigin(Fts5Index *p, i64 *piOrigin); |
| 226477 | +static int sqlite3Fts5IndexContentlessDelete(Fts5Index *p, i64 iOrigin, i64 iRowid); |
| 226478 | + |
| 225570 | 226479 | /* |
| 225571 | 226480 | ** End of interface to code in fts5_index.c. |
| 225572 | 226481 | **************************************************************************/ |
| 225573 | 226482 | |
| 225574 | 226483 | /************************************************************************** |
| | @@ -225649,10 +226558,15 @@ |
| 225649 | 226558 | /* |
| 225650 | 226559 | ** Empty (but do not delete) a hash table. |
| 225651 | 226560 | */ |
| 225652 | 226561 | static void sqlite3Fts5HashClear(Fts5Hash*); |
| 225653 | 226562 | |
| 226563 | +/* |
| 226564 | +** Return true if the hash is empty, false otherwise. |
| 226565 | +*/ |
| 226566 | +static int sqlite3Fts5HashIsEmpty(Fts5Hash*); |
| 226567 | + |
| 225654 | 226568 | static int sqlite3Fts5HashQuery( |
| 225655 | 226569 | Fts5Hash*, /* Hash table to query */ |
| 225656 | 226570 | int nPre, |
| 225657 | 226571 | const char *pTerm, int nTerm, /* Query term */ |
| 225658 | 226572 | void **ppObj, /* OUT: Pointer to doclist for pTerm */ |
| | @@ -225668,10 +226582,11 @@ |
| 225668 | 226582 | static void sqlite3Fts5HashScanEntry(Fts5Hash *, |
| 225669 | 226583 | const char **pzTerm, /* OUT: term (nul-terminated) */ |
| 225670 | 226584 | const u8 **ppDoclist, /* OUT: pointer to doclist */ |
| 225671 | 226585 | int *pnDoclist /* OUT: size of doclist in bytes */ |
| 225672 | 226586 | ); |
| 226587 | + |
| 225673 | 226588 | |
| 225674 | 226589 | |
| 225675 | 226590 | /* |
| 225676 | 226591 | ** End of interface to code in fts5_hash.c. |
| 225677 | 226592 | **************************************************************************/ |
| | @@ -228542,10 +229457,12 @@ |
| 228542 | 229457 | #define FTS5_DEFAULT_AUTOMERGE 4 |
| 228543 | 229458 | #define FTS5_DEFAULT_USERMERGE 4 |
| 228544 | 229459 | #define FTS5_DEFAULT_CRISISMERGE 16 |
| 228545 | 229460 | #define FTS5_DEFAULT_HASHSIZE (1024*1024) |
| 228546 | 229461 | |
| 229462 | +#define FTS5_DEFAULT_DELETE_AUTOMERGE 10 /* default 10% */ |
| 229463 | + |
| 228547 | 229464 | /* Maximum allowed page size */ |
| 228548 | 229465 | #define FTS5_MAX_PAGE_SIZE (64*1024) |
| 228549 | 229466 | |
| 228550 | 229467 | static int fts5_iswhitespace(char x){ |
| 228551 | 229468 | return (x==' '); |
| | @@ -228871,10 +229788,20 @@ |
| 228871 | 229788 | pConfig->eContent = FTS5_CONTENT_NONE; |
| 228872 | 229789 | } |
| 228873 | 229790 | } |
| 228874 | 229791 | return rc; |
| 228875 | 229792 | } |
| 229793 | + |
| 229794 | + if( sqlite3_strnicmp("contentless_delete", zCmd, nCmd)==0 ){ |
| 229795 | + if( (zArg[0]!='0' && zArg[0]!='1') || zArg[1]!='\0' ){ |
| 229796 | + *pzErr = sqlite3_mprintf("malformed contentless_delete=... directive"); |
| 229797 | + rc = SQLITE_ERROR; |
| 229798 | + }else{ |
| 229799 | + pConfig->bContentlessDelete = (zArg[0]=='1'); |
| 229800 | + } |
| 229801 | + return rc; |
| 229802 | + } |
| 228876 | 229803 | |
| 228877 | 229804 | if( sqlite3_strnicmp("content_rowid", zCmd, nCmd)==0 ){ |
| 228878 | 229805 | if( pConfig->zContentRowid ){ |
| 228879 | 229806 | *pzErr = sqlite3_mprintf("multiple content_rowid=... directives"); |
| 228880 | 229807 | rc = SQLITE_ERROR; |
| | @@ -229115,10 +230042,32 @@ |
| 229115 | 230042 | } |
| 229116 | 230043 | |
| 229117 | 230044 | sqlite3_free(zOne); |
| 229118 | 230045 | sqlite3_free(zTwo); |
| 229119 | 230046 | } |
| 230047 | + |
| 230048 | + /* We only allow contentless_delete=1 if the table is indeed contentless. */ |
| 230049 | + if( rc==SQLITE_OK |
| 230050 | + && pRet->bContentlessDelete |
| 230051 | + && pRet->eContent!=FTS5_CONTENT_NONE |
| 230052 | + ){ |
| 230053 | + *pzErr = sqlite3_mprintf( |
| 230054 | + "contentless_delete=1 requires a contentless table" |
| 230055 | + ); |
| 230056 | + rc = SQLITE_ERROR; |
| 230057 | + } |
| 230058 | + |
| 230059 | + /* We only allow contentless_delete=1 if columnsize=0 is not present. |
| 230060 | + ** |
| 230061 | + ** This restriction may be removed at some point. |
| 230062 | + */ |
| 230063 | + if( rc==SQLITE_OK && pRet->bContentlessDelete && pRet->bColumnsize==0 ){ |
| 230064 | + *pzErr = sqlite3_mprintf( |
| 230065 | + "contentless_delete=1 is incompatible with columnsize=0" |
| 230066 | + ); |
| 230067 | + rc = SQLITE_ERROR; |
| 230068 | + } |
| 229120 | 230069 | |
| 229121 | 230070 | /* If a tokenizer= option was successfully parsed, the tokenizer has |
| 229122 | 230071 | ** already been allocated. Otherwise, allocate an instance of the default |
| 229123 | 230072 | ** tokenizer (unicode61) now. */ |
| 229124 | 230073 | if( rc==SQLITE_OK && pRet->pTok==0 ){ |
| | @@ -229409,10 +230358,22 @@ |
| 229409 | 230358 | if( nCrisisMerge<=1 ) nCrisisMerge = FTS5_DEFAULT_CRISISMERGE; |
| 229410 | 230359 | if( nCrisisMerge>=FTS5_MAX_SEGMENT ) nCrisisMerge = FTS5_MAX_SEGMENT-1; |
| 229411 | 230360 | pConfig->nCrisisMerge = nCrisisMerge; |
| 229412 | 230361 | } |
| 229413 | 230362 | } |
| 230363 | + |
| 230364 | + else if( 0==sqlite3_stricmp(zKey, "deletemerge") ){ |
| 230365 | + int nVal = -1; |
| 230366 | + if( SQLITE_INTEGER==sqlite3_value_numeric_type(pVal) ){ |
| 230367 | + nVal = sqlite3_value_int(pVal); |
| 230368 | + }else{ |
| 230369 | + *pbBadkey = 1; |
| 230370 | + } |
| 230371 | + if( nVal<0 ) nVal = FTS5_DEFAULT_DELETE_AUTOMERGE; |
| 230372 | + if( nVal>100 ) nVal = 0; |
| 230373 | + pConfig->nDeleteMerge = nVal; |
| 230374 | + } |
| 229414 | 230375 | |
| 229415 | 230376 | else if( 0==sqlite3_stricmp(zKey, "rank") ){ |
| 229416 | 230377 | const char *zIn = (const char*)sqlite3_value_text(pVal); |
| 229417 | 230378 | char *zRank; |
| 229418 | 230379 | char *zRankArgs; |
| | @@ -229458,10 +230419,11 @@ |
| 229458 | 230419 | pConfig->pgsz = FTS5_DEFAULT_PAGE_SIZE; |
| 229459 | 230420 | pConfig->nAutomerge = FTS5_DEFAULT_AUTOMERGE; |
| 229460 | 230421 | pConfig->nUsermerge = FTS5_DEFAULT_USERMERGE; |
| 229461 | 230422 | pConfig->nCrisisMerge = FTS5_DEFAULT_CRISISMERGE; |
| 229462 | 230423 | pConfig->nHashSize = FTS5_DEFAULT_HASHSIZE; |
| 230424 | + pConfig->nDeleteMerge = FTS5_DEFAULT_DELETE_AUTOMERGE; |
| 229463 | 230425 | |
| 229464 | 230426 | zSql = sqlite3Fts5Mprintf(&rc, zSelect, pConfig->zDb, pConfig->zName); |
| 229465 | 230427 | if( zSql ){ |
| 229466 | 230428 | rc = sqlite3_prepare_v2(pConfig->db, zSql, -1, &p, 0); |
| 229467 | 230429 | sqlite3_free(zSql); |
| | @@ -231981,11 +232943,11 @@ |
| 231981 | 232943 | } |
| 231982 | 232944 | |
| 231983 | 232945 | return pRet; |
| 231984 | 232946 | } |
| 231985 | 232947 | |
| 231986 | | -#ifdef SQLITE_TEST |
| 232948 | +#if defined(SQLITE_TEST) || defined(SQLITE_FTS5_DEBUG) |
| 231987 | 232949 | static char *fts5ExprTermPrint(Fts5ExprTerm *pTerm){ |
| 231988 | 232950 | sqlite3_int64 nByte = 0; |
| 231989 | 232951 | Fts5ExprTerm *p; |
| 231990 | 232952 | char *zQuoted; |
| 231991 | 232953 | |
| | @@ -232348,18 +233310,18 @@ |
| 232348 | 233310 | iCode = sqlite3_value_int(apVal[0]); |
| 232349 | 233311 | if( nArg==2 ) bRemoveDiacritics = sqlite3_value_int(apVal[1]); |
| 232350 | 233312 | sqlite3_result_int(pCtx, sqlite3Fts5UnicodeFold(iCode, bRemoveDiacritics)); |
| 232351 | 233313 | } |
| 232352 | 233314 | } |
| 232353 | | -#endif /* ifdef SQLITE_TEST */ |
| 233315 | +#endif /* if SQLITE_TEST || SQLITE_FTS5_DEBUG */ |
| 232354 | 233316 | |
| 232355 | 233317 | /* |
| 232356 | 233318 | ** This is called during initialization to register the fts5_expr() scalar |
| 232357 | 233319 | ** UDF with the SQLite handle passed as the only argument. |
| 232358 | 233320 | */ |
| 232359 | 233321 | static int sqlite3Fts5ExprInit(Fts5Global *pGlobal, sqlite3 *db){ |
| 232360 | | -#ifdef SQLITE_TEST |
| 233322 | +#if defined(SQLITE_TEST) || defined(SQLITE_FTS5_DEBUG) |
| 232361 | 233323 | struct Fts5ExprFunc { |
| 232362 | 233324 | const char *z; |
| 232363 | 233325 | void (*x)(sqlite3_context*,int,sqlite3_value**); |
| 232364 | 233326 | } aFunc[] = { |
| 232365 | 233327 | { "fts5_expr", fts5ExprFunctionHr }, |
| | @@ -233115,11 +234077,10 @@ |
| 233115 | 234077 | pList = 0; |
| 233116 | 234078 | for(i=0; i<nMergeSlot; i++){ |
| 233117 | 234079 | pList = fts5HashEntryMerge(pList, ap[i]); |
| 233118 | 234080 | } |
| 233119 | 234081 | |
| 233120 | | - pHash->nEntry = 0; |
| 233121 | 234082 | sqlite3_free(ap); |
| 233122 | 234083 | *ppSorted = pList; |
| 233123 | 234084 | return SQLITE_OK; |
| 233124 | 234085 | } |
| 233125 | 234086 | |
| | @@ -233168,10 +234129,32 @@ |
| 233168 | 234129 | Fts5Hash *p, /* Hash table to query */ |
| 233169 | 234130 | const char *pTerm, int nTerm /* Query prefix */ |
| 233170 | 234131 | ){ |
| 233171 | 234132 | return fts5HashEntrySort(p, pTerm, nTerm, &p->pScan); |
| 233172 | 234133 | } |
| 234134 | + |
| 234135 | +#ifdef SQLITE_DEBUG |
| 234136 | +static int fts5HashCount(Fts5Hash *pHash){ |
| 234137 | + int nEntry = 0; |
| 234138 | + int ii; |
| 234139 | + for(ii=0; ii<pHash->nSlot; ii++){ |
| 234140 | + Fts5HashEntry *p = 0; |
| 234141 | + for(p=pHash->aSlot[ii]; p; p=p->pHashNext){ |
| 234142 | + nEntry++; |
| 234143 | + } |
| 234144 | + } |
| 234145 | + return nEntry; |
| 234146 | +} |
| 234147 | +#endif |
| 234148 | + |
| 234149 | +/* |
| 234150 | +** Return true if the hash table is empty, false otherwise. |
| 234151 | +*/ |
| 234152 | +static int sqlite3Fts5HashIsEmpty(Fts5Hash *pHash){ |
| 234153 | + assert( pHash->nEntry==fts5HashCount(pHash) ); |
| 234154 | + return pHash->nEntry==0; |
| 234155 | +} |
| 233173 | 234156 | |
| 233174 | 234157 | static void sqlite3Fts5HashScanNext(Fts5Hash *p){ |
| 233175 | 234158 | assert( !sqlite3Fts5HashScanEof(p) ); |
| 233176 | 234159 | p->pScan = p->pScan->pScanNext; |
| 233177 | 234160 | } |
| | @@ -233257,32 +234240,50 @@ |
| 233257 | 234240 | # error "FTS5_MAX_PREFIX_INDEXES is too large" |
| 233258 | 234241 | #endif |
| 233259 | 234242 | |
| 233260 | 234243 | #define FTS5_MAX_LEVEL 64 |
| 233261 | 234244 | |
| 234245 | +/* |
| 234246 | +** There are two versions of the format used for the structure record: |
| 234247 | +** |
| 234248 | +** 1. the legacy format, that may be read by all fts5 versions, and |
| 234249 | +** |
| 234250 | +** 2. the V2 format, which is used by contentless_delete=1 databases. |
| 234251 | +** |
| 234252 | +** Both begin with a 4-byte "configuration cookie" value. Then, a legacy |
| 234253 | +** format structure record contains a varint - the number of levels in |
| 234254 | +** the structure. Whereas a V2 structure record contains the constant |
| 234255 | +** 4 bytes [0xff 0x00 0x00 0x01]. This is unambiguous as the value of a |
| 234256 | +** varint has to be at least 16256 to begin with "0xFF". And the default |
| 234257 | +** maximum number of levels is 64. |
| 234258 | +** |
| 234259 | +** See below for more on structure record formats. |
| 234260 | +*/ |
| 234261 | +#define FTS5_STRUCTURE_V2 "\xFF\x00\x00\x01" |
| 234262 | + |
| 233262 | 234263 | /* |
| 233263 | 234264 | ** Details: |
| 233264 | 234265 | ** |
| 233265 | 234266 | ** The %_data table managed by this module, |
| 233266 | 234267 | ** |
| 233267 | 234268 | ** CREATE TABLE %_data(id INTEGER PRIMARY KEY, block BLOB); |
| 233268 | 234269 | ** |
| 233269 | | -** , contains the following 5 types of records. See the comments surrounding |
| 234270 | +** , contains the following 6 types of records. See the comments surrounding |
| 233270 | 234271 | ** the FTS5_*_ROWID macros below for a description of how %_data rowids are |
| 233271 | 234272 | ** assigned to each fo them. |
| 233272 | 234273 | ** |
| 233273 | 234274 | ** 1. Structure Records: |
| 233274 | 234275 | ** |
| 233275 | 234276 | ** The set of segments that make up an index - the index structure - are |
| 233276 | 234277 | ** recorded in a single record within the %_data table. The record consists |
| 233277 | 234278 | ** of a single 32-bit configuration cookie value followed by a list of |
| 233278 | | -** SQLite varints. If the FTS table features more than one index (because |
| 233279 | | -** there are one or more prefix indexes), it is guaranteed that all share |
| 233280 | | -** the same cookie value. |
| 234279 | +** SQLite varints. |
| 233281 | 234280 | ** |
| 233282 | | -** Immediately following the configuration cookie, the record begins with |
| 233283 | | -** three varints: |
| 234281 | +** If the structure record is a V2 record, the configuration cookie is |
| 234282 | +** followed by the following 4 bytes: [0xFF 0x00 0x00 0x01]. |
| 234283 | +** |
| 234284 | +** Next, the record continues with three varints: |
| 233284 | 234285 | ** |
| 233285 | 234286 | ** + number of levels, |
| 233286 | 234287 | ** + total number of segments on all levels, |
| 233287 | 234288 | ** + value of write counter. |
| 233288 | 234289 | ** |
| | @@ -233293,10 +234294,16 @@ |
| 233293 | 234294 | ** + for each segment from oldest to newest: |
| 233294 | 234295 | ** + segment id (always > 0) |
| 233295 | 234296 | ** + first leaf page number (often 1, always greater than 0) |
| 233296 | 234297 | ** + final leaf page number |
| 233297 | 234298 | ** |
| 234299 | +** Then, for V2 structures only: |
| 234300 | +** |
| 234301 | +** + lower origin counter value, |
| 234302 | +** + upper origin counter value, |
| 234303 | +** + the number of tombstone hash pages. |
| 234304 | +** |
| 233298 | 234305 | ** 2. The Averages Record: |
| 233299 | 234306 | ** |
| 233300 | 234307 | ** A single record within the %_data table. The data is a list of varints. |
| 233301 | 234308 | ** The first value is the number of rows in the index. Then, for each column |
| 233302 | 234309 | ** from left to right, the total number of tokens in the column for all |
| | @@ -233408,10 +234415,42 @@ |
| 233408 | 234415 | ** * Copy of first rowid on page indicated by previous field. As a varint. |
| 233409 | 234416 | ** |
| 233410 | 234417 | ** * A list of delta-encoded varints - the first rowid on each subsequent |
| 233411 | 234418 | ** child page. |
| 233412 | 234419 | ** |
| 234420 | +** 6. Tombstone Hash Page |
| 234421 | +** |
| 234422 | +** These records are only ever present in contentless_delete=1 tables. |
| 234423 | +** There are zero or more of these associated with each segment. They |
| 234424 | +** are used to store the tombstone rowids for rows contained in the |
| 234425 | +** associated segments. |
| 234426 | +** |
| 234427 | +** The set of nHashPg tombstone hash pages associated with a single |
| 234428 | +** segment together form a single hash table containing tombstone rowids. |
| 234429 | +** To find the page of the hash on which a key might be stored: |
| 234430 | +** |
| 234431 | +** iPg = (rowid % nHashPg) |
| 234432 | +** |
| 234433 | +** Then, within page iPg, which has nSlot slots: |
| 234434 | +** |
| 234435 | +** iSlot = (rowid / nHashPg) % nSlot |
| 234436 | +** |
| 234437 | +** Each tombstone hash page begins with an 8 byte header: |
| 234438 | +** |
| 234439 | +** 1-byte: Key-size (the size in bytes of each slot). Either 4 or 8. |
| 234440 | +** 1-byte: rowid-0-tombstone flag. This flag is only valid on the |
| 234441 | +** first tombstone hash page for each segment (iPg=0). If set, |
| 234442 | +** the hash table contains rowid 0. If clear, it does not. |
| 234443 | +** Rowid 0 is handled specially. |
| 234444 | +** 2-bytes: unused. |
| 234445 | +** 4-bytes: Big-endian integer containing number of entries on page. |
| 234446 | +** |
| 234447 | +** Following this are nSlot 4 or 8 byte slots (depending on the key-size |
| 234448 | +** in the first byte of the page header). The number of slots may be |
| 234449 | +** determined based on the size of the page record and the key-size: |
| 234450 | +** |
| 234451 | +** nSlot = (nByte - 8) / key-size |
| 233413 | 234452 | */ |
| 233414 | 234453 | |
| 233415 | 234454 | /* |
| 233416 | 234455 | ** Rowids for the averages and structure records in the %_data table. |
| 233417 | 234456 | */ |
| | @@ -233441,10 +234480,11 @@ |
| 233441 | 234480 | ((i64)(pgno)) \ |
| 233442 | 234481 | ) |
| 233443 | 234482 | |
| 233444 | 234483 | #define FTS5_SEGMENT_ROWID(segid, pgno) fts5_dri(segid, 0, 0, pgno) |
| 233445 | 234484 | #define FTS5_DLIDX_ROWID(segid, height, pgno) fts5_dri(segid, 1, height, pgno) |
| 234485 | +#define FTS5_TOMBSTONE_ROWID(segid,ipg) fts5_dri(segid+(1<<16), 0, 0, ipg) |
| 233446 | 234486 | |
| 233447 | 234487 | #ifdef SQLITE_DEBUG |
| 233448 | 234488 | static int sqlite3Fts5Corrupt() { return SQLITE_CORRUPT_VTAB; } |
| 233449 | 234489 | #endif |
| 233450 | 234490 | |
| | @@ -233476,10 +234516,16 @@ |
| 233476 | 234516 | int szLeaf; /* Size of leaf without page-index */ |
| 233477 | 234517 | }; |
| 233478 | 234518 | |
| 233479 | 234519 | /* |
| 233480 | 234520 | ** One object per %_data table. |
| 234521 | +** |
| 234522 | +** nContentlessDelete: |
| 234523 | +** The number of contentless delete operations since the most recent |
| 234524 | +** call to fts5IndexFlush() or fts5IndexDiscardData(). This is tracked |
| 234525 | +** so that extra auto-merge work can be done by fts5IndexFlush() to |
| 234526 | +** account for the delete operations. |
| 233481 | 234527 | */ |
| 233482 | 234528 | struct Fts5Index { |
| 233483 | 234529 | Fts5Config *pConfig; /* Virtual table configuration */ |
| 233484 | 234530 | char *zDataTbl; /* Name of %_data table */ |
| 233485 | 234531 | int nWorkUnit; /* Leaf pages in a "unit" of work */ |
| | @@ -233490,10 +234536,12 @@ |
| 233490 | 234536 | */ |
| 233491 | 234537 | Fts5Hash *pHash; /* Hash table for in-memory data */ |
| 233492 | 234538 | int nPendingData; /* Current bytes of pending data */ |
| 233493 | 234539 | i64 iWriteRowid; /* Rowid for current doc being written */ |
| 233494 | 234540 | int bDelete; /* Current write is a delete */ |
| 234541 | + int nContentlessDelete; /* Number of contentless delete ops */ |
| 234542 | + int nPendingRow; /* Number of INSERT in hash table */ |
| 233495 | 234543 | |
| 233496 | 234544 | /* Error state. */ |
| 233497 | 234545 | int rc; /* Current error code */ |
| 233498 | 234546 | |
| 233499 | 234547 | /* State used by the fts5DataXXX() functions. */ |
| | @@ -233524,24 +234572,37 @@ |
| 233524 | 234572 | |
| 233525 | 234573 | /* |
| 233526 | 234574 | ** The contents of the "structure" record for each index are represented |
| 233527 | 234575 | ** using an Fts5Structure record in memory. Which uses instances of the |
| 233528 | 234576 | ** other Fts5StructureXXX types as components. |
| 234577 | +** |
| 234578 | +** nOriginCntr: |
| 234579 | +** This value is set to non-zero for structure records created for |
| 234580 | +** contentlessdelete=1 tables only. In that case it represents the |
| 234581 | +** origin value to apply to the next top-level segment created. |
| 233529 | 234582 | */ |
| 233530 | 234583 | struct Fts5StructureSegment { |
| 233531 | 234584 | int iSegid; /* Segment id */ |
| 233532 | 234585 | int pgnoFirst; /* First leaf page number in segment */ |
| 233533 | 234586 | int pgnoLast; /* Last leaf page number in segment */ |
| 234587 | + |
| 234588 | + /* contentlessdelete=1 tables only: */ |
| 234589 | + u64 iOrigin1; |
| 234590 | + u64 iOrigin2; |
| 234591 | + int nPgTombstone; /* Number of tombstone hash table pages */ |
| 234592 | + u64 nEntryTombstone; /* Number of tombstone entries that "count" */ |
| 234593 | + u64 nEntry; /* Number of rows in this segment */ |
| 233534 | 234594 | }; |
| 233535 | 234595 | struct Fts5StructureLevel { |
| 233536 | 234596 | int nMerge; /* Number of segments in incr-merge */ |
| 233537 | 234597 | int nSeg; /* Total number of segments on level */ |
| 233538 | 234598 | Fts5StructureSegment *aSeg; /* Array of segments. aSeg[0] is oldest. */ |
| 233539 | 234599 | }; |
| 233540 | 234600 | struct Fts5Structure { |
| 233541 | 234601 | int nRef; /* Object reference count */ |
| 233542 | 234602 | u64 nWriteCounter; /* Total leaves written to level 0 */ |
| 234603 | + u64 nOriginCntr; /* Origin value for next top-level segment */ |
| 233543 | 234604 | int nSegment; /* Total segments in this structure */ |
| 233544 | 234605 | int nLevel; /* Number of levels in this index */ |
| 233545 | 234606 | Fts5StructureLevel aLevel[1]; /* Array of nLevel level objects */ |
| 233546 | 234607 | }; |
| 233547 | 234608 | |
| | @@ -233626,18 +234687,27 @@ |
| 233626 | 234687 | ** corresponding aRowidOffset[] entry is set to the byte offset of the |
| 233627 | 234688 | ** start of the "position-list-size" field within the page. |
| 233628 | 234689 | ** |
| 233629 | 234690 | ** iTermIdx: |
| 233630 | 234691 | ** Index of current term on iTermLeafPgno. |
| 234692 | +** |
| 234693 | +** apTombstone/nTombstone: |
| 234694 | +** These are used for contentless_delete=1 tables only. When the cursor |
| 234695 | +** is first allocated, the apTombstone[] array is allocated so that it |
| 234696 | +** is large enough for all tombstones hash pages associated with the |
| 234697 | +** segment. The pages themselves are loaded lazily from the database as |
| 234698 | +** they are required. |
| 233631 | 234699 | */ |
| 233632 | 234700 | struct Fts5SegIter { |
| 233633 | 234701 | Fts5StructureSegment *pSeg; /* Segment to iterate through */ |
| 233634 | 234702 | int flags; /* Mask of configuration flags */ |
| 233635 | 234703 | int iLeafPgno; /* Current leaf page number */ |
| 233636 | 234704 | Fts5Data *pLeaf; /* Current leaf data */ |
| 233637 | 234705 | Fts5Data *pNextLeaf; /* Leaf page (iLeafPgno+1) */ |
| 233638 | 234706 | i64 iLeafOffset; /* Byte offset within current leaf */ |
| 234707 | + Fts5Data **apTombstone; /* Array of tombstone pages */ |
| 234708 | + int nTombstone; |
| 233639 | 234709 | |
| 233640 | 234710 | /* Next method */ |
| 233641 | 234711 | void (*xNext)(Fts5Index*, Fts5SegIter*, int*); |
| 233642 | 234712 | |
| 233643 | 234713 | /* The page and offset from which the current term was read. The offset |
| | @@ -233762,10 +234832,64 @@ |
| 233762 | 234832 | } |
| 233763 | 234833 | |
| 233764 | 234834 | static u16 fts5GetU16(const u8 *aIn){ |
| 233765 | 234835 | return ((u16)aIn[0] << 8) + aIn[1]; |
| 233766 | 234836 | } |
| 234837 | + |
| 234838 | +/* |
| 234839 | +** The only argument points to a buffer at least 8 bytes in size. This |
| 234840 | +** function interprets the first 8 bytes of the buffer as a 64-bit big-endian |
| 234841 | +** unsigned integer and returns the result. |
| 234842 | +*/ |
| 234843 | +static u64 fts5GetU64(u8 *a){ |
| 234844 | + return ((u64)a[0] << 56) |
| 234845 | + + ((u64)a[1] << 48) |
| 234846 | + + ((u64)a[2] << 40) |
| 234847 | + + ((u64)a[3] << 32) |
| 234848 | + + ((u64)a[4] << 24) |
| 234849 | + + ((u64)a[5] << 16) |
| 234850 | + + ((u64)a[6] << 8) |
| 234851 | + + ((u64)a[7] << 0); |
| 234852 | +} |
| 234853 | + |
| 234854 | +/* |
| 234855 | +** The only argument points to a buffer at least 4 bytes in size. This |
| 234856 | +** function interprets the first 4 bytes of the buffer as a 32-bit big-endian |
| 234857 | +** unsigned integer and returns the result. |
| 234858 | +*/ |
| 234859 | +static u32 fts5GetU32(const u8 *a){ |
| 234860 | + return ((u32)a[0] << 24) |
| 234861 | + + ((u32)a[1] << 16) |
| 234862 | + + ((u32)a[2] << 8) |
| 234863 | + + ((u32)a[3] << 0); |
| 234864 | +} |
| 234865 | + |
| 234866 | +/* |
| 234867 | +** Write iVal, formated as a 64-bit big-endian unsigned integer, to the |
| 234868 | +** buffer indicated by the first argument. |
| 234869 | +*/ |
| 234870 | +static void fts5PutU64(u8 *a, u64 iVal){ |
| 234871 | + a[0] = ((iVal >> 56) & 0xFF); |
| 234872 | + a[1] = ((iVal >> 48) & 0xFF); |
| 234873 | + a[2] = ((iVal >> 40) & 0xFF); |
| 234874 | + a[3] = ((iVal >> 32) & 0xFF); |
| 234875 | + a[4] = ((iVal >> 24) & 0xFF); |
| 234876 | + a[5] = ((iVal >> 16) & 0xFF); |
| 234877 | + a[6] = ((iVal >> 8) & 0xFF); |
| 234878 | + a[7] = ((iVal >> 0) & 0xFF); |
| 234879 | +} |
| 234880 | + |
| 234881 | +/* |
| 234882 | +** Write iVal, formated as a 32-bit big-endian unsigned integer, to the |
| 234883 | +** buffer indicated by the first argument. |
| 234884 | +*/ |
| 234885 | +static void fts5PutU32(u8 *a, u32 iVal){ |
| 234886 | + a[0] = ((iVal >> 24) & 0xFF); |
| 234887 | + a[1] = ((iVal >> 16) & 0xFF); |
| 234888 | + a[2] = ((iVal >> 8) & 0xFF); |
| 234889 | + a[3] = ((iVal >> 0) & 0xFF); |
| 234890 | +} |
| 233767 | 234891 | |
| 233768 | 234892 | /* |
| 233769 | 234893 | ** Allocate and return a buffer at least nByte bytes in size. |
| 233770 | 234894 | ** |
| 233771 | 234895 | ** If an OOM error is encountered, return NULL and set the error code in |
| | @@ -233990,14 +235114,21 @@ |
| 233990 | 235114 | } |
| 233991 | 235115 | |
| 233992 | 235116 | /* |
| 233993 | 235117 | ** Remove all records associated with segment iSegid. |
| 233994 | 235118 | */ |
| 233995 | | -static void fts5DataRemoveSegment(Fts5Index *p, int iSegid){ |
| 235119 | +static void fts5DataRemoveSegment(Fts5Index *p, Fts5StructureSegment *pSeg){ |
| 235120 | + int iSegid = pSeg->iSegid; |
| 233996 | 235121 | i64 iFirst = FTS5_SEGMENT_ROWID(iSegid, 0); |
| 233997 | 235122 | i64 iLast = FTS5_SEGMENT_ROWID(iSegid+1, 0)-1; |
| 233998 | 235123 | fts5DataDelete(p, iFirst, iLast); |
| 235124 | + |
| 235125 | + if( pSeg->nPgTombstone ){ |
| 235126 | + i64 iTomb1 = FTS5_TOMBSTONE_ROWID(iSegid, 0); |
| 235127 | + i64 iTomb2 = FTS5_TOMBSTONE_ROWID(iSegid, pSeg->nPgTombstone-1); |
| 235128 | + fts5DataDelete(p, iTomb1, iTomb2); |
| 235129 | + } |
| 233999 | 235130 | if( p->pIdxDeleter==0 ){ |
| 234000 | 235131 | Fts5Config *pConfig = p->pConfig; |
| 234001 | 235132 | fts5IndexPrepareStmt(p, &p->pIdxDeleter, sqlite3_mprintf( |
| 234002 | 235133 | "DELETE FROM '%q'.'%q_idx' WHERE segid=?", |
| 234003 | 235134 | pConfig->zDb, pConfig->zName |
| | @@ -234104,14 +235235,22 @@ |
| 234104 | 235235 | int iLvl; |
| 234105 | 235236 | int nLevel = 0; |
| 234106 | 235237 | int nSegment = 0; |
| 234107 | 235238 | sqlite3_int64 nByte; /* Bytes of space to allocate at pRet */ |
| 234108 | 235239 | Fts5Structure *pRet = 0; /* Structure object to return */ |
| 235240 | + int bStructureV2 = 0; /* True for FTS5_STRUCTURE_V2 */ |
| 235241 | + u64 nOriginCntr = 0; /* Largest origin value seen so far */ |
| 234109 | 235242 | |
| 234110 | 235243 | /* Grab the cookie value */ |
| 234111 | 235244 | if( piCookie ) *piCookie = sqlite3Fts5Get32(pData); |
| 234112 | 235245 | i = 4; |
| 235246 | + |
| 235247 | + /* Check if this is a V2 structure record. Set bStructureV2 if it is. */ |
| 235248 | + if( 0==memcmp(&pData[i], FTS5_STRUCTURE_V2, 4) ){ |
| 235249 | + i += 4; |
| 235250 | + bStructureV2 = 1; |
| 235251 | + } |
| 234113 | 235252 | |
| 234114 | 235253 | /* Read the total number of levels and segments from the start of the |
| 234115 | 235254 | ** structure record. */ |
| 234116 | 235255 | i += fts5GetVarint32(&pData[i], nLevel); |
| 234117 | 235256 | i += fts5GetVarint32(&pData[i], nSegment); |
| | @@ -234159,10 +235298,18 @@ |
| 234159 | 235298 | } |
| 234160 | 235299 | assert( pSeg!=0 ); |
| 234161 | 235300 | i += fts5GetVarint32(&pData[i], pSeg->iSegid); |
| 234162 | 235301 | i += fts5GetVarint32(&pData[i], pSeg->pgnoFirst); |
| 234163 | 235302 | i += fts5GetVarint32(&pData[i], pSeg->pgnoLast); |
| 235303 | + if( bStructureV2 ){ |
| 235304 | + i += fts5GetVarint(&pData[i], &pSeg->iOrigin1); |
| 235305 | + i += fts5GetVarint(&pData[i], &pSeg->iOrigin2); |
| 235306 | + i += fts5GetVarint32(&pData[i], pSeg->nPgTombstone); |
| 235307 | + i += fts5GetVarint(&pData[i], &pSeg->nEntryTombstone); |
| 235308 | + i += fts5GetVarint(&pData[i], &pSeg->nEntry); |
| 235309 | + nOriginCntr = MAX(nOriginCntr, pSeg->iOrigin2); |
| 235310 | + } |
| 234164 | 235311 | if( pSeg->pgnoLast<pSeg->pgnoFirst ){ |
| 234165 | 235312 | rc = FTS5_CORRUPT; |
| 234166 | 235313 | break; |
| 234167 | 235314 | } |
| 234168 | 235315 | } |
| | @@ -234169,10 +235316,13 @@ |
| 234169 | 235316 | if( iLvl>0 && pLvl[-1].nMerge && nTotal==0 ) rc = FTS5_CORRUPT; |
| 234170 | 235317 | if( iLvl==nLevel-1 && pLvl->nMerge ) rc = FTS5_CORRUPT; |
| 234171 | 235318 | } |
| 234172 | 235319 | } |
| 234173 | 235320 | if( nSegment!=0 && rc==SQLITE_OK ) rc = FTS5_CORRUPT; |
| 235321 | + if( bStructureV2 ){ |
| 235322 | + pRet->nOriginCntr = nOriginCntr+1; |
| 235323 | + } |
| 234174 | 235324 | |
| 234175 | 235325 | if( rc!=SQLITE_OK ){ |
| 234176 | 235326 | fts5StructureRelease(pRet); |
| 234177 | 235327 | pRet = 0; |
| 234178 | 235328 | } |
| | @@ -234381,21 +235531,25 @@ |
| 234381 | 235531 | static void fts5StructureWrite(Fts5Index *p, Fts5Structure *pStruct){ |
| 234382 | 235532 | if( p->rc==SQLITE_OK ){ |
| 234383 | 235533 | Fts5Buffer buf; /* Buffer to serialize record into */ |
| 234384 | 235534 | int iLvl; /* Used to iterate through levels */ |
| 234385 | 235535 | int iCookie; /* Cookie value to store */ |
| 235536 | + int nHdr = (pStruct->nOriginCntr>0 ? (4+4+9+9+9) : (4+9+9)); |
| 234386 | 235537 | |
| 234387 | 235538 | assert( pStruct->nSegment==fts5StructureCountSegments(pStruct) ); |
| 234388 | 235539 | memset(&buf, 0, sizeof(Fts5Buffer)); |
| 234389 | 235540 | |
| 234390 | 235541 | /* Append the current configuration cookie */ |
| 234391 | 235542 | iCookie = p->pConfig->iCookie; |
| 234392 | 235543 | if( iCookie<0 ) iCookie = 0; |
| 234393 | 235544 | |
| 234394 | | - if( 0==sqlite3Fts5BufferSize(&p->rc, &buf, 4+9+9+9) ){ |
| 235545 | + if( 0==sqlite3Fts5BufferSize(&p->rc, &buf, nHdr) ){ |
| 234395 | 235546 | sqlite3Fts5Put32(buf.p, iCookie); |
| 234396 | 235547 | buf.n = 4; |
| 235548 | + if( pStruct->nOriginCntr>0 ){ |
| 235549 | + fts5BufferSafeAppendBlob(&buf, FTS5_STRUCTURE_V2, 4); |
| 235550 | + } |
| 234397 | 235551 | fts5BufferSafeAppendVarint(&buf, pStruct->nLevel); |
| 234398 | 235552 | fts5BufferSafeAppendVarint(&buf, pStruct->nSegment); |
| 234399 | 235553 | fts5BufferSafeAppendVarint(&buf, (i64)pStruct->nWriteCounter); |
| 234400 | 235554 | } |
| 234401 | 235555 | |
| | @@ -234405,13 +235559,21 @@ |
| 234405 | 235559 | fts5BufferAppendVarint(&p->rc, &buf, pLvl->nMerge); |
| 234406 | 235560 | fts5BufferAppendVarint(&p->rc, &buf, pLvl->nSeg); |
| 234407 | 235561 | assert( pLvl->nMerge<=pLvl->nSeg ); |
| 234408 | 235562 | |
| 234409 | 235563 | for(iSeg=0; iSeg<pLvl->nSeg; iSeg++){ |
| 234410 | | - fts5BufferAppendVarint(&p->rc, &buf, pLvl->aSeg[iSeg].iSegid); |
| 234411 | | - fts5BufferAppendVarint(&p->rc, &buf, pLvl->aSeg[iSeg].pgnoFirst); |
| 234412 | | - fts5BufferAppendVarint(&p->rc, &buf, pLvl->aSeg[iSeg].pgnoLast); |
| 235564 | + Fts5StructureSegment *pSeg = &pLvl->aSeg[iSeg]; |
| 235565 | + fts5BufferAppendVarint(&p->rc, &buf, pSeg->iSegid); |
| 235566 | + fts5BufferAppendVarint(&p->rc, &buf, pSeg->pgnoFirst); |
| 235567 | + fts5BufferAppendVarint(&p->rc, &buf, pSeg->pgnoLast); |
| 235568 | + if( pStruct->nOriginCntr>0 ){ |
| 235569 | + fts5BufferAppendVarint(&p->rc, &buf, pSeg->iOrigin1); |
| 235570 | + fts5BufferAppendVarint(&p->rc, &buf, pSeg->iOrigin2); |
| 235571 | + fts5BufferAppendVarint(&p->rc, &buf, pSeg->nPgTombstone); |
| 235572 | + fts5BufferAppendVarint(&p->rc, &buf, pSeg->nEntryTombstone); |
| 235573 | + fts5BufferAppendVarint(&p->rc, &buf, pSeg->nEntry); |
| 235574 | + } |
| 234413 | 235575 | } |
| 234414 | 235576 | } |
| 234415 | 235577 | |
| 234416 | 235578 | fts5DataWrite(p, FTS5_STRUCTURE_ROWID, buf.p, buf.n); |
| 234417 | 235579 | fts5BufferFree(&buf); |
| | @@ -234929,10 +236091,27 @@ |
| 234929 | 236091 | pIter->xNext = fts5SegIterNext_None; |
| 234930 | 236092 | }else{ |
| 234931 | 236093 | pIter->xNext = fts5SegIterNext; |
| 234932 | 236094 | } |
| 234933 | 236095 | } |
| 236096 | + |
| 236097 | +/* |
| 236098 | +** Allocate a tombstone hash page array (pIter->apTombstone) for the |
| 236099 | +** iterator passed as the second argument. If an OOM error occurs, leave |
| 236100 | +** an error in the Fts5Index object. |
| 236101 | +*/ |
| 236102 | +static void fts5SegIterAllocTombstone(Fts5Index *p, Fts5SegIter *pIter){ |
| 236103 | + const int nTomb = pIter->pSeg->nPgTombstone; |
| 236104 | + if( nTomb>0 ){ |
| 236105 | + Fts5Data **apTomb = 0; |
| 236106 | + apTomb = (Fts5Data**)sqlite3Fts5MallocZero(&p->rc, sizeof(Fts5Data)*nTomb); |
| 236107 | + if( apTomb ){ |
| 236108 | + pIter->apTombstone = apTomb; |
| 236109 | + pIter->nTombstone = nTomb; |
| 236110 | + } |
| 236111 | + } |
| 236112 | +} |
| 234934 | 236113 | |
| 234935 | 236114 | /* |
| 234936 | 236115 | ** Initialize the iterator object pIter to iterate through the entries in |
| 234937 | 236116 | ** segment pSeg. The iterator is left pointing to the first entry when |
| 234938 | 236117 | ** this function returns. |
| | @@ -234971,10 +236150,11 @@ |
| 234971 | 236150 | assert_nc( pIter->pLeaf->nn>4 ); |
| 234972 | 236151 | assert_nc( fts5LeafFirstTermOff(pIter->pLeaf)==4 ); |
| 234973 | 236152 | pIter->iPgidxOff = pIter->pLeaf->szLeaf+1; |
| 234974 | 236153 | fts5SegIterLoadTerm(p, pIter, 0); |
| 234975 | 236154 | fts5SegIterLoadNPos(p, pIter); |
| 236155 | + fts5SegIterAllocTombstone(p, pIter); |
| 234976 | 236156 | } |
| 234977 | 236157 | } |
| 234978 | 236158 | |
| 234979 | 236159 | /* |
| 234980 | 236160 | ** This function is only ever called on iterators created by calls to |
| | @@ -235672,10 +236852,11 @@ |
| 235672 | 236852 | } |
| 235673 | 236853 | } |
| 235674 | 236854 | } |
| 235675 | 236855 | |
| 235676 | 236856 | fts5SegIterSetNext(p, pIter); |
| 236857 | + fts5SegIterAllocTombstone(p, pIter); |
| 235677 | 236858 | |
| 235678 | 236859 | /* Either: |
| 235679 | 236860 | ** |
| 235680 | 236861 | ** 1) an error has occurred, or |
| 235681 | 236862 | ** 2) the iterator points to EOF, or |
| | @@ -235751,18 +236932,33 @@ |
| 235751 | 236932 | } |
| 235752 | 236933 | } |
| 235753 | 236934 | |
| 235754 | 236935 | fts5SegIterSetNext(p, pIter); |
| 235755 | 236936 | } |
| 236937 | + |
| 236938 | +/* |
| 236939 | +** Array ap[] contains n elements. Release each of these elements using |
| 236940 | +** fts5DataRelease(). Then free the array itself using sqlite3_free(). |
| 236941 | +*/ |
| 236942 | +static void fts5IndexFreeArray(Fts5Data **ap, int n){ |
| 236943 | + if( ap ){ |
| 236944 | + int ii; |
| 236945 | + for(ii=0; ii<n; ii++){ |
| 236946 | + fts5DataRelease(ap[ii]); |
| 236947 | + } |
| 236948 | + sqlite3_free(ap); |
| 236949 | + } |
| 236950 | +} |
| 235756 | 236951 | |
| 235757 | 236952 | /* |
| 235758 | 236953 | ** Zero the iterator passed as the only argument. |
| 235759 | 236954 | */ |
| 235760 | 236955 | static void fts5SegIterClear(Fts5SegIter *pIter){ |
| 235761 | 236956 | fts5BufferFree(&pIter->term); |
| 235762 | 236957 | fts5DataRelease(pIter->pLeaf); |
| 235763 | 236958 | fts5DataRelease(pIter->pNextLeaf); |
| 236959 | + fts5IndexFreeArray(pIter->apTombstone, pIter->nTombstone); |
| 235764 | 236960 | fts5DlidxIterFree(pIter->pDlidx); |
| 235765 | 236961 | sqlite3_free(pIter->aRowidOffset); |
| 235766 | 236962 | memset(pIter, 0, sizeof(Fts5SegIter)); |
| 235767 | 236963 | } |
| 235768 | 236964 | |
| | @@ -236095,10 +237291,88 @@ |
| 236095 | 237291 | static void fts5MultiIterSetEof(Fts5Iter *pIter){ |
| 236096 | 237292 | Fts5SegIter *pSeg = &pIter->aSeg[ pIter->aFirst[1].iFirst ]; |
| 236097 | 237293 | pIter->base.bEof = pSeg->pLeaf==0; |
| 236098 | 237294 | pIter->iSwitchRowid = pSeg->iRowid; |
| 236099 | 237295 | } |
| 237296 | + |
| 237297 | +/* |
| 237298 | +** The argument to this macro must be an Fts5Data structure containing a |
| 237299 | +** tombstone hash page. This macro returns the key-size of the hash-page. |
| 237300 | +*/ |
| 237301 | +#define TOMBSTONE_KEYSIZE(pPg) (pPg->p[0]==4 ? 4 : 8) |
| 237302 | + |
| 237303 | +#define TOMBSTONE_NSLOT(pPg) \ |
| 237304 | + ((pPg->nn > 16) ? ((pPg->nn-8) / TOMBSTONE_KEYSIZE(pPg)) : 1) |
| 237305 | + |
| 237306 | +/* |
| 237307 | +** Query a single tombstone hash table for rowid iRowid. Return true if |
| 237308 | +** it is found or false otherwise. The tombstone hash table is one of |
| 237309 | +** nHashTable tables. |
| 237310 | +*/ |
| 237311 | +static int fts5IndexTombstoneQuery( |
| 237312 | + Fts5Data *pHash, /* Hash table page to query */ |
| 237313 | + int nHashTable, /* Number of pages attached to segment */ |
| 237314 | + u64 iRowid /* Rowid to query hash for */ |
| 237315 | +){ |
| 237316 | + const int szKey = TOMBSTONE_KEYSIZE(pHash); |
| 237317 | + const int nSlot = TOMBSTONE_NSLOT(pHash); |
| 237318 | + int iSlot = (iRowid / nHashTable) % nSlot; |
| 237319 | + int nCollide = nSlot; |
| 237320 | + |
| 237321 | + if( iRowid==0 ){ |
| 237322 | + return pHash->p[1]; |
| 237323 | + }else if( szKey==4 ){ |
| 237324 | + u32 *aSlot = (u32*)&pHash->p[8]; |
| 237325 | + while( aSlot[iSlot] ){ |
| 237326 | + if( fts5GetU32((u8*)&aSlot[iSlot])==iRowid ) return 1; |
| 237327 | + if( nCollide--==0 ) break; |
| 237328 | + iSlot = (iSlot+1)%nSlot; |
| 237329 | + } |
| 237330 | + }else{ |
| 237331 | + u64 *aSlot = (u64*)&pHash->p[8]; |
| 237332 | + while( aSlot[iSlot] ){ |
| 237333 | + if( fts5GetU64((u8*)&aSlot[iSlot])==iRowid ) return 1; |
| 237334 | + if( nCollide--==0 ) break; |
| 237335 | + iSlot = (iSlot+1)%nSlot; |
| 237336 | + } |
| 237337 | + } |
| 237338 | + |
| 237339 | + return 0; |
| 237340 | +} |
| 237341 | + |
| 237342 | +/* |
| 237343 | +** Return true if the iterator passed as the only argument points |
| 237344 | +** to an segment entry for which there is a tombstone. Return false |
| 237345 | +** if there is no tombstone or if the iterator is already at EOF. |
| 237346 | +*/ |
| 237347 | +static int fts5MultiIterIsDeleted(Fts5Iter *pIter){ |
| 237348 | + int iFirst = pIter->aFirst[1].iFirst; |
| 237349 | + Fts5SegIter *pSeg = &pIter->aSeg[iFirst]; |
| 237350 | + |
| 237351 | + if( pSeg->pLeaf && pSeg->nTombstone ){ |
| 237352 | + /* Figure out which page the rowid might be present on. */ |
| 237353 | + int iPg = ((u64)pSeg->iRowid) % pSeg->nTombstone; |
| 237354 | + assert( iPg>=0 ); |
| 237355 | + |
| 237356 | + /* If tombstone hash page iPg has not yet been loaded from the |
| 237357 | + ** database, load it now. */ |
| 237358 | + if( pSeg->apTombstone[iPg]==0 ){ |
| 237359 | + pSeg->apTombstone[iPg] = fts5DataRead(pIter->pIndex, |
| 237360 | + FTS5_TOMBSTONE_ROWID(pSeg->pSeg->iSegid, iPg) |
| 237361 | + ); |
| 237362 | + if( pSeg->apTombstone[iPg]==0 ) return 0; |
| 237363 | + } |
| 237364 | + |
| 237365 | + return fts5IndexTombstoneQuery( |
| 237366 | + pSeg->apTombstone[iPg], |
| 237367 | + pSeg->nTombstone, |
| 237368 | + pSeg->iRowid |
| 237369 | + ); |
| 237370 | + } |
| 237371 | + |
| 237372 | + return 0; |
| 237373 | +} |
| 236100 | 237374 | |
| 236101 | 237375 | /* |
| 236102 | 237376 | ** Move the iterator to the next entry. |
| 236103 | 237377 | ** |
| 236104 | 237378 | ** If an error occurs, an error code is left in Fts5Index.rc. It is not |
| | @@ -236133,11 +237407,13 @@ |
| 236133 | 237407 | if( pSeg->pLeaf==0 ) return; |
| 236134 | 237408 | } |
| 236135 | 237409 | |
| 236136 | 237410 | fts5AssertMultiIterSetup(p, pIter); |
| 236137 | 237411 | assert( pSeg==&pIter->aSeg[pIter->aFirst[1].iFirst] && pSeg->pLeaf ); |
| 236138 | | - if( pIter->bSkipEmpty==0 || pSeg->nPos ){ |
| 237412 | + if( (pIter->bSkipEmpty==0 || pSeg->nPos) |
| 237413 | + && 0==fts5MultiIterIsDeleted(pIter) |
| 237414 | + ){ |
| 236139 | 237415 | pIter->xSetOutputs(pIter, pSeg); |
| 236140 | 237416 | return; |
| 236141 | 237417 | } |
| 236142 | 237418 | bUseFrom = 0; |
| 236143 | 237419 | } |
| | @@ -236165,11 +237441,13 @@ |
| 236165 | 237441 | fts5MultiIterSetEof(pIter); |
| 236166 | 237442 | *pbNewTerm = 1; |
| 236167 | 237443 | } |
| 236168 | 237444 | fts5AssertMultiIterSetup(p, pIter); |
| 236169 | 237445 | |
| 236170 | | - }while( fts5MultiIterIsEmpty(p, pIter) ); |
| 237446 | + }while( (fts5MultiIterIsEmpty(p, pIter) || fts5MultiIterIsDeleted(pIter)) |
| 237447 | + && (p->rc==SQLITE_OK) |
| 237448 | + ); |
| 236171 | 237449 | } |
| 236172 | 237450 | } |
| 236173 | 237451 | |
| 236174 | 237452 | static void fts5IterSetOutputs_Noop(Fts5Iter *pUnused1, Fts5SegIter *pUnused2){ |
| 236175 | 237453 | UNUSED_PARAM2(pUnused1, pUnused2); |
| | @@ -236720,11 +237998,13 @@ |
| 236720 | 237998 | } |
| 236721 | 237999 | } |
| 236722 | 238000 | fts5MultiIterSetEof(pNew); |
| 236723 | 238001 | fts5AssertMultiIterSetup(p, pNew); |
| 236724 | 238002 | |
| 236725 | | - if( pNew->bSkipEmpty && fts5MultiIterIsEmpty(p, pNew) ){ |
| 238003 | + if( (pNew->bSkipEmpty && fts5MultiIterIsEmpty(p, pNew)) |
| 238004 | + || fts5MultiIterIsDeleted(pNew) |
| 238005 | + ){ |
| 236726 | 238006 | fts5MultiIterNext(p, pNew, 0, 0); |
| 236727 | 238007 | }else if( pNew->base.bEof==0 ){ |
| 236728 | 238008 | Fts5SegIter *pSeg = &pNew->aSeg[pNew->aFirst[1].iFirst]; |
| 236729 | 238009 | pNew->xSetOutputs(pNew, pSeg); |
| 236730 | 238010 | } |
| | @@ -236898,11 +238178,13 @@ |
| 236898 | 238178 | static void fts5IndexDiscardData(Fts5Index *p){ |
| 236899 | 238179 | assert( p->pHash || p->nPendingData==0 ); |
| 236900 | 238180 | if( p->pHash ){ |
| 236901 | 238181 | sqlite3Fts5HashClear(p->pHash); |
| 236902 | 238182 | p->nPendingData = 0; |
| 238183 | + p->nPendingRow = 0; |
| 236903 | 238184 | } |
| 238185 | + p->nContentlessDelete = 0; |
| 236904 | 238186 | } |
| 236905 | 238187 | |
| 236906 | 238188 | /* |
| 236907 | 238189 | ** Return the size of the prefix, in bytes, that buffer |
| 236908 | 238190 | ** (pNew/<length-unknown>) shares with buffer (pOld/nOld). |
| | @@ -237535,10 +238817,16 @@ |
| 237535 | 238817 | pSeg->iSegid = iSegid; |
| 237536 | 238818 | pStruct->nSegment++; |
| 237537 | 238819 | |
| 237538 | 238820 | /* Read input from all segments in the input level */ |
| 237539 | 238821 | nInput = pLvl->nSeg; |
| 238822 | + |
| 238823 | + /* Set the range of origins that will go into the output segment. */ |
| 238824 | + if( pStruct->nOriginCntr>0 ){ |
| 238825 | + pSeg->iOrigin1 = pLvl->aSeg[0].iOrigin1; |
| 238826 | + pSeg->iOrigin2 = pLvl->aSeg[pLvl->nSeg-1].iOrigin2; |
| 238827 | + } |
| 237540 | 238828 | } |
| 237541 | 238829 | bOldest = (pLvlOut->nSeg==1 && pStruct->nLevel==iLvl+2); |
| 237542 | 238830 | |
| 237543 | 238831 | assert( iLvl>=0 ); |
| 237544 | 238832 | for(fts5MultiIterNew(p, pStruct, flags, 0, 0, 0, iLvl, nInput, &pIter); |
| | @@ -237594,12 +238882,15 @@ |
| 237594 | 238882 | assert( pIter!=0 || p->rc!=SQLITE_OK ); |
| 237595 | 238883 | if( fts5MultiIterEof(p, pIter) ){ |
| 237596 | 238884 | int i; |
| 237597 | 238885 | |
| 237598 | 238886 | /* Remove the redundant segments from the %_data table */ |
| 238887 | + assert( pSeg->nEntry==0 ); |
| 237599 | 238888 | for(i=0; i<nInput; i++){ |
| 237600 | | - fts5DataRemoveSegment(p, pLvl->aSeg[i].iSegid); |
| 238889 | + Fts5StructureSegment *pOld = &pLvl->aSeg[i]; |
| 238890 | + pSeg->nEntry += (pOld->nEntry - pOld->nEntryTombstone); |
| 238891 | + fts5DataRemoveSegment(p, pOld); |
| 237601 | 238892 | } |
| 237602 | 238893 | |
| 237603 | 238894 | /* Remove the redundant segments from the input level */ |
| 237604 | 238895 | if( pLvl->nSeg!=nInput ){ |
| 237605 | 238896 | int nMove = (pLvl->nSeg - nInput) * sizeof(Fts5StructureSegment); |
| | @@ -237620,10 +238911,47 @@ |
| 237620 | 238911 | |
| 237621 | 238912 | fts5MultiIterFree(pIter); |
| 237622 | 238913 | fts5BufferFree(&term); |
| 237623 | 238914 | if( pnRem ) *pnRem -= writer.nLeafWritten; |
| 237624 | 238915 | } |
| 238916 | + |
| 238917 | +/* |
| 238918 | +** If this is not a contentless_delete=1 table, or if the 'deletemerge' |
| 238919 | +** configuration option is set to 0, then this function always returns -1. |
| 238920 | +** Otherwise, it searches the structure object passed as the second argument |
| 238921 | +** for a level suitable for merging due to having a large number of |
| 238922 | +** tombstones in the tombstone hash. If one is found, its index is returned. |
| 238923 | +** Otherwise, if there is no suitable level, -1. |
| 238924 | +*/ |
| 238925 | +static int fts5IndexFindDeleteMerge(Fts5Index *p, Fts5Structure *pStruct){ |
| 238926 | + Fts5Config *pConfig = p->pConfig; |
| 238927 | + int iRet = -1; |
| 238928 | + if( pConfig->bContentlessDelete && pConfig->nDeleteMerge>0 ){ |
| 238929 | + int ii; |
| 238930 | + int nBest = 0; |
| 238931 | + |
| 238932 | + for(ii=0; ii<pStruct->nLevel; ii++){ |
| 238933 | + Fts5StructureLevel *pLvl = &pStruct->aLevel[ii]; |
| 238934 | + i64 nEntry = 0; |
| 238935 | + i64 nTomb = 0; |
| 238936 | + int iSeg; |
| 238937 | + for(iSeg=0; iSeg<pLvl->nSeg; iSeg++){ |
| 238938 | + nEntry += pLvl->aSeg[iSeg].nEntry; |
| 238939 | + nTomb += pLvl->aSeg[iSeg].nEntryTombstone; |
| 238940 | + } |
| 238941 | + assert_nc( nEntry>0 || pLvl->nSeg==0 ); |
| 238942 | + if( nEntry>0 ){ |
| 238943 | + int nPercent = (nTomb * 100) / nEntry; |
| 238944 | + if( nPercent>=pConfig->nDeleteMerge && nPercent>nBest ){ |
| 238945 | + iRet = ii; |
| 238946 | + nBest = nPercent; |
| 238947 | + } |
| 238948 | + } |
| 238949 | + } |
| 238950 | + } |
| 238951 | + return iRet; |
| 238952 | +} |
| 237625 | 238953 | |
| 237626 | 238954 | /* |
| 237627 | 238955 | ** Do up to nPg pages of automerge work on the index. |
| 237628 | 238956 | ** |
| 237629 | 238957 | ** Return true if any changes were actually made, or false otherwise. |
| | @@ -237640,42 +238968,39 @@ |
| 237640 | 238968 | while( nRem>0 && p->rc==SQLITE_OK ){ |
| 237641 | 238969 | int iLvl; /* To iterate through levels */ |
| 237642 | 238970 | int iBestLvl = 0; /* Level offering the most input segments */ |
| 237643 | 238971 | int nBest = 0; /* Number of input segments on best level */ |
| 237644 | 238972 | |
| 237645 | | - /* Set iBestLvl to the level to read input segments from. */ |
| 238973 | + /* Set iBestLvl to the level to read input segments from. Or to -1 if |
| 238974 | + ** there is no level suitable to merge segments from. */ |
| 237646 | 238975 | assert( pStruct->nLevel>0 ); |
| 237647 | 238976 | for(iLvl=0; iLvl<pStruct->nLevel; iLvl++){ |
| 237648 | 238977 | Fts5StructureLevel *pLvl = &pStruct->aLevel[iLvl]; |
| 237649 | 238978 | if( pLvl->nMerge ){ |
| 237650 | 238979 | if( pLvl->nMerge>nBest ){ |
| 237651 | 238980 | iBestLvl = iLvl; |
| 237652 | | - nBest = pLvl->nMerge; |
| 238981 | + nBest = nMin; |
| 237653 | 238982 | } |
| 237654 | 238983 | break; |
| 237655 | 238984 | } |
| 237656 | 238985 | if( pLvl->nSeg>nBest ){ |
| 237657 | 238986 | nBest = pLvl->nSeg; |
| 237658 | 238987 | iBestLvl = iLvl; |
| 237659 | 238988 | } |
| 237660 | 238989 | } |
| 237661 | | - |
| 237662 | | - /* If nBest is still 0, then the index must be empty. */ |
| 237663 | | -#ifdef SQLITE_DEBUG |
| 237664 | | - for(iLvl=0; nBest==0 && iLvl<pStruct->nLevel; iLvl++){ |
| 237665 | | - assert( pStruct->aLevel[iLvl].nSeg==0 ); |
| 237666 | | - } |
| 237667 | | -#endif |
| 237668 | | - |
| 237669 | | - if( nBest<nMin && pStruct->aLevel[iBestLvl].nMerge==0 ){ |
| 237670 | | - break; |
| 237671 | | - } |
| 238990 | + if( nBest<nMin ){ |
| 238991 | + iBestLvl = fts5IndexFindDeleteMerge(p, pStruct); |
| 238992 | + } |
| 238993 | + |
| 238994 | + if( iBestLvl<0 ) break; |
| 237672 | 238995 | bRet = 1; |
| 237673 | 238996 | fts5IndexMergeLevel(p, &pStruct, iBestLvl, &nRem); |
| 237674 | 238997 | if( p->rc==SQLITE_OK && pStruct->aLevel[iBestLvl].nMerge==0 ){ |
| 237675 | 238998 | fts5StructurePromote(p, iBestLvl+1, pStruct); |
| 237676 | 238999 | } |
| 239000 | + |
| 239001 | + if( nMin==1 ) nMin = 2; |
| 237677 | 239002 | } |
| 237678 | 239003 | *ppStruct = pStruct; |
| 237679 | 239004 | return bRet; |
| 237680 | 239005 | } |
| 237681 | 239006 | |
| | @@ -237856,13 +239181,17 @@ |
| 237856 | 239181 | if( pLeaf->nn>pLeaf->szLeaf ){ |
| 237857 | 239182 | int iFirst = 0; |
| 237858 | 239183 | int i1 = pLeaf->szLeaf; |
| 237859 | 239184 | int i2 = 0; |
| 237860 | 239185 | |
| 239186 | + i1 += fts5GetVarint32(&aPg[i1], iFirst); |
| 239187 | + if( iFirst<iNext ){ |
| 239188 | + p->rc = FTS5_CORRUPT; |
| 239189 | + break; |
| 239190 | + } |
| 237861 | 239191 | aIdx = sqlite3Fts5MallocZero(&p->rc, (pLeaf->nn-pLeaf->szLeaf)+2); |
| 237862 | 239192 | if( aIdx==0 ) break; |
| 237863 | | - i1 += fts5GetVarint32(&aPg[i1], iFirst); |
| 237864 | 239193 | i2 = sqlite3Fts5PutVarint(aIdx, iFirst-nShift); |
| 237865 | 239194 | if( i1<pLeaf->nn ){ |
| 237866 | 239195 | memcpy(&aIdx[i2], &aPg[i1], pLeaf->nn-i1); |
| 237867 | 239196 | i2 += (pLeaf->nn-i1); |
| 237868 | 239197 | } |
| | @@ -238180,202 +239509,212 @@ |
| 238180 | 239509 | int pgnoLast = 0; /* Last leaf page number in segment */ |
| 238181 | 239510 | |
| 238182 | 239511 | /* Obtain a reference to the index structure and allocate a new segment-id |
| 238183 | 239512 | ** for the new level-0 segment. */ |
| 238184 | 239513 | pStruct = fts5StructureRead(p); |
| 238185 | | - iSegid = fts5AllocateSegid(p, pStruct); |
| 238186 | | - fts5StructureInvalidate(p); |
| 238187 | | - |
| 238188 | | - if( iSegid ){ |
| 238189 | | - const int pgsz = p->pConfig->pgsz; |
| 238190 | | - int eDetail = p->pConfig->eDetail; |
| 238191 | | - int bSecureDelete = p->pConfig->bSecureDelete; |
| 238192 | | - Fts5StructureSegment *pSeg; /* New segment within pStruct */ |
| 238193 | | - Fts5Buffer *pBuf; /* Buffer in which to assemble leaf page */ |
| 238194 | | - Fts5Buffer *pPgidx; /* Buffer in which to assemble pgidx */ |
| 238195 | | - |
| 238196 | | - Fts5SegWriter writer; |
| 238197 | | - fts5WriteInit(p, &writer, iSegid); |
| 238198 | | - |
| 238199 | | - pBuf = &writer.writer.buf; |
| 238200 | | - pPgidx = &writer.writer.pgidx; |
| 238201 | | - |
| 238202 | | - /* fts5WriteInit() should have initialized the buffers to (most likely) |
| 238203 | | - ** the maximum space required. */ |
| 238204 | | - assert( p->rc || pBuf->nSpace>=(pgsz + FTS5_DATA_PADDING) ); |
| 238205 | | - assert( p->rc || pPgidx->nSpace>=(pgsz + FTS5_DATA_PADDING) ); |
| 238206 | | - |
| 238207 | | - /* Begin scanning through hash table entries. This loop runs once for each |
| 238208 | | - ** term/doclist currently stored within the hash table. */ |
| 238209 | | - if( p->rc==SQLITE_OK ){ |
| 238210 | | - p->rc = sqlite3Fts5HashScanInit(pHash, 0, 0); |
| 238211 | | - } |
| 238212 | | - while( p->rc==SQLITE_OK && 0==sqlite3Fts5HashScanEof(pHash) ){ |
| 238213 | | - const char *zTerm; /* Buffer containing term */ |
| 238214 | | - int nTerm; /* Size of zTerm in bytes */ |
| 238215 | | - const u8 *pDoclist; /* Pointer to doclist for this term */ |
| 238216 | | - int nDoclist; /* Size of doclist in bytes */ |
| 238217 | | - |
| 238218 | | - /* Get the term and doclist for this entry. */ |
| 238219 | | - sqlite3Fts5HashScanEntry(pHash, &zTerm, &pDoclist, &nDoclist); |
| 238220 | | - nTerm = (int)strlen(zTerm); |
| 238221 | | - if( bSecureDelete==0 ){ |
| 238222 | | - fts5WriteAppendTerm(p, &writer, nTerm, (const u8*)zTerm); |
| 238223 | | - if( p->rc!=SQLITE_OK ) break; |
| 238224 | | - assert( writer.bFirstRowidInPage==0 ); |
| 238225 | | - } |
| 238226 | | - |
| 238227 | | - if( !bSecureDelete && pgsz>=(pBuf->n + pPgidx->n + nDoclist + 1) ){ |
| 238228 | | - /* The entire doclist will fit on the current leaf. */ |
| 238229 | | - fts5BufferSafeAppendBlob(pBuf, pDoclist, nDoclist); |
| 238230 | | - }else{ |
| 238231 | | - int bTermWritten = !bSecureDelete; |
| 238232 | | - i64 iRowid = 0; |
| 238233 | | - i64 iPrev = 0; |
| 238234 | | - int iOff = 0; |
| 238235 | | - |
| 238236 | | - /* The entire doclist will not fit on this leaf. The following |
| 238237 | | - ** loop iterates through the poslists that make up the current |
| 238238 | | - ** doclist. */ |
| 238239 | | - while( p->rc==SQLITE_OK && iOff<nDoclist ){ |
| 238240 | | - u64 iDelta = 0; |
| 238241 | | - iOff += fts5GetVarint(&pDoclist[iOff], &iDelta); |
| 238242 | | - iRowid += iDelta; |
| 238243 | | - |
| 238244 | | - /* If in secure delete mode, and if this entry in the poslist is |
| 238245 | | - ** in fact a delete, then edit the existing segments directly |
| 238246 | | - ** using fts5FlushSecureDelete(). */ |
| 238247 | | - if( bSecureDelete ){ |
| 238248 | | - if( eDetail==FTS5_DETAIL_NONE ){ |
| 238249 | | - if( iOff<nDoclist && pDoclist[iOff]==0x00 ){ |
| 238250 | | - fts5FlushSecureDelete(p, pStruct, zTerm, iRowid); |
| 238251 | | - iOff++; |
| 238252 | | - if( iOff<nDoclist && pDoclist[iOff]==0x00 ){ |
| 238253 | | - iOff++; |
| 238254 | | - nDoclist = 0; |
| 238255 | | - }else{ |
| 238256 | | - continue; |
| 238257 | | - } |
| 238258 | | - } |
| 238259 | | - }else if( (pDoclist[iOff] & 0x01) ){ |
| 238260 | | - fts5FlushSecureDelete(p, pStruct, zTerm, iRowid); |
| 238261 | | - if( p->rc!=SQLITE_OK || pDoclist[iOff]==0x01 ){ |
| 238262 | | - iOff++; |
| 238263 | | - continue; |
| 238264 | | - } |
| 238265 | | - } |
| 238266 | | - } |
| 238267 | | - |
| 238268 | | - if( p->rc==SQLITE_OK && bTermWritten==0 ){ |
| 238269 | | - fts5WriteAppendTerm(p, &writer, nTerm, (const u8*)zTerm); |
| 238270 | | - bTermWritten = 1; |
| 238271 | | - assert( p->rc!=SQLITE_OK || writer.bFirstRowidInPage==0 ); |
| 238272 | | - } |
| 238273 | | - |
| 238274 | | - if( writer.bFirstRowidInPage ){ |
| 238275 | | - fts5PutU16(&pBuf->p[0], (u16)pBuf->n); /* first rowid on page */ |
| 238276 | | - pBuf->n += sqlite3Fts5PutVarint(&pBuf->p[pBuf->n], iRowid); |
| 238277 | | - writer.bFirstRowidInPage = 0; |
| 238278 | | - fts5WriteDlidxAppend(p, &writer, iRowid); |
| 238279 | | - }else{ |
| 238280 | | - pBuf->n += sqlite3Fts5PutVarint(&pBuf->p[pBuf->n], iRowid-iPrev); |
| 238281 | | - } |
| 238282 | | - if( p->rc!=SQLITE_OK ) break; |
| 238283 | | - assert( pBuf->n<=pBuf->nSpace ); |
| 238284 | | - iPrev = iRowid; |
| 238285 | | - |
| 238286 | | - if( eDetail==FTS5_DETAIL_NONE ){ |
| 238287 | | - if( iOff<nDoclist && pDoclist[iOff]==0 ){ |
| 238288 | | - pBuf->p[pBuf->n++] = 0; |
| 238289 | | - iOff++; |
| 238290 | | - if( iOff<nDoclist && pDoclist[iOff]==0 ){ |
| 238291 | | - pBuf->p[pBuf->n++] = 0; |
| 238292 | | - iOff++; |
| 238293 | | - } |
| 238294 | | - } |
| 238295 | | - if( (pBuf->n + pPgidx->n)>=pgsz ){ |
| 238296 | | - fts5WriteFlushLeaf(p, &writer); |
| 238297 | | - } |
| 238298 | | - }else{ |
| 238299 | | - int bDummy; |
| 238300 | | - int nPos; |
| 238301 | | - int nCopy = fts5GetPoslistSize(&pDoclist[iOff], &nPos, &bDummy); |
| 238302 | | - nCopy += nPos; |
| 238303 | | - if( (pBuf->n + pPgidx->n + nCopy) <= pgsz ){ |
| 238304 | | - /* The entire poslist will fit on the current leaf. So copy |
| 238305 | | - ** it in one go. */ |
| 238306 | | - fts5BufferSafeAppendBlob(pBuf, &pDoclist[iOff], nCopy); |
| 238307 | | - }else{ |
| 238308 | | - /* The entire poslist will not fit on this leaf. So it needs |
| 238309 | | - ** to be broken into sections. The only qualification being |
| 238310 | | - ** that each varint must be stored contiguously. */ |
| 238311 | | - const u8 *pPoslist = &pDoclist[iOff]; |
| 238312 | | - int iPos = 0; |
| 238313 | | - while( p->rc==SQLITE_OK ){ |
| 238314 | | - int nSpace = pgsz - pBuf->n - pPgidx->n; |
| 238315 | | - int n = 0; |
| 238316 | | - if( (nCopy - iPos)<=nSpace ){ |
| 238317 | | - n = nCopy - iPos; |
| 238318 | | - }else{ |
| 238319 | | - n = fts5PoslistPrefix(&pPoslist[iPos], nSpace); |
| 238320 | | - } |
| 238321 | | - assert( n>0 ); |
| 238322 | | - fts5BufferSafeAppendBlob(pBuf, &pPoslist[iPos], n); |
| 238323 | | - iPos += n; |
| 238324 | | - if( (pBuf->n + pPgidx->n)>=pgsz ){ |
| 238325 | | - fts5WriteFlushLeaf(p, &writer); |
| 238326 | | - } |
| 238327 | | - if( iPos>=nCopy ) break; |
| 238328 | | - } |
| 238329 | | - } |
| 238330 | | - iOff += nCopy; |
| 238331 | | - } |
| 238332 | | - } |
| 238333 | | - } |
| 238334 | | - |
| 238335 | | - /* TODO2: Doclist terminator written here. */ |
| 238336 | | - /* pBuf->p[pBuf->n++] = '\0'; */ |
| 238337 | | - assert( pBuf->n<=pBuf->nSpace ); |
| 238338 | | - if( p->rc==SQLITE_OK ) sqlite3Fts5HashScanNext(pHash); |
| 238339 | | - } |
| 238340 | | - sqlite3Fts5HashClear(pHash); |
| 238341 | | - fts5WriteFinish(p, &writer, &pgnoLast); |
| 238342 | | - |
| 238343 | | - assert( p->rc!=SQLITE_OK || bSecureDelete || pgnoLast>0 ); |
| 238344 | | - if( pgnoLast>0 ){ |
| 238345 | | - /* Update the Fts5Structure. It is written back to the database by the |
| 238346 | | - ** fts5StructureRelease() call below. */ |
| 238347 | | - if( pStruct->nLevel==0 ){ |
| 238348 | | - fts5StructureAddLevel(&p->rc, &pStruct); |
| 238349 | | - } |
| 238350 | | - fts5StructureExtendLevel(&p->rc, pStruct, 0, 1, 0); |
| 238351 | | - if( p->rc==SQLITE_OK ){ |
| 238352 | | - pSeg = &pStruct->aLevel[0].aSeg[ pStruct->aLevel[0].nSeg++ ]; |
| 238353 | | - pSeg->iSegid = iSegid; |
| 238354 | | - pSeg->pgnoFirst = 1; |
| 238355 | | - pSeg->pgnoLast = pgnoLast; |
| 238356 | | - pStruct->nSegment++; |
| 238357 | | - } |
| 238358 | | - fts5StructurePromote(p, 0, pStruct); |
| 238359 | | - } |
| 238360 | | - } |
| 238361 | | - |
| 238362 | | - fts5IndexAutomerge(p, &pStruct, pgnoLast); |
| 238363 | | - fts5IndexCrisismerge(p, &pStruct); |
| 238364 | | - fts5StructureWrite(p, pStruct); |
| 238365 | | - fts5StructureRelease(pStruct); |
| 239514 | + fts5StructureInvalidate(p); |
| 239515 | + |
| 239516 | + if( sqlite3Fts5HashIsEmpty(pHash)==0 ){ |
| 239517 | + iSegid = fts5AllocateSegid(p, pStruct); |
| 239518 | + if( iSegid ){ |
| 239519 | + const int pgsz = p->pConfig->pgsz; |
| 239520 | + int eDetail = p->pConfig->eDetail; |
| 239521 | + int bSecureDelete = p->pConfig->bSecureDelete; |
| 239522 | + Fts5StructureSegment *pSeg; /* New segment within pStruct */ |
| 239523 | + Fts5Buffer *pBuf; /* Buffer in which to assemble leaf page */ |
| 239524 | + Fts5Buffer *pPgidx; /* Buffer in which to assemble pgidx */ |
| 239525 | + |
| 239526 | + Fts5SegWriter writer; |
| 239527 | + fts5WriteInit(p, &writer, iSegid); |
| 239528 | + |
| 239529 | + pBuf = &writer.writer.buf; |
| 239530 | + pPgidx = &writer.writer.pgidx; |
| 239531 | + |
| 239532 | + /* fts5WriteInit() should have initialized the buffers to (most likely) |
| 239533 | + ** the maximum space required. */ |
| 239534 | + assert( p->rc || pBuf->nSpace>=(pgsz + FTS5_DATA_PADDING) ); |
| 239535 | + assert( p->rc || pPgidx->nSpace>=(pgsz + FTS5_DATA_PADDING) ); |
| 239536 | + |
| 239537 | + /* Begin scanning through hash table entries. This loop runs once for each |
| 239538 | + ** term/doclist currently stored within the hash table. */ |
| 239539 | + if( p->rc==SQLITE_OK ){ |
| 239540 | + p->rc = sqlite3Fts5HashScanInit(pHash, 0, 0); |
| 239541 | + } |
| 239542 | + while( p->rc==SQLITE_OK && 0==sqlite3Fts5HashScanEof(pHash) ){ |
| 239543 | + const char *zTerm; /* Buffer containing term */ |
| 239544 | + int nTerm; /* Size of zTerm in bytes */ |
| 239545 | + const u8 *pDoclist; /* Pointer to doclist for this term */ |
| 239546 | + int nDoclist; /* Size of doclist in bytes */ |
| 239547 | + |
| 239548 | + /* Get the term and doclist for this entry. */ |
| 239549 | + sqlite3Fts5HashScanEntry(pHash, &zTerm, &pDoclist, &nDoclist); |
| 239550 | + nTerm = (int)strlen(zTerm); |
| 239551 | + if( bSecureDelete==0 ){ |
| 239552 | + fts5WriteAppendTerm(p, &writer, nTerm, (const u8*)zTerm); |
| 239553 | + if( p->rc!=SQLITE_OK ) break; |
| 239554 | + assert( writer.bFirstRowidInPage==0 ); |
| 239555 | + } |
| 239556 | + |
| 239557 | + if( !bSecureDelete && pgsz>=(pBuf->n + pPgidx->n + nDoclist + 1) ){ |
| 239558 | + /* The entire doclist will fit on the current leaf. */ |
| 239559 | + fts5BufferSafeAppendBlob(pBuf, pDoclist, nDoclist); |
| 239560 | + }else{ |
| 239561 | + int bTermWritten = !bSecureDelete; |
| 239562 | + i64 iRowid = 0; |
| 239563 | + i64 iPrev = 0; |
| 239564 | + int iOff = 0; |
| 239565 | + |
| 239566 | + /* The entire doclist will not fit on this leaf. The following |
| 239567 | + ** loop iterates through the poslists that make up the current |
| 239568 | + ** doclist. */ |
| 239569 | + while( p->rc==SQLITE_OK && iOff<nDoclist ){ |
| 239570 | + u64 iDelta = 0; |
| 239571 | + iOff += fts5GetVarint(&pDoclist[iOff], &iDelta); |
| 239572 | + iRowid += iDelta; |
| 239573 | + |
| 239574 | + /* If in secure delete mode, and if this entry in the poslist is |
| 239575 | + ** in fact a delete, then edit the existing segments directly |
| 239576 | + ** using fts5FlushSecureDelete(). */ |
| 239577 | + if( bSecureDelete ){ |
| 239578 | + if( eDetail==FTS5_DETAIL_NONE ){ |
| 239579 | + if( iOff<nDoclist && pDoclist[iOff]==0x00 ){ |
| 239580 | + fts5FlushSecureDelete(p, pStruct, zTerm, iRowid); |
| 239581 | + iOff++; |
| 239582 | + if( iOff<nDoclist && pDoclist[iOff]==0x00 ){ |
| 239583 | + iOff++; |
| 239584 | + nDoclist = 0; |
| 239585 | + }else{ |
| 239586 | + continue; |
| 239587 | + } |
| 239588 | + } |
| 239589 | + }else if( (pDoclist[iOff] & 0x01) ){ |
| 239590 | + fts5FlushSecureDelete(p, pStruct, zTerm, iRowid); |
| 239591 | + if( p->rc!=SQLITE_OK || pDoclist[iOff]==0x01 ){ |
| 239592 | + iOff++; |
| 239593 | + continue; |
| 239594 | + } |
| 239595 | + } |
| 239596 | + } |
| 239597 | + |
| 239598 | + if( p->rc==SQLITE_OK && bTermWritten==0 ){ |
| 239599 | + fts5WriteAppendTerm(p, &writer, nTerm, (const u8*)zTerm); |
| 239600 | + bTermWritten = 1; |
| 239601 | + assert( p->rc!=SQLITE_OK || writer.bFirstRowidInPage==0 ); |
| 239602 | + } |
| 239603 | + |
| 239604 | + if( writer.bFirstRowidInPage ){ |
| 239605 | + fts5PutU16(&pBuf->p[0], (u16)pBuf->n); /* first rowid on page */ |
| 239606 | + pBuf->n += sqlite3Fts5PutVarint(&pBuf->p[pBuf->n], iRowid); |
| 239607 | + writer.bFirstRowidInPage = 0; |
| 239608 | + fts5WriteDlidxAppend(p, &writer, iRowid); |
| 239609 | + }else{ |
| 239610 | + pBuf->n += sqlite3Fts5PutVarint(&pBuf->p[pBuf->n], iRowid-iPrev); |
| 239611 | + } |
| 239612 | + if( p->rc!=SQLITE_OK ) break; |
| 239613 | + assert( pBuf->n<=pBuf->nSpace ); |
| 239614 | + iPrev = iRowid; |
| 239615 | + |
| 239616 | + if( eDetail==FTS5_DETAIL_NONE ){ |
| 239617 | + if( iOff<nDoclist && pDoclist[iOff]==0 ){ |
| 239618 | + pBuf->p[pBuf->n++] = 0; |
| 239619 | + iOff++; |
| 239620 | + if( iOff<nDoclist && pDoclist[iOff]==0 ){ |
| 239621 | + pBuf->p[pBuf->n++] = 0; |
| 239622 | + iOff++; |
| 239623 | + } |
| 239624 | + } |
| 239625 | + if( (pBuf->n + pPgidx->n)>=pgsz ){ |
| 239626 | + fts5WriteFlushLeaf(p, &writer); |
| 239627 | + } |
| 239628 | + }else{ |
| 239629 | + int bDummy; |
| 239630 | + int nPos; |
| 239631 | + int nCopy = fts5GetPoslistSize(&pDoclist[iOff], &nPos, &bDummy); |
| 239632 | + nCopy += nPos; |
| 239633 | + if( (pBuf->n + pPgidx->n + nCopy) <= pgsz ){ |
| 239634 | + /* The entire poslist will fit on the current leaf. So copy |
| 239635 | + ** it in one go. */ |
| 239636 | + fts5BufferSafeAppendBlob(pBuf, &pDoclist[iOff], nCopy); |
| 239637 | + }else{ |
| 239638 | + /* The entire poslist will not fit on this leaf. So it needs |
| 239639 | + ** to be broken into sections. The only qualification being |
| 239640 | + ** that each varint must be stored contiguously. */ |
| 239641 | + const u8 *pPoslist = &pDoclist[iOff]; |
| 239642 | + int iPos = 0; |
| 239643 | + while( p->rc==SQLITE_OK ){ |
| 239644 | + int nSpace = pgsz - pBuf->n - pPgidx->n; |
| 239645 | + int n = 0; |
| 239646 | + if( (nCopy - iPos)<=nSpace ){ |
| 239647 | + n = nCopy - iPos; |
| 239648 | + }else{ |
| 239649 | + n = fts5PoslistPrefix(&pPoslist[iPos], nSpace); |
| 239650 | + } |
| 239651 | + assert( n>0 ); |
| 239652 | + fts5BufferSafeAppendBlob(pBuf, &pPoslist[iPos], n); |
| 239653 | + iPos += n; |
| 239654 | + if( (pBuf->n + pPgidx->n)>=pgsz ){ |
| 239655 | + fts5WriteFlushLeaf(p, &writer); |
| 239656 | + } |
| 239657 | + if( iPos>=nCopy ) break; |
| 239658 | + } |
| 239659 | + } |
| 239660 | + iOff += nCopy; |
| 239661 | + } |
| 239662 | + } |
| 239663 | + } |
| 239664 | + |
| 239665 | + /* TODO2: Doclist terminator written here. */ |
| 239666 | + /* pBuf->p[pBuf->n++] = '\0'; */ |
| 239667 | + assert( pBuf->n<=pBuf->nSpace ); |
| 239668 | + if( p->rc==SQLITE_OK ) sqlite3Fts5HashScanNext(pHash); |
| 239669 | + } |
| 239670 | + sqlite3Fts5HashClear(pHash); |
| 239671 | + fts5WriteFinish(p, &writer, &pgnoLast); |
| 239672 | + |
| 239673 | + assert( p->rc!=SQLITE_OK || bSecureDelete || pgnoLast>0 ); |
| 239674 | + if( pgnoLast>0 ){ |
| 239675 | + /* Update the Fts5Structure. It is written back to the database by the |
| 239676 | + ** fts5StructureRelease() call below. */ |
| 239677 | + if( pStruct->nLevel==0 ){ |
| 239678 | + fts5StructureAddLevel(&p->rc, &pStruct); |
| 239679 | + } |
| 239680 | + fts5StructureExtendLevel(&p->rc, pStruct, 0, 1, 0); |
| 239681 | + if( p->rc==SQLITE_OK ){ |
| 239682 | + pSeg = &pStruct->aLevel[0].aSeg[ pStruct->aLevel[0].nSeg++ ]; |
| 239683 | + pSeg->iSegid = iSegid; |
| 239684 | + pSeg->pgnoFirst = 1; |
| 239685 | + pSeg->pgnoLast = pgnoLast; |
| 239686 | + if( pStruct->nOriginCntr>0 ){ |
| 239687 | + pSeg->iOrigin1 = pStruct->nOriginCntr; |
| 239688 | + pSeg->iOrigin2 = pStruct->nOriginCntr; |
| 239689 | + pSeg->nEntry = p->nPendingRow; |
| 239690 | + pStruct->nOriginCntr++; |
| 239691 | + } |
| 239692 | + pStruct->nSegment++; |
| 239693 | + } |
| 239694 | + fts5StructurePromote(p, 0, pStruct); |
| 239695 | + } |
| 239696 | + } |
| 239697 | + } |
| 239698 | + |
| 239699 | + fts5IndexAutomerge(p, &pStruct, pgnoLast + p->nContentlessDelete); |
| 239700 | + fts5IndexCrisismerge(p, &pStruct); |
| 239701 | + fts5StructureWrite(p, pStruct); |
| 239702 | + fts5StructureRelease(pStruct); |
| 239703 | + p->nContentlessDelete = 0; |
| 238366 | 239704 | } |
| 238367 | 239705 | |
| 238368 | 239706 | /* |
| 238369 | 239707 | ** Flush any data stored in the in-memory hash tables to the database. |
| 238370 | 239708 | */ |
| 238371 | 239709 | static void fts5IndexFlush(Fts5Index *p){ |
| 238372 | 239710 | /* Unless it is empty, flush the hash table to disk */ |
| 238373 | | - if( p->nPendingData ){ |
| 239711 | + if( p->nPendingData || p->nContentlessDelete ){ |
| 238374 | 239712 | assert( p->pHash ); |
| 238375 | | - p->nPendingData = 0; |
| 238376 | 239713 | fts5FlushOneHash(p); |
| 239714 | + p->nPendingData = 0; |
| 239715 | + p->nPendingRow = 0; |
| 238377 | 239716 | } |
| 238378 | 239717 | } |
| 238379 | 239718 | |
| 238380 | 239719 | static Fts5Structure *fts5IndexOptimizeStruct( |
| 238381 | 239720 | Fts5Index *p, |
| | @@ -238387,21 +239726,26 @@ |
| 238387 | 239726 | int i; |
| 238388 | 239727 | |
| 238389 | 239728 | /* Figure out if this structure requires optimization. A structure does |
| 238390 | 239729 | ** not require optimization if either: |
| 238391 | 239730 | ** |
| 238392 | | - ** + it consists of fewer than two segments, or |
| 238393 | | - ** + all segments are on the same level, or |
| 238394 | | - ** + all segments except one are currently inputs to a merge operation. |
| 239731 | + ** 1. it consists of fewer than two segments, or |
| 239732 | + ** 2. all segments are on the same level, or |
| 239733 | + ** 3. all segments except one are currently inputs to a merge operation. |
| 238395 | 239734 | ** |
| 238396 | | - ** In the first case, return NULL. In the second, increment the ref-count |
| 238397 | | - ** on *pStruct and return a copy of the pointer to it. |
| 239735 | + ** In the first case, if there are no tombstone hash pages, return NULL. In |
| 239736 | + ** the second, increment the ref-count on *pStruct and return a copy of the |
| 239737 | + ** pointer to it. |
| 238398 | 239738 | */ |
| 238399 | | - if( nSeg<2 ) return 0; |
| 239739 | + if( nSeg==0 ) return 0; |
| 238400 | 239740 | for(i=0; i<pStruct->nLevel; i++){ |
| 238401 | 239741 | int nThis = pStruct->aLevel[i].nSeg; |
| 238402 | | - if( nThis==nSeg || (nThis==nSeg-1 && pStruct->aLevel[i].nMerge==nThis) ){ |
| 239742 | + int nMerge = pStruct->aLevel[i].nMerge; |
| 239743 | + if( nThis>0 && (nThis==nSeg || (nThis==nSeg-1 && nMerge==nThis)) ){ |
| 239744 | + if( nSeg==1 && nThis==1 && pStruct->aLevel[i].aSeg[0].nPgTombstone==0 ){ |
| 239745 | + return 0; |
| 239746 | + } |
| 238403 | 239747 | fts5StructureRef(pStruct); |
| 238404 | 239748 | return pStruct; |
| 238405 | 239749 | } |
| 238406 | 239750 | assert( pStruct->aLevel[i].nMerge<=nThis ); |
| 238407 | 239751 | } |
| | @@ -238413,10 +239757,11 @@ |
| 238413 | 239757 | Fts5StructureLevel *pLvl; |
| 238414 | 239758 | nByte = nSeg * sizeof(Fts5StructureSegment); |
| 238415 | 239759 | pNew->nLevel = MIN(pStruct->nLevel+1, FTS5_MAX_LEVEL); |
| 238416 | 239760 | pNew->nRef = 1; |
| 238417 | 239761 | pNew->nWriteCounter = pStruct->nWriteCounter; |
| 239762 | + pNew->nOriginCntr = pStruct->nOriginCntr; |
| 238418 | 239763 | pLvl = &pNew->aLevel[pNew->nLevel-1]; |
| 238419 | 239764 | pLvl->aSeg = (Fts5StructureSegment*)sqlite3Fts5MallocZero(&p->rc, nByte); |
| 238420 | 239765 | if( pLvl->aSeg ){ |
| 238421 | 239766 | int iLvl, iSeg; |
| 238422 | 239767 | int iSegOut = 0; |
| | @@ -238443,10 +239788,11 @@ |
| 238443 | 239788 | Fts5Structure *pStruct; |
| 238444 | 239789 | Fts5Structure *pNew = 0; |
| 238445 | 239790 | |
| 238446 | 239791 | assert( p->rc==SQLITE_OK ); |
| 238447 | 239792 | fts5IndexFlush(p); |
| 239793 | + assert( p->nContentlessDelete==0 ); |
| 238448 | 239794 | pStruct = fts5StructureRead(p); |
| 238449 | 239795 | fts5StructureInvalidate(p); |
| 238450 | 239796 | |
| 238451 | 239797 | if( pStruct ){ |
| 238452 | 239798 | pNew = fts5IndexOptimizeStruct(p, pStruct); |
| | @@ -238472,19 +239818,22 @@ |
| 238472 | 239818 | /* |
| 238473 | 239819 | ** This is called to implement the special "VALUES('merge', $nMerge)" |
| 238474 | 239820 | ** INSERT command. |
| 238475 | 239821 | */ |
| 238476 | 239822 | static int sqlite3Fts5IndexMerge(Fts5Index *p, int nMerge){ |
| 238477 | | - Fts5Structure *pStruct = fts5StructureRead(p); |
| 239823 | + Fts5Structure *pStruct = 0; |
| 239824 | + |
| 239825 | + fts5IndexFlush(p); |
| 239826 | + pStruct = fts5StructureRead(p); |
| 238478 | 239827 | if( pStruct ){ |
| 238479 | 239828 | int nMin = p->pConfig->nUsermerge; |
| 238480 | 239829 | fts5StructureInvalidate(p); |
| 238481 | 239830 | if( nMerge<0 ){ |
| 238482 | 239831 | Fts5Structure *pNew = fts5IndexOptimizeStruct(p, pStruct); |
| 238483 | 239832 | fts5StructureRelease(pStruct); |
| 238484 | 239833 | pStruct = pNew; |
| 238485 | | - nMin = 2; |
| 239834 | + nMin = 1; |
| 238486 | 239835 | nMerge = nMerge*-1; |
| 238487 | 239836 | } |
| 238488 | 239837 | if( pStruct && pStruct->nLevel ){ |
| 238489 | 239838 | if( fts5IndexMerge(p, &pStruct, nMerge, nMin) ){ |
| 238490 | 239839 | fts5StructureWrite(p, pStruct); |
| | @@ -238994,10 +240343,13 @@ |
| 238994 | 240343 | fts5IndexFlush(p); |
| 238995 | 240344 | } |
| 238996 | 240345 | |
| 238997 | 240346 | p->iWriteRowid = iRowid; |
| 238998 | 240347 | p->bDelete = bDelete; |
| 240348 | + if( bDelete==0 ){ |
| 240349 | + p->nPendingRow++; |
| 240350 | + } |
| 238999 | 240351 | return fts5IndexReturn(p); |
| 239000 | 240352 | } |
| 239001 | 240353 | |
| 239002 | 240354 | /* |
| 239003 | 240355 | ** Commit data to disk. |
| | @@ -239031,10 +240383,13 @@ |
| 239031 | 240383 | static int sqlite3Fts5IndexReinit(Fts5Index *p){ |
| 239032 | 240384 | Fts5Structure s; |
| 239033 | 240385 | fts5StructureInvalidate(p); |
| 239034 | 240386 | fts5IndexDiscardData(p); |
| 239035 | 240387 | memset(&s, 0, sizeof(Fts5Structure)); |
| 240388 | + if( p->pConfig->bContentlessDelete ){ |
| 240389 | + s.nOriginCntr = 1; |
| 240390 | + } |
| 239036 | 240391 | fts5DataWrite(p, FTS5_AVERAGES_ROWID, (const u8*)"", 0); |
| 239037 | 240392 | fts5StructureWrite(p, &s); |
| 239038 | 240393 | return fts5IndexReturn(p); |
| 239039 | 240394 | } |
| 239040 | 240395 | |
| | @@ -239422,10 +240777,351 @@ |
| 239422 | 240777 | pStruct = fts5StructureRead(p); |
| 239423 | 240778 | fts5StructureRelease(pStruct); |
| 239424 | 240779 | return fts5IndexReturn(p); |
| 239425 | 240780 | } |
| 239426 | 240781 | |
| 240782 | +/* |
| 240783 | +** Retrieve the origin value that will be used for the segment currently |
| 240784 | +** being accumulated in the in-memory hash table when it is flushed to |
| 240785 | +** disk. If successful, SQLITE_OK is returned and (*piOrigin) set to |
| 240786 | +** the queried value. Or, if an error occurs, an error code is returned |
| 240787 | +** and the final value of (*piOrigin) is undefined. |
| 240788 | +*/ |
| 240789 | +static int sqlite3Fts5IndexGetOrigin(Fts5Index *p, i64 *piOrigin){ |
| 240790 | + Fts5Structure *pStruct; |
| 240791 | + pStruct = fts5StructureRead(p); |
| 240792 | + if( pStruct ){ |
| 240793 | + *piOrigin = pStruct->nOriginCntr; |
| 240794 | + fts5StructureRelease(pStruct); |
| 240795 | + } |
| 240796 | + return fts5IndexReturn(p); |
| 240797 | +} |
| 240798 | + |
| 240799 | +/* |
| 240800 | +** Buffer pPg contains a page of a tombstone hash table - one of nPg pages |
| 240801 | +** associated with the same segment. This function adds rowid iRowid to |
| 240802 | +** the hash table. The caller is required to guarantee that there is at |
| 240803 | +** least one free slot on the page. |
| 240804 | +** |
| 240805 | +** If parameter bForce is false and the hash table is deemed to be full |
| 240806 | +** (more than half of the slots are occupied), then non-zero is returned |
| 240807 | +** and iRowid not inserted. Or, if bForce is true or if the hash table page |
| 240808 | +** is not full, iRowid is inserted and zero returned. |
| 240809 | +*/ |
| 240810 | +static int fts5IndexTombstoneAddToPage( |
| 240811 | + Fts5Data *pPg, |
| 240812 | + int bForce, |
| 240813 | + int nPg, |
| 240814 | + u64 iRowid |
| 240815 | +){ |
| 240816 | + const int szKey = TOMBSTONE_KEYSIZE(pPg); |
| 240817 | + const int nSlot = TOMBSTONE_NSLOT(pPg); |
| 240818 | + const int nElem = fts5GetU32(&pPg->p[4]); |
| 240819 | + int iSlot = (iRowid / nPg) % nSlot; |
| 240820 | + int nCollide = nSlot; |
| 240821 | + |
| 240822 | + if( szKey==4 && iRowid>0xFFFFFFFF ) return 2; |
| 240823 | + if( iRowid==0 ){ |
| 240824 | + pPg->p[1] = 0x01; |
| 240825 | + return 0; |
| 240826 | + } |
| 240827 | + |
| 240828 | + if( bForce==0 && nElem>=(nSlot/2) ){ |
| 240829 | + return 1; |
| 240830 | + } |
| 240831 | + |
| 240832 | + fts5PutU32(&pPg->p[4], nElem+1); |
| 240833 | + if( szKey==4 ){ |
| 240834 | + u32 *aSlot = (u32*)&pPg->p[8]; |
| 240835 | + while( aSlot[iSlot] ){ |
| 240836 | + iSlot = (iSlot + 1) % nSlot; |
| 240837 | + if( nCollide--==0 ) return 0; |
| 240838 | + } |
| 240839 | + fts5PutU32((u8*)&aSlot[iSlot], (u32)iRowid); |
| 240840 | + }else{ |
| 240841 | + u64 *aSlot = (u64*)&pPg->p[8]; |
| 240842 | + while( aSlot[iSlot] ){ |
| 240843 | + iSlot = (iSlot + 1) % nSlot; |
| 240844 | + if( nCollide--==0 ) return 0; |
| 240845 | + } |
| 240846 | + fts5PutU64((u8*)&aSlot[iSlot], iRowid); |
| 240847 | + } |
| 240848 | + |
| 240849 | + return 0; |
| 240850 | +} |
| 240851 | + |
| 240852 | +/* |
| 240853 | +** This function attempts to build a new hash containing all the keys |
| 240854 | +** currently in the tombstone hash table for segment pSeg. The new |
| 240855 | +** hash will be stored in the nOut buffers passed in array apOut[]. |
| 240856 | +** All pages of the new hash use key-size szKey (4 or 8). |
| 240857 | +** |
| 240858 | +** Return 0 if the hash is successfully rebuilt into the nOut pages. |
| 240859 | +** Or non-zero if it is not (because one page became overfull). In this |
| 240860 | +** case the caller should retry with a larger nOut parameter. |
| 240861 | +** |
| 240862 | +** Parameter pData1 is page iPg1 of the hash table being rebuilt. |
| 240863 | +*/ |
| 240864 | +static int fts5IndexTombstoneRehash( |
| 240865 | + Fts5Index *p, |
| 240866 | + Fts5StructureSegment *pSeg, /* Segment to rebuild hash of */ |
| 240867 | + Fts5Data *pData1, /* One page of current hash - or NULL */ |
| 240868 | + int iPg1, /* Which page of the current hash is pData1 */ |
| 240869 | + int szKey, /* 4 or 8, the keysize */ |
| 240870 | + int nOut, /* Number of output pages */ |
| 240871 | + Fts5Data **apOut /* Array of output hash pages */ |
| 240872 | +){ |
| 240873 | + int ii; |
| 240874 | + int res = 0; |
| 240875 | + |
| 240876 | + /* Initialize the headers of all the output pages */ |
| 240877 | + for(ii=0; ii<nOut; ii++){ |
| 240878 | + apOut[ii]->p[0] = szKey; |
| 240879 | + fts5PutU32(&apOut[ii]->p[4], 0); |
| 240880 | + } |
| 240881 | + |
| 240882 | + /* Loop through the current pages of the hash table. */ |
| 240883 | + for(ii=0; res==0 && ii<pSeg->nPgTombstone; ii++){ |
| 240884 | + Fts5Data *pData = 0; /* Page ii of the current hash table */ |
| 240885 | + Fts5Data *pFree = 0; /* Free this at the end of the loop */ |
| 240886 | + |
| 240887 | + if( iPg1==ii ){ |
| 240888 | + pData = pData1; |
| 240889 | + }else{ |
| 240890 | + pFree = pData = fts5DataRead(p, FTS5_TOMBSTONE_ROWID(pSeg->iSegid, ii)); |
| 240891 | + } |
| 240892 | + |
| 240893 | + if( pData ){ |
| 240894 | + int szKeyIn = TOMBSTONE_KEYSIZE(pData); |
| 240895 | + int nSlotIn = (pData->nn - 8) / szKeyIn; |
| 240896 | + int iIn; |
| 240897 | + for(iIn=0; iIn<nSlotIn; iIn++){ |
| 240898 | + u64 iVal = 0; |
| 240899 | + |
| 240900 | + /* Read the value from slot iIn of the input page into iVal. */ |
| 240901 | + if( szKeyIn==4 ){ |
| 240902 | + u32 *aSlot = (u32*)&pData->p[8]; |
| 240903 | + if( aSlot[iIn] ) iVal = fts5GetU32((u8*)&aSlot[iIn]); |
| 240904 | + }else{ |
| 240905 | + u64 *aSlot = (u64*)&pData->p[8]; |
| 240906 | + if( aSlot[iIn] ) iVal = fts5GetU64((u8*)&aSlot[iIn]); |
| 240907 | + } |
| 240908 | + |
| 240909 | + /* If iVal is not 0 at this point, insert it into the new hash table */ |
| 240910 | + if( iVal ){ |
| 240911 | + Fts5Data *pPg = apOut[(iVal % nOut)]; |
| 240912 | + res = fts5IndexTombstoneAddToPage(pPg, 0, nOut, iVal); |
| 240913 | + if( res ) break; |
| 240914 | + } |
| 240915 | + } |
| 240916 | + |
| 240917 | + /* If this is page 0 of the old hash, copy the rowid-0-flag from the |
| 240918 | + ** old hash to the new. */ |
| 240919 | + if( ii==0 ){ |
| 240920 | + apOut[0]->p[1] = pData->p[1]; |
| 240921 | + } |
| 240922 | + } |
| 240923 | + fts5DataRelease(pFree); |
| 240924 | + } |
| 240925 | + |
| 240926 | + return res; |
| 240927 | +} |
| 240928 | + |
| 240929 | +/* |
| 240930 | +** This is called to rebuild the hash table belonging to segment pSeg. |
| 240931 | +** If parameter pData1 is not NULL, then one page of the existing hash table |
| 240932 | +** has already been loaded - pData1, which is page iPg1. The key-size for |
| 240933 | +** the new hash table is szKey (4 or 8). |
| 240934 | +** |
| 240935 | +** If successful, the new hash table is not written to disk. Instead, |
| 240936 | +** output parameter (*pnOut) is set to the number of pages in the new |
| 240937 | +** hash table, and (*papOut) to point to an array of buffers containing |
| 240938 | +** the new page data. |
| 240939 | +** |
| 240940 | +** If an error occurs, an error code is left in the Fts5Index object and |
| 240941 | +** both output parameters set to 0 before returning. |
| 240942 | +*/ |
| 240943 | +static void fts5IndexTombstoneRebuild( |
| 240944 | + Fts5Index *p, |
| 240945 | + Fts5StructureSegment *pSeg, /* Segment to rebuild hash of */ |
| 240946 | + Fts5Data *pData1, /* One page of current hash - or NULL */ |
| 240947 | + int iPg1, /* Which page of the current hash is pData1 */ |
| 240948 | + int szKey, /* 4 or 8, the keysize */ |
| 240949 | + int *pnOut, /* OUT: Number of output pages */ |
| 240950 | + Fts5Data ***papOut /* OUT: Output hash pages */ |
| 240951 | +){ |
| 240952 | + const int MINSLOT = 32; |
| 240953 | + int nSlotPerPage = MAX(MINSLOT, (p->pConfig->pgsz - 8) / szKey); |
| 240954 | + int nSlot = 0; /* Number of slots in each output page */ |
| 240955 | + int nOut = 0; |
| 240956 | + |
| 240957 | + /* Figure out how many output pages (nOut) and how many slots per |
| 240958 | + ** page (nSlot). There are three possibilities: |
| 240959 | + ** |
| 240960 | + ** 1. The hash table does not yet exist. In this case the new hash |
| 240961 | + ** table will consist of a single page with MINSLOT slots. |
| 240962 | + ** |
| 240963 | + ** 2. The hash table exists but is currently a single page. In this |
| 240964 | + ** case an attempt is made to grow the page to accommodate the new |
| 240965 | + ** entry. The page is allowed to grow up to nSlotPerPage (see above) |
| 240966 | + ** slots. |
| 240967 | + ** |
| 240968 | + ** 3. The hash table already consists of more than one page, or of |
| 240969 | + ** a single page already so large that it cannot be grown. In this |
| 240970 | + ** case the new hash consists of (nPg*2+1) pages of nSlotPerPage |
| 240971 | + ** slots each, where nPg is the current number of pages in the |
| 240972 | + ** hash table. |
| 240973 | + */ |
| 240974 | + if( pSeg->nPgTombstone==0 ){ |
| 240975 | + /* Case 1. */ |
| 240976 | + nOut = 1; |
| 240977 | + nSlot = MINSLOT; |
| 240978 | + }else if( pSeg->nPgTombstone==1 ){ |
| 240979 | + /* Case 2. */ |
| 240980 | + int nElem = (int)fts5GetU32(&pData1->p[4]); |
| 240981 | + assert( pData1 && iPg1==0 ); |
| 240982 | + nOut = 1; |
| 240983 | + nSlot = MAX(nElem*4, MINSLOT); |
| 240984 | + if( nSlot>nSlotPerPage ) nOut = 0; |
| 240985 | + } |
| 240986 | + if( nOut==0 ){ |
| 240987 | + /* Case 3. */ |
| 240988 | + nOut = (pSeg->nPgTombstone * 2 + 1); |
| 240989 | + nSlot = nSlotPerPage; |
| 240990 | + } |
| 240991 | + |
| 240992 | + /* Allocate the required array and output pages */ |
| 240993 | + while( 1 ){ |
| 240994 | + int res = 0; |
| 240995 | + int ii = 0; |
| 240996 | + int szPage = 0; |
| 240997 | + Fts5Data **apOut = 0; |
| 240998 | + |
| 240999 | + /* Allocate space for the new hash table */ |
| 241000 | + assert( nSlot>=MINSLOT ); |
| 241001 | + apOut = (Fts5Data**)sqlite3Fts5MallocZero(&p->rc, sizeof(Fts5Data*) * nOut); |
| 241002 | + szPage = 8 + nSlot*szKey; |
| 241003 | + for(ii=0; ii<nOut; ii++){ |
| 241004 | + Fts5Data *pNew = (Fts5Data*)sqlite3Fts5MallocZero(&p->rc, |
| 241005 | + sizeof(Fts5Data)+szPage |
| 241006 | + ); |
| 241007 | + if( pNew ){ |
| 241008 | + pNew->nn = szPage; |
| 241009 | + pNew->p = (u8*)&pNew[1]; |
| 241010 | + apOut[ii] = pNew; |
| 241011 | + } |
| 241012 | + } |
| 241013 | + |
| 241014 | + /* Rebuild the hash table. */ |
| 241015 | + if( p->rc==SQLITE_OK ){ |
| 241016 | + res = fts5IndexTombstoneRehash(p, pSeg, pData1, iPg1, szKey, nOut, apOut); |
| 241017 | + } |
| 241018 | + if( res==0 ){ |
| 241019 | + if( p->rc ){ |
| 241020 | + fts5IndexFreeArray(apOut, nOut); |
| 241021 | + apOut = 0; |
| 241022 | + nOut = 0; |
| 241023 | + } |
| 241024 | + *pnOut = nOut; |
| 241025 | + *papOut = apOut; |
| 241026 | + break; |
| 241027 | + } |
| 241028 | + |
| 241029 | + /* If control flows to here, it was not possible to rebuild the hash |
| 241030 | + ** table. Free all buffers and then try again with more pages. */ |
| 241031 | + assert( p->rc==SQLITE_OK ); |
| 241032 | + fts5IndexFreeArray(apOut, nOut); |
| 241033 | + nSlot = nSlotPerPage; |
| 241034 | + nOut = nOut*2 + 1; |
| 241035 | + } |
| 241036 | +} |
| 241037 | + |
| 241038 | + |
| 241039 | +/* |
| 241040 | +** Add a tombstone for rowid iRowid to segment pSeg. |
| 241041 | +*/ |
| 241042 | +static void fts5IndexTombstoneAdd( |
| 241043 | + Fts5Index *p, |
| 241044 | + Fts5StructureSegment *pSeg, |
| 241045 | + u64 iRowid |
| 241046 | +){ |
| 241047 | + Fts5Data *pPg = 0; |
| 241048 | + int iPg = -1; |
| 241049 | + int szKey = 0; |
| 241050 | + int nHash = 0; |
| 241051 | + Fts5Data **apHash = 0; |
| 241052 | + |
| 241053 | + p->nContentlessDelete++; |
| 241054 | + |
| 241055 | + if( pSeg->nPgTombstone>0 ){ |
| 241056 | + iPg = iRowid % pSeg->nPgTombstone; |
| 241057 | + pPg = fts5DataRead(p, FTS5_TOMBSTONE_ROWID(pSeg->iSegid,iPg)); |
| 241058 | + if( pPg==0 ){ |
| 241059 | + assert( p->rc!=SQLITE_OK ); |
| 241060 | + return; |
| 241061 | + } |
| 241062 | + |
| 241063 | + if( 0==fts5IndexTombstoneAddToPage(pPg, 0, pSeg->nPgTombstone, iRowid) ){ |
| 241064 | + fts5DataWrite(p, FTS5_TOMBSTONE_ROWID(pSeg->iSegid,iPg), pPg->p, pPg->nn); |
| 241065 | + fts5DataRelease(pPg); |
| 241066 | + return; |
| 241067 | + } |
| 241068 | + } |
| 241069 | + |
| 241070 | + /* Have to rebuild the hash table. First figure out the key-size (4 or 8). */ |
| 241071 | + szKey = pPg ? TOMBSTONE_KEYSIZE(pPg) : 4; |
| 241072 | + if( iRowid>0xFFFFFFFF ) szKey = 8; |
| 241073 | + |
| 241074 | + /* Rebuild the hash table */ |
| 241075 | + fts5IndexTombstoneRebuild(p, pSeg, pPg, iPg, szKey, &nHash, &apHash); |
| 241076 | + assert( p->rc==SQLITE_OK || (nHash==0 && apHash==0) ); |
| 241077 | + |
| 241078 | + /* If all has succeeded, write the new rowid into one of the new hash |
| 241079 | + ** table pages, then write them all out to disk. */ |
| 241080 | + if( nHash ){ |
| 241081 | + int ii = 0; |
| 241082 | + fts5IndexTombstoneAddToPage(apHash[iRowid % nHash], 1, nHash, iRowid); |
| 241083 | + for(ii=0; ii<nHash; ii++){ |
| 241084 | + i64 iTombstoneRowid = FTS5_TOMBSTONE_ROWID(pSeg->iSegid, ii); |
| 241085 | + fts5DataWrite(p, iTombstoneRowid, apHash[ii]->p, apHash[ii]->nn); |
| 241086 | + } |
| 241087 | + pSeg->nPgTombstone = nHash; |
| 241088 | + fts5StructureWrite(p, p->pStruct); |
| 241089 | + } |
| 241090 | + |
| 241091 | + fts5DataRelease(pPg); |
| 241092 | + fts5IndexFreeArray(apHash, nHash); |
| 241093 | +} |
| 241094 | + |
| 241095 | +/* |
| 241096 | +** Add iRowid to the tombstone list of the segment or segments that contain |
| 241097 | +** rows from origin iOrigin. Return SQLITE_OK if successful, or an SQLite |
| 241098 | +** error code otherwise. |
| 241099 | +*/ |
| 241100 | +static int sqlite3Fts5IndexContentlessDelete(Fts5Index *p, i64 iOrigin, i64 iRowid){ |
| 241101 | + Fts5Structure *pStruct; |
| 241102 | + pStruct = fts5StructureRead(p); |
| 241103 | + if( pStruct ){ |
| 241104 | + int bFound = 0; /* True after pSeg->nEntryTombstone incr. */ |
| 241105 | + int iLvl; |
| 241106 | + for(iLvl=pStruct->nLevel-1; iLvl>=0; iLvl--){ |
| 241107 | + int iSeg; |
| 241108 | + for(iSeg=pStruct->aLevel[iLvl].nSeg-1; iSeg>=0; iSeg--){ |
| 241109 | + Fts5StructureSegment *pSeg = &pStruct->aLevel[iLvl].aSeg[iSeg]; |
| 241110 | + if( pSeg->iOrigin1<=(u64)iOrigin && pSeg->iOrigin2>=(u64)iOrigin ){ |
| 241111 | + if( bFound==0 ){ |
| 241112 | + pSeg->nEntryTombstone++; |
| 241113 | + bFound = 1; |
| 241114 | + } |
| 241115 | + fts5IndexTombstoneAdd(p, pSeg, iRowid); |
| 241116 | + } |
| 241117 | + } |
| 241118 | + } |
| 241119 | + fts5StructureRelease(pStruct); |
| 241120 | + } |
| 241121 | + return fts5IndexReturn(p); |
| 241122 | +} |
| 239427 | 241123 | |
| 239428 | 241124 | /************************************************************************* |
| 239429 | 241125 | ************************************************************************** |
| 239430 | 241126 | ** Below this point is the implementation of the integrity-check |
| 239431 | 241127 | ** functionality. |
| | @@ -239973,17 +241669,18 @@ |
| 239973 | 241669 | ************************************************************************** |
| 239974 | 241670 | ** Below this point is the implementation of the fts5_decode() scalar |
| 239975 | 241671 | ** function only. |
| 239976 | 241672 | */ |
| 239977 | 241673 | |
| 239978 | | -#ifdef SQLITE_TEST |
| 241674 | +#if defined(SQLITE_TEST) || defined(SQLITE_FTS5_DEBUG) |
| 239979 | 241675 | /* |
| 239980 | 241676 | ** Decode a segment-data rowid from the %_data table. This function is |
| 239981 | 241677 | ** the opposite of macro FTS5_SEGMENT_ROWID(). |
| 239982 | 241678 | */ |
| 239983 | 241679 | static void fts5DecodeRowid( |
| 239984 | 241680 | i64 iRowid, /* Rowid from %_data table */ |
| 241681 | + int *pbTombstone, /* OUT: Tombstone hash flag */ |
| 239985 | 241682 | int *piSegid, /* OUT: Segment id */ |
| 239986 | 241683 | int *pbDlidx, /* OUT: Dlidx flag */ |
| 239987 | 241684 | int *piHeight, /* OUT: Height */ |
| 239988 | 241685 | int *piPgno /* OUT: Page number */ |
| 239989 | 241686 | ){ |
| | @@ -239995,34 +241692,39 @@ |
| 239995 | 241692 | |
| 239996 | 241693 | *pbDlidx = (int)(iRowid & 0x0001); |
| 239997 | 241694 | iRowid >>= FTS5_DATA_DLI_B; |
| 239998 | 241695 | |
| 239999 | 241696 | *piSegid = (int)(iRowid & (((i64)1 << FTS5_DATA_ID_B) - 1)); |
| 241697 | + iRowid >>= FTS5_DATA_ID_B; |
| 241698 | + |
| 241699 | + *pbTombstone = (int)(iRowid & 0x0001); |
| 240000 | 241700 | } |
| 240001 | | -#endif /* SQLITE_TEST */ |
| 241701 | +#endif /* SQLITE_TEST || SQLITE_FTS5_DEBUG */ |
| 240002 | 241702 | |
| 240003 | | -#ifdef SQLITE_TEST |
| 241703 | +#if defined(SQLITE_TEST) || defined(SQLITE_FTS5_DEBUG) |
| 240004 | 241704 | static void fts5DebugRowid(int *pRc, Fts5Buffer *pBuf, i64 iKey){ |
| 240005 | | - int iSegid, iHeight, iPgno, bDlidx; /* Rowid compenents */ |
| 240006 | | - fts5DecodeRowid(iKey, &iSegid, &bDlidx, &iHeight, &iPgno); |
| 241705 | + int iSegid, iHeight, iPgno, bDlidx, bTomb; /* Rowid compenents */ |
| 241706 | + fts5DecodeRowid(iKey, &bTomb, &iSegid, &bDlidx, &iHeight, &iPgno); |
| 240007 | 241707 | |
| 240008 | 241708 | if( iSegid==0 ){ |
| 240009 | 241709 | if( iKey==FTS5_AVERAGES_ROWID ){ |
| 240010 | 241710 | sqlite3Fts5BufferAppendPrintf(pRc, pBuf, "{averages} "); |
| 240011 | 241711 | }else{ |
| 240012 | 241712 | sqlite3Fts5BufferAppendPrintf(pRc, pBuf, "{structure}"); |
| 240013 | 241713 | } |
| 240014 | 241714 | } |
| 240015 | 241715 | else{ |
| 240016 | | - sqlite3Fts5BufferAppendPrintf(pRc, pBuf, "{%ssegid=%d h=%d pgno=%d}", |
| 240017 | | - bDlidx ? "dlidx " : "", iSegid, iHeight, iPgno |
| 241716 | + sqlite3Fts5BufferAppendPrintf(pRc, pBuf, "{%s%ssegid=%d h=%d pgno=%d}", |
| 241717 | + bDlidx ? "dlidx " : "", |
| 241718 | + bTomb ? "tombstone " : "", |
| 241719 | + iSegid, iHeight, iPgno |
| 240018 | 241720 | ); |
| 240019 | 241721 | } |
| 240020 | 241722 | } |
| 240021 | | -#endif /* SQLITE_TEST */ |
| 241723 | +#endif /* SQLITE_TEST || SQLITE_FTS5_DEBUG */ |
| 240022 | 241724 | |
| 240023 | | -#ifdef SQLITE_TEST |
| 241725 | +#if defined(SQLITE_TEST) || defined(SQLITE_FTS5_DEBUG) |
| 240024 | 241726 | static void fts5DebugStructure( |
| 240025 | 241727 | int *pRc, /* IN/OUT: error code */ |
| 240026 | 241728 | Fts5Buffer *pBuf, |
| 240027 | 241729 | Fts5Structure *p |
| 240028 | 241730 | ){ |
| | @@ -240033,20 +241735,26 @@ |
| 240033 | 241735 | sqlite3Fts5BufferAppendPrintf(pRc, pBuf, |
| 240034 | 241736 | " {lvl=%d nMerge=%d nSeg=%d", iLvl, pLvl->nMerge, pLvl->nSeg |
| 240035 | 241737 | ); |
| 240036 | 241738 | for(iSeg=0; iSeg<pLvl->nSeg; iSeg++){ |
| 240037 | 241739 | Fts5StructureSegment *pSeg = &pLvl->aSeg[iSeg]; |
| 240038 | | - sqlite3Fts5BufferAppendPrintf(pRc, pBuf, " {id=%d leaves=%d..%d}", |
| 241740 | + sqlite3Fts5BufferAppendPrintf(pRc, pBuf, " {id=%d leaves=%d..%d", |
| 240039 | 241741 | pSeg->iSegid, pSeg->pgnoFirst, pSeg->pgnoLast |
| 240040 | 241742 | ); |
| 241743 | + if( pSeg->iOrigin1>0 ){ |
| 241744 | + sqlite3Fts5BufferAppendPrintf(pRc, pBuf, " origin=%lld..%lld", |
| 241745 | + pSeg->iOrigin1, pSeg->iOrigin2 |
| 241746 | + ); |
| 241747 | + } |
| 241748 | + sqlite3Fts5BufferAppendPrintf(pRc, pBuf, "}"); |
| 240041 | 241749 | } |
| 240042 | 241750 | sqlite3Fts5BufferAppendPrintf(pRc, pBuf, "}"); |
| 240043 | 241751 | } |
| 240044 | 241752 | } |
| 240045 | | -#endif /* SQLITE_TEST */ |
| 241753 | +#endif /* SQLITE_TEST || SQLITE_FTS5_DEBUG */ |
| 240046 | 241754 | |
| 240047 | | -#ifdef SQLITE_TEST |
| 241755 | +#if defined(SQLITE_TEST) || defined(SQLITE_FTS5_DEBUG) |
| 240048 | 241756 | /* |
| 240049 | 241757 | ** This is part of the fts5_decode() debugging aid. |
| 240050 | 241758 | ** |
| 240051 | 241759 | ** Arguments pBlob/nBlob contain a serialized Fts5Structure object. This |
| 240052 | 241760 | ** function appends a human-readable representation of the same object |
| | @@ -240067,13 +241775,13 @@ |
| 240067 | 241775 | } |
| 240068 | 241776 | |
| 240069 | 241777 | fts5DebugStructure(pRc, pBuf, p); |
| 240070 | 241778 | fts5StructureRelease(p); |
| 240071 | 241779 | } |
| 240072 | | -#endif /* SQLITE_TEST */ |
| 241780 | +#endif /* SQLITE_TEST || SQLITE_FTS5_DEBUG */ |
| 240073 | 241781 | |
| 240074 | | -#ifdef SQLITE_TEST |
| 241782 | +#if defined(SQLITE_TEST) || defined(SQLITE_FTS5_DEBUG) |
| 240075 | 241783 | /* |
| 240076 | 241784 | ** This is part of the fts5_decode() debugging aid. |
| 240077 | 241785 | ** |
| 240078 | 241786 | ** Arguments pBlob/nBlob contain an "averages" record. This function |
| 240079 | 241787 | ** appends a human-readable representation of record to the buffer passed |
| | @@ -240092,13 +241800,13 @@ |
| 240092 | 241800 | i += sqlite3Fts5GetVarint(&pBlob[i], &iVal); |
| 240093 | 241801 | sqlite3Fts5BufferAppendPrintf(pRc, pBuf, "%s%d", zSpace, (int)iVal); |
| 240094 | 241802 | zSpace = " "; |
| 240095 | 241803 | } |
| 240096 | 241804 | } |
| 240097 | | -#endif /* SQLITE_TEST */ |
| 241805 | +#endif /* SQLITE_TEST || SQLITE_FTS5_DEBUG */ |
| 240098 | 241806 | |
| 240099 | | -#ifdef SQLITE_TEST |
| 241807 | +#if defined(SQLITE_TEST) || defined(SQLITE_FTS5_DEBUG) |
| 240100 | 241808 | /* |
| 240101 | 241809 | ** Buffer (a/n) is assumed to contain a list of serialized varints. Read |
| 240102 | 241810 | ** each varint and append its string representation to buffer pBuf. Return |
| 240103 | 241811 | ** after either the input buffer is exhausted or a 0 value is read. |
| 240104 | 241812 | ** |
| | @@ -240111,13 +241819,13 @@ |
| 240111 | 241819 | iOff += fts5GetVarint32(&a[iOff], iVal); |
| 240112 | 241820 | sqlite3Fts5BufferAppendPrintf(pRc, pBuf, " %d", iVal); |
| 240113 | 241821 | } |
| 240114 | 241822 | return iOff; |
| 240115 | 241823 | } |
| 240116 | | -#endif /* SQLITE_TEST */ |
| 241824 | +#endif /* SQLITE_TEST || SQLITE_FTS5_DEBUG */ |
| 240117 | 241825 | |
| 240118 | | -#ifdef SQLITE_TEST |
| 241826 | +#if defined(SQLITE_TEST) || defined(SQLITE_FTS5_DEBUG) |
| 240119 | 241827 | /* |
| 240120 | 241828 | ** The start of buffer (a/n) contains the start of a doclist. The doclist |
| 240121 | 241829 | ** may or may not finish within the buffer. This function appends a text |
| 240122 | 241830 | ** representation of the part of the doclist that is present to buffer |
| 240123 | 241831 | ** pBuf. |
| | @@ -240146,13 +241854,13 @@ |
| 240146 | 241854 | } |
| 240147 | 241855 | } |
| 240148 | 241856 | |
| 240149 | 241857 | return iOff; |
| 240150 | 241858 | } |
| 240151 | | -#endif /* SQLITE_TEST */ |
| 241859 | +#endif /* SQLITE_TEST || SQLITE_FTS5_DEBUG */ |
| 240152 | 241860 | |
| 240153 | | -#ifdef SQLITE_TEST |
| 241861 | +#if defined(SQLITE_TEST) || defined(SQLITE_FTS5_DEBUG) |
| 240154 | 241862 | /* |
| 240155 | 241863 | ** This function is part of the fts5_decode() debugging function. It is |
| 240156 | 241864 | ** only ever used with detail=none tables. |
| 240157 | 241865 | ** |
| 240158 | 241866 | ** Buffer (pData/nData) contains a doclist in the format used by detail=none |
| | @@ -240189,13 +241897,13 @@ |
| 240189 | 241897 | } |
| 240190 | 241898 | |
| 240191 | 241899 | sqlite3Fts5BufferAppendPrintf(pRc, pBuf, " %lld%s", iRowid, zApp); |
| 240192 | 241900 | } |
| 240193 | 241901 | } |
| 240194 | | -#endif /* SQLITE_TEST */ |
| 241902 | +#endif /* SQLITE_TEST || SQLITE_FTS5_DEBUG */ |
| 240195 | 241903 | |
| 240196 | | -#ifdef SQLITE_TEST |
| 241904 | +#if defined(SQLITE_TEST) || defined(SQLITE_FTS5_DEBUG) |
| 240197 | 241905 | /* |
| 240198 | 241906 | ** The implementation of user-defined scalar function fts5_decode(). |
| 240199 | 241907 | */ |
| 240200 | 241908 | static void fts5DecodeFunction( |
| 240201 | 241909 | sqlite3_context *pCtx, /* Function call context */ |
| | @@ -240202,10 +241910,11 @@ |
| 240202 | 241910 | int nArg, /* Number of args (always 2) */ |
| 240203 | 241911 | sqlite3_value **apVal /* Function arguments */ |
| 240204 | 241912 | ){ |
| 240205 | 241913 | i64 iRowid; /* Rowid for record being decoded */ |
| 240206 | 241914 | int iSegid,iHeight,iPgno,bDlidx;/* Rowid components */ |
| 241915 | + int bTomb; |
| 240207 | 241916 | const u8 *aBlob; int n; /* Record to decode */ |
| 240208 | 241917 | u8 *a = 0; |
| 240209 | 241918 | Fts5Buffer s; /* Build up text to return here */ |
| 240210 | 241919 | int rc = SQLITE_OK; /* Return code */ |
| 240211 | 241920 | sqlite3_int64 nSpace = 0; |
| | @@ -240224,11 +241933,11 @@ |
| 240224 | 241933 | nSpace = n + FTS5_DATA_ZERO_PADDING; |
| 240225 | 241934 | a = (u8*)sqlite3Fts5MallocZero(&rc, nSpace); |
| 240226 | 241935 | if( a==0 ) goto decode_out; |
| 240227 | 241936 | if( n>0 ) memcpy(a, aBlob, n); |
| 240228 | 241937 | |
| 240229 | | - fts5DecodeRowid(iRowid, &iSegid, &bDlidx, &iHeight, &iPgno); |
| 241938 | + fts5DecodeRowid(iRowid, &bTomb, &iSegid, &bDlidx, &iHeight, &iPgno); |
| 240230 | 241939 | |
| 240231 | 241940 | fts5DebugRowid(&rc, &s, iRowid); |
| 240232 | 241941 | if( bDlidx ){ |
| 240233 | 241942 | Fts5Data dlidx; |
| 240234 | 241943 | Fts5DlidxLvl lvl; |
| | @@ -240242,10 +241951,32 @@ |
| 240242 | 241951 | |
| 240243 | 241952 | for(fts5DlidxLvlNext(&lvl); lvl.bEof==0; fts5DlidxLvlNext(&lvl)){ |
| 240244 | 241953 | sqlite3Fts5BufferAppendPrintf(&rc, &s, |
| 240245 | 241954 | " %d(%lld)", lvl.iLeafPgno, lvl.iRowid |
| 240246 | 241955 | ); |
| 241956 | + } |
| 241957 | + }else if( bTomb ){ |
| 241958 | + u32 nElem = fts5GetU32(&a[4]); |
| 241959 | + int szKey = (aBlob[0]==4 || aBlob[0]==8) ? aBlob[0] : 8; |
| 241960 | + int nSlot = (n - 8) / szKey; |
| 241961 | + int ii; |
| 241962 | + sqlite3Fts5BufferAppendPrintf(&rc, &s, " nElem=%d", (int)nElem); |
| 241963 | + if( aBlob[1] ){ |
| 241964 | + sqlite3Fts5BufferAppendPrintf(&rc, &s, " 0"); |
| 241965 | + } |
| 241966 | + for(ii=0; ii<nSlot; ii++){ |
| 241967 | + u64 iVal = 0; |
| 241968 | + if( szKey==4 ){ |
| 241969 | + u32 *aSlot = (u32*)&aBlob[8]; |
| 241970 | + if( aSlot[ii] ) iVal = fts5GetU32((u8*)&aSlot[ii]); |
| 241971 | + }else{ |
| 241972 | + u64 *aSlot = (u64*)&aBlob[8]; |
| 241973 | + if( aSlot[ii] ) iVal = fts5GetU64((u8*)&aSlot[ii]); |
| 241974 | + } |
| 241975 | + if( iVal!=0 ){ |
| 241976 | + sqlite3Fts5BufferAppendPrintf(&rc, &s, " %lld", (i64)iVal); |
| 241977 | + } |
| 240247 | 241978 | } |
| 240248 | 241979 | }else if( iSegid==0 ){ |
| 240249 | 241980 | if( iRowid==FTS5_AVERAGES_ROWID ){ |
| 240250 | 241981 | fts5DecodeAverages(&rc, &s, a, n); |
| 240251 | 241982 | }else{ |
| | @@ -240400,13 +242131,13 @@ |
| 240400 | 242131 | }else{ |
| 240401 | 242132 | sqlite3_result_error_code(pCtx, rc); |
| 240402 | 242133 | } |
| 240403 | 242134 | fts5BufferFree(&s); |
| 240404 | 242135 | } |
| 240405 | | -#endif /* SQLITE_TEST */ |
| 242136 | +#endif /* SQLITE_TEST || SQLITE_FTS5_DEBUG */ |
| 240406 | 242137 | |
| 240407 | | -#ifdef SQLITE_TEST |
| 242138 | +#if defined(SQLITE_TEST) || defined(SQLITE_FTS5_DEBUG) |
| 240408 | 242139 | /* |
| 240409 | 242140 | ** The implementation of user-defined scalar function fts5_rowid(). |
| 240410 | 242141 | */ |
| 240411 | 242142 | static void fts5RowidFunction( |
| 240412 | 242143 | sqlite3_context *pCtx, /* Function call context */ |
| | @@ -240436,11 +242167,239 @@ |
| 240436 | 242167 | "first arg to fts5_rowid() must be 'segment'" , -1 |
| 240437 | 242168 | ); |
| 240438 | 242169 | } |
| 240439 | 242170 | } |
| 240440 | 242171 | } |
| 240441 | | -#endif /* SQLITE_TEST */ |
| 242172 | +#endif /* SQLITE_TEST || SQLITE_FTS5_DEBUG */ |
| 242173 | + |
| 242174 | +#if defined(SQLITE_TEST) || defined(SQLITE_FTS5_DEBUG) |
| 242175 | + |
| 242176 | +typedef struct Fts5StructVtab Fts5StructVtab; |
| 242177 | +struct Fts5StructVtab { |
| 242178 | + sqlite3_vtab base; |
| 242179 | +}; |
| 242180 | + |
| 242181 | +typedef struct Fts5StructVcsr Fts5StructVcsr; |
| 242182 | +struct Fts5StructVcsr { |
| 242183 | + sqlite3_vtab_cursor base; |
| 242184 | + Fts5Structure *pStruct; |
| 242185 | + int iLevel; |
| 242186 | + int iSeg; |
| 242187 | + int iRowid; |
| 242188 | +}; |
| 242189 | + |
| 242190 | +/* |
| 242191 | +** Create a new fts5_structure() table-valued function. |
| 242192 | +*/ |
| 242193 | +static int fts5structConnectMethod( |
| 242194 | + sqlite3 *db, |
| 242195 | + void *pAux, |
| 242196 | + int argc, const char *const*argv, |
| 242197 | + sqlite3_vtab **ppVtab, |
| 242198 | + char **pzErr |
| 242199 | +){ |
| 242200 | + Fts5StructVtab *pNew = 0; |
| 242201 | + int rc = SQLITE_OK; |
| 242202 | + |
| 242203 | + rc = sqlite3_declare_vtab(db, |
| 242204 | + "CREATE TABLE xyz(" |
| 242205 | + "level, segment, merge, segid, leaf1, leaf2, loc1, loc2, " |
| 242206 | + "npgtombstone, nentrytombstone, nentry, struct HIDDEN);" |
| 242207 | + ); |
| 242208 | + if( rc==SQLITE_OK ){ |
| 242209 | + pNew = sqlite3Fts5MallocZero(&rc, sizeof(*pNew)); |
| 242210 | + } |
| 242211 | + |
| 242212 | + *ppVtab = (sqlite3_vtab*)pNew; |
| 242213 | + return rc; |
| 242214 | +} |
| 242215 | + |
| 242216 | +/* |
| 242217 | +** We must have a single struct=? constraint that will be passed through |
| 242218 | +** into the xFilter method. If there is no valid stmt=? constraint, |
| 242219 | +** then return an SQLITE_CONSTRAINT error. |
| 242220 | +*/ |
| 242221 | +static int fts5structBestIndexMethod( |
| 242222 | + sqlite3_vtab *tab, |
| 242223 | + sqlite3_index_info *pIdxInfo |
| 242224 | +){ |
| 242225 | + int i; |
| 242226 | + int rc = SQLITE_CONSTRAINT; |
| 242227 | + struct sqlite3_index_constraint *p; |
| 242228 | + pIdxInfo->estimatedCost = (double)100; |
| 242229 | + pIdxInfo->estimatedRows = 100; |
| 242230 | + pIdxInfo->idxNum = 0; |
| 242231 | + for(i=0, p=pIdxInfo->aConstraint; i<pIdxInfo->nConstraint; i++, p++){ |
| 242232 | + if( p->usable==0 ) continue; |
| 242233 | + if( p->op==SQLITE_INDEX_CONSTRAINT_EQ && p->iColumn==11 ){ |
| 242234 | + rc = SQLITE_OK; |
| 242235 | + pIdxInfo->aConstraintUsage[i].omit = 1; |
| 242236 | + pIdxInfo->aConstraintUsage[i].argvIndex = 1; |
| 242237 | + break; |
| 242238 | + } |
| 242239 | + } |
| 242240 | + return rc; |
| 242241 | +} |
| 242242 | + |
| 242243 | +/* |
| 242244 | +** This method is the destructor for bytecodevtab objects. |
| 242245 | +*/ |
| 242246 | +static int fts5structDisconnectMethod(sqlite3_vtab *pVtab){ |
| 242247 | + Fts5StructVtab *p = (Fts5StructVtab*)pVtab; |
| 242248 | + sqlite3_free(p); |
| 242249 | + return SQLITE_OK; |
| 242250 | +} |
| 242251 | + |
| 242252 | +/* |
| 242253 | +** Constructor for a new bytecodevtab_cursor object. |
| 242254 | +*/ |
| 242255 | +static int fts5structOpenMethod(sqlite3_vtab *p, sqlite3_vtab_cursor **ppCsr){ |
| 242256 | + int rc = SQLITE_OK; |
| 242257 | + Fts5StructVcsr *pNew = 0; |
| 242258 | + |
| 242259 | + pNew = sqlite3Fts5MallocZero(&rc, sizeof(*pNew)); |
| 242260 | + *ppCsr = (sqlite3_vtab_cursor*)pNew; |
| 242261 | + |
| 242262 | + return SQLITE_OK; |
| 242263 | +} |
| 242264 | + |
| 242265 | +/* |
| 242266 | +** Destructor for a bytecodevtab_cursor. |
| 242267 | +*/ |
| 242268 | +static int fts5structCloseMethod(sqlite3_vtab_cursor *cur){ |
| 242269 | + Fts5StructVcsr *pCsr = (Fts5StructVcsr*)cur; |
| 242270 | + fts5StructureRelease(pCsr->pStruct); |
| 242271 | + sqlite3_free(pCsr); |
| 242272 | + return SQLITE_OK; |
| 242273 | +} |
| 242274 | + |
| 242275 | + |
| 242276 | +/* |
| 242277 | +** Advance a bytecodevtab_cursor to its next row of output. |
| 242278 | +*/ |
| 242279 | +static int fts5structNextMethod(sqlite3_vtab_cursor *cur){ |
| 242280 | + Fts5StructVcsr *pCsr = (Fts5StructVcsr*)cur; |
| 242281 | + Fts5Structure *p = pCsr->pStruct; |
| 242282 | + |
| 242283 | + assert( pCsr->pStruct ); |
| 242284 | + pCsr->iSeg++; |
| 242285 | + pCsr->iRowid++; |
| 242286 | + while( pCsr->iLevel<p->nLevel && pCsr->iSeg>=p->aLevel[pCsr->iLevel].nSeg ){ |
| 242287 | + pCsr->iLevel++; |
| 242288 | + pCsr->iSeg = 0; |
| 242289 | + } |
| 242290 | + if( pCsr->iLevel>=p->nLevel ){ |
| 242291 | + fts5StructureRelease(pCsr->pStruct); |
| 242292 | + pCsr->pStruct = 0; |
| 242293 | + } |
| 242294 | + return SQLITE_OK; |
| 242295 | +} |
| 242296 | + |
| 242297 | +/* |
| 242298 | +** Return TRUE if the cursor has been moved off of the last |
| 242299 | +** row of output. |
| 242300 | +*/ |
| 242301 | +static int fts5structEofMethod(sqlite3_vtab_cursor *cur){ |
| 242302 | + Fts5StructVcsr *pCsr = (Fts5StructVcsr*)cur; |
| 242303 | + return pCsr->pStruct==0; |
| 242304 | +} |
| 242305 | + |
| 242306 | +static int fts5structRowidMethod( |
| 242307 | + sqlite3_vtab_cursor *cur, |
| 242308 | + sqlite_int64 *piRowid |
| 242309 | +){ |
| 242310 | + Fts5StructVcsr *pCsr = (Fts5StructVcsr*)cur; |
| 242311 | + *piRowid = pCsr->iRowid; |
| 242312 | + return SQLITE_OK; |
| 242313 | +} |
| 242314 | + |
| 242315 | +/* |
| 242316 | +** Return values of columns for the row at which the bytecodevtab_cursor |
| 242317 | +** is currently pointing. |
| 242318 | +*/ |
| 242319 | +static int fts5structColumnMethod( |
| 242320 | + sqlite3_vtab_cursor *cur, /* The cursor */ |
| 242321 | + sqlite3_context *ctx, /* First argument to sqlite3_result_...() */ |
| 242322 | + int i /* Which column to return */ |
| 242323 | +){ |
| 242324 | + Fts5StructVcsr *pCsr = (Fts5StructVcsr*)cur; |
| 242325 | + Fts5Structure *p = pCsr->pStruct; |
| 242326 | + Fts5StructureSegment *pSeg = &p->aLevel[pCsr->iLevel].aSeg[pCsr->iSeg]; |
| 242327 | + |
| 242328 | + switch( i ){ |
| 242329 | + case 0: /* level */ |
| 242330 | + sqlite3_result_int(ctx, pCsr->iLevel); |
| 242331 | + break; |
| 242332 | + case 1: /* segment */ |
| 242333 | + sqlite3_result_int(ctx, pCsr->iSeg); |
| 242334 | + break; |
| 242335 | + case 2: /* merge */ |
| 242336 | + sqlite3_result_int(ctx, pCsr->iSeg < p->aLevel[pCsr->iLevel].nMerge); |
| 242337 | + break; |
| 242338 | + case 3: /* segid */ |
| 242339 | + sqlite3_result_int(ctx, pSeg->iSegid); |
| 242340 | + break; |
| 242341 | + case 4: /* leaf1 */ |
| 242342 | + sqlite3_result_int(ctx, pSeg->pgnoFirst); |
| 242343 | + break; |
| 242344 | + case 5: /* leaf2 */ |
| 242345 | + sqlite3_result_int(ctx, pSeg->pgnoLast); |
| 242346 | + break; |
| 242347 | + case 6: /* origin1 */ |
| 242348 | + sqlite3_result_int64(ctx, pSeg->iOrigin1); |
| 242349 | + break; |
| 242350 | + case 7: /* origin2 */ |
| 242351 | + sqlite3_result_int64(ctx, pSeg->iOrigin2); |
| 242352 | + break; |
| 242353 | + case 8: /* npgtombstone */ |
| 242354 | + sqlite3_result_int(ctx, pSeg->nPgTombstone); |
| 242355 | + break; |
| 242356 | + case 9: /* nentrytombstone */ |
| 242357 | + sqlite3_result_int64(ctx, pSeg->nEntryTombstone); |
| 242358 | + break; |
| 242359 | + case 10: /* nentry */ |
| 242360 | + sqlite3_result_int64(ctx, pSeg->nEntry); |
| 242361 | + break; |
| 242362 | + } |
| 242363 | + return SQLITE_OK; |
| 242364 | +} |
| 242365 | + |
| 242366 | +/* |
| 242367 | +** Initialize a cursor. |
| 242368 | +** |
| 242369 | +** idxNum==0 means show all subprograms |
| 242370 | +** idxNum==1 means show only the main bytecode and omit subprograms. |
| 242371 | +*/ |
| 242372 | +static int fts5structFilterMethod( |
| 242373 | + sqlite3_vtab_cursor *pVtabCursor, |
| 242374 | + int idxNum, const char *idxStr, |
| 242375 | + int argc, sqlite3_value **argv |
| 242376 | +){ |
| 242377 | + Fts5StructVcsr *pCsr = (Fts5StructVcsr *)pVtabCursor; |
| 242378 | + int rc = SQLITE_OK; |
| 242379 | + |
| 242380 | + const u8 *aBlob = 0; |
| 242381 | + int nBlob = 0; |
| 242382 | + |
| 242383 | + assert( argc==1 ); |
| 242384 | + fts5StructureRelease(pCsr->pStruct); |
| 242385 | + pCsr->pStruct = 0; |
| 242386 | + |
| 242387 | + nBlob = sqlite3_value_bytes(argv[0]); |
| 242388 | + aBlob = (const u8*)sqlite3_value_blob(argv[0]); |
| 242389 | + rc = fts5StructureDecode(aBlob, nBlob, 0, &pCsr->pStruct); |
| 242390 | + if( rc==SQLITE_OK ){ |
| 242391 | + pCsr->iLevel = 0; |
| 242392 | + pCsr->iRowid = 0; |
| 242393 | + pCsr->iSeg = -1; |
| 242394 | + rc = fts5structNextMethod(pVtabCursor); |
| 242395 | + } |
| 242396 | + |
| 242397 | + return rc; |
| 242398 | +} |
| 242399 | + |
| 242400 | +#endif /* SQLITE_TEST || SQLITE_FTS5_DEBUG */ |
| 240442 | 242401 | |
| 240443 | 242402 | /* |
| 240444 | 242403 | ** This is called as part of registering the FTS5 module with database |
| 240445 | 242404 | ** connection db. It registers several user-defined scalar functions useful |
| 240446 | 242405 | ** with FTS5. |
| | @@ -240447,11 +242406,11 @@ |
| 240447 | 242406 | ** |
| 240448 | 242407 | ** If successful, SQLITE_OK is returned. If an error occurs, some other |
| 240449 | 242408 | ** SQLite error code is returned instead. |
| 240450 | 242409 | */ |
| 240451 | 242410 | static int sqlite3Fts5IndexInit(sqlite3 *db){ |
| 240452 | | -#ifdef SQLITE_TEST |
| 242411 | +#if defined(SQLITE_TEST) || defined(SQLITE_FTS5_DEBUG) |
| 240453 | 242412 | int rc = sqlite3_create_function( |
| 240454 | 242413 | db, "fts5_decode", 2, SQLITE_UTF8, 0, fts5DecodeFunction, 0, 0 |
| 240455 | 242414 | ); |
| 240456 | 242415 | |
| 240457 | 242416 | if( rc==SQLITE_OK ){ |
| | @@ -240464,10 +242423,40 @@ |
| 240464 | 242423 | if( rc==SQLITE_OK ){ |
| 240465 | 242424 | rc = sqlite3_create_function( |
| 240466 | 242425 | db, "fts5_rowid", -1, SQLITE_UTF8, 0, fts5RowidFunction, 0, 0 |
| 240467 | 242426 | ); |
| 240468 | 242427 | } |
| 242428 | + |
| 242429 | + if( rc==SQLITE_OK ){ |
| 242430 | + static const sqlite3_module fts5structure_module = { |
| 242431 | + 0, /* iVersion */ |
| 242432 | + 0, /* xCreate */ |
| 242433 | + fts5structConnectMethod, /* xConnect */ |
| 242434 | + fts5structBestIndexMethod, /* xBestIndex */ |
| 242435 | + fts5structDisconnectMethod, /* xDisconnect */ |
| 242436 | + 0, /* xDestroy */ |
| 242437 | + fts5structOpenMethod, /* xOpen */ |
| 242438 | + fts5structCloseMethod, /* xClose */ |
| 242439 | + fts5structFilterMethod, /* xFilter */ |
| 242440 | + fts5structNextMethod, /* xNext */ |
| 242441 | + fts5structEofMethod, /* xEof */ |
| 242442 | + fts5structColumnMethod, /* xColumn */ |
| 242443 | + fts5structRowidMethod, /* xRowid */ |
| 242444 | + 0, /* xUpdate */ |
| 242445 | + 0, /* xBegin */ |
| 242446 | + 0, /* xSync */ |
| 242447 | + 0, /* xCommit */ |
| 242448 | + 0, /* xRollback */ |
| 242449 | + 0, /* xFindFunction */ |
| 242450 | + 0, /* xRename */ |
| 242451 | + 0, /* xSavepoint */ |
| 242452 | + 0, /* xRelease */ |
| 242453 | + 0, /* xRollbackTo */ |
| 242454 | + 0 /* xShadowName */ |
| 242455 | + }; |
| 242456 | + rc = sqlite3_create_module(db, "fts5_structure", &fts5structure_module, 0); |
| 242457 | + } |
| 240469 | 242458 | return rc; |
| 240470 | 242459 | #else |
| 240471 | 242460 | return SQLITE_OK; |
| 240472 | 242461 | UNUSED_PARAM(db); |
| 240473 | 242462 | #endif |
| | @@ -242107,11 +244096,10 @@ |
| 242107 | 244096 | Fts5Config *pConfig = pTab->p.pConfig; |
| 242108 | 244097 | int eType0; /* value_type() of apVal[0] */ |
| 242109 | 244098 | int rc = SQLITE_OK; /* Return code */ |
| 242110 | 244099 | int bUpdateOrDelete = 0; |
| 242111 | 244100 | |
| 242112 | | - |
| 242113 | 244101 | /* A transaction must be open when this is called. */ |
| 242114 | 244102 | assert( pTab->ts.eState==1 || pTab->ts.eState==2 ); |
| 242115 | 244103 | |
| 242116 | 244104 | assert( pVtab->zErrMsg==0 ); |
| 242117 | 244105 | assert( nArg==1 || nArg==(2+pConfig->nCol+2) ); |
| | @@ -242136,11 +244124,18 @@ |
| 242136 | 244124 | /* A "special" INSERT op. These are handled separately. */ |
| 242137 | 244125 | const char *z = (const char*)sqlite3_value_text(apVal[2+pConfig->nCol]); |
| 242138 | 244126 | if( pConfig->eContent!=FTS5_CONTENT_NORMAL |
| 242139 | 244127 | && 0==sqlite3_stricmp("delete", z) |
| 242140 | 244128 | ){ |
| 242141 | | - rc = fts5SpecialDelete(pTab, apVal); |
| 244129 | + if( pConfig->bContentlessDelete ){ |
| 244130 | + fts5SetVtabError(pTab, |
| 244131 | + "'delete' may not be used with a contentless_delete=1 table" |
| 244132 | + ); |
| 244133 | + rc = SQLITE_ERROR; |
| 244134 | + }else{ |
| 244135 | + rc = fts5SpecialDelete(pTab, apVal); |
| 244136 | + } |
| 242142 | 244137 | }else{ |
| 242143 | 244138 | rc = fts5SpecialInsert(pTab, z, apVal[2 + pConfig->nCol + 1]); |
| 242144 | 244139 | } |
| 242145 | 244140 | }else{ |
| 242146 | 244141 | /* A regular INSERT, UPDATE or DELETE statement. The trick here is that |
| | @@ -242153,20 +244148,24 @@ |
| 242153 | 244148 | ** 4) INSERT |
| 242154 | 244149 | ** |
| 242155 | 244150 | ** Cases 3 and 4 may violate the rowid constraint. |
| 242156 | 244151 | */ |
| 242157 | 244152 | int eConflict = SQLITE_ABORT; |
| 242158 | | - if( pConfig->eContent==FTS5_CONTENT_NORMAL ){ |
| 244153 | + if( pConfig->eContent==FTS5_CONTENT_NORMAL || pConfig->bContentlessDelete ){ |
| 242159 | 244154 | eConflict = sqlite3_vtab_on_conflict(pConfig->db); |
| 242160 | 244155 | } |
| 242161 | 244156 | |
| 242162 | 244157 | assert( eType0==SQLITE_INTEGER || eType0==SQLITE_NULL ); |
| 242163 | 244158 | assert( nArg!=1 || eType0==SQLITE_INTEGER ); |
| 242164 | 244159 | |
| 242165 | 244160 | /* Filter out attempts to run UPDATE or DELETE on contentless tables. |
| 242166 | | - ** This is not suported. */ |
| 242167 | | - if( eType0==SQLITE_INTEGER && fts5IsContentless(pTab) ){ |
| 244161 | + ** This is not suported. Except - DELETE is supported if the CREATE |
| 244162 | + ** VIRTUAL TABLE statement contained "contentless_delete=1". */ |
| 244163 | + if( eType0==SQLITE_INTEGER |
| 244164 | + && pConfig->eContent==FTS5_CONTENT_NONE |
| 244165 | + && (nArg>1 || pConfig->bContentlessDelete==0) |
| 244166 | + ){ |
| 242168 | 244167 | pTab->p.base.zErrMsg = sqlite3_mprintf( |
| 242169 | 244168 | "cannot %s contentless fts5 table: %s", |
| 242170 | 244169 | (nArg>1 ? "UPDATE" : "DELETE from"), pConfig->zName |
| 242171 | 244170 | ); |
| 242172 | 244171 | rc = SQLITE_ERROR; |
| | @@ -242249,12 +244248,11 @@ |
| 242249 | 244248 | static int fts5SyncMethod(sqlite3_vtab *pVtab){ |
| 242250 | 244249 | int rc; |
| 242251 | 244250 | Fts5FullTable *pTab = (Fts5FullTable*)pVtab; |
| 242252 | 244251 | fts5CheckTransactionState(pTab, FTS5_SYNC, 0); |
| 242253 | 244252 | pTab->p.pConfig->pzErrmsg = &pTab->p.base.zErrMsg; |
| 242254 | | - fts5TripCursors(pTab); |
| 242255 | | - rc = sqlite3Fts5StorageSync(pTab->pStorage); |
| 244253 | + rc = sqlite3Fts5FlushToDisk(&pTab->p); |
| 242256 | 244254 | pTab->p.pConfig->pzErrmsg = 0; |
| 242257 | 244255 | return rc; |
| 242258 | 244256 | } |
| 242259 | 244257 | |
| 242260 | 244258 | /* |
| | @@ -243299,11 +245297,11 @@ |
| 243299 | 245297 | int nArg, /* Number of args */ |
| 243300 | 245298 | sqlite3_value **apUnused /* Function arguments */ |
| 243301 | 245299 | ){ |
| 243302 | 245300 | assert( nArg==0 ); |
| 243303 | 245301 | UNUSED_PARAM2(nArg, apUnused); |
| 243304 | | - sqlite3_result_text(pCtx, "fts5: 2023-07-08 17:42:24 07d95ed60f0a17ea13b4bc19c2ab2ec9052fedd27c9e1e57a1ec6e3a6470e5b7", -1, SQLITE_TRANSIENT); |
| 245302 | + sqlite3_result_text(pCtx, "fts5: 2023-08-01 11:03:06 aa55c83f35c2ab134e0842201e46e021079283f9c65595c86664060b3aa8d715", -1, SQLITE_TRANSIENT); |
| 243305 | 245303 | } |
| 243306 | 245304 | |
| 243307 | 245305 | /* |
| 243308 | 245306 | ** Return true if zName is the extension on one of the shadow tables used |
| 243309 | 245307 | ** by this module. |
| | @@ -243512,14 +245510,14 @@ |
| 243512 | 245510 | "SELECT %s FROM %s T WHERE T.%Q=?", /* LOOKUP */ |
| 243513 | 245511 | |
| 243514 | 245512 | "INSERT INTO %Q.'%q_content' VALUES(%s)", /* INSERT_CONTENT */ |
| 243515 | 245513 | "REPLACE INTO %Q.'%q_content' VALUES(%s)", /* REPLACE_CONTENT */ |
| 243516 | 245514 | "DELETE FROM %Q.'%q_content' WHERE id=?", /* DELETE_CONTENT */ |
| 243517 | | - "REPLACE INTO %Q.'%q_docsize' VALUES(?,?)", /* REPLACE_DOCSIZE */ |
| 245515 | + "REPLACE INTO %Q.'%q_docsize' VALUES(?,?%s)", /* REPLACE_DOCSIZE */ |
| 243518 | 245516 | "DELETE FROM %Q.'%q_docsize' WHERE id=?", /* DELETE_DOCSIZE */ |
| 243519 | 245517 | |
| 243520 | | - "SELECT sz FROM %Q.'%q_docsize' WHERE id=?", /* LOOKUP_DOCSIZE */ |
| 245518 | + "SELECT sz%s FROM %Q.'%q_docsize' WHERE id=?", /* LOOKUP_DOCSIZE */ |
| 243521 | 245519 | |
| 243522 | 245520 | "REPLACE INTO %Q.'%q_config' VALUES(?,?)", /* REPLACE_CONFIG */ |
| 243523 | 245521 | "SELECT %s FROM %s AS T", /* SCAN */ |
| 243524 | 245522 | }; |
| 243525 | 245523 | Fts5Config *pC = p->pConfig; |
| | @@ -243562,10 +245560,23 @@ |
| 243562 | 245560 | zSql = sqlite3_mprintf(azStmt[eStmt], pC->zDb, pC->zName, zBind); |
| 243563 | 245561 | sqlite3_free(zBind); |
| 243564 | 245562 | } |
| 243565 | 245563 | break; |
| 243566 | 245564 | } |
| 245565 | + |
| 245566 | + case FTS5_STMT_REPLACE_DOCSIZE: |
| 245567 | + zSql = sqlite3_mprintf(azStmt[eStmt], pC->zDb, pC->zName, |
| 245568 | + (pC->bContentlessDelete ? ",?" : "") |
| 245569 | + ); |
| 245570 | + break; |
| 245571 | + |
| 245572 | + case FTS5_STMT_LOOKUP_DOCSIZE: |
| 245573 | + zSql = sqlite3_mprintf(azStmt[eStmt], |
| 245574 | + (pC->bContentlessDelete ? ",origin" : ""), |
| 245575 | + pC->zDb, pC->zName |
| 245576 | + ); |
| 245577 | + break; |
| 243567 | 245578 | |
| 243568 | 245579 | default: |
| 243569 | 245580 | zSql = sqlite3_mprintf(azStmt[eStmt], pC->zDb, pC->zName); |
| 243570 | 245581 | break; |
| 243571 | 245582 | } |
| | @@ -243752,13 +245763,15 @@ |
| 243752 | 245763 | } |
| 243753 | 245764 | sqlite3_free(zDefn); |
| 243754 | 245765 | } |
| 243755 | 245766 | |
| 243756 | 245767 | if( rc==SQLITE_OK && pConfig->bColumnsize ){ |
| 243757 | | - rc = sqlite3Fts5CreateTable( |
| 243758 | | - pConfig, "docsize", "id INTEGER PRIMARY KEY, sz BLOB", 0, pzErr |
| 243759 | | - ); |
| 245768 | + const char *zCols = "id INTEGER PRIMARY KEY, sz BLOB"; |
| 245769 | + if( pConfig->bContentlessDelete ){ |
| 245770 | + zCols = "id INTEGER PRIMARY KEY, sz BLOB, origin INTEGER"; |
| 245771 | + } |
| 245772 | + rc = sqlite3Fts5CreateTable(pConfig, "docsize", zCols, 0, pzErr); |
| 243760 | 245773 | } |
| 243761 | 245774 | if( rc==SQLITE_OK ){ |
| 243762 | 245775 | rc = sqlite3Fts5CreateTable( |
| 243763 | 245776 | pConfig, "config", "k PRIMARY KEY, v", 1, pzErr |
| 243764 | 245777 | ); |
| | @@ -243831,11 +245844,11 @@ |
| 243831 | 245844 | i64 iDel, |
| 243832 | 245845 | sqlite3_value **apVal |
| 243833 | 245846 | ){ |
| 243834 | 245847 | Fts5Config *pConfig = p->pConfig; |
| 243835 | 245848 | sqlite3_stmt *pSeek = 0; /* SELECT to read row iDel from %_data */ |
| 243836 | | - int rc; /* Return code */ |
| 245849 | + int rc = SQLITE_OK; /* Return code */ |
| 243837 | 245850 | int rc2; /* sqlite3_reset() return code */ |
| 243838 | 245851 | int iCol; |
| 243839 | 245852 | Fts5InsertCtx ctx; |
| 243840 | 245853 | |
| 243841 | 245854 | if( apVal==0 ){ |
| | @@ -243847,11 +245860,10 @@ |
| 243847 | 245860 | } |
| 243848 | 245861 | } |
| 243849 | 245862 | |
| 243850 | 245863 | ctx.pStorage = p; |
| 243851 | 245864 | ctx.iCol = -1; |
| 243852 | | - rc = sqlite3Fts5IndexBeginWrite(p->pIndex, 1, iDel); |
| 243853 | 245865 | for(iCol=1; rc==SQLITE_OK && iCol<=pConfig->nCol; iCol++){ |
| 243854 | 245866 | if( pConfig->abUnindexed[iCol-1]==0 ){ |
| 243855 | 245867 | const char *zText; |
| 243856 | 245868 | int nText; |
| 243857 | 245869 | assert( pSeek==0 || apVal==0 ); |
| | @@ -243884,10 +245896,41 @@ |
| 243884 | 245896 | rc2 = sqlite3_reset(pSeek); |
| 243885 | 245897 | if( rc==SQLITE_OK ) rc = rc2; |
| 243886 | 245898 | return rc; |
| 243887 | 245899 | } |
| 243888 | 245900 | |
| 245901 | +/* |
| 245902 | +** This function is called to process a DELETE on a contentless_delete=1 |
| 245903 | +** table. It adds the tombstone required to delete the entry with rowid |
| 245904 | +** iDel. If successful, SQLITE_OK is returned. Or, if an error occurs, |
| 245905 | +** an SQLite error code. |
| 245906 | +*/ |
| 245907 | +static int fts5StorageContentlessDelete(Fts5Storage *p, i64 iDel){ |
| 245908 | + i64 iOrigin = 0; |
| 245909 | + sqlite3_stmt *pLookup = 0; |
| 245910 | + int rc = SQLITE_OK; |
| 245911 | + |
| 245912 | + assert( p->pConfig->bContentlessDelete ); |
| 245913 | + assert( p->pConfig->eContent==FTS5_CONTENT_NONE ); |
| 245914 | + |
| 245915 | + /* Look up the origin of the document in the %_docsize table. Store |
| 245916 | + ** this in stack variable iOrigin. */ |
| 245917 | + rc = fts5StorageGetStmt(p, FTS5_STMT_LOOKUP_DOCSIZE, &pLookup, 0); |
| 245918 | + if( rc==SQLITE_OK ){ |
| 245919 | + sqlite3_bind_int64(pLookup, 1, iDel); |
| 245920 | + if( SQLITE_ROW==sqlite3_step(pLookup) ){ |
| 245921 | + iOrigin = sqlite3_column_int64(pLookup, 1); |
| 245922 | + } |
| 245923 | + rc = sqlite3_reset(pLookup); |
| 245924 | + } |
| 245925 | + |
| 245926 | + if( rc==SQLITE_OK && iOrigin!=0 ){ |
| 245927 | + rc = sqlite3Fts5IndexContentlessDelete(p->pIndex, iOrigin, iDel); |
| 245928 | + } |
| 245929 | + |
| 245930 | + return rc; |
| 245931 | +} |
| 243889 | 245932 | |
| 243890 | 245933 | /* |
| 243891 | 245934 | ** Insert a record into the %_docsize table. Specifically, do: |
| 243892 | 245935 | ** |
| 243893 | 245936 | ** INSERT OR REPLACE INTO %_docsize(id, sz) VALUES(iRowid, pBuf); |
| | @@ -243904,14 +245947,21 @@ |
| 243904 | 245947 | if( p->pConfig->bColumnsize ){ |
| 243905 | 245948 | sqlite3_stmt *pReplace = 0; |
| 243906 | 245949 | rc = fts5StorageGetStmt(p, FTS5_STMT_REPLACE_DOCSIZE, &pReplace, 0); |
| 243907 | 245950 | if( rc==SQLITE_OK ){ |
| 243908 | 245951 | sqlite3_bind_int64(pReplace, 1, iRowid); |
| 243909 | | - sqlite3_bind_blob(pReplace, 2, pBuf->p, pBuf->n, SQLITE_STATIC); |
| 243910 | | - sqlite3_step(pReplace); |
| 243911 | | - rc = sqlite3_reset(pReplace); |
| 243912 | | - sqlite3_bind_null(pReplace, 2); |
| 245952 | + if( p->pConfig->bContentlessDelete ){ |
| 245953 | + i64 iOrigin = 0; |
| 245954 | + rc = sqlite3Fts5IndexGetOrigin(p->pIndex, &iOrigin); |
| 245955 | + sqlite3_bind_int64(pReplace, 3, iOrigin); |
| 245956 | + } |
| 245957 | + if( rc==SQLITE_OK ){ |
| 245958 | + sqlite3_bind_blob(pReplace, 2, pBuf->p, pBuf->n, SQLITE_STATIC); |
| 245959 | + sqlite3_step(pReplace); |
| 245960 | + rc = sqlite3_reset(pReplace); |
| 245961 | + sqlite3_bind_null(pReplace, 2); |
| 245962 | + } |
| 243913 | 245963 | } |
| 243914 | 245964 | } |
| 243915 | 245965 | return rc; |
| 243916 | 245966 | } |
| 243917 | 245967 | |
| | @@ -243971,11 +246021,19 @@ |
| 243971 | 246021 | assert( pConfig->eContent!=FTS5_CONTENT_NORMAL || apVal==0 ); |
| 243972 | 246022 | rc = fts5StorageLoadTotals(p, 1); |
| 243973 | 246023 | |
| 243974 | 246024 | /* Delete the index records */ |
| 243975 | 246025 | if( rc==SQLITE_OK ){ |
| 243976 | | - rc = fts5StorageDeleteFromIndex(p, iDel, apVal); |
| 246026 | + rc = sqlite3Fts5IndexBeginWrite(p->pIndex, 1, iDel); |
| 246027 | + } |
| 246028 | + |
| 246029 | + if( rc==SQLITE_OK ){ |
| 246030 | + if( p->pConfig->bContentlessDelete ){ |
| 246031 | + rc = fts5StorageContentlessDelete(p, iDel); |
| 246032 | + }else{ |
| 246033 | + rc = fts5StorageDeleteFromIndex(p, iDel, apVal); |
| 246034 | + } |
| 243977 | 246035 | } |
| 243978 | 246036 | |
| 243979 | 246037 | /* Delete the %_docsize record */ |
| 243980 | 246038 | if( rc==SQLITE_OK && pConfig->bColumnsize ){ |
| 243981 | 246039 | rc = fts5StorageGetStmt(p, FTS5_STMT_DELETE_DOCSIZE, &pDel, 0); |
| 243982 | 246040 | |