Fossil SCM

New version of tmstmpvfs.c that fixes the WAL-checksum bug.

drh 2026-01-27 16:20 timestamp-vfs
Commit 6cb48bf66adf3a8496549ec56cd7079634b2c6c0c70c910e8d8a38d177bc2fa7
Parent 7e63541cddb482c…
1 file changed +74 -14
~ extsrc/tmstmpvfs.c
M extsrc/tmstmpvfs.c
+74 -14
--- extsrc/tmstmpvfs.c
+++ extsrc/tmstmpvfs.c
@@ -10,13 +10,11 @@
1010
 **
1111
 ******************************************************************************
1212
 **
1313
 ** This file implements a VFS shim that writes a timestamp and other tracing
1414
 ** information into 16 byts of reserved space at the end of each page of the
15
-** database file. The additional data is written as the page is added to
16
-** the WAL file for databases in WAL mode, or as the database file itself
17
-** is modified in rollback modes.
15
+** database file.
1816
 **
1917
 ** The VFS also tries to generate log-files with names of the form:
2018
 **
2119
 ** $(DATABASE)-tmstmp/$(TIME)-$(PID)-$(ID)
2220
 **
@@ -42,12 +40,11 @@
4240
 ** You may want to add additional compiler options, of course,
4341
 ** according to the needs of your project.
4442
 **
4543
 ** Another option is to statically link both SQLite and this extension
4644
 ** into your application. If both this file and "sqlite3.c" are statically
47
-** linked, and if "sqlite3.c" is compiled with -DSQLITE_EXTRA_INIT=
48
-** SQLite amalgamation "sqlite3.c" file with the option like:
45
+** linked, and if "sqlite3.c" is compiled with an option like:
4946
 **
5047
 ** -DSQLITE_EXTRA_INIT=sqlite3_register_tmstmpvfs
5148
 **
5249
 ** Then SQLite will use the tmstmp VFS by default throughout your
5350
 ** application.
@@ -137,16 +134,21 @@
137134
 ** The timestamp layout is as follows:
138135
 **
139136
 ** bytes 0,1 Zero. Reserved for future expansion
140137
 ** bytes 2-7 Milliseconds since the Unix Epoch
141138
 ** bytes 8-11 WAL frame number
142
-** bytes 12 0: WAL write 1: WAL txn 2: rollback write
139
+** bytes 12 0: WAL write 2: rollback write
143140
 ** bytes 13-15 Lower 24 bits of Salt-1
144141
 **
145142
 ** For transactions that occur in rollback mode, only the timestamp
146143
 ** in bytes 2-7 and byte 12 are non-zero. Byte 12 is set to 2 for
147144
 ** rollback writes.
145
+**
146
+** The 16-byte tag is added to each database page when the content
147
+** is written into the database file itself. This shim does not make
148
+** any changes to the page as it is written to the WAL file, since
149
+** that would mess up the WAL checksum.
148150
 **
149151
 ** LOGGING
150152
 **
151153
 ** An open database connection that attempts to write to the database
152154
 ** will create a log file if a directory name $(DATABASE)-tmstmp exists.
@@ -204,10 +206,66 @@
204206
 ** ELOG_CLOSE_WAL "Close the WAL file connection"
205207
 ** op = 0x0e
206208
 **
207209
 ** ELOG_CLOSE_DB "Close the DB connection"
208210
 ** op = 0x0f
211
+**
212
+** VIEWING TIMESTAMPS AND LOGS
213
+**
214
+** The command-line utility at tool/showtmlog.c will read and display
215
+** the content of one or more tmstmpvfs.c log files. If all of the
216
+** log files are stored in directory $(DATABASE)-tmstmp, then you can
217
+** view them all using a command like shown below (with an extra "?"
218
+** inserted on the wildcard to avoid closing the C-language comment
219
+** that contains this text):
220
+**
221
+** showtmlog $(DATABASE)-tmstmp/?*
222
+**
223
+** The command-line utility at tools/showdb.c can be used to show the
224
+** timestamps on pages of a database file, using a command like this:
225
+**
226
+** showdb --tmstmp $(DATABASE) pgidx
227
+*
228
+** The command above shows the timestamp and the intended use of every
229
+** pages in the database, in human-readable form. If you also add
230
+** the --csv option to the command above, then the command generates
231
+** a Comma-Separated-Value (CSV) file as output, which contains a
232
+** decoding of the complete timestamp tag on each page of the database.
233
+** This CVS file can be easily imported into another SQLite database
234
+** using a CLI command like the following:
235
+**
236
+** .import --csv '|showdb --tmstmp -csv orig.db pgidx' ts_table
237
+**
238
+** In the command above, the database containing the timestamps is
239
+** "orig.db" and the content is imported into a new table named "ts_table".
240
+** The "ts_table" is created automatically, using the column names found
241
+** in the first line of the CSV file. All columns of the automatically
242
+** created ts_table are of type TEXT. It might make more sense to
243
+** create the table yourself, using more sensible datatypes, like this:
244
+**
245
+** CREATE TABLE ts_table (
246
+** pgno INT, -- page number
247
+** tm REAL, -- seconds since 1970-01-01
248
+** frame INT, -- WAL frame number
249
+** flg INT, -- flag (tag byte 12)
250
+** salt INT, -- WAL salt (tag bytes 13-15)
251
+** parent INT, -- Parent page number
252
+** child INT, -- Index of this page in its parent
253
+** ovfl INT, -- Index of this page on the overflow chain
254
+** txt TEXT -- Description of this page
255
+** );
256
+**
257
+** Then import using:
258
+**
259
+** .import --csv --skip 1 '|showdb --tmstmp --csv orig.db pgidx' ts_table
260
+**
261
+** Note the addition of the "--skip 1" option on ".import" to bypass the
262
+** first line of the CSV file that contains the column names.
263
+**
264
+** Both programs "showdb" and "showtmlog" can be built by running
265
+** "make showtmlog showdb" from the top-level of a recent SQLite
266
+** source tree.
209267
 */
210268
 #if defined(SQLITE_AMALGAMATION) && !defined(SQLITE_TMSTMPVFS_STATIC)
211269
 # define SQLITE_TMSTMPVFS_STATIC
212270
 #endif
213271
 #ifdef SQLITE_TMSTMPVFS_STATIC
@@ -559,27 +617,29 @@
559617
 if( iAmt==24 ){
560618
 /* A frame header */
561619
 u32 x = 0;
562620
 p->iFrame = (iOfst - 32)/(p->pgsz+24)+1;
563621
 p->pgno = tmstmpGetU32((const u8*)zBuf);
564
- p->salt1 = tmstmpGetU32(((const u8*)zBuf)+8);
622
+ p->salt1 = tmstmpGetU32(((const u8*)zBuf)+16);
565623
 memcpy(&x, ((const u8*)zBuf)+4, 4);
566624
 p->isCommit = (x!=0);
567625
 p->iOfst = iOfst;
568626
 }else if( iAmt>=512 && iOfst==p->iOfst+24 ){
569
- unsigned char *s = (unsigned char*)zBuf+iAmt-TMSTMP_RESERVE;
627
+ unsigned char s[TMSTMP_RESERVE];
570628
 memset(s, 0, TMSTMP_RESERVE);
571629
 tmstmpPutTS(p, s+2);
572
- tmstmpPutU32(p->iFrame, s+8);
573
- tmstmpPutU32(p->salt1, s+12);
574
- s[12] = p->isCommit ? 1 : 0;
575
- tmstmpEvent(p, ELOG_WAL_PAGE, s[12], p->pgno, p->iFrame, s+2);
630
+ tmstmpEvent(p, ELOG_WAL_PAGE, p->isCommit, p->pgno, p->iFrame, s+2);
576631
 }else if( iAmt==32 && iOfst==0 ){
577
- u32 salt1 = tmstmpGetU32(((const u8*)zBuf)+16);
578
- tmstmpEvent(p, ELOG_WAL_RESET, 0, 0, salt1, 0);
632
+ p->salt1 = tmstmpGetU32(((const u8*)zBuf)+16);
633
+ tmstmpEvent(p, ELOG_WAL_RESET, 0, 0, p->salt1, 0);
579634
 }
580635
 }else if( p->inCkpt ){
636
+ unsigned char *s = (unsigned char*)zBuf+iAmt-TMSTMP_RESERVE;
637
+ memset(s, 0, TMSTMP_RESERVE);
638
+ tmstmpPutTS(p, s+2);
639
+ tmstmpPutU32(p->iFrame, s+8);
640
+ tmstmpPutU32(p->pPartner->salt1, s+12);
581641
 assert( p->pgsz>0 );
582642
 tmstmpEvent(p, ELOG_CKPT_PAGE, 0, (iOfst/p->pgsz)+1, p->iFrame, 0);
583643
 }else if( p->pPartner==0 ){
584644
 /* Writing into a database in rollback mode */
585645
 unsigned char *s = (unsigned char*)zBuf+iAmt-TMSTMP_RESERVE;
586646
--- extsrc/tmstmpvfs.c
+++ extsrc/tmstmpvfs.c
@@ -10,13 +10,11 @@
10 **
11 ******************************************************************************
12 **
13 ** This file implements a VFS shim that writes a timestamp and other tracing
14 ** information into 16 byts of reserved space at the end of each page of the
15 ** database file. The additional data is written as the page is added to
16 ** the WAL file for databases in WAL mode, or as the database file itself
17 ** is modified in rollback modes.
18 **
19 ** The VFS also tries to generate log-files with names of the form:
20 **
21 ** $(DATABASE)-tmstmp/$(TIME)-$(PID)-$(ID)
22 **
@@ -42,12 +40,11 @@
42 ** You may want to add additional compiler options, of course,
43 ** according to the needs of your project.
44 **
45 ** Another option is to statically link both SQLite and this extension
46 ** into your application. If both this file and "sqlite3.c" are statically
47 ** linked, and if "sqlite3.c" is compiled with -DSQLITE_EXTRA_INIT=
48 ** SQLite amalgamation "sqlite3.c" file with the option like:
49 **
50 ** -DSQLITE_EXTRA_INIT=sqlite3_register_tmstmpvfs
51 **
52 ** Then SQLite will use the tmstmp VFS by default throughout your
53 ** application.
@@ -137,16 +134,21 @@
137 ** The timestamp layout is as follows:
138 **
139 ** bytes 0,1 Zero. Reserved for future expansion
140 ** bytes 2-7 Milliseconds since the Unix Epoch
141 ** bytes 8-11 WAL frame number
142 ** bytes 12 0: WAL write 1: WAL txn 2: rollback write
143 ** bytes 13-15 Lower 24 bits of Salt-1
144 **
145 ** For transactions that occur in rollback mode, only the timestamp
146 ** in bytes 2-7 and byte 12 are non-zero. Byte 12 is set to 2 for
147 ** rollback writes.
 
 
 
 
 
148 **
149 ** LOGGING
150 **
151 ** An open database connection that attempts to write to the database
152 ** will create a log file if a directory name $(DATABASE)-tmstmp exists.
@@ -204,10 +206,66 @@
204 ** ELOG_CLOSE_WAL "Close the WAL file connection"
205 ** op = 0x0e
206 **
207 ** ELOG_CLOSE_DB "Close the DB connection"
208 ** op = 0x0f
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
209 */
210 #if defined(SQLITE_AMALGAMATION) && !defined(SQLITE_TMSTMPVFS_STATIC)
211 # define SQLITE_TMSTMPVFS_STATIC
212 #endif
213 #ifdef SQLITE_TMSTMPVFS_STATIC
@@ -559,27 +617,29 @@
559 if( iAmt==24 ){
560 /* A frame header */
561 u32 x = 0;
562 p->iFrame = (iOfst - 32)/(p->pgsz+24)+1;
563 p->pgno = tmstmpGetU32((const u8*)zBuf);
564 p->salt1 = tmstmpGetU32(((const u8*)zBuf)+8);
565 memcpy(&x, ((const u8*)zBuf)+4, 4);
566 p->isCommit = (x!=0);
567 p->iOfst = iOfst;
568 }else if( iAmt>=512 && iOfst==p->iOfst+24 ){
569 unsigned char *s = (unsigned char*)zBuf+iAmt-TMSTMP_RESERVE;
570 memset(s, 0, TMSTMP_RESERVE);
571 tmstmpPutTS(p, s+2);
572 tmstmpPutU32(p->iFrame, s+8);
573 tmstmpPutU32(p->salt1, s+12);
574 s[12] = p->isCommit ? 1 : 0;
575 tmstmpEvent(p, ELOG_WAL_PAGE, s[12], p->pgno, p->iFrame, s+2);
576 }else if( iAmt==32 && iOfst==0 ){
577 u32 salt1 = tmstmpGetU32(((const u8*)zBuf)+16);
578 tmstmpEvent(p, ELOG_WAL_RESET, 0, 0, salt1, 0);
579 }
580 }else if( p->inCkpt ){
 
 
 
 
 
581 assert( p->pgsz>0 );
582 tmstmpEvent(p, ELOG_CKPT_PAGE, 0, (iOfst/p->pgsz)+1, p->iFrame, 0);
583 }else if( p->pPartner==0 ){
584 /* Writing into a database in rollback mode */
585 unsigned char *s = (unsigned char*)zBuf+iAmt-TMSTMP_RESERVE;
586
--- extsrc/tmstmpvfs.c
+++ extsrc/tmstmpvfs.c
@@ -10,13 +10,11 @@
10 **
11 ******************************************************************************
12 **
13 ** This file implements a VFS shim that writes a timestamp and other tracing
14 ** information into 16 byts of reserved space at the end of each page of the
15 ** database file.
 
 
16 **
17 ** The VFS also tries to generate log-files with names of the form:
18 **
19 ** $(DATABASE)-tmstmp/$(TIME)-$(PID)-$(ID)
20 **
@@ -42,12 +40,11 @@
40 ** You may want to add additional compiler options, of course,
41 ** according to the needs of your project.
42 **
43 ** Another option is to statically link both SQLite and this extension
44 ** into your application. If both this file and "sqlite3.c" are statically
45 ** linked, and if "sqlite3.c" is compiled with an option like:
 
46 **
47 ** -DSQLITE_EXTRA_INIT=sqlite3_register_tmstmpvfs
48 **
49 ** Then SQLite will use the tmstmp VFS by default throughout your
50 ** application.
@@ -137,16 +134,21 @@
134 ** The timestamp layout is as follows:
135 **
136 ** bytes 0,1 Zero. Reserved for future expansion
137 ** bytes 2-7 Milliseconds since the Unix Epoch
138 ** bytes 8-11 WAL frame number
139 ** bytes 12 0: WAL write 2: rollback write
140 ** bytes 13-15 Lower 24 bits of Salt-1
141 **
142 ** For transactions that occur in rollback mode, only the timestamp
143 ** in bytes 2-7 and byte 12 are non-zero. Byte 12 is set to 2 for
144 ** rollback writes.
145 **
146 ** The 16-byte tag is added to each database page when the content
147 ** is written into the database file itself. This shim does not make
148 ** any changes to the page as it is written to the WAL file, since
149 ** that would mess up the WAL checksum.
150 **
151 ** LOGGING
152 **
153 ** An open database connection that attempts to write to the database
154 ** will create a log file if a directory name $(DATABASE)-tmstmp exists.
@@ -204,10 +206,66 @@
206 ** ELOG_CLOSE_WAL "Close the WAL file connection"
207 ** op = 0x0e
208 **
209 ** ELOG_CLOSE_DB "Close the DB connection"
210 ** op = 0x0f
211 **
212 ** VIEWING TIMESTAMPS AND LOGS
213 **
214 ** The command-line utility at tool/showtmlog.c will read and display
215 ** the content of one or more tmstmpvfs.c log files. If all of the
216 ** log files are stored in directory $(DATABASE)-tmstmp, then you can
217 ** view them all using a command like shown below (with an extra "?"
218 ** inserted on the wildcard to avoid closing the C-language comment
219 ** that contains this text):
220 **
221 ** showtmlog $(DATABASE)-tmstmp/?*
222 **
223 ** The command-line utility at tools/showdb.c can be used to show the
224 ** timestamps on pages of a database file, using a command like this:
225 **
226 ** showdb --tmstmp $(DATABASE) pgidx
227 *
228 ** The command above shows the timestamp and the intended use of every
229 ** pages in the database, in human-readable form. If you also add
230 ** the --csv option to the command above, then the command generates
231 ** a Comma-Separated-Value (CSV) file as output, which contains a
232 ** decoding of the complete timestamp tag on each page of the database.
233 ** This CVS file can be easily imported into another SQLite database
234 ** using a CLI command like the following:
235 **
236 ** .import --csv '|showdb --tmstmp -csv orig.db pgidx' ts_table
237 **
238 ** In the command above, the database containing the timestamps is
239 ** "orig.db" and the content is imported into a new table named "ts_table".
240 ** The "ts_table" is created automatically, using the column names found
241 ** in the first line of the CSV file. All columns of the automatically
242 ** created ts_table are of type TEXT. It might make more sense to
243 ** create the table yourself, using more sensible datatypes, like this:
244 **
245 ** CREATE TABLE ts_table (
246 ** pgno INT, -- page number
247 ** tm REAL, -- seconds since 1970-01-01
248 ** frame INT, -- WAL frame number
249 ** flg INT, -- flag (tag byte 12)
250 ** salt INT, -- WAL salt (tag bytes 13-15)
251 ** parent INT, -- Parent page number
252 ** child INT, -- Index of this page in its parent
253 ** ovfl INT, -- Index of this page on the overflow chain
254 ** txt TEXT -- Description of this page
255 ** );
256 **
257 ** Then import using:
258 **
259 ** .import --csv --skip 1 '|showdb --tmstmp --csv orig.db pgidx' ts_table
260 **
261 ** Note the addition of the "--skip 1" option on ".import" to bypass the
262 ** first line of the CSV file that contains the column names.
263 **
264 ** Both programs "showdb" and "showtmlog" can be built by running
265 ** "make showtmlog showdb" from the top-level of a recent SQLite
266 ** source tree.
267 */
268 #if defined(SQLITE_AMALGAMATION) && !defined(SQLITE_TMSTMPVFS_STATIC)
269 # define SQLITE_TMSTMPVFS_STATIC
270 #endif
271 #ifdef SQLITE_TMSTMPVFS_STATIC
@@ -559,27 +617,29 @@
617 if( iAmt==24 ){
618 /* A frame header */
619 u32 x = 0;
620 p->iFrame = (iOfst - 32)/(p->pgsz+24)+1;
621 p->pgno = tmstmpGetU32((const u8*)zBuf);
622 p->salt1 = tmstmpGetU32(((const u8*)zBuf)+16);
623 memcpy(&x, ((const u8*)zBuf)+4, 4);
624 p->isCommit = (x!=0);
625 p->iOfst = iOfst;
626 }else if( iAmt>=512 && iOfst==p->iOfst+24 ){
627 unsigned char s[TMSTMP_RESERVE];
628 memset(s, 0, TMSTMP_RESERVE);
629 tmstmpPutTS(p, s+2);
630 tmstmpEvent(p, ELOG_WAL_PAGE, p->isCommit, p->pgno, p->iFrame, s+2);
 
 
 
631 }else if( iAmt==32 && iOfst==0 ){
632 p->salt1 = tmstmpGetU32(((const u8*)zBuf)+16);
633 tmstmpEvent(p, ELOG_WAL_RESET, 0, 0, p->salt1, 0);
634 }
635 }else if( p->inCkpt ){
636 unsigned char *s = (unsigned char*)zBuf+iAmt-TMSTMP_RESERVE;
637 memset(s, 0, TMSTMP_RESERVE);
638 tmstmpPutTS(p, s+2);
639 tmstmpPutU32(p->iFrame, s+8);
640 tmstmpPutU32(p->pPartner->salt1, s+12);
641 assert( p->pgsz>0 );
642 tmstmpEvent(p, ELOG_CKPT_PAGE, 0, (iOfst/p->pgsz)+1, p->iFrame, 0);
643 }else if( p->pPartner==0 ){
644 /* Writing into a database in rollback mode */
645 unsigned char *s = (unsigned char*)zBuf+iAmt-TMSTMP_RESERVE;
646

Keyboard Shortcuts

Open search /
Next entry (timeline) j
Previous entry (timeline) k
Open focused entry Enter
Show this help ?
Toggle theme Top nav button