1 # 2010 May 24 2 # 3 # The author disclaims copyright to this source code. In place of 4 # a legal notice, here is a blessing: 5 # 6 # May you do good and not evil. 7 # May you find forgiveness for yourself and forgive others. 8 # May you share freely, never taking more than you give. 9 # 10 #*********************************************************************** 11 # 12 13 set testdir [file dirname $argv0] 14 source $testdir/tester.tcl 15 source $testdir/lock_common.tcl 16 source $testdir/wal_common.tcl 17 18 ifcapable !wal {finish_test ; return } 19 20 # Read and return the contents of file $filename. Treat the content as 21 # binary data. 22 # 23 proc readfile {filename} { 24 set fd [open $filename] 25 fconfigure $fd -encoding binary 26 fconfigure $fd -translation binary 27 set data [read $fd] 28 close $fd 29 return $data 30 } 31 32 # 33 # File $filename must be a WAL file on disk. Check that the checksum of frame 34 # $iFrame in the file is correct when interpreting data as $endian-endian 35 # integers ($endian must be either "big" or "little"). If the checksum looks 36 # correct, return 1. Otherwise 0. 37 # 38 proc log_checksum_verify {filename iFrame endian} { 39 set data [readfile $filename] 40 41 foreach {offset c1 c2} [log_checksum_calc $data $iFrame $endian] {} 42 43 binary scan [string range $data $offset [expr $offset+7]] II expect1 expect2 44 set expect1 [expr $expect1&0xFFFFFFFF] 45 set expect2 [expr $expect2&0xFFFFFFFF] 46 47 expr {$c1==$expect1 && $c2==$expect2} 48 } 49 50 # File $filename must be a WAL file on disk. Compute the checksum for frame 51 # $iFrame in the file by interpreting data as $endian-endian integers 52 # ($endian must be either "big" or "little"). Then write the computed 53 # checksum into the file. 54 # 55 proc log_checksum_write {filename iFrame endian} { 56 set data [readfile $filename] 57 58 foreach {offset c1 c2} [log_checksum_calc $data $iFrame $endian] {} 59 60 set bin [binary format II $c1 $c2] 61 set fd [open $filename r+] 62 fconfigure $fd -encoding binary 63 fconfigure $fd -translation binary 64 seek $fd $offset 65 puts -nonewline $fd $bin 66 close $fd 67 } 68 69 # Calculate and return the checksum for a particular frame in a WAL. 70 # 71 # Arguments are: 72 # 73 # $data Blob containing the entire contents of a WAL. 74 # 75 # $iFrame Frame number within the $data WAL. Frames are numbered 76 # starting at 1. 77 # 78 # $endian One of "big" or "little". 79 # 80 # Returns a list of three elements, as follows: 81 # 82 # * The byte offset of the checksum belonging to frame $iFrame in the WAL. 83 # * The first integer in the calculated version of the checksum. 84 # * The second integer in the calculated version of the checksum. 85 # 86 proc log_checksum_calc {data iFrame endian} { 87 88 binary scan [string range $data 8 11] I pgsz 89 if {$iFrame > 1} { 90 set n [wal_file_size [expr $iFrame-2] $pgsz] 91 binary scan [string range $data [expr $n+16] [expr $n+23]] II c1 c2 92 } else { 93 set c1 0 94 set c2 0 95 wal_cksum $endian c1 c2 [string range $data 0 23] 96 } 97 98 set n [wal_file_size [expr $iFrame-1] $pgsz] 99 wal_cksum $endian c1 c2 [string range $data $n [expr $n+7]] 100 wal_cksum $endian c1 c2 [string range $data [expr $n+24] [expr $n+24+$pgsz-1]] 101 102 list [expr $n+16] $c1 $c2 103 } 104 105 # 106 # File $filename must be a WAL file on disk. Set the 'magic' field of the 107 # WAL header to indicate that checksums are $endian-endian ($endian must be 108 # either "big" or "little"). 109 # 110 # Also update the wal header checksum (since the wal header contents may 111 # have changed). 112 # 113 proc log_checksum_writemagic {filename endian} { 114 set val [expr {0x377f0682 | ($endian == "big" ? 1 : 0)}] 115 set bin [binary format I $val] 116 set fd [open $filename r+] 117 fconfigure $fd -encoding binary 118 fconfigure $fd -translation binary 119 puts -nonewline $fd $bin 120 121 seek $fd 0 122 set blob [read $fd 24] 123 set c1 0 124 set c2 0 125 wal_cksum $endian c1 c2 $blob 126 seek $fd 24 127 puts -nonewline $fd [binary format II $c1 $c2] 128 129 close $fd 130 } 131 132 #------------------------------------------------------------------------- 133 # Test cases walcksum-1.* attempt to verify the following: 134 # 135 # * That both native and non-native order checksum log files can 136 # be recovered. 137 # 138 # * That when appending to native or non-native checksum log files 139 # SQLite continues to use the right kind of checksums. 140 # 141 # * Test point 2 when the appending process is not one that recovered 142 # the log file. 143 # 144 # * Test that both native and non-native checksum log files can be 145 # checkpointed. And that after doing so the next write to the log 146 # file occurs using native byte-order checksums. 147 # 148 set native "big" 149 if {$::tcl_platform(byteOrder) == "littleEndian"} { set native "little" } 150 foreach endian {big little} { 151 152 # Create a database. Leave some data in the log file. 153 # 154 do_test walcksum-1.$endian.1 { 155 catch { db close } 156 file delete -force test.db test.db-wal test.db-journal 157 sqlite3 db test.db 158 execsql { 159 PRAGMA page_size = 1024; 160 PRAGMA auto_vacuum = 0; 161 PRAGMA synchronous = NORMAL; 162 163 CREATE TABLE t1(a PRIMARY KEY, b); 164 INSERT INTO t1 VALUES(1, 'one'); 165 INSERT INTO t1 VALUES(2, 'two'); 166 INSERT INTO t1 VALUES(3, 'three'); 167 INSERT INTO t1 VALUES(5, 'five'); 168 169 PRAGMA journal_mode = WAL; 170 INSERT INTO t1 VALUES(8, 'eight'); 171 INSERT INTO t1 VALUES(13, 'thirteen'); 172 INSERT INTO t1 VALUES(21, 'twentyone'); 173 } 174 175 file copy -force test.db test2.db 176 file copy -force test.db-wal test2.db-wal 177 db close 178 179 list [file size test2.db] [file size test2.db-wal] 180 } [list [expr 1024*3] [wal_file_size 6 1024]] 181 182 # Verify that the checksums are valid for all frames and that they 183 # are calculated by interpreting data in native byte-order. 184 # 185 for {set f 1} {$f <= 6} {incr f} { 186 do_test walcksum-1.$endian.2.$f { 187 log_checksum_verify test2.db-wal $f $native 188 } 1 189 } 190 191 # Replace all checksums in the current WAL file with $endian versions. 192 # Then check that it is still possible to recover and read the database. 193 # 194 log_checksum_writemagic test2.db-wal $endian 195 for {set f 1} {$f <= 6} {incr f} { 196 do_test walcksum-1.$endian.3.$f { 197 log_checksum_write test2.db-wal $f $endian 198 log_checksum_verify test2.db-wal $f $endian 199 } {1} 200 } 201 do_test walcksum-1.$endian.4.1 { 202 file copy -force test2.db test.db 203 file copy -force test2.db-wal test.db-wal 204 sqlite3 db test.db 205 execsql { SELECT a FROM t1 } 206 } {1 2 3 5 8 13 21} 207 208 # Following recovery, any frames written to the log should use the same 209 # endianness as the existing frames. Check that this is the case. 210 # 211 do_test walcksum-1.$endian.5.0 { 212 execsql { 213 PRAGMA synchronous = NORMAL; 214 INSERT INTO t1 VALUES(34, 'thirtyfour'); 215 } 216 list [file size test.db] [file size test.db-wal] 217 } [list [expr 1024*3] [wal_file_size 8 1024]] 218 for {set f 1} {$f <= 8} {incr f} { 219 do_test walcksum-1.$endian.5.$f { 220 log_checksum_verify test.db-wal $f $endian 221 } {1} 222 } 223 224 # Now connect a second connection to the database. Check that this one 225 # (not the one that did recovery) also appends frames to the log using 226 # the same endianness for checksums as the existing frames. 227 # 228 do_test walcksum-1.$endian.6 { 229 sqlite3 db2 test.db 230 execsql { 231 PRAGMA integrity_check; 232 SELECT a FROM t1; 233 } db2 234 } {ok 1 2 3 5 8 13 21 34} 235 do_test walcksum-1.$endian.7.0 { 236 execsql { 237 PRAGMA synchronous = NORMAL; 238 INSERT INTO t1 VALUES(55, 'fiftyfive'); 239 } db2 240 list [file size test.db] [file size test.db-wal] 241 } [list [expr 1024*3] [wal_file_size 10 1024]] 242 for {set f 1} {$f <= 10} {incr f} { 243 do_test walcksum-1.$endian.7.$f { 244 log_checksum_verify test.db-wal $f $endian 245 } {1} 246 } 247 248 # Now that both the recoverer and non-recoverer have added frames to the 249 # log file, check that it can still be recovered. 250 # 251 file copy -force test.db test2.db 252 file copy -force test.db-wal test2.db-wal 253 do_test walcksum-1.$endian.7.11 { 254 sqlite3 db3 test2.db 255 execsql { 256 PRAGMA integrity_check; 257 SELECT a FROM t1; 258 } db3 259 } {ok 1 2 3 5 8 13 21 34 55} 260 db3 close 261 262 # Run a checkpoint on the database file. Then, check that any frames written 263 # to the start of the log use native byte-order checksums. 264 # 265 do_test walcksum-1.$endian.8.1 { 266 execsql { 267 PRAGMA wal_checkpoint; 268 INSERT INTO t1 VALUES(89, 'eightynine'); 269 } 270 log_checksum_verify test.db-wal 1 $native 271 } {1} 272 do_test walcksum-1.$endian.8.2 { 273 log_checksum_verify test.db-wal 2 $native 274 } {1} 275 do_test walcksum-1.$endian.8.3 { 276 log_checksum_verify test.db-wal 3 $native 277 } {0} 278 279 do_test walcksum-1.$endian.9 { 280 execsql { 281 PRAGMA integrity_check; 282 SELECT a FROM t1; 283 } db2 284 } {ok 1 2 3 5 8 13 21 34 55 89} 285 286 catch { db close } 287 catch { db2 close } 288 } 289 290 #------------------------------------------------------------------------- 291 # Test case walcksum-2.* tests that if a statement transaction is rolled 292 # back after frames are written to the WAL, and then (after writing some 293 # more) the outer transaction is committed, the WAL file is still correctly 294 # formatted (and can be recovered by a second process if required). 295 # 296 do_test walcksum-2.1 { 297 file delete -force test.db test.db-wal test.db-journal 298 sqlite3 db test.db 299 execsql { 300 PRAGMA synchronous = NORMAL; 301 PRAGMA page_size = 1024; 302 PRAGMA journal_mode = WAL; 303 PRAGMA cache_size = 10; 304 CREATE TABLE t1(x PRIMARY KEY); 305 PRAGMA wal_checkpoint; 306 INSERT INTO t1 VALUES(randomblob(800)); 307 BEGIN; 308 INSERT INTO t1 SELECT randomblob(800) FROM t1; /* 2 */ 309 INSERT INTO t1 SELECT randomblob(800) FROM t1; /* 4 */ 310 INSERT INTO t1 SELECT randomblob(800) FROM t1; /* 8 */ 311 INSERT INTO t1 SELECT randomblob(800) FROM t1; /* 16 */ 312 SAVEPOINT one; 313 INSERT INTO t1 SELECT randomblob(800) FROM t1; /* 32 */ 314 INSERT INTO t1 SELECT randomblob(800) FROM t1; /* 64 */ 315 INSERT INTO t1 SELECT randomblob(800) FROM t1; /* 128 */ 316 INSERT INTO t1 SELECT randomblob(800) FROM t1; /* 256 */ 317 ROLLBACK TO one; 318 INSERT INTO t1 SELECT randomblob(800) FROM t1; /* 32 */ 319 INSERT INTO t1 SELECT randomblob(800) FROM t1; /* 64 */ 320 INSERT INTO t1 SELECT randomblob(800) FROM t1; /* 128 */ 321 INSERT INTO t1 SELECT randomblob(800) FROM t1; /* 256 */ 322 COMMIT; 323 } 324 325 file copy -force test.db test2.db 326 file copy -force test.db-wal test2.db-wal 327 328 sqlite3 db2 test2.db 329 execsql { 330 PRAGMA integrity_check; 331 SELECT count(*) FROM t1; 332 } db2 333 } {ok 256} 334 catch { db close } 335 catch { db2 close } 336 337 #------------------------------------------------------------------------- 338 # Test case walcksum-3.* tests that the checksum calculation detects single 339 # byte changes to frame or frame-header data and considers the frame 340 # invalid as a result. 341 # 342 do_test walcksum-3.1 { 343 file delete -force test.db test.db-wal test.db-journal 344 sqlite3 db test.db 345 346 execsql { 347 PRAGMA synchronous = NORMAL; 348 PRAGMA page_size = 1024; 349 CREATE TABLE t1(a, b); 350 INSERT INTO t1 VALUES(1, randomblob(300)); 351 INSERT INTO t1 VALUES(2, randomblob(300)); 352 PRAGMA journal_mode = WAL; 353 INSERT INTO t1 VALUES(3, randomblob(300)); 354 } 355 356 file size test.db-wal 357 } [wal_file_size 1 1024] 358 do_test walcksum-3.2 { 359 file copy -force test.db-wal test2.db-wal 360 file copy -force test.db test2.db 361 sqlite3 db2 test2.db 362 execsql { SELECT a FROM t1 } db2 363 } {1 2 3} 364 db2 close 365 file copy -force test.db test2.db 366 367 368 foreach incr {1 2 3 20 40 60 80 100 120 140 160 180 200 220 240 253 254 255} { 369 do_test walcksum-3.3.$incr { 370 set FAIL 0 371 for {set iOff 0} {$iOff < [wal_file_size 1 1024]} {incr iOff} { 372 373 file copy -force test.db-wal test2.db-wal 374 set fd [open test2.db-wal r+] 375 fconfigure $fd -encoding binary 376 fconfigure $fd -translation binary 377 378 seek $fd $iOff 379 binary scan [read $fd 1] c x 380 seek $fd $iOff 381 puts -nonewline $fd [binary format c [expr {($x+$incr)&0xFF}]] 382 close $fd 383 384 sqlite3 db2 test2.db 385 if { [execsql { SELECT a FROM t1 } db2] != "1 2" } {set FAIL 1} 386 db2 close 387 } 388 set FAIL 389 } {0} 390 } 391 392 finish_test 393 394
