1 # 2007 March 28 2 # 3 # The author disclaims copyright to this source code. 4 # 5 #************************************************************************* 6 # This file implements regression tests for SQLite library. The focus 7 # of this script is testing isspace/isalnum/tolower problems with the 8 # FTS2 module. Unfortunately, this code isn't a really principled set 9 # of tests, because it is impossible to know where new uses of these 10 # functions might appear. 11 # 12 # $Id: fts2l.test,v 1.2 2007/12/13 21:54:11 drh Exp $ 13 # 14 15 set testdir [file dirname $argv0] 16 source $testdir/tester.tcl 17 18 # If SQLITE_ENABLE_FTS2 is defined, omit this file. 19 ifcapable !fts2 { 20 finish_test 21 return 22 } 23 24 # Tests that startsWith() (calls isspace, tolower, isalnum) can handle 25 # hi-bit chars. parseSpec() also calls isalnum here. 26 do_test fts2l-1.1 { 27 execsql "CREATE VIRTUAL TABLE t1 USING fts2(content, \x80)" 28 } {} 29 30 # Additionally tests isspace() call in getToken(), and isalnum() call 31 # in tokenListToIdList(). 32 do_test fts2l-1.2 { 33 catch { 34 execsql "CREATE VIRTUAL TABLE t2 USING fts2(content, tokenize \x80)" 35 } 36 sqlite3_errmsg $DB 37 } "unknown tokenizer: \x80" 38 39 # Additionally test final isalnum() in startsWith(). 40 do_test fts2l-1.3 { 41 execsql "CREATE VIRTUAL TABLE t3 USING fts2(content, tokenize\x80)" 42 } {} 43 44 # The snippet-generation code has calls to isspace() which are sort of 45 # hard to get to. It finds convenient breakpoints by starting ~40 46 # chars before and after the matched term, and scanning ~10 chars 47 # around that position for isspace() characters. The long word with 48 # embedded hi-bit chars causes one of these isspace() calls to be 49 # exercised. The version with a couple extra spaces should cause the 50 # other isspace() call to be exercised. [Both cases have been tested 51 # in the debugger, but I'm hoping to continue to catch it if simple 52 # constant changes change things slightly. 53 # 54 # The trailing and leading hi-bit chars help with code which tests for 55 # isspace() to coalesce multiple spaces. 56 57 set word "\x80xxxxx\x80xxxxx\x80xxxxx\x80xxxxx\x80xxxxx\x80xxxxx\x80" 58 set phrase1 "$word $word $word target $word $word $word" 59 set phrase2 "$word $word $word target $word $word $word" 60 61 db eval {CREATE VIRTUAL TABLE t4 USING fts2(content)} 62 db eval "INSERT INTO t4 (content) VALUES ('$phrase1')" 63 db eval "INSERT INTO t4 (content) VALUES ('$phrase2')" 64 65 do_test fts2l-1.4 { 66 execsql {SELECT rowid, length(snippet(t4)) FROM t4 WHERE t4 MATCH 'target'} 67 } {1 111 2 117} 68 69 finish_test 70
