1 # 2007 June 21 2 # 3 # The author disclaims copyright to this source code. In place of 4 # a legal notice, here is a blessing: 5 # 6 # May you do good and not evil. 7 # May you find forgiveness for yourself and forgive others. 8 # May you share freely, never taking more than you give. 9 # 10 #************************************************************************* 11 # This file implements regression tests for SQLite library. The focus 12 # of this script is testing the pluggable tokeniser feature of the 13 # FTS3 module. 14 # 15 # $Id: fts3atoken.test,v 1.1 2007/08/20 17:38:42 shess Exp $ 16 # 17 18 set testdir [file dirname $argv0] 19 source $testdir/tester.tcl 20 21 # If SQLITE_ENABLE_FTS3 is defined, omit this file. 22 ifcapable !fts3 { 23 finish_test 24 return 25 } 26 27 proc escape_string {str} { 28 set out "" 29 foreach char [split $str ""] { 30 scan $char %c i 31 if {$i<=127} { 32 append out $char 33 } else { 34 append out [format {\x%.4x} $i] 35 } 36 } 37 set out 38 } 39 40 #-------------------------------------------------------------------------- 41 # Test cases fts3token-1.* are the warm-body test for the SQL scalar 42 # function fts3_tokenizer(). The procedure is as follows: 43 # 44 # 1: Verify that there is no such fts3 tokenizer as 'blah'. 45 # 46 # 2: Query for the built-in tokenizer 'simple'. Insert a copy of the 47 # retrieved value as tokenizer 'blah'. 48 # 49 # 3: Test that the value returned for tokenizer 'blah' is now the 50 # same as that retrieved for 'simple'. 51 # 52 # 4: Test that it is now possible to create an fts3 table using 53 # tokenizer 'blah' (it was not possible in step 1). 54 # 55 # 5: Test that the table created to use tokenizer 'blah' is usable. 56 # 57 do_test fts3token-1.1 { 58 catchsql { 59 CREATE VIRTUAL TABLE t1 USING fts3(content, tokenize blah); 60 } 61 } {1 {unknown tokenizer: blah}} 62 do_test fts3token-1.2 { 63 execsql { 64 SELECT fts3_tokenizer('blah', fts3_tokenizer('simple')) IS NULL; 65 } 66 } {0} 67 do_test fts3token-1.3 { 68 execsql { 69 SELECT fts3_tokenizer('blah') == fts3_tokenizer('simple'); 70 } 71 } {1} 72 do_test fts3token-1.4 { 73 catchsql { 74 CREATE VIRTUAL TABLE t1 USING fts3(content, tokenize blah); 75 } 76 } {0 {}} 77 do_test fts3token-1.5 { 78 execsql { 79 INSERT INTO t1(content) VALUES('There was movement at the station'); 80 INSERT INTO t1(content) VALUES('For the word has passed around'); 81 INSERT INTO t1(content) VALUES('That the colt from ol regret had got away'); 82 SELECT content FROM t1 WHERE content MATCH 'movement' 83 } 84 } {{There was movement at the station}} 85 86 #-------------------------------------------------------------------------- 87 # Test cases fts3token-2.* test error cases in the scalar function based 88 # API for getting and setting tokenizers. 89 # 90 do_test fts3token-2.1 { 91 catchsql { 92 SELECT fts3_tokenizer('nosuchtokenizer'); 93 } 94 } {1 {unknown tokenizer: nosuchtokenizer}} 95 96 #-------------------------------------------------------------------------- 97 # Test cases fts3token-3.* test the three built-in tokenizers with a 98 # simple input string via the built-in test function. This is as much 99 # to test the test function as the tokenizer implementations. 100 # 101 do_test fts3token-3.1 { 102 execsql { 103 SELECT fts3_tokenizer_test('simple', 'I don''t see how'); 104 } 105 } {{0 i I 1 don don 2 t t 3 see see 4 how how}} 106 do_test fts3token-3.2 { 107 execsql { 108 SELECT fts3_tokenizer_test('porter', 'I don''t see how'); 109 } 110 } {{0 i I 1 don don 2 t t 3 see see 4 how how}} 111 ifcapable icu { 112 do_test fts3token-3.3 { 113 execsql { 114 SELECT fts3_tokenizer_test('icu', 'I don''t see how'); 115 } 116 } {{0 i I 1 don't don't 2 see see 3 how how}} 117 } 118 119 #-------------------------------------------------------------------------- 120 # Test cases fts3token-4.* test the ICU tokenizer. In practice, this 121 # tokenizer only has two modes - "thai" and "everybody else". Some other 122 # Asian languages (Lao, Khmer etc.) require the same special treatment as 123 # Thai, but ICU doesn't support them yet. 124 # 125 ifcapable icu { 126 127 proc do_icu_test {name locale input output} { 128 set ::out [db eval { SELECT fts3_tokenizer_test('icu', $locale, $input) }] 129 do_test $name { 130 lindex $::out 0 131 } $output 132 } 133 134 do_icu_test fts3token-4.1 en_US {} {} 135 do_icu_test fts3token-4.2 en_US {Test cases fts3} [list \ 136 0 test Test 1 cases cases 2 fts3 fts3 137 ] 138 139 # The following test shows that ICU is smart enough to recognise 140 # Thai chararacters, even when the locale is set to English/United 141 # States. 142 # 143 set input "\u0e2d\u0e30\u0e44\u0e23\u0e19\u0e30\u0e04\u0e23\u0e31\u0e1a" 144 set output "0 \u0e2d\u0e30\u0e44\u0e23 \u0e2d\u0e30\u0e44\u0e23 " 145 append output "1 \u0e19\u0e30 \u0e19\u0e30 " 146 append output "2 \u0e04\u0e23\u0e31\u0e1a \u0e04\u0e23\u0e31\u0e1a" 147 148 do_icu_test fts3token-4.3 th_TH $input $output 149 do_icu_test fts3token-4.4 en_US $input $output 150 151 # ICU handles an unknown locale by falling back to the default. 152 # So this is not an error. 153 do_icu_test fts3token-4.5 MiddleOfTheOcean $input $output 154 155 set longtoken "AReallyReallyLongTokenOneThatWillSurelyRequire" 156 append longtoken "AReallocInTheIcuTokenizerCode" 157 158 set input "short tokens then " 159 append input $longtoken 160 set output "0 short short " 161 append output "1 tokens tokens " 162 append output "2 then then " 163 append output "3 [string tolower $longtoken] $longtoken" 164 165 do_icu_test fts3token-4.6 MiddleOfTheOcean $input $output 166 do_icu_test fts3token-4.7 th_TH $input $output 167 do_icu_test fts3token-4.8 en_US $input $output 168 } 169 170 do_test fts3token-internal { 171 execsql { SELECT fts3_tokenizer_internal_test() } 172 } {ok} 173 174 finish_test 175
