1 l=8, c=1, stat=0, len=64, token="-option" 2 l=8, c=9, stat=0, len=64, token="key" 3 l=9, c=1, stat=0, len=64, token="+reset" 4 l=9, c=10, stat=0, len=64, token="imbedded#hash" 5 l=11, c=1, stat=0, len=64, token="This is a single token" 6 l=13, c=1, stat=0, len=64, token="And\ 'even '"more "complex" 7 l=15, c=1, stat=0, len=64, token="Backslash chars \n are returned as is" 8 l=16, c=1, stat=0, len=64, token="regardless \n of quoting" 9 l=18, c=1, stat=0, len=64, token="Single quote escapes" 10 l=19, c=2, stat=0, len=64, token="'" 11 l=19, c=4, stat=0, len=64, token="'" 12 l=21, c=1, stat=0, len=64, token="Double quote escapes" 13 l=22, c=2, stat=0, len=64, token=""" 14 l=22, c=4, stat=0, len=64, token=""" 15 l=22, c=8, stat=0, len=64, token=""" 16 l=24, c=1, stat=0, len=64, token="Back slash escapes" 17 l=25, c=2, stat=0, len=64, token="\" 18 l=25, c=4, stat=0, len=64, token="\" 19 l=25, c=9, stat=0, len=64, token="\" 20 l=27, c=1, stat=0, len=64, token="Space Character Escapes" 21 l=28, c=2, stat=0, len=64, token=" " 22 l=28, c=4, stat=0, len=64, token=" " 23 l=28, c=9, stat=0, len=64, token=" " 24 l=30, c=1, stat=0, len=64, token="Empty Tokens, using quotes" 25 l=31, c=1, stat=0, len=64, token="" 26 l=31, c=6, stat=0, len=64, token="" 27 l=33, c=1, stat=0, len=64, token="Unicode charcaters are handled" 28 l=34, c=1, stat=0, len=64, token=" ' ` " 29 l=35, c=1, stat=0, len=64, token=" " 30 l=36, c=1, stat=0, len=64, token="" 31 l=38, c=1, stat=0, len=64, token="Line____Continuation" 32 l=41, c=1, stat=0, len=64, token="Double_Quoted_Line____Continuation" 33 l=44, c=1, stat=0, len=64, token="Single_Quoted_Line__\ 34 __Continuation_NOT!" 35 l=47, c=1, stat=0, len=64, token="double_quoted_newlines__ 36 __are_part_of_token" 37 l=50, c=1, stat=0, len=64, token="single_quoted_newlines__ 38 __are_part_of_token" 39 l=53, c=1, stat=0, len=64, token="Last Token before EOF" 40 EOF Found 41 42 l=1, c=1, stat=0, len=64, token="Next token bad quotes" 43 Bad Quotes l=1, c=25 token="unfinished quotes ->" 44 45 l=1, c=1, stat=0, len=64, token="Binary input follows" 46 Binary Char at l=2, c=4 47 48 l=1, c=1, stat=0, len=64, token="Very BIG Token Tests" 49 l=2, c=1, stat=0, len=256, token="aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa..." 50 l=3, c=1, stat=0, len=1024, token="bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb..." 51 l=4, c=1, stat=0, len=4096, token="cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc..." 52 l=5, c=1, stat=0, len=8192, token="dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd..." 53 l=6, c=1, stat=0, len=12288, token="eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee..." 54 l=7, c=1, stat=0, len=16384, token="ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff..." 55 l=8, c=1, stat=0, len=8392704, token="eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee..." 56 l=9, c=1, stat=0, len=8392704, token="and all is well!" 57 EOF Found 58 59
