1 // Copyright 2011 The Go Authors. All rights reserved. 2 // Use of this source code is governed by a BSD-style 3 // license that can be found in the LICENSE file. 4 5 package parse 6 7 import ( 8 "fmt" 9 "testing" 10 ) 11 12 // Make the types prettyprint. 13 var itemName = map[itemType]string{ 14 itemError: "error", 15 itemBool: "bool", 16 itemChar: "char", 17 itemCharConstant: "charconst", 18 itemComplex: "complex", 19 itemColonEquals: ":=", 20 itemEOF: "EOF", 21 itemField: "field", 22 itemIdentifier: "identifier", 23 itemLeftDelim: "left delim", 24 itemLeftParen: "(", 25 itemNumber: "number", 26 itemPipe: "pipe", 27 itemRawString: "raw string", 28 itemRightDelim: "right delim", 29 itemRightParen: ")", 30 itemSpace: "space", 31 itemString: "string", 32 itemVariable: "variable", 33 34 // keywords 35 itemDot: ".", 36 itemDefine: "define", 37 itemElse: "else", 38 itemIf: "if", 39 itemEnd: "end", 40 itemNil: "nil", 41 itemRange: "range", 42 itemTemplate: "template", 43 itemWith: "with", 44 } 45 46 func (i itemType) String() string { 47 s := itemName[i] 48 if s == "" { 49 return fmt.Sprintf("item%d", int(i)) 50 } 51 return s 52 } 53 54 type lexTest struct { 55 name string 56 input string 57 items []item 58 } 59 60 var ( 61 tEOF = item{itemEOF, 0, ""} 62 tFor = item{itemIdentifier, 0, "for"} 63 tLeft = item{itemLeftDelim, 0, "{{"} 64 tLpar = item{itemLeftParen, 0, "("} 65 tPipe = item{itemPipe, 0, "|"} 66 tQuote = item{itemString, 0, `"abc \n\t\" "`} 67 tRange = item{itemRange, 0, "range"} 68 tRight = item{itemRightDelim, 0, "}}"} 69 tRpar = item{itemRightParen, 0, ")"} 70 tSpace = item{itemSpace, 0, " "} 71 raw = "`" + `abc\n\t\" ` + "`" 72 rawNL = "`now is{{\n}}the time`" // Contains newline inside raw quote. 73 tRawQuote = item{itemRawString, 0, raw} 74 tRawQuoteNL = item{itemRawString, 0, rawNL} 75 ) 76 77 var lexTests = []lexTest{ 78 {"empty", "", []item{tEOF}}, 79 {"spaces", " \t\n", []item{{itemText, 0, " \t\n"}, tEOF}}, 80 {"text", `now is the time`, []item{{itemText, 0, "now is the time"}, tEOF}}, 81 {"text with comment", "hello-{{/* this is a comment */}}-world", []item{ 82 {itemText, 0, "hello-"}, 83 {itemText, 0, "-world"}, 84 tEOF, 85 }}, 86 {"punctuation", "{{,@% }}", []item{ 87 tLeft, 88 {itemChar, 0, ","}, 89 {itemChar, 0, "@"}, 90 {itemChar, 0, "%"}, 91 tSpace, 92 tRight, 93 tEOF, 94 }}, 95 {"parens", "{{((3))}}", []item{ 96 tLeft, 97 tLpar, 98 tLpar, 99 {itemNumber, 0, "3"}, 100 tRpar, 101 tRpar, 102 tRight, 103 tEOF, 104 }}, 105 {"empty action", `{{}}`, []item{tLeft, tRight, tEOF}}, 106 {"for", `{{for}}`, []item{tLeft, tFor, tRight, tEOF}}, 107 {"quote", `{{"abc \n\t\" "}}`, []item{tLeft, tQuote, tRight, tEOF}}, 108 {"raw quote", "{{" + raw + "}}", []item{tLeft, tRawQuote, tRight, tEOF}}, 109 {"raw quote with newline", "{{" + rawNL + "}}", []item{tLeft, tRawQuoteNL, tRight, tEOF}}, 110 {"numbers", "{{1 02 0x14 -7.2i 1e3 +1.2e-4 4.2i 1+2i}}", []item{ 111 tLeft, 112 {itemNumber, 0, "1"}, 113 tSpace, 114 {itemNumber, 0, "02"}, 115 tSpace, 116 {itemNumber, 0, "0x14"}, 117 tSpace, 118 {itemNumber, 0, "-7.2i"}, 119 tSpace, 120 {itemNumber, 0, "1e3"}, 121 tSpace, 122 {itemNumber, 0, "+1.2e-4"}, 123 tSpace, 124 {itemNumber, 0, "4.2i"}, 125 tSpace, 126 {itemComplex, 0, "1+2i"}, 127 tRight, 128 tEOF, 129 }}, 130 {"characters", `{{'a' '\n' '\'' '\\' '\u00FF' '\xFF' ''}}`, []item{ 131 tLeft, 132 {itemCharConstant, 0, `'a'`}, 133 tSpace, 134 {itemCharConstant, 0, `'\n'`}, 135 tSpace, 136 {itemCharConstant, 0, `'\''`}, 137 tSpace, 138 {itemCharConstant, 0, `'\\'`}, 139 tSpace, 140 {itemCharConstant, 0, `'\u00FF'`}, 141 tSpace, 142 {itemCharConstant, 0, `'\xFF'`}, 143 tSpace, 144 {itemCharConstant, 0, `''`}, 145 tRight, 146 tEOF, 147 }}, 148 {"bools", "{{true false}}", []item{ 149 tLeft, 150 {itemBool, 0, "true"}, 151 tSpace, 152 {itemBool, 0, "false"}, 153 tRight, 154 tEOF, 155 }}, 156 {"dot", "{{.}}", []item{ 157 tLeft, 158 {itemDot, 0, "."}, 159 tRight, 160 tEOF, 161 }}, 162 {"nil", "{{nil}}", []item{ 163 tLeft, 164 {itemNil, 0, "nil"}, 165 tRight, 166 tEOF, 167 }}, 168 {"dots", "{{.x . .2 .x.y.z}}", []item{ 169 tLeft, 170 {itemField, 0, ".x"}, 171 tSpace, 172 {itemDot, 0, "."}, 173 tSpace, 174 {itemNumber, 0, ".2"}, 175 tSpace, 176 {itemField, 0, ".x"}, 177 {itemField, 0, ".y"}, 178 {itemField, 0, ".z"}, 179 tRight, 180 tEOF, 181 }}, 182 {"keywords", "{{range if else end with}}", []item{ 183 tLeft, 184 {itemRange, 0, "range"}, 185 tSpace, 186 {itemIf, 0, "if"}, 187 tSpace, 188 {itemElse, 0, "else"}, 189 tSpace, 190 {itemEnd, 0, "end"}, 191 tSpace, 192 {itemWith, 0, "with"}, 193 tRight, 194 tEOF, 195 }}, 196 {"variables", "{{$c := printf $ $hello $23 $ $var.Field .Method}}", []item{ 197 tLeft, 198 {itemVariable, 0, "$c"}, 199 tSpace, 200 {itemColonEquals, 0, ":="}, 201 tSpace, 202 {itemIdentifier, 0, "printf"}, 203 tSpace, 204 {itemVariable, 0, "$"}, 205 tSpace, 206 {itemVariable, 0, "$hello"}, 207 tSpace, 208 {itemVariable, 0, "$23"}, 209 tSpace, 210 {itemVariable, 0, "$"}, 211 tSpace, 212 {itemVariable, 0, "$var"}, 213 {itemField, 0, ".Field"}, 214 tSpace, 215 {itemField, 0, ".Method"}, 216 tRight, 217 tEOF, 218 }}, 219 {"variable invocation", "{{$x 23}}", []item{ 220 tLeft, 221 {itemVariable, 0, "$x"}, 222 tSpace, 223 {itemNumber, 0, "23"}, 224 tRight, 225 tEOF, 226 }}, 227 {"pipeline", `intro {{echo hi 1.2 |noargs|args 1 "hi"}} outro`, []item{ 228 {itemText, 0, "intro "}, 229 tLeft, 230 {itemIdentifier, 0, "echo"}, 231 tSpace, 232 {itemIdentifier, 0, "hi"}, 233 tSpace, 234 {itemNumber, 0, "1.2"}, 235 tSpace, 236 tPipe, 237 {itemIdentifier, 0, "noargs"}, 238 tPipe, 239 {itemIdentifier, 0, "args"}, 240 tSpace, 241 {itemNumber, 0, "1"}, 242 tSpace, 243 {itemString, 0, `"hi"`}, 244 tRight, 245 {itemText, 0, " outro"}, 246 tEOF, 247 }}, 248 {"declaration", "{{$v := 3}}", []item{ 249 tLeft, 250 {itemVariable, 0, "$v"}, 251 tSpace, 252 {itemColonEquals, 0, ":="}, 253 tSpace, 254 {itemNumber, 0, "3"}, 255 tRight, 256 tEOF, 257 }}, 258 {"2 declarations", "{{$v , $w := 3}}", []item{ 259 tLeft, 260 {itemVariable, 0, "$v"}, 261 tSpace, 262 {itemChar, 0, ","}, 263 tSpace, 264 {itemVariable, 0, "$w"}, 265 tSpace, 266 {itemColonEquals, 0, ":="}, 267 tSpace, 268 {itemNumber, 0, "3"}, 269 tRight, 270 tEOF, 271 }}, 272 {"field of parenthesized expression", "{{(.X).Y}}", []item{ 273 tLeft, 274 tLpar, 275 {itemField, 0, ".X"}, 276 tRpar, 277 {itemField, 0, ".Y"}, 278 tRight, 279 tEOF, 280 }}, 281 // errors 282 {"badchar", "#{{\x01}}", []item{ 283 {itemText, 0, "#"}, 284 tLeft, 285 {itemError, 0, "unrecognized character in action: U+0001"}, 286 }}, 287 {"unclosed action", "{{\n}}", []item{ 288 tLeft, 289 {itemError, 0, "unclosed action"}, 290 }}, 291 {"EOF in action", "{{range", []item{ 292 tLeft, 293 tRange, 294 {itemError, 0, "unclosed action"}, 295 }}, 296 {"unclosed quote", "{{\"\n\"}}", []item{ 297 tLeft, 298 {itemError, 0, "unterminated quoted string"}, 299 }}, 300 {"unclosed raw quote", "{{`xx}}", []item{ 301 tLeft, 302 {itemError, 0, "unterminated raw quoted string"}, 303 }}, 304 {"unclosed char constant", "{{'\n}}", []item{ 305 tLeft, 306 {itemError, 0, "unterminated character constant"}, 307 }}, 308 {"bad number", "{{3k}}", []item{ 309 tLeft, 310 {itemError, 0, `bad number syntax: "3k"`}, 311 }}, 312 {"unclosed paren", "{{(3}}", []item{ 313 tLeft, 314 tLpar, 315 {itemNumber, 0, "3"}, 316 {itemError, 0, `unclosed left paren`}, 317 }}, 318 {"extra right paren", "{{3)}}", []item{ 319 tLeft, 320 {itemNumber, 0, "3"}, 321 tRpar, 322 {itemError, 0, `unexpected right paren U+0029 ')'`}, 323 }}, 324 325 // Fixed bugs 326 // Many elements in an action blew the lookahead until 327 // we made lexInsideAction not loop. 328 {"long pipeline deadlock", "{{|||||}}", []item{ 329 tLeft, 330 tPipe, 331 tPipe, 332 tPipe, 333 tPipe, 334 tPipe, 335 tRight, 336 tEOF, 337 }}, 338 {"text with bad comment", "hello-{{/*/}}-world", []item{ 339 {itemText, 0, "hello-"}, 340 {itemError, 0, `unclosed comment`}, 341 }}, 342 {"text with comment close separted from delim", "hello-{{/* */ }}-world", []item{ 343 {itemText, 0, "hello-"}, 344 {itemError, 0, `comment ends before closing delimiter`}, 345 }}, 346 // This one is an error that we can't catch because it breaks templates with 347 // minimized JavaScript. Should have fixed it before Go 1.1. 348 {"unmatched right delimiter", "hello-{.}}-world", []item{ 349 {itemText, 0, "hello-{.}}-world"}, 350 tEOF, 351 }}, 352 } 353 354 // collect gathers the emitted items into a slice. 355 func collect(t *lexTest, left, right string) (items []item) { 356 l := lex(t.name, t.input, left, right) 357 for { 358 item := l.nextItem() 359 items = append(items, item) 360 if item.typ == itemEOF || item.typ == itemError { 361 break 362 } 363 } 364 return 365 } 366 367 func equal(i1, i2 []item, checkPos bool) bool { 368 if len(i1) != len(i2) { 369 return false 370 } 371 for k := range i1 { 372 if i1[k].typ != i2[k].typ { 373 return false 374 } 375 if i1[k].val != i2[k].val { 376 return false 377 } 378 if checkPos && i1[k].pos != i2[k].pos { 379 return false 380 } 381 } 382 return true 383 } 384 385 func TestLex(t *testing.T) { 386 for _, test := range lexTests { 387 items := collect(&test, "", "") 388 if !equal(items, test.items, false) { 389 t.Errorf("%s: got\n\t%+v\nexpected\n\t%v", test.name, items, test.items) 390 } 391 } 392 } 393 394 // Some easy cases from above, but with delimiters $$ and @@ 395 var lexDelimTests = []lexTest{ 396 {"punctuation", "$$,@%{{}}@@", []item{ 397 tLeftDelim, 398 {itemChar, 0, ","}, 399 {itemChar, 0, "@"}, 400 {itemChar, 0, "%"}, 401 {itemChar, 0, "{"}, 402 {itemChar, 0, "{"}, 403 {itemChar, 0, "}"}, 404 {itemChar, 0, "}"}, 405 tRightDelim, 406 tEOF, 407 }}, 408 {"empty action", `$$@@`, []item{tLeftDelim, tRightDelim, tEOF}}, 409 {"for", `$$for@@`, []item{tLeftDelim, tFor, tRightDelim, tEOF}}, 410 {"quote", `$$"abc \n\t\" "@@`, []item{tLeftDelim, tQuote, tRightDelim, tEOF}}, 411 {"raw quote", "$$" + raw + "@@", []item{tLeftDelim, tRawQuote, tRightDelim, tEOF}}, 412 } 413 414 var ( 415 tLeftDelim = item{itemLeftDelim, 0, "$$"} 416 tRightDelim = item{itemRightDelim, 0, "@@"} 417 ) 418 419 func TestDelims(t *testing.T) { 420 for _, test := range lexDelimTests { 421 items := collect(&test, "$$", "@@") 422 if !equal(items, test.items, false) { 423 t.Errorf("%s: got\n\t%v\nexpected\n\t%v", test.name, items, test.items) 424 } 425 } 426 } 427 428 var lexPosTests = []lexTest{ 429 {"empty", "", []item{tEOF}}, 430 {"punctuation", "{{,@%#}}", []item{ 431 {itemLeftDelim, 0, "{{"}, 432 {itemChar, 2, ","}, 433 {itemChar, 3, "@"}, 434 {itemChar, 4, "%"}, 435 {itemChar, 5, "#"}, 436 {itemRightDelim, 6, "}}"}, 437 {itemEOF, 8, ""}, 438 }}, 439 {"sample", "0123{{hello}}xyz", []item{ 440 {itemText, 0, "0123"}, 441 {itemLeftDelim, 4, "{{"}, 442 {itemIdentifier, 6, "hello"}, 443 {itemRightDelim, 11, "}}"}, 444 {itemText, 13, "xyz"}, 445 {itemEOF, 16, ""}, 446 }}, 447 } 448 449 // The other tests don't check position, to make the test cases easier to construct. 450 // This one does. 451 func TestPos(t *testing.T) { 452 for _, test := range lexPosTests { 453 items := collect(&test, "", "") 454 if !equal(items, test.items, true) { 455 t.Errorf("%s: got\n\t%v\nexpected\n\t%v", test.name, items, test.items) 456 if len(items) == len(test.items) { 457 // Detailed print; avoid item.String() to expose the position value. 458 for i := range items { 459 if !equal(items[i:i+1], test.items[i:i+1], true) { 460 i1 := items[i] 461 i2 := test.items[i] 462 t.Errorf("\t#%d: got {%v %d %q} expected {%v %d %q}", i, i1.typ, i1.pos, i1.val, i2.typ, i2.pos, i2.val) 463 } 464 } 465 } 466 } 467 } 468 } 469 470 // Test that an error shuts down the lexing goroutine. 471 func TestShutdown(t *testing.T) { 472 // We need to duplicate template.Parse here to hold on to the lexer. 473 const text = "erroneous{{define}}{{else}}1234" 474 lexer := lex("foo", text, "{{", "}}") 475 _, err := New("root").parseLexer(lexer, text) 476 if err == nil { 477 t.Fatalf("expected error") 478 } 479 // The error should have drained the input. Therefore, the lexer should be shut down. 480 token, ok := <-lexer.items 481 if ok { 482 t.Fatalf("input was not drained; got %v", token) 483 } 484 } 485 486 // parseLexer is a local version of parse that lets us pass in the lexer instead of building it. 487 // We expect an error, so the tree set and funcs list are explicitly nil. 488 func (t *Tree) parseLexer(lex *lexer, text string) (tree *Tree, err error) { 489 defer t.recover(&err) 490 t.ParseName = t.Name 491 t.startParse(nil, lex) 492 t.parse(nil) 493 t.add(nil) 494 t.stopParse() 495 return t, nil 496 } 497
