1 // Copyright 2011 The Go Authors. All rights reserved. 2 // Use of this source code is governed by a BSD-style 3 // license that can be found in the LICENSE file. 4 5 package parse 6 7 import ( 8 "fmt" 9 "testing" 10 ) 11 12 // Make the types prettyprint. 13 var itemName = map[itemType]string{ 14 itemError: "error", 15 itemBool: "bool", 16 itemChar: "char", 17 itemCharConstant: "charconst", 18 itemComplex: "complex", 19 itemColonEquals: ":=", 20 itemEOF: "EOF", 21 itemField: "field", 22 itemIdentifier: "identifier", 23 itemLeftDelim: "left delim", 24 itemLeftParen: "(", 25 itemNumber: "number", 26 itemPipe: "pipe", 27 itemRawString: "raw string", 28 itemRightDelim: "right delim", 29 itemRightParen: ")", 30 itemSpace: "space", 31 itemString: "string", 32 itemVariable: "variable", 33 34 // keywords 35 itemDot: ".", 36 itemBlock: "block", 37 itemDefine: "define", 38 itemElse: "else", 39 itemIf: "if", 40 itemEnd: "end", 41 itemNil: "nil", 42 itemRange: "range", 43 itemTemplate: "template", 44 itemWith: "with", 45 } 46 47 func (i itemType) String() string { 48 s := itemName[i] 49 if s == "" { 50 return fmt.Sprintf("item%d", int(i)) 51 } 52 return s 53 } 54 55 type lexTest struct { 56 name string 57 input string 58 items []item 59 } 60 61 func mkItem(typ itemType, text string) item { 62 return item{ 63 typ: typ, 64 val: text, 65 } 66 } 67 68 var ( 69 tDot = mkItem(itemDot, ".") 70 tBlock = mkItem(itemBlock, "block") 71 tEOF = mkItem(itemEOF, "") 72 tFor = mkItem(itemIdentifier, "for") 73 tLeft = mkItem(itemLeftDelim, "{{") 74 tLpar = mkItem(itemLeftParen, "(") 75 tPipe = mkItem(itemPipe, "|") 76 tQuote = mkItem(itemString, `"abc \n\t\" "`) 77 tRange = mkItem(itemRange, "range") 78 tRight = mkItem(itemRightDelim, "}}") 79 tRpar = mkItem(itemRightParen, ")") 80 tSpace = mkItem(itemSpace, " ") 81 raw = "`" + `abc\n\t\" ` + "`" 82 rawNL = "`now is{{\n}}the time`" // Contains newline inside raw quote. 83 tRawQuote = mkItem(itemRawString, raw) 84 tRawQuoteNL = mkItem(itemRawString, rawNL) 85 ) 86 87 var lexTests = []lexTest{ 88 {"empty", "", []item{tEOF}}, 89 {"spaces", " \t\n", []item{mkItem(itemText, " \t\n"), tEOF}}, 90 {"text", `now is the time`, []item{mkItem(itemText, "now is the time"), tEOF}}, 91 {"text with comment", "hello-{{/* this is a comment */}}-world", []item{ 92 mkItem(itemText, "hello-"), 93 mkItem(itemText, "-world"), 94 tEOF, 95 }}, 96 {"punctuation", "{{,@% }}", []item{ 97 tLeft, 98 mkItem(itemChar, ","), 99 mkItem(itemChar, "@"), 100 mkItem(itemChar, "%"), 101 tSpace, 102 tRight, 103 tEOF, 104 }}, 105 {"parens", "{{((3))}}", []item{ 106 tLeft, 107 tLpar, 108 tLpar, 109 mkItem(itemNumber, "3"), 110 tRpar, 111 tRpar, 112 tRight, 113 tEOF, 114 }}, 115 {"empty action", `{{}}`, []item{tLeft, tRight, tEOF}}, 116 {"for", `{{for}}`, []item{tLeft, tFor, tRight, tEOF}}, 117 {"block", `{{block "foo" .}}`, []item{ 118 tLeft, tBlock, tSpace, mkItem(itemString, `"foo"`), tSpace, tDot, tRight, tEOF, 119 }}, 120 {"quote", `{{"abc \n\t\" "}}`, []item{tLeft, tQuote, tRight, tEOF}}, 121 {"raw quote", "{{" + raw + "}}", []item{tLeft, tRawQuote, tRight, tEOF}}, 122 {"raw quote with newline", "{{" + rawNL + "}}", []item{tLeft, tRawQuoteNL, tRight, tEOF}}, 123 {"numbers", "{{1 02 0x14 -7.2i 1e3 +1.2e-4 4.2i 1+2i}}", []item{ 124 tLeft, 125 mkItem(itemNumber, "1"), 126 tSpace, 127 mkItem(itemNumber, "02"), 128 tSpace, 129 mkItem(itemNumber, "0x14"), 130 tSpace, 131 mkItem(itemNumber, "-7.2i"), 132 tSpace, 133 mkItem(itemNumber, "1e3"), 134 tSpace, 135 mkItem(itemNumber, "+1.2e-4"), 136 tSpace, 137 mkItem(itemNumber, "4.2i"), 138 tSpace, 139 mkItem(itemComplex, "1+2i"), 140 tRight, 141 tEOF, 142 }}, 143 {"characters", `{{'a' '\n' '\'' '\\' '\u00FF' '\xFF' ''}}`, []item{ 144 tLeft, 145 mkItem(itemCharConstant, `'a'`), 146 tSpace, 147 mkItem(itemCharConstant, `'\n'`), 148 tSpace, 149 mkItem(itemCharConstant, `'\''`), 150 tSpace, 151 mkItem(itemCharConstant, `'\\'`), 152 tSpace, 153 mkItem(itemCharConstant, `'\u00FF'`), 154 tSpace, 155 mkItem(itemCharConstant, `'\xFF'`), 156 tSpace, 157 mkItem(itemCharConstant, `''`), 158 tRight, 159 tEOF, 160 }}, 161 {"bools", "{{true false}}", []item{ 162 tLeft, 163 mkItem(itemBool, "true"), 164 tSpace, 165 mkItem(itemBool, "false"), 166 tRight, 167 tEOF, 168 }}, 169 {"dot", "{{.}}", []item{ 170 tLeft, 171 tDot, 172 tRight, 173 tEOF, 174 }}, 175 {"nil", "{{nil}}", []item{ 176 tLeft, 177 mkItem(itemNil, "nil"), 178 tRight, 179 tEOF, 180 }}, 181 {"dots", "{{.x . .2 .x.y.z}}", []item{ 182 tLeft, 183 mkItem(itemField, ".x"), 184 tSpace, 185 tDot, 186 tSpace, 187 mkItem(itemNumber, ".2"), 188 tSpace, 189 mkItem(itemField, ".x"), 190 mkItem(itemField, ".y"), 191 mkItem(itemField, ".z"), 192 tRight, 193 tEOF, 194 }}, 195 {"keywords", "{{range if else end with}}", []item{ 196 tLeft, 197 mkItem(itemRange, "range"), 198 tSpace, 199 mkItem(itemIf, "if"), 200 tSpace, 201 mkItem(itemElse, "else"), 202 tSpace, 203 mkItem(itemEnd, "end"), 204 tSpace, 205 mkItem(itemWith, "with"), 206 tRight, 207 tEOF, 208 }}, 209 {"variables", "{{$c := printf $ $hello $23 $ $var.Field .Method}}", []item{ 210 tLeft, 211 mkItem(itemVariable, "$c"), 212 tSpace, 213 mkItem(itemColonEquals, ":="), 214 tSpace, 215 mkItem(itemIdentifier, "printf"), 216 tSpace, 217 mkItem(itemVariable, "$"), 218 tSpace, 219 mkItem(itemVariable, "$hello"), 220 tSpace, 221 mkItem(itemVariable, "$23"), 222 tSpace, 223 mkItem(itemVariable, "$"), 224 tSpace, 225 mkItem(itemVariable, "$var"), 226 mkItem(itemField, ".Field"), 227 tSpace, 228 mkItem(itemField, ".Method"), 229 tRight, 230 tEOF, 231 }}, 232 {"variable invocation", "{{$x 23}}", []item{ 233 tLeft, 234 mkItem(itemVariable, "$x"), 235 tSpace, 236 mkItem(itemNumber, "23"), 237 tRight, 238 tEOF, 239 }}, 240 {"pipeline", `intro {{echo hi 1.2 |noargs|args 1 "hi"}} outro`, []item{ 241 mkItem(itemText, "intro "), 242 tLeft, 243 mkItem(itemIdentifier, "echo"), 244 tSpace, 245 mkItem(itemIdentifier, "hi"), 246 tSpace, 247 mkItem(itemNumber, "1.2"), 248 tSpace, 249 tPipe, 250 mkItem(itemIdentifier, "noargs"), 251 tPipe, 252 mkItem(itemIdentifier, "args"), 253 tSpace, 254 mkItem(itemNumber, "1"), 255 tSpace, 256 mkItem(itemString, `"hi"`), 257 tRight, 258 mkItem(itemText, " outro"), 259 tEOF, 260 }}, 261 {"declaration", "{{$v := 3}}", []item{ 262 tLeft, 263 mkItem(itemVariable, "$v"), 264 tSpace, 265 mkItem(itemColonEquals, ":="), 266 tSpace, 267 mkItem(itemNumber, "3"), 268 tRight, 269 tEOF, 270 }}, 271 {"2 declarations", "{{$v , $w := 3}}", []item{ 272 tLeft, 273 mkItem(itemVariable, "$v"), 274 tSpace, 275 mkItem(itemChar, ","), 276 tSpace, 277 mkItem(itemVariable, "$w"), 278 tSpace, 279 mkItem(itemColonEquals, ":="), 280 tSpace, 281 mkItem(itemNumber, "3"), 282 tRight, 283 tEOF, 284 }}, 285 {"field of parenthesized expression", "{{(.X).Y}}", []item{ 286 tLeft, 287 tLpar, 288 mkItem(itemField, ".X"), 289 tRpar, 290 mkItem(itemField, ".Y"), 291 tRight, 292 tEOF, 293 }}, 294 {"trimming spaces before and after", "hello- {{- 3 -}} -world", []item{ 295 mkItem(itemText, "hello-"), 296 tLeft, 297 mkItem(itemNumber, "3"), 298 tRight, 299 mkItem(itemText, "-world"), 300 tEOF, 301 }}, 302 {"trimming spaces before and after comment", "hello- {{- /* hello */ -}} -world", []item{ 303 mkItem(itemText, "hello-"), 304 mkItem(itemText, "-world"), 305 tEOF, 306 }}, 307 // errors 308 {"badchar", "#{{\x01}}", []item{ 309 mkItem(itemText, "#"), 310 tLeft, 311 mkItem(itemError, "unrecognized character in action: U+0001"), 312 }}, 313 {"unclosed action", "{{\n}}", []item{ 314 tLeft, 315 mkItem(itemError, "unclosed action"), 316 }}, 317 {"EOF in action", "{{range", []item{ 318 tLeft, 319 tRange, 320 mkItem(itemError, "unclosed action"), 321 }}, 322 {"unclosed quote", "{{\"\n\"}}", []item{ 323 tLeft, 324 mkItem(itemError, "unterminated quoted string"), 325 }}, 326 {"unclosed raw quote", "{{`xx}}", []item{ 327 tLeft, 328 mkItem(itemError, "unterminated raw quoted string"), 329 }}, 330 {"unclosed char constant", "{{'\n}}", []item{ 331 tLeft, 332 mkItem(itemError, "unterminated character constant"), 333 }}, 334 {"bad number", "{{3k}}", []item{ 335 tLeft, 336 mkItem(itemError, `bad number syntax: "3k"`), 337 }}, 338 {"unclosed paren", "{{(3}}", []item{ 339 tLeft, 340 tLpar, 341 mkItem(itemNumber, "3"), 342 mkItem(itemError, `unclosed left paren`), 343 }}, 344 {"extra right paren", "{{3)}}", []item{ 345 tLeft, 346 mkItem(itemNumber, "3"), 347 tRpar, 348 mkItem(itemError, `unexpected right paren U+0029 ')'`), 349 }}, 350 351 // Fixed bugs 352 // Many elements in an action blew the lookahead until 353 // we made lexInsideAction not loop. 354 {"long pipeline deadlock", "{{|||||}}", []item{ 355 tLeft, 356 tPipe, 357 tPipe, 358 tPipe, 359 tPipe, 360 tPipe, 361 tRight, 362 tEOF, 363 }}, 364 {"text with bad comment", "hello-{{/*/}}-world", []item{ 365 mkItem(itemText, "hello-"), 366 mkItem(itemError, `unclosed comment`), 367 }}, 368 {"text with comment close separated from delim", "hello-{{/* */ }}-world", []item{ 369 mkItem(itemText, "hello-"), 370 mkItem(itemError, `comment ends before closing delimiter`), 371 }}, 372 // This one is an error that we can't catch because it breaks templates with 373 // minimized JavaScript. Should have fixed it before Go 1.1. 374 {"unmatched right delimiter", "hello-{.}}-world", []item{ 375 mkItem(itemText, "hello-{.}}-world"), 376 tEOF, 377 }}, 378 } 379 380 // collect gathers the emitted items into a slice. 381 func collect(t *lexTest, left, right string) (items []item) { 382 l := lex(t.name, t.input, left, right) 383 for { 384 item := l.nextItem() 385 items = append(items, item) 386 if item.typ == itemEOF || item.typ == itemError { 387 break 388 } 389 } 390 return 391 } 392 393 func equal(i1, i2 []item, checkPos bool) bool { 394 if len(i1) != len(i2) { 395 return false 396 } 397 for k := range i1 { 398 if i1[k].typ != i2[k].typ { 399 return false 400 } 401 if i1[k].val != i2[k].val { 402 return false 403 } 404 if checkPos && i1[k].pos != i2[k].pos { 405 return false 406 } 407 if checkPos && i1[k].line != i2[k].line { 408 return false 409 } 410 } 411 return true 412 } 413 414 func TestLex(t *testing.T) { 415 for _, test := range lexTests { 416 items := collect(&test, "", "") 417 if !equal(items, test.items, false) { 418 t.Errorf("%s: got\n\t%+v\nexpected\n\t%v", test.name, items, test.items) 419 } 420 } 421 } 422 423 // Some easy cases from above, but with delimiters $$ and @@ 424 var lexDelimTests = []lexTest{ 425 {"punctuation", "$$,@%{{}}@@", []item{ 426 tLeftDelim, 427 mkItem(itemChar, ","), 428 mkItem(itemChar, "@"), 429 mkItem(itemChar, "%"), 430 mkItem(itemChar, "{"), 431 mkItem(itemChar, "{"), 432 mkItem(itemChar, "}"), 433 mkItem(itemChar, "}"), 434 tRightDelim, 435 tEOF, 436 }}, 437 {"empty action", `$$@@`, []item{tLeftDelim, tRightDelim, tEOF}}, 438 {"for", `$$for@@`, []item{tLeftDelim, tFor, tRightDelim, tEOF}}, 439 {"quote", `$$"abc \n\t\" "@@`, []item{tLeftDelim, tQuote, tRightDelim, tEOF}}, 440 {"raw quote", "$$" + raw + "@@", []item{tLeftDelim, tRawQuote, tRightDelim, tEOF}}, 441 } 442 443 var ( 444 tLeftDelim = mkItem(itemLeftDelim, "$$") 445 tRightDelim = mkItem(itemRightDelim, "@@") 446 ) 447 448 func TestDelims(t *testing.T) { 449 for _, test := range lexDelimTests { 450 items := collect(&test, "$$", "@@") 451 if !equal(items, test.items, false) { 452 t.Errorf("%s: got\n\t%v\nexpected\n\t%v", test.name, items, test.items) 453 } 454 } 455 } 456 457 var lexPosTests = []lexTest{ 458 {"empty", "", []item{{itemEOF, 0, "", 1}}}, 459 {"punctuation", "{{,@%#}}", []item{ 460 {itemLeftDelim, 0, "{{", 1}, 461 {itemChar, 2, ",", 1}, 462 {itemChar, 3, "@", 1}, 463 {itemChar, 4, "%", 1}, 464 {itemChar, 5, "#", 1}, 465 {itemRightDelim, 6, "}}", 1}, 466 {itemEOF, 8, "", 1}, 467 }}, 468 {"sample", "0123{{hello}}xyz", []item{ 469 {itemText, 0, "0123", 1}, 470 {itemLeftDelim, 4, "{{", 1}, 471 {itemIdentifier, 6, "hello", 1}, 472 {itemRightDelim, 11, "}}", 1}, 473 {itemText, 13, "xyz", 1}, 474 {itemEOF, 16, "", 1}, 475 }}, 476 {"trimafter", "{{x -}}\n{{y}}", []item{ 477 {itemLeftDelim, 0, "{{", 1}, 478 {itemIdentifier, 2, "x", 1}, 479 {itemRightDelim, 5, "}}", 1}, 480 {itemLeftDelim, 8, "{{", 2}, 481 {itemIdentifier, 10, "y", 2}, 482 {itemRightDelim, 11, "}}", 2}, 483 {itemEOF, 13, "", 2}, 484 }}, 485 {"trimbefore", "{{x}}\n{{- y}}", []item{ 486 {itemLeftDelim, 0, "{{", 1}, 487 {itemIdentifier, 2, "x", 1}, 488 {itemRightDelim, 3, "}}", 1}, 489 {itemLeftDelim, 6, "{{", 2}, 490 {itemIdentifier, 10, "y", 2}, 491 {itemRightDelim, 11, "}}", 2}, 492 {itemEOF, 13, "", 2}, 493 }}, 494 } 495 496 // The other tests don't check position, to make the test cases easier to construct. 497 // This one does. 498 func TestPos(t *testing.T) { 499 for _, test := range lexPosTests { 500 items := collect(&test, "", "") 501 if !equal(items, test.items, true) { 502 t.Errorf("%s: got\n\t%v\nexpected\n\t%v", test.name, items, test.items) 503 if len(items) == len(test.items) { 504 // Detailed print; avoid item.String() to expose the position value. 505 for i := range items { 506 if !equal(items[i:i+1], test.items[i:i+1], true) { 507 i1 := items[i] 508 i2 := test.items[i] 509 t.Errorf("\t#%d: got {%v %d %q %d} expected {%v %d %q %d}", 510 i, i1.typ, i1.pos, i1.val, i1.line, i2.typ, i2.pos, i2.val, i2.line) 511 } 512 } 513 } 514 } 515 } 516 } 517 518 // Test that an error shuts down the lexing goroutine. 519 func TestShutdown(t *testing.T) { 520 // We need to duplicate template.Parse here to hold on to the lexer. 521 const text = "erroneous{{define}}{{else}}1234" 522 lexer := lex("foo", text, "{{", "}}") 523 _, err := New("root").parseLexer(lexer) 524 if err == nil { 525 t.Fatalf("expected error") 526 } 527 // The error should have drained the input. Therefore, the lexer should be shut down. 528 token, ok := <-lexer.items 529 if ok { 530 t.Fatalf("input was not drained; got %v", token) 531 } 532 } 533 534 // parseLexer is a local version of parse that lets us pass in the lexer instead of building it. 535 // We expect an error, so the tree set and funcs list are explicitly nil. 536 func (t *Tree) parseLexer(lex *lexer) (tree *Tree, err error) { 537 defer t.recover(&err) 538 t.ParseName = t.Name 539 t.startParse(nil, lex, map[string]*Tree{}) 540 t.parse() 541 t.add() 542 t.stopParse() 543 return t, nil 544 } 545
