1 // Copright 2014 PDFium Authors. All rights reserved. 2 // Use of this source code is governed by a BSD-style license that can be 3 // found in the LICENSE file. 4 5 // Original code copyright 2014 Foxit Software Inc. http://www.foxitsoftware.com 6 7 #include "xfa/fxfa/fm2js/cxfa_fmlexer.h" 8 9 #include <algorithm> 10 11 #include "core/fxcrt/fx_extension.h" 12 #include "third_party/base/ptr_util.h" 13 #include "third_party/icu/source/common/unicode/uchar.h" 14 15 namespace { 16 17 bool IsFormCalcCharacter(wchar_t c) { 18 return (c >= 0x09 && c <= 0x0D) || (c >= 0x20 && c <= 0xd7FF) || 19 (c >= 0xE000 && c <= 0xFFFD); 20 } 21 22 bool IsIdentifierCharacter(wchar_t c) { 23 return u_isalnum(c) || c == 0x005F || // '_' 24 c == 0x0024; // '$' 25 } 26 27 bool IsInitialIdentifierCharacter(wchar_t c) { 28 return u_isalpha(c) || c == 0x005F || // '_' 29 c == 0x0024 || // '$' 30 c == 0x0021; // '!' 31 } 32 33 bool IsWhitespaceCharacter(wchar_t c) { 34 return c == 0x0009 || // Horizontal tab 35 c == 0x000B || // Vertical tab 36 c == 0x000C || // Form feed 37 c == 0x0020; // Space 38 } 39 40 const XFA_FMKeyword keyWords[] = { 41 {TOKand, 0x00000026, L"&"}, 42 {TOKlparen, 0x00000028, L"("}, 43 {TOKrparen, 0x00000029, L")"}, 44 {TOKmul, 0x0000002a, L"*"}, 45 {TOKplus, 0x0000002b, L"+"}, 46 {TOKcomma, 0x0000002c, L","}, 47 {TOKminus, 0x0000002d, L"-"}, 48 {TOKdot, 0x0000002e, L"."}, 49 {TOKdiv, 0x0000002f, L"/"}, 50 {TOKlt, 0x0000003c, L"<"}, 51 {TOKassign, 0x0000003d, L"="}, 52 {TOKgt, 0x0000003e, L">"}, 53 {TOKlbracket, 0x0000005b, L"["}, 54 {TOKrbracket, 0x0000005d, L"]"}, 55 {TOKor, 0x0000007c, L"|"}, 56 {TOKdotscream, 0x0000ec11, L".#"}, 57 {TOKdotstar, 0x0000ec18, L".*"}, 58 {TOKdotdot, 0x0000ec1c, L".."}, 59 {TOKle, 0x000133f9, L"<="}, 60 {TOKne, 0x000133fa, L"<>"}, 61 {TOKeq, 0x0001391a, L"=="}, 62 {TOKge, 0x00013e3b, L">="}, 63 {TOKdo, 0x00020153, L"do"}, 64 {TOKkseq, 0x00020676, L"eq"}, 65 {TOKksge, 0x000210ac, L"ge"}, 66 {TOKksgt, 0x000210bb, L"gt"}, 67 {TOKif, 0x00021aef, L"if"}, 68 {TOKin, 0x00021af7, L"in"}, 69 {TOKksle, 0x00022a51, L"le"}, 70 {TOKkslt, 0x00022a60, L"lt"}, 71 {TOKksne, 0x00023493, L"ne"}, 72 {TOKksor, 0x000239c1, L"or"}, 73 {TOKnull, 0x052931bb, L"null"}, 74 {TOKbreak, 0x05518c25, L"break"}, 75 {TOKksand, 0x09f9db33, L"and"}, 76 {TOKend, 0x0a631437, L"end"}, 77 {TOKeof, 0x0a63195a, L"eof"}, 78 {TOKfor, 0x0a7d67a7, L"for"}, 79 {TOKnan, 0x0b4f91dd, L"nan"}, 80 {TOKksnot, 0x0b4fd9b1, L"not"}, 81 {TOKvar, 0x0c2203e9, L"var"}, 82 {TOKthen, 0x2d5738cf, L"then"}, 83 {TOKelse, 0x45f65ee9, L"else"}, 84 {TOKexit, 0x4731d6ba, L"exit"}, 85 {TOKdownto, 0x4caadc3b, L"downto"}, 86 {TOKreturn, 0x4db8bd60, L"return"}, 87 {TOKinfinity, 0x5c0a010a, L"infinity"}, 88 {TOKendwhile, 0x5c64bff0, L"endwhile"}, 89 {TOKforeach, 0x67e31f38, L"foreach"}, 90 {TOKendfunc, 0x68f984a3, L"endfunc"}, 91 {TOKelseif, 0x78253218, L"elseif"}, 92 {TOKwhile, 0x84229259, L"while"}, 93 {TOKendfor, 0x8ab49d7e, L"endfor"}, 94 {TOKthrow, 0x8db05c94, L"throw"}, 95 {TOKstep, 0xa7a7887c, L"step"}, 96 {TOKupto, 0xb5155328, L"upto"}, 97 {TOKcontinue, 0xc0340685, L"continue"}, 98 {TOKfunc, 0xcdce60ec, L"func"}, 99 {TOKendif, 0xe0e8fee6, L"endif"}, 100 }; 101 102 const XFA_FM_TOKEN KEYWORD_START = TOKdo; 103 const XFA_FM_TOKEN KEYWORD_END = TOKendif; 104 105 const wchar_t* tokenStrings[] = { 106 L"TOKand", L"TOKlparen", L"TOKrparen", L"TOKmul", 107 L"TOKplus", L"TOKcomma", L"TOKminus", L"TOKdot", 108 L"TOKdiv", L"TOKlt", L"TOKassign", L"TOKgt", 109 L"TOKlbracket", L"TOKrbracket", L"TOKor", L"TOKdotscream", 110 L"TOKdotstar", L"TOKdotdot", L"TOKle", L"TOKne", 111 L"TOKeq", L"TOKge", L"TOKdo", L"TOKkseq", 112 L"TOKksge", L"TOKksgt", L"TOKif", L"TOKin", 113 L"TOKksle", L"TOKkslt", L"TOKksne", L"TOKksor", 114 L"TOKnull", L"TOKbreak", L"TOKksand", L"TOKend", 115 L"TOKeof", L"TOKfor", L"TOKnan", L"TOKksnot", 116 L"TOKvar", L"TOKthen", L"TOKelse", L"TOKexit", 117 L"TOKdownto", L"TOKreturn", L"TOKinfinity", L"TOKendwhile", 118 L"TOKforeach", L"TOKendfunc", L"TOKelseif", L"TOKwhile", 119 L"TOKendfor", L"TOKthrow", L"TOKstep", L"TOKupto", 120 L"TOKcontinue", L"TOKfunc", L"TOKendif", L"TOKstar", 121 L"TOKidentifier", L"TOKunderscore", L"TOKdollar", L"TOKexclamation", 122 L"TOKcall", L"TOKstring", L"TOKnumber", L"TOKreserver", 123 }; 124 125 XFA_FM_TOKEN TokenizeIdentifier(const WideStringView& str) { 126 uint32_t key = FX_HashCode_GetW(str, true); 127 128 const XFA_FMKeyword* end = std::begin(keyWords) + KEYWORD_END + 1; 129 const XFA_FMKeyword* result = 130 std::lower_bound(std::begin(keyWords) + KEYWORD_START, end, key, 131 [](const XFA_FMKeyword& iter, const uint32_t& val) { 132 return iter.m_hash < val; 133 }); 134 if (result != end && result->m_hash == key) 135 return result->m_type; 136 return TOKidentifier; 137 } 138 139 } // namespace 140 141 CXFA_FMToken::CXFA_FMToken() : m_type(TOKreserver), m_line_num(1) {} 142 143 CXFA_FMToken::CXFA_FMToken(uint32_t line_num) 144 : m_type(TOKreserver), m_line_num(line_num) {} 145 146 CXFA_FMToken::~CXFA_FMToken() {} 147 148 WideString CXFA_FMToken::ToDebugString() const { 149 WideString str(L"type = "); 150 str += tokenStrings[m_type]; 151 str += L", string = "; 152 str += m_string; 153 str += L", line_num = "; 154 str += std::to_wstring(m_line_num).c_str(); 155 return str; 156 } 157 158 CXFA_FMLexer::CXFA_FMLexer(const WideStringView& wsFormCalc) 159 : m_cursor(wsFormCalc.unterminated_c_str()), 160 m_end(m_cursor + wsFormCalc.GetLength() - 1), 161 m_current_line(1), 162 m_lexer_error(false) {} 163 164 CXFA_FMLexer::~CXFA_FMLexer() {} 165 166 std::unique_ptr<CXFA_FMToken> CXFA_FMLexer::NextToken() { 167 if (m_lexer_error) 168 return nullptr; 169 170 m_token = pdfium::MakeUnique<CXFA_FMToken>(m_current_line); 171 while (m_cursor <= m_end && *m_cursor) { 172 if (!IsFormCalcCharacter(*m_cursor)) { 173 RaiseError(); 174 return nullptr; 175 } 176 177 switch (*m_cursor) { 178 case '\n': 179 ++m_current_line; 180 m_token->m_line_num = m_current_line; 181 ++m_cursor; 182 break; 183 case '\r': 184 ++m_cursor; 185 break; 186 case ';': 187 AdvanceForComment(); 188 break; 189 case '"': 190 m_token->m_type = TOKstring; 191 AdvanceForString(); 192 return std::move(m_token); 193 case '0': 194 case '1': 195 case '2': 196 case '3': 197 case '4': 198 case '5': 199 case '6': 200 case '7': 201 case '8': 202 case '9': 203 m_token->m_type = TOKnumber; 204 AdvanceForNumber(); 205 return std::move(m_token); 206 case '=': 207 ++m_cursor; 208 if (m_cursor > m_end) { 209 m_token->m_type = TOKassign; 210 return std::move(m_token); 211 } 212 213 if (!IsFormCalcCharacter(*m_cursor)) { 214 RaiseError(); 215 return nullptr; 216 } 217 if (*m_cursor == '=') { 218 m_token->m_type = TOKeq; 219 ++m_cursor; 220 } else { 221 m_token->m_type = TOKassign; 222 } 223 return std::move(m_token); 224 case '<': 225 ++m_cursor; 226 if (m_cursor > m_end) { 227 m_token->m_type = TOKlt; 228 return std::move(m_token); 229 } 230 231 if (!IsFormCalcCharacter(*m_cursor)) { 232 RaiseError(); 233 return nullptr; 234 } 235 if (*m_cursor == '=') { 236 m_token->m_type = TOKle; 237 ++m_cursor; 238 } else if (*m_cursor == '>') { 239 m_token->m_type = TOKne; 240 ++m_cursor; 241 } else { 242 m_token->m_type = TOKlt; 243 } 244 return std::move(m_token); 245 case '>': 246 ++m_cursor; 247 if (m_cursor > m_end) { 248 m_token->m_type = TOKgt; 249 return std::move(m_token); 250 } 251 252 if (!IsFormCalcCharacter(*m_cursor)) { 253 RaiseError(); 254 return nullptr; 255 } 256 if (*m_cursor == '=') { 257 m_token->m_type = TOKge; 258 ++m_cursor; 259 } else { 260 m_token->m_type = TOKgt; 261 } 262 return std::move(m_token); 263 case ',': 264 m_token->m_type = TOKcomma; 265 ++m_cursor; 266 return std::move(m_token); 267 case '(': 268 m_token->m_type = TOKlparen; 269 ++m_cursor; 270 return std::move(m_token); 271 case ')': 272 m_token->m_type = TOKrparen; 273 ++m_cursor; 274 return std::move(m_token); 275 case '[': 276 m_token->m_type = TOKlbracket; 277 ++m_cursor; 278 return std::move(m_token); 279 case ']': 280 m_token->m_type = TOKrbracket; 281 ++m_cursor; 282 return std::move(m_token); 283 case '&': 284 ++m_cursor; 285 m_token->m_type = TOKand; 286 return std::move(m_token); 287 case '|': 288 ++m_cursor; 289 m_token->m_type = TOKor; 290 return std::move(m_token); 291 case '+': 292 ++m_cursor; 293 m_token->m_type = TOKplus; 294 return std::move(m_token); 295 case '-': 296 ++m_cursor; 297 m_token->m_type = TOKminus; 298 return std::move(m_token); 299 case '*': 300 ++m_cursor; 301 m_token->m_type = TOKmul; 302 return std::move(m_token); 303 case '/': { 304 ++m_cursor; 305 if (m_cursor > m_end) { 306 m_token->m_type = TOKdiv; 307 return std::move(m_token); 308 } 309 310 if (!IsFormCalcCharacter(*m_cursor)) { 311 RaiseError(); 312 return nullptr; 313 } 314 if (*m_cursor != '/') { 315 m_token->m_type = TOKdiv; 316 return std::move(m_token); 317 } 318 AdvanceForComment(); 319 break; 320 } 321 case '.': 322 ++m_cursor; 323 if (m_cursor > m_end) { 324 m_token->m_type = TOKdot; 325 return std::move(m_token); 326 } 327 328 if (!IsFormCalcCharacter(*m_cursor)) { 329 RaiseError(); 330 return nullptr; 331 } 332 333 if (*m_cursor == '.') { 334 m_token->m_type = TOKdotdot; 335 ++m_cursor; 336 } else if (*m_cursor == '*') { 337 m_token->m_type = TOKdotstar; 338 ++m_cursor; 339 } else if (*m_cursor == '#') { 340 m_token->m_type = TOKdotscream; 341 ++m_cursor; 342 } else if (*m_cursor <= '9' && *m_cursor >= '0') { 343 m_token->m_type = TOKnumber; 344 --m_cursor; 345 AdvanceForNumber(); 346 } else { 347 m_token->m_type = TOKdot; 348 } 349 return std::move(m_token); 350 default: 351 if (IsWhitespaceCharacter(*m_cursor)) { 352 ++m_cursor; 353 break; 354 } 355 if (!IsInitialIdentifierCharacter(*m_cursor)) { 356 RaiseError(); 357 return nullptr; 358 } 359 AdvanceForIdentifier(); 360 return std::move(m_token); 361 } 362 } 363 364 // If there isn't currently a token type then mark it EOF. 365 if (m_token->m_type == TOKreserver) 366 m_token->m_type = TOKeof; 367 return std::move(m_token); 368 } 369 370 void CXFA_FMLexer::AdvanceForNumber() { 371 // This will set end to the character after the end of the number. 372 wchar_t* end = nullptr; 373 if (m_cursor) 374 wcstod(const_cast<wchar_t*>(m_cursor), &end); 375 if (!end || FXSYS_iswalpha(*end)) { 376 RaiseError(); 377 return; 378 } 379 380 m_token->m_string = 381 WideStringView(m_cursor, static_cast<size_t>(end - m_cursor)); 382 m_cursor = end; 383 } 384 385 void CXFA_FMLexer::AdvanceForString() { 386 const wchar_t* start = m_cursor; 387 ++m_cursor; 388 while (m_cursor <= m_end && *m_cursor) { 389 if (!IsFormCalcCharacter(*m_cursor)) 390 break; 391 392 if (*m_cursor == '"') { 393 // Check for escaped "s, i.e. "". 394 ++m_cursor; 395 // If the end of the input has been reached it was not escaped. 396 if (m_cursor > m_end) { 397 m_token->m_string = 398 WideStringView(start, static_cast<size_t>(m_cursor - start)); 399 return; 400 } 401 // If the next character is not a " then the end of the string has been 402 // found. 403 if (*m_cursor != '"') { 404 if (!IsFormCalcCharacter(*m_cursor)) { 405 break; 406 } 407 m_token->m_string = WideStringView(start, (m_cursor - start)); 408 return; 409 } 410 } 411 ++m_cursor; 412 } 413 414 // Didn't find the end of the string. 415 RaiseError(); 416 } 417 418 void CXFA_FMLexer::AdvanceForIdentifier() { 419 const wchar_t* start = m_cursor; 420 ++m_cursor; 421 while (m_cursor <= m_end && *m_cursor) { 422 if (!IsFormCalcCharacter(*m_cursor)) { 423 RaiseError(); 424 return; 425 } 426 427 if (!IsIdentifierCharacter(*m_cursor)) { 428 break; 429 } 430 ++m_cursor; 431 } 432 m_token->m_string = 433 WideStringView(start, static_cast<size_t>(m_cursor - start)); 434 m_token->m_type = TokenizeIdentifier(m_token->m_string); 435 } 436 437 void CXFA_FMLexer::AdvanceForComment() { 438 m_cursor++; 439 while (m_cursor <= m_end && *m_cursor) { 440 if (!IsFormCalcCharacter(*m_cursor)) { 441 RaiseError(); 442 return; 443 } 444 445 if (*m_cursor == L'\r') { 446 ++m_cursor; 447 return; 448 } 449 if (*m_cursor == L'\n') { 450 ++m_current_line; 451 ++m_cursor; 452 return; 453 } 454 ++m_cursor; 455 } 456 } 457
