1 ## @file 2 # This file is used to define comment parsing interface 3 # 4 # Copyright (c) 2011 - 2014, Intel Corporation. All rights reserved.<BR> 5 # 6 # This program and the accompanying materials are licensed and made available 7 # under the terms and conditions of the BSD License which accompanies this 8 # distribution. The full text of the license may be found at 9 # http://opensource.org/licenses/bsd-license.php 10 # 11 # THE PROGRAM IS DISTRIBUTED UNDER THE BSD LICENSE ON AN "AS IS" BASIS, 12 # WITHOUT WARRANTIES OR REPRESENTATIONS OF ANY KIND, EITHER EXPRESS OR IMPLIED. 13 # 14 15 ''' 16 CommentParsing 17 ''' 18 19 ## 20 # Import Modules 21 # 22 import re 23 24 from Library.String import GetSplitValueList 25 from Library.String import CleanString2 26 from Library.DataType import HEADER_COMMENT_NOT_STARTED 27 from Library.DataType import TAB_COMMENT_SPLIT 28 from Library.DataType import HEADER_COMMENT_LICENSE 29 from Library.DataType import HEADER_COMMENT_ABSTRACT 30 from Library.DataType import HEADER_COMMENT_COPYRIGHT 31 from Library.DataType import HEADER_COMMENT_DESCRIPTION 32 from Library.DataType import TAB_SPACE_SPLIT 33 from Library.DataType import TAB_COMMA_SPLIT 34 from Library.DataType import SUP_MODULE_LIST 35 from Library.DataType import TAB_VALUE_SPLIT 36 from Library.DataType import TAB_PCD_VALIDRANGE 37 from Library.DataType import TAB_PCD_VALIDLIST 38 from Library.DataType import TAB_PCD_EXPRESSION 39 from Library.DataType import TAB_PCD_PROMPT 40 from Library.DataType import TAB_CAPHEX_START 41 from Library.DataType import TAB_HEX_START 42 from Library.DataType import PCD_ERR_CODE_MAX_SIZE 43 from Library.ExpressionValidate import IsValidRangeExpr 44 from Library.ExpressionValidate import IsValidListExpr 45 from Library.ExpressionValidate import IsValidLogicalExpr 46 from Object.POM.CommonObject import TextObject 47 from Object.POM.CommonObject import PcdErrorObject 48 import Logger.Log as Logger 49 from Logger.ToolError import FORMAT_INVALID 50 from Logger.ToolError import FORMAT_NOT_SUPPORTED 51 from Logger import StringTable as ST 52 53 ## ParseHeaderCommentSection 54 # 55 # Parse Header comment section lines, extract Abstract, Description, Copyright 56 # , License lines 57 # 58 # @param CommentList: List of (Comment, LineNumber) 59 # @param FileName: FileName of the comment 60 # 61 def ParseHeaderCommentSection(CommentList, FileName = None, IsBinaryHeader = False): 62 Abstract = '' 63 Description = '' 64 Copyright = '' 65 License = '' 66 EndOfLine = "\n" 67 if IsBinaryHeader: 68 STR_HEADER_COMMENT_START = "@BinaryHeader" 69 else: 70 STR_HEADER_COMMENT_START = "@file" 71 HeaderCommentStage = HEADER_COMMENT_NOT_STARTED 72 73 # 74 # first find the last copyright line 75 # 76 Last = 0 77 for Index in xrange(len(CommentList)-1, 0, -1): 78 Line = CommentList[Index][0] 79 if _IsCopyrightLine(Line): 80 Last = Index 81 break 82 83 for Item in CommentList: 84 Line = Item[0] 85 LineNo = Item[1] 86 87 if not Line.startswith(TAB_COMMENT_SPLIT) and Line: 88 Logger.Error("\nUPT", FORMAT_INVALID, ST.ERR_INVALID_COMMENT_FORMAT, FileName, Item[1]) 89 Comment = CleanString2(Line)[1] 90 Comment = Comment.strip() 91 # 92 # if there are blank lines between License or Description, keep them as they would be 93 # indication of different block; or in the position that Abstract should be, also keep it 94 # as it indicates that no abstract 95 # 96 if not Comment and HeaderCommentStage not in [HEADER_COMMENT_LICENSE, \ 97 HEADER_COMMENT_DESCRIPTION, HEADER_COMMENT_ABSTRACT]: 98 continue 99 100 if HeaderCommentStage == HEADER_COMMENT_NOT_STARTED: 101 if Comment.startswith(STR_HEADER_COMMENT_START): 102 HeaderCommentStage = HEADER_COMMENT_ABSTRACT 103 else: 104 License += Comment + EndOfLine 105 else: 106 if HeaderCommentStage == HEADER_COMMENT_ABSTRACT: 107 # 108 # in case there is no abstract and description 109 # 110 if not Comment: 111 HeaderCommentStage = HEADER_COMMENT_DESCRIPTION 112 elif _IsCopyrightLine(Comment): 113 Result, ErrMsg = _ValidateCopyright(Comment) 114 ValidateCopyright(Result, ST.WRN_INVALID_COPYRIGHT, FileName, LineNo, ErrMsg) 115 Copyright += Comment + EndOfLine 116 HeaderCommentStage = HEADER_COMMENT_COPYRIGHT 117 else: 118 Abstract += Comment + EndOfLine 119 HeaderCommentStage = HEADER_COMMENT_DESCRIPTION 120 elif HeaderCommentStage == HEADER_COMMENT_DESCRIPTION: 121 # 122 # in case there is no description 123 # 124 if _IsCopyrightLine(Comment): 125 Result, ErrMsg = _ValidateCopyright(Comment) 126 ValidateCopyright(Result, ST.WRN_INVALID_COPYRIGHT, FileName, LineNo, ErrMsg) 127 Copyright += Comment + EndOfLine 128 HeaderCommentStage = HEADER_COMMENT_COPYRIGHT 129 else: 130 Description += Comment + EndOfLine 131 elif HeaderCommentStage == HEADER_COMMENT_COPYRIGHT: 132 if _IsCopyrightLine(Comment): 133 Result, ErrMsg = _ValidateCopyright(Comment) 134 ValidateCopyright(Result, ST.WRN_INVALID_COPYRIGHT, FileName, LineNo, ErrMsg) 135 Copyright += Comment + EndOfLine 136 else: 137 # 138 # Contents after copyright line are license, those non-copyright lines in between 139 # copyright line will be discarded 140 # 141 if LineNo > Last: 142 if License: 143 License += EndOfLine 144 License += Comment + EndOfLine 145 HeaderCommentStage = HEADER_COMMENT_LICENSE 146 else: 147 if not Comment and not License: 148 continue 149 License += Comment + EndOfLine 150 151 return Abstract.strip(), Description.strip(), Copyright.strip(), License.strip() 152 153 ## _IsCopyrightLine 154 # check whether current line is copyright line, the criteria is whether there is case insensitive keyword "Copyright" 155 # followed by zero or more white space characters followed by a "(" character 156 # 157 # @param LineContent: the line need to be checked 158 # @return: True if current line is copyright line, False else 159 # 160 def _IsCopyrightLine (LineContent): 161 LineContent = LineContent.upper() 162 Result = False 163 164 ReIsCopyrightRe = re.compile(r"""(^|\s)COPYRIGHT *\(""", re.DOTALL) 165 if ReIsCopyrightRe.search(LineContent): 166 Result = True 167 168 return Result 169 170 ## ParseGenericComment 171 # 172 # @param GenericComment: Generic comment list, element of 173 # (CommentLine, LineNum) 174 # @param ContainerFile: Input value for filename of Dec file 175 # 176 def ParseGenericComment (GenericComment, ContainerFile=None, SkipTag=None): 177 if ContainerFile: 178 pass 179 HelpTxt = None 180 HelpStr = '' 181 182 for Item in GenericComment: 183 CommentLine = Item[0] 184 Comment = CleanString2(CommentLine)[1] 185 if SkipTag is not None and Comment.startswith(SkipTag): 186 Comment = Comment.replace(SkipTag, '', 1) 187 HelpStr += Comment + '\n' 188 189 if HelpStr: 190 HelpTxt = TextObject() 191 if HelpStr.endswith('\n') and not HelpStr.endswith('\n\n') and HelpStr != '\n': 192 HelpStr = HelpStr[:-1] 193 HelpTxt.SetString(HelpStr) 194 195 return HelpTxt 196 197 ## ParsePcdErrorCode 198 # 199 # @param Value: original ErrorCode value 200 # @param ContainerFile: Input value for filename of Dec file 201 # @param LineNum: Line Num 202 # 203 def ParsePcdErrorCode (Value = None, ContainerFile = None, LineNum = None): 204 try: 205 if Value.strip().startswith((TAB_HEX_START, TAB_CAPHEX_START)): 206 Base = 16 207 else: 208 Base = 10 209 ErrorCode = long(Value, Base) 210 if ErrorCode > PCD_ERR_CODE_MAX_SIZE or ErrorCode < 0: 211 Logger.Error('Parser', 212 FORMAT_NOT_SUPPORTED, 213 "The format %s of ErrorCode is not valid, should be UNIT32 type or long type" % Value, 214 File = ContainerFile, 215 Line = LineNum) 216 # 217 # To delete the tailing 'L' 218 # 219 return hex(ErrorCode)[:-1] 220 except ValueError, XStr: 221 if XStr: 222 pass 223 Logger.Error('Parser', 224 FORMAT_NOT_SUPPORTED, 225 "The format %s of ErrorCode is not valid, should be UNIT32 type or long type" % Value, 226 File = ContainerFile, 227 Line = LineNum) 228 229 ## ParseDecPcdGenericComment 230 # 231 # @param GenericComment: Generic comment list, element of (CommentLine, 232 # LineNum) 233 # @param ContainerFile: Input value for filename of Dec file 234 # 235 def ParseDecPcdGenericComment (GenericComment, ContainerFile, TokenSpaceGuidCName, CName, MacroReplaceDict): 236 HelpStr = '' 237 PromptStr = '' 238 PcdErr = None 239 PcdErrList = [] 240 ValidValueNum = 0 241 ValidRangeNum = 0 242 ExpressionNum = 0 243 244 for (CommentLine, LineNum) in GenericComment: 245 Comment = CleanString2(CommentLine)[1] 246 # 247 # To replace Macro 248 # 249 MACRO_PATTERN = '[\t\s]*\$\([A-Z][_A-Z0-9]*\)' 250 MatchedStrs = re.findall(MACRO_PATTERN, Comment) 251 for MatchedStr in MatchedStrs: 252 if MatchedStr: 253 Macro = MatchedStr.strip().lstrip('$(').rstrip(')').strip() 254 if Macro in MacroReplaceDict: 255 Comment = Comment.replace(MatchedStr, MacroReplaceDict[Macro]) 256 if Comment.startswith(TAB_PCD_VALIDRANGE): 257 if ValidValueNum > 0 or ExpressionNum > 0: 258 Logger.Error('Parser', 259 FORMAT_NOT_SUPPORTED, 260 ST.WRN_MULTI_PCD_RANGES, 261 File = ContainerFile, 262 Line = LineNum) 263 else: 264 PcdErr = PcdErrorObject() 265 PcdErr.SetTokenSpaceGuidCName(TokenSpaceGuidCName) 266 PcdErr.SetCName(CName) 267 PcdErr.SetFileLine(Comment) 268 PcdErr.SetLineNum(LineNum) 269 ValidRangeNum += 1 270 ValidRange = Comment.replace(TAB_PCD_VALIDRANGE, "", 1).strip() 271 Valid, Cause = _CheckRangeExpression(ValidRange) 272 if Valid: 273 ValueList = ValidRange.split(TAB_VALUE_SPLIT) 274 if len(ValueList) > 1: 275 PcdErr.SetValidValueRange((TAB_VALUE_SPLIT.join(ValueList[1:])).strip()) 276 PcdErr.SetErrorNumber(ParsePcdErrorCode(ValueList[0], ContainerFile, LineNum)) 277 else: 278 PcdErr.SetValidValueRange(ValidRange) 279 PcdErrList.append(PcdErr) 280 else: 281 Logger.Error("Parser", 282 FORMAT_NOT_SUPPORTED, 283 Cause, 284 ContainerFile, 285 LineNum) 286 elif Comment.startswith(TAB_PCD_VALIDLIST): 287 if ValidRangeNum > 0 or ExpressionNum > 0: 288 Logger.Error('Parser', 289 FORMAT_NOT_SUPPORTED, 290 ST.WRN_MULTI_PCD_RANGES, 291 File = ContainerFile, 292 Line = LineNum) 293 elif ValidValueNum > 0: 294 Logger.Error('Parser', 295 FORMAT_NOT_SUPPORTED, 296 ST.WRN_MULTI_PCD_VALIDVALUE, 297 File = ContainerFile, 298 Line = LineNum) 299 else: 300 PcdErr = PcdErrorObject() 301 PcdErr.SetTokenSpaceGuidCName(TokenSpaceGuidCName) 302 PcdErr.SetCName(CName) 303 PcdErr.SetFileLine(Comment) 304 PcdErr.SetLineNum(LineNum) 305 ValidValueNum += 1 306 ValidValueExpr = Comment.replace(TAB_PCD_VALIDLIST, "", 1).strip() 307 Valid, Cause = _CheckListExpression(ValidValueExpr) 308 if Valid: 309 ValidValue = Comment.replace(TAB_PCD_VALIDLIST, "", 1).replace(TAB_COMMA_SPLIT, TAB_SPACE_SPLIT) 310 ValueList = ValidValue.split(TAB_VALUE_SPLIT) 311 if len(ValueList) > 1: 312 PcdErr.SetValidValue((TAB_VALUE_SPLIT.join(ValueList[1:])).strip()) 313 PcdErr.SetErrorNumber(ParsePcdErrorCode(ValueList[0], ContainerFile, LineNum)) 314 else: 315 PcdErr.SetValidValue(ValidValue) 316 PcdErrList.append(PcdErr) 317 else: 318 Logger.Error("Parser", 319 FORMAT_NOT_SUPPORTED, 320 Cause, 321 ContainerFile, 322 LineNum) 323 elif Comment.startswith(TAB_PCD_EXPRESSION): 324 if ValidRangeNum > 0 or ValidValueNum > 0: 325 Logger.Error('Parser', 326 FORMAT_NOT_SUPPORTED, 327 ST.WRN_MULTI_PCD_RANGES, 328 File = ContainerFile, 329 Line = LineNum) 330 else: 331 PcdErr = PcdErrorObject() 332 PcdErr.SetTokenSpaceGuidCName(TokenSpaceGuidCName) 333 PcdErr.SetCName(CName) 334 PcdErr.SetFileLine(Comment) 335 PcdErr.SetLineNum(LineNum) 336 ExpressionNum += 1 337 Expression = Comment.replace(TAB_PCD_EXPRESSION, "", 1).strip() 338 Valid, Cause = _CheckExpression(Expression) 339 if Valid: 340 ValueList = Expression.split(TAB_VALUE_SPLIT) 341 if len(ValueList) > 1: 342 PcdErr.SetExpression((TAB_VALUE_SPLIT.join(ValueList[1:])).strip()) 343 PcdErr.SetErrorNumber(ParsePcdErrorCode(ValueList[0], ContainerFile, LineNum)) 344 else: 345 PcdErr.SetExpression(Expression) 346 PcdErrList.append(PcdErr) 347 else: 348 Logger.Error("Parser", 349 FORMAT_NOT_SUPPORTED, 350 Cause, 351 ContainerFile, 352 LineNum) 353 elif Comment.startswith(TAB_PCD_PROMPT): 354 if PromptStr: 355 Logger.Error('Parser', 356 FORMAT_NOT_SUPPORTED, 357 ST.WRN_MULTI_PCD_PROMPT, 358 File = ContainerFile, 359 Line = LineNum) 360 PromptStr = Comment.replace(TAB_PCD_PROMPT, "", 1).strip() 361 else: 362 if Comment: 363 HelpStr += Comment + '\n' 364 365 # 366 # remove the last EOL if the comment is of format 'FOO\n' 367 # 368 if HelpStr.endswith('\n'): 369 if HelpStr != '\n' and not HelpStr.endswith('\n\n'): 370 HelpStr = HelpStr[:-1] 371 372 return HelpStr, PcdErrList, PromptStr 373 374 ## ParseDecPcdTailComment 375 # 376 # @param TailCommentList: Tail comment list of Pcd, item of format (Comment, LineNum) 377 # @param ContainerFile: Input value for filename of Dec file 378 # @retVal SupModuleList: The supported module type list detected 379 # @retVal HelpStr: The generic help text string detected 380 # 381 def ParseDecPcdTailComment (TailCommentList, ContainerFile): 382 assert(len(TailCommentList) == 1) 383 TailComment = TailCommentList[0][0] 384 LineNum = TailCommentList[0][1] 385 386 Comment = TailComment.lstrip(" #") 387 388 ReFindFirstWordRe = re.compile(r"""^([^ #]*)""", re.DOTALL) 389 390 # 391 # get first word and compare with SUP_MODULE_LIST 392 # 393 MatchObject = ReFindFirstWordRe.match(Comment) 394 if not (MatchObject and MatchObject.group(1) in SUP_MODULE_LIST): 395 return None, Comment 396 397 # 398 # parse line, it must have supported module type specified 399 # 400 if Comment.find(TAB_COMMENT_SPLIT) == -1: 401 Comment += TAB_COMMENT_SPLIT 402 SupMode, HelpStr = GetSplitValueList(Comment, TAB_COMMENT_SPLIT, 1) 403 SupModuleList = [] 404 for Mod in GetSplitValueList(SupMode, TAB_SPACE_SPLIT): 405 if not Mod: 406 continue 407 elif Mod not in SUP_MODULE_LIST: 408 Logger.Error("UPT", 409 FORMAT_INVALID, 410 ST.WRN_INVALID_MODULE_TYPE%Mod, 411 ContainerFile, 412 LineNum) 413 else: 414 SupModuleList.append(Mod) 415 416 return SupModuleList, HelpStr 417 418 ## _CheckListExpression 419 # 420 # @param Expression: Pcd value list expression 421 # 422 def _CheckListExpression(Expression): 423 ListExpr = '' 424 if TAB_VALUE_SPLIT in Expression: 425 ListExpr = Expression[Expression.find(TAB_VALUE_SPLIT)+1:] 426 else: 427 ListExpr = Expression 428 429 return IsValidListExpr(ListExpr) 430 431 ## _CheckExpreesion 432 # 433 # @param Expression: Pcd value expression 434 # 435 def _CheckExpression(Expression): 436 Expr = '' 437 if TAB_VALUE_SPLIT in Expression: 438 Expr = Expression[Expression.find(TAB_VALUE_SPLIT)+1:] 439 else: 440 Expr = Expression 441 return IsValidLogicalExpr(Expr, True) 442 443 ## _CheckRangeExpression 444 # 445 # @param Expression: Pcd range expression 446 # 447 def _CheckRangeExpression(Expression): 448 RangeExpr = '' 449 if TAB_VALUE_SPLIT in Expression: 450 RangeExpr = Expression[Expression.find(TAB_VALUE_SPLIT)+1:] 451 else: 452 RangeExpr = Expression 453 454 return IsValidRangeExpr(RangeExpr) 455 456 ## ValidateCopyright 457 # 458 # 459 # 460 def ValidateCopyright(Result, ErrType, FileName, LineNo, ErrMsg): 461 if not Result: 462 Logger.Warn("\nUPT", ErrType, FileName, LineNo, ErrMsg) 463 464 ## _ValidateCopyright 465 # 466 # @param Line: Line that contains copyright information, # stripped 467 # 468 # @retval Result: True if line is conformed to Spec format, False else 469 # @retval ErrMsg: the detailed error description 470 # 471 def _ValidateCopyright(Line): 472 if Line: 473 pass 474 Result = True 475 ErrMsg = '' 476 477 return Result, ErrMsg 478 479 def GenerateTokenList (Comment): 480 # 481 # Tokenize Comment using '#' and ' ' as token seperators 482 # 483 RelplacedComment = None 484 while Comment != RelplacedComment: 485 RelplacedComment = Comment 486 Comment = Comment.replace('##', '#').replace(' ', ' ').replace(' ', '#').strip('# ') 487 return Comment.split('#') 488 489 490 # 491 # Comment - Comment to parse 492 # TypeTokens - A dictionary of type token synonyms 493 # RemoveTokens - A list of tokens to remove from help text 494 # ParseVariable - True for parsing [Guids]. Otherwise False 495 # 496 def ParseComment (Comment, UsageTokens, TypeTokens, RemoveTokens, ParseVariable): 497 # 498 # Initialize return values 499 # 500 Usage = None 501 Type = None 502 String = None 503 504 Comment = Comment[0] 505 506 NumTokens = 2 507 if ParseVariable: 508 # 509 # Remove white space around first instance of ':' from Comment if 'Variable' 510 # is in front of ':' and Variable is the 1st or 2nd token in Comment. 511 # 512 List = Comment.split(':', 1) 513 if len(List) > 1: 514 SubList = GenerateTokenList (List[0].strip()) 515 if len(SubList) in [1, 2] and SubList[-1] == 'Variable': 516 if List[1].strip().find('L"') == 0: 517 Comment = List[0].strip() + ':' + List[1].strip() 518 519 # 520 # Remove first instance of L"<VariableName> from Comment and put into String 521 # if and only if L"<VariableName>" is the 1st token, the 2nd token. Or 522 # L"<VariableName>" is the third token immediately following 'Variable:'. 523 # 524 End = -1 525 Start = Comment.find('Variable:L"') 526 if Start >= 0: 527 String = Comment[Start + 9:] 528 End = String[2:].find('"') 529 else: 530 Start = Comment.find('L"') 531 if Start >= 0: 532 String = Comment[Start:] 533 End = String[2:].find('"') 534 if End >= 0: 535 SubList = GenerateTokenList (Comment[:Start]) 536 if len(SubList) < 2: 537 Comment = Comment[:Start] + String[End + 3:] 538 String = String[:End + 3] 539 Type = 'Variable' 540 NumTokens = 1 541 542 # 543 # Initialze HelpText to Comment. 544 # Content will be remove from HelpText as matching tokens are found 545 # 546 HelpText = Comment 547 548 # 549 # Tokenize Comment using '#' and ' ' as token seperators 550 # 551 List = GenerateTokenList (Comment) 552 553 # 554 # Search first two tokens for Usage and Type and remove any matching tokens 555 # from HelpText 556 # 557 for Token in List[0:NumTokens]: 558 if Usage == None and Token in UsageTokens: 559 Usage = UsageTokens[Token] 560 HelpText = HelpText.replace(Token, '') 561 if Usage != None or not ParseVariable: 562 for Token in List[0:NumTokens]: 563 if Type == None and Token in TypeTokens: 564 Type = TypeTokens[Token] 565 HelpText = HelpText.replace(Token, '') 566 if Usage != None: 567 for Token in List[0:NumTokens]: 568 if Token in RemoveTokens: 569 HelpText = HelpText.replace(Token, '') 570 571 # 572 # If no Usage token is present and set Usage to UNDEFINED 573 # 574 if Usage == None: 575 Usage = 'UNDEFINED' 576 577 # 578 # If no Type token is present and set Type to UNDEFINED 579 # 580 if Type == None: 581 Type = 'UNDEFINED' 582 583 # 584 # If Type is not 'Variable:', then set String to None 585 # 586 if Type != 'Variable': 587 String = None 588 589 # 590 # Strip ' ' and '#' from the beginning of HelpText 591 # If HelpText is an empty string after all parsing is 592 # complete then set HelpText to None 593 # 594 HelpText = HelpText.lstrip('# ') 595 if HelpText == '': 596 HelpText = None 597 598 # 599 # Return parsing results 600 # 601 return Usage, Type, String, HelpText 602
