1 ## @file 2 # preprocess source file 3 # 4 # Copyright (c) 2007 - 2014, Intel Corporation. All rights reserved.<BR> 5 # 6 # This program and the accompanying materials 7 # are licensed and made available under the terms and conditions of the BSD License 8 # which accompanies this distribution. The full text of the license may be found at 9 # http://opensource.org/licenses/bsd-license.php 10 # 11 # THE PROGRAM IS DISTRIBUTED UNDER THE BSD LICENSE ON AN "AS IS" BASIS, 12 # WITHOUT WARRANTIES OR REPRESENTATIONS OF ANY KIND, EITHER EXPRESS OR IMPLIED. 13 # 14 15 ## 16 # Import Modules 17 # 18 import sys 19 import Common.LongFilePathOs as os 20 import re 21 import CodeFragmentCollector 22 import FileProfile 23 from CommonDataClass import DataClass 24 from Common import EdkLogger 25 from EotToolError import * 26 import EotGlobalData 27 28 # Global Dicts 29 IncludeFileListDict = {} 30 IncludePathListDict = {} 31 ComplexTypeDict = {} 32 SUDict = {} 33 34 ## GetFuncDeclPattern() method 35 # 36 # Get the pattern of function declaration 37 # 38 # @return p: the pattern of function declaration 39 # 40 def GetFuncDeclPattern(): 41 p = re.compile(r'(EFIAPI|EFI_BOOT_SERVICE|EFI_RUNTIME_SERVICE)?\s*[_\w]+\s*\(.*\).*', re.DOTALL) 42 return p 43 44 ## GetArrayPattern() method 45 # 46 # Get the pattern of array 47 # 48 # @return p: the pattern of array 49 # 50 def GetArrayPattern(): 51 p = re.compile(r'[_\w]*\s*[\[.*\]]+') 52 return p 53 54 ## GetTypedefFuncPointerPattern() method 55 # 56 # Get the pattern of function pointer 57 # 58 # @return p: the pattern of function pointer 59 # 60 def GetTypedefFuncPointerPattern(): 61 p = re.compile('[_\w\s]*\([\w\s]*\*+\s*[_\w]+\s*\)\s*\(.*\)', re.DOTALL) 62 return p 63 64 ## GetDB() method 65 # 66 # Get global database instance 67 # 68 # @return EotGlobalData.gDb: the global database instance 69 # 70 def GetDB(): 71 return EotGlobalData.gDb 72 73 ## PrintErrorMsg() method 74 # 75 # print error message 76 # 77 # @param ErrorType: Type of error 78 # @param Msg: Error message 79 # @param TableName: table name of error found 80 # @param ItemId: id of item 81 # 82 def PrintErrorMsg(ErrorType, Msg, TableName, ItemId): 83 Msg = Msg.replace('\n', '').replace('\r', '') 84 MsgPartList = Msg.split() 85 Msg = '' 86 for Part in MsgPartList: 87 Msg += Part 88 Msg += ' ' 89 GetDB().TblReport.Insert(ErrorType, OtherMsg = Msg, BelongsToTable = TableName, BelongsToItem = ItemId) 90 91 ## GetIdType() method 92 # 93 # Find type of input string 94 # 95 # @param Str: String to be parsed 96 # 97 # @return Type: The type of the string 98 # 99 def GetIdType(Str): 100 Type = DataClass.MODEL_UNKNOWN 101 Str = Str.replace('#', '# ') 102 List = Str.split() 103 if List[1] == 'include': 104 Type = DataClass.MODEL_IDENTIFIER_INCLUDE 105 elif List[1] == 'define': 106 Type = DataClass.MODEL_IDENTIFIER_MACRO_DEFINE 107 elif List[1] == 'ifdef': 108 Type = DataClass.MODEL_IDENTIFIER_MACRO_IFDEF 109 elif List[1] == 'ifndef': 110 Type = DataClass.MODEL_IDENTIFIER_MACRO_IFNDEF 111 elif List[1] == 'endif': 112 Type = DataClass.MODEL_IDENTIFIER_MACRO_ENDIF 113 elif List[1] == 'pragma': 114 Type = DataClass.MODEL_IDENTIFIER_MACRO_PROGMA 115 else: 116 Type = DataClass.MODEL_UNKNOWN 117 return Type 118 119 ## GetIdentifierList() method 120 # 121 # Get id of all files 122 # 123 # @return IdList: The list of all id of files 124 # 125 def GetIdentifierList(): 126 IdList = [] 127 128 for pp in FileProfile.PPDirectiveList: 129 Type = GetIdType(pp.Content) 130 IdPP = DataClass.IdentifierClass(-1, '', '', '', pp.Content, Type, -1, -1, pp.StartPos[0],pp.StartPos[1],pp.EndPos[0],pp.EndPos[1]) 131 IdList.append(IdPP) 132 133 for ae in FileProfile.AssignmentExpressionList: 134 IdAE = DataClass.IdentifierClass(-1, ae.Operator, '', ae.Name, ae.Value, DataClass.MODEL_IDENTIFIER_ASSIGNMENT_EXPRESSION, -1, -1, ae.StartPos[0],ae.StartPos[1],ae.EndPos[0],ae.EndPos[1]) 135 IdList.append(IdAE) 136 137 FuncDeclPattern = GetFuncDeclPattern() 138 ArrayPattern = GetArrayPattern() 139 for var in FileProfile.VariableDeclarationList: 140 DeclText = var.Declarator.strip() 141 while DeclText.startswith('*'): 142 var.Modifier += '*' 143 DeclText = DeclText.lstrip('*').strip() 144 var.Declarator = DeclText 145 if FuncDeclPattern.match(var.Declarator): 146 DeclSplitList = var.Declarator.split('(') 147 FuncName = DeclSplitList[0] 148 FuncNamePartList = FuncName.split() 149 if len(FuncNamePartList) > 1: 150 FuncName = FuncNamePartList[-1] 151 Index = 0 152 while Index < len(FuncNamePartList) - 1: 153 var.Modifier += ' ' + FuncNamePartList[Index] 154 var.Declarator = var.Declarator.lstrip().lstrip(FuncNamePartList[Index]) 155 Index += 1 156 IdVar = DataClass.IdentifierClass(-1, var.Modifier, '', var.Declarator, '', DataClass.MODEL_IDENTIFIER_FUNCTION_DECLARATION, -1, -1, var.StartPos[0],var.StartPos[1],var.EndPos[0],var.EndPos[1]) 157 IdList.append(IdVar) 158 continue 159 160 if var.Declarator.find('{') == -1: 161 for decl in var.Declarator.split(','): 162 DeclList = decl.split('=') 163 Name = DeclList[0].strip() 164 if ArrayPattern.match(Name): 165 LSBPos = var.Declarator.find('[') 166 var.Modifier += ' ' + Name[LSBPos:] 167 Name = Name[0:LSBPos] 168 169 IdVar = DataClass.IdentifierClass(-1, var.Modifier, '', Name, (len(DeclList) > 1 and [DeclList[1]]or [''])[0], DataClass.MODEL_IDENTIFIER_VARIABLE, -1, -1, var.StartPos[0],var.StartPos[1],var.EndPos[0],var.EndPos[1]) 170 IdList.append(IdVar) 171 else: 172 DeclList = var.Declarator.split('=') 173 Name = DeclList[0].strip() 174 if ArrayPattern.match(Name): 175 LSBPos = var.Declarator.find('[') 176 var.Modifier += ' ' + Name[LSBPos:] 177 Name = Name[0:LSBPos] 178 IdVar = DataClass.IdentifierClass(-1, var.Modifier, '', Name, (len(DeclList) > 1 and [DeclList[1]]or [''])[0], DataClass.MODEL_IDENTIFIER_VARIABLE, -1, -1, var.StartPos[0],var.StartPos[1],var.EndPos[0],var.EndPos[1]) 179 IdList.append(IdVar) 180 181 for enum in FileProfile.EnumerationDefinitionList: 182 LBPos = enum.Content.find('{') 183 RBPos = enum.Content.find('}') 184 Name = enum.Content[4:LBPos].strip() 185 Value = enum.Content[LBPos+1:RBPos] 186 IdEnum = DataClass.IdentifierClass(-1, '', '', Name, Value, DataClass.MODEL_IDENTIFIER_ENUMERATE, -1, -1, enum.StartPos[0],enum.StartPos[1],enum.EndPos[0],enum.EndPos[1]) 187 IdList.append(IdEnum) 188 189 for su in FileProfile.StructUnionDefinitionList: 190 Type = DataClass.MODEL_IDENTIFIER_STRUCTURE 191 SkipLen = 6 192 if su.Content.startswith('union'): 193 Type = DataClass.MODEL_IDENTIFIER_UNION 194 SkipLen = 5 195 LBPos = su.Content.find('{') 196 RBPos = su.Content.find('}') 197 if LBPos == -1 or RBPos == -1: 198 Name = su.Content[SkipLen:].strip() 199 Value = '' 200 else: 201 Name = su.Content[SkipLen:LBPos].strip() 202 Value = su.Content[LBPos+1:RBPos] 203 IdPE = DataClass.IdentifierClass(-1, '', '', Name, Value, Type, -1, -1, su.StartPos[0],su.StartPos[1],su.EndPos[0],su.EndPos[1]) 204 IdList.append(IdPE) 205 206 TdFuncPointerPattern = GetTypedefFuncPointerPattern() 207 for td in FileProfile.TypedefDefinitionList: 208 Modifier = '' 209 Name = td.ToType 210 Value = td.FromType 211 if TdFuncPointerPattern.match(td.ToType): 212 Modifier = td.FromType 213 LBPos = td.ToType.find('(') 214 TmpStr = td.ToType[LBPos+1:].strip() 215 StarPos = TmpStr.find('*') 216 if StarPos != -1: 217 Modifier += ' ' + TmpStr[0:StarPos] 218 while TmpStr[StarPos] == '*': 219 Modifier += ' ' + '*' 220 StarPos += 1 221 TmpStr = TmpStr[StarPos:].strip() 222 RBPos = TmpStr.find(')') 223 Name = TmpStr[0:RBPos] 224 Value = 'FP' + TmpStr[RBPos + 1:] 225 226 IdTd = DataClass.IdentifierClass(-1, Modifier, '', Name, Value, DataClass.MODEL_IDENTIFIER_TYPEDEF, -1, -1, td.StartPos[0],td.StartPos[1],td.EndPos[0],td.EndPos[1]) 227 IdList.append(IdTd) 228 229 for funcCall in FileProfile.FunctionCallingList: 230 IdFC = DataClass.IdentifierClass(-1, '', '', funcCall.FuncName, funcCall.ParamList, DataClass.MODEL_IDENTIFIER_FUNCTION_CALLING, -1, -1, funcCall.StartPos[0],funcCall.StartPos[1],funcCall.EndPos[0],funcCall.EndPos[1]) 231 IdList.append(IdFC) 232 return IdList 233 234 ## GetParamList() method 235 # 236 # Get a list of parameters 237 # 238 # @param FuncDeclarator: Function declarator 239 # @param FuncNameLine: Line number of function name 240 # @param FuncNameOffset: Offset of function name 241 # 242 # @return ParamIdList: A list of parameters 243 # 244 def GetParamList(FuncDeclarator, FuncNameLine = 0, FuncNameOffset = 0): 245 ParamIdList = [] 246 DeclSplitList = FuncDeclarator.split('(') 247 if len(DeclSplitList) < 2: 248 return ParamIdList 249 FuncName = DeclSplitList[0] 250 ParamStr = DeclSplitList[1].rstrip(')') 251 LineSkipped = 0 252 OffsetSkipped = 0 253 Start = 0 254 while FuncName.find('\n', Start) != -1: 255 LineSkipped += 1 256 OffsetSkipped = 0 257 Start += FuncName.find('\n', Start) 258 Start += 1 259 OffsetSkipped += len(FuncName[Start:]) 260 OffsetSkipped += 1 #skip '(' 261 ParamBeginLine = FuncNameLine + LineSkipped 262 ParamBeginOffset = OffsetSkipped 263 for p in ParamStr.split(','): 264 ListP = p.split() 265 if len(ListP) == 0: 266 continue 267 ParamName = ListP[-1] 268 DeclText = ParamName.strip() 269 RightSpacePos = p.rfind(ParamName) 270 ParamModifier = p[0:RightSpacePos] 271 if ParamName == 'OPTIONAL': 272 if ParamModifier == '': 273 ParamModifier += ' ' + 'OPTIONAL' 274 DeclText = '' 275 else: 276 ParamName = ListP[-2] 277 DeclText = ParamName.strip() 278 RightSpacePos = p.rfind(ParamName) 279 ParamModifier = p[0:RightSpacePos] 280 ParamModifier += 'OPTIONAL' 281 while DeclText.startswith('*'): 282 ParamModifier += ' ' + '*' 283 DeclText = DeclText.lstrip('*').strip() 284 ParamName = DeclText 285 286 Start = 0 287 while p.find('\n', Start) != -1: 288 LineSkipped += 1 289 OffsetSkipped = 0 290 Start += p.find('\n', Start) 291 Start += 1 292 OffsetSkipped += len(p[Start:]) 293 294 ParamEndLine = ParamBeginLine + LineSkipped 295 ParamEndOffset = OffsetSkipped 296 IdParam = DataClass.IdentifierClass(-1, ParamModifier, '', ParamName, '', DataClass.MODEL_IDENTIFIER_PARAMETER, -1, -1, ParamBeginLine, ParamBeginOffset, ParamEndLine, ParamEndOffset) 297 ParamIdList.append(IdParam) 298 ParamBeginLine = ParamEndLine 299 ParamBeginOffset = OffsetSkipped + 1 #skip ',' 300 301 return ParamIdList 302 303 ## GetFunctionList() 304 # 305 # Get a list of functions 306 # 307 # @return FuncObjList: A list of function objects 308 # 309 def GetFunctionList(): 310 FuncObjList = [] 311 for FuncDef in FileProfile.FunctionDefinitionList: 312 ParamIdList = [] 313 DeclText = FuncDef.Declarator.strip() 314 while DeclText.startswith('*'): 315 FuncDef.Modifier += '*' 316 DeclText = DeclText.lstrip('*').strip() 317 318 FuncDef.Declarator = FuncDef.Declarator.lstrip('*') 319 DeclSplitList = FuncDef.Declarator.split('(') 320 if len(DeclSplitList) < 2: 321 continue 322 323 FuncName = DeclSplitList[0] 324 FuncNamePartList = FuncName.split() 325 if len(FuncNamePartList) > 1: 326 FuncName = FuncNamePartList[-1] 327 Index = 0 328 while Index < len(FuncNamePartList) - 1: 329 FuncDef.Modifier += ' ' + FuncNamePartList[Index] 330 Index += 1 331 332 FuncObj = DataClass.FunctionClass(-1, FuncDef.Declarator, FuncDef.Modifier, FuncName.strip(), '', FuncDef.StartPos[0],FuncDef.StartPos[1],FuncDef.EndPos[0],FuncDef.EndPos[1], FuncDef.LeftBracePos[0], FuncDef.LeftBracePos[1], -1, ParamIdList, []) 333 FuncObjList.append(FuncObj) 334 335 return FuncObjList 336 337 ## CreateCCodeDB() method 338 # 339 # Create database for all c code 340 # 341 # @param FileNameList: A list of all c code file names 342 # 343 def CreateCCodeDB(FileNameList): 344 FileObjList = [] 345 ParseErrorFileList = [] 346 ParsedFiles = {} 347 for FullName in FileNameList: 348 if os.path.splitext(FullName)[1] in ('.h', '.c'): 349 if FullName.lower() in ParsedFiles: 350 continue 351 ParsedFiles[FullName.lower()] = 1 352 EdkLogger.info("Parsing " + FullName) 353 model = FullName.endswith('c') and DataClass.MODEL_FILE_C or DataClass.MODEL_FILE_H 354 collector = CodeFragmentCollector.CodeFragmentCollector(FullName) 355 try: 356 collector.ParseFile() 357 except: 358 ParseErrorFileList.append(FullName) 359 BaseName = os.path.basename(FullName) 360 DirName = os.path.dirname(FullName) 361 Ext = os.path.splitext(BaseName)[1].lstrip('.') 362 ModifiedTime = os.path.getmtime(FullName) 363 FileObj = DataClass.FileClass(-1, BaseName, Ext, DirName, FullName, model, ModifiedTime, GetFunctionList(), GetIdentifierList(), []) 364 FileObjList.append(FileObj) 365 collector.CleanFileProfileBuffer() 366 367 if len(ParseErrorFileList) > 0: 368 EdkLogger.info("Found unrecoverable error during parsing:\n\t%s\n" % "\n\t".join(ParseErrorFileList)) 369 370 Db = EotGlobalData.gDb 371 for file in FileObjList: 372 Db.InsertOneFile(file) 373 374 Db.UpdateIdentifierBelongsToFunction() 375 376 ## 377 # 378 # This acts like the main() function for the script, unless it is 'import'ed into another 379 # script. 380 # 381 if __name__ == '__main__': 382 383 EdkLogger.Initialize() 384 EdkLogger.SetLevel(EdkLogger.QUIET) 385 CollectSourceCodeDataIntoDB(sys.argv[1]) 386 387 print 'Done!' 388
