1 ## @file 2 # Collect all defined strings in multiple uni files. 3 # 4 # Copyright (c) 2014 - 2016, Intel Corporation. All rights reserved.<BR> 5 # 6 # This program and the accompanying materials are licensed and made available 7 # under the terms and conditions of the BSD License which accompanies this 8 # distribution. The full text of the license may be found at 9 # http://opensource.org/licenses/bsd-license.php 10 # 11 # THE PROGRAM IS DISTRIBUTED UNDER THE BSD LICENSE ON AN "AS IS" BASIS, 12 # WITHOUT WARRANTIES OR REPRESENTATIONS OF ANY KIND, EITHER EXPRESS OR IMPLIED. 13 # 14 """ 15 Collect all defined strings in multiple uni files 16 """ 17 18 ## 19 # Import Modules 20 # 21 import os, codecs, re 22 import distutils.util 23 from Logger import ToolError 24 from Logger import Log as EdkLogger 25 from Logger import StringTable as ST 26 from Library.String import GetLineNo 27 from Library.Misc import PathClass 28 from Library.Misc import GetCharIndexOutStr 29 from Library import DataType as DT 30 from Library.ParserValidate import CheckUTF16FileHeader 31 32 ## 33 # Static definitions 34 # 35 UNICODE_WIDE_CHAR = u'\\wide' 36 UNICODE_NARROW_CHAR = u'\\narrow' 37 UNICODE_NON_BREAKING_CHAR = u'\\nbr' 38 UNICODE_UNICODE_CR = '\r' 39 UNICODE_UNICODE_LF = '\n' 40 41 NARROW_CHAR = u'\uFFF0' 42 WIDE_CHAR = u'\uFFF1' 43 NON_BREAKING_CHAR = u'\uFFF2' 44 CR = u'\u000D' 45 LF = u'\u000A' 46 NULL = u'\u0000' 47 TAB = u'\t' 48 BACK_SPLASH = u'\\' 49 50 gINCLUDE_PATTERN = re.compile("^!include[\s]+([\S]+)[\s]*$", re.MULTILINE | re.UNICODE) 51 52 gLANG_CONV_TABLE = {'eng':'en', 'fra':'fr', \ 53 'aar':'aa', 'abk':'ab', 'ave':'ae', 'afr':'af', 'aka':'ak', 'amh':'am', \ 54 'arg':'an', 'ara':'ar', 'asm':'as', 'ava':'av', 'aym':'ay', 'aze':'az', \ 55 'bak':'ba', 'bel':'be', 'bul':'bg', 'bih':'bh', 'bis':'bi', 'bam':'bm', \ 56 'ben':'bn', 'bod':'bo', 'bre':'br', 'bos':'bs', 'cat':'ca', 'che':'ce', \ 57 'cha':'ch', 'cos':'co', 'cre':'cr', 'ces':'cs', 'chu':'cu', 'chv':'cv', \ 58 'cym':'cy', 'dan':'da', 'deu':'de', 'div':'dv', 'dzo':'dz', 'ewe':'ee', \ 59 'ell':'el', 'epo':'eo', 'spa':'es', 'est':'et', 'eus':'eu', 'fas':'fa', \ 60 'ful':'ff', 'fin':'fi', 'fij':'fj', 'fao':'fo', 'fry':'fy', 'gle':'ga', \ 61 'gla':'gd', 'glg':'gl', 'grn':'gn', 'guj':'gu', 'glv':'gv', 'hau':'ha', \ 62 'heb':'he', 'hin':'hi', 'hmo':'ho', 'hrv':'hr', 'hat':'ht', 'hun':'hu', \ 63 'hye':'hy', 'her':'hz', 'ina':'ia', 'ind':'id', 'ile':'ie', 'ibo':'ig', \ 64 'iii':'ii', 'ipk':'ik', 'ido':'io', 'isl':'is', 'ita':'it', 'iku':'iu', \ 65 'jpn':'ja', 'jav':'jv', 'kat':'ka', 'kon':'kg', 'kik':'ki', 'kua':'kj', \ 66 'kaz':'kk', 'kal':'kl', 'khm':'km', 'kan':'kn', 'kor':'ko', 'kau':'kr', \ 67 'kas':'ks', 'kur':'ku', 'kom':'kv', 'cor':'kw', 'kir':'ky', 'lat':'la', \ 68 'ltz':'lb', 'lug':'lg', 'lim':'li', 'lin':'ln', 'lao':'lo', 'lit':'lt', \ 69 'lub':'lu', 'lav':'lv', 'mlg':'mg', 'mah':'mh', 'mri':'mi', 'mkd':'mk', \ 70 'mal':'ml', 'mon':'mn', 'mar':'mr', 'msa':'ms', 'mlt':'mt', 'mya':'my', \ 71 'nau':'na', 'nob':'nb', 'nde':'nd', 'nep':'ne', 'ndo':'ng', 'nld':'nl', \ 72 'nno':'nn', 'nor':'no', 'nbl':'nr', 'nav':'nv', 'nya':'ny', 'oci':'oc', \ 73 'oji':'oj', 'orm':'om', 'ori':'or', 'oss':'os', 'pan':'pa', 'pli':'pi', \ 74 'pol':'pl', 'pus':'ps', 'por':'pt', 'que':'qu', 'roh':'rm', 'run':'rn', \ 75 'ron':'ro', 'rus':'ru', 'kin':'rw', 'san':'sa', 'srd':'sc', 'snd':'sd', \ 76 'sme':'se', 'sag':'sg', 'sin':'si', 'slk':'sk', 'slv':'sl', 'smo':'sm', \ 77 'sna':'sn', 'som':'so', 'sqi':'sq', 'srp':'sr', 'ssw':'ss', 'sot':'st', \ 78 'sun':'su', 'swe':'sv', 'swa':'sw', 'tam':'ta', 'tel':'te', 'tgk':'tg', \ 79 'tha':'th', 'tir':'ti', 'tuk':'tk', 'tgl':'tl', 'tsn':'tn', 'ton':'to', \ 80 'tur':'tr', 'tso':'ts', 'tat':'tt', 'twi':'tw', 'tah':'ty', 'uig':'ug', \ 81 'ukr':'uk', 'urd':'ur', 'uzb':'uz', 'ven':'ve', 'vie':'vi', 'vol':'vo', \ 82 'wln':'wa', 'wol':'wo', 'xho':'xh', 'yid':'yi', 'yor':'yo', 'zha':'za', \ 83 'zho':'zh', 'zul':'zu'} 84 85 ## Convert a python unicode string to a normal string 86 # 87 # Convert a python unicode string to a normal string 88 # UniToStr(u'I am a string') is 'I am a string' 89 # 90 # @param Uni: The python unicode string 91 # 92 # @retval: The formatted normal string 93 # 94 def UniToStr(Uni): 95 return repr(Uni)[2:-1] 96 97 ## Convert a unicode string to a Hex list 98 # 99 # Convert a unicode string to a Hex list 100 # UniToHexList('ABC') is ['0x41', '0x00', '0x42', '0x00', '0x43', '0x00'] 101 # 102 # @param Uni: The python unicode string 103 # 104 # @retval List: The formatted hex list 105 # 106 def UniToHexList(Uni): 107 List = [] 108 for Item in Uni: 109 Temp = '%04X' % ord(Item) 110 List.append('0x' + Temp[2:4]) 111 List.append('0x' + Temp[0:2]) 112 return List 113 114 ## Convert special unicode characters 115 # 116 # Convert special characters to (c), (r) and (tm). 117 # 118 # @param Uni: The python unicode string 119 # 120 # @retval NewUni: The converted unicode string 121 # 122 def ConvertSpecialUnicodes(Uni): 123 NewUni = Uni 124 NewUni = NewUni.replace(u'\u00A9', '(c)') 125 NewUni = NewUni.replace(u'\u00AE', '(r)') 126 NewUni = NewUni.replace(u'\u2122', '(tm)') 127 return NewUni 128 129 ## GetLanguageCode1766 130 # 131 # Check the language code read from .UNI file and convert RFC 4646 codes to RFC 1766 codes 132 # RFC 1766 language codes supported in compatiblity mode 133 # RFC 4646 language codes supported in native mode 134 # 135 # @param LangName: Language codes read from .UNI file 136 # 137 # @retval LangName: Valid lanugage code in RFC 1766 format or None 138 # 139 def GetLanguageCode1766(LangName, File=None): 140 return LangName 141 142 length = len(LangName) 143 if length == 2: 144 if LangName.isalpha(): 145 for Key in gLANG_CONV_TABLE.keys(): 146 if gLANG_CONV_TABLE.get(Key) == LangName.lower(): 147 return Key 148 elif length == 3: 149 if LangName.isalpha() and gLANG_CONV_TABLE.get(LangName.lower()): 150 return LangName 151 else: 152 EdkLogger.Error("Unicode File Parser", 153 ToolError.FORMAT_INVALID, 154 "Invalid RFC 1766 language code : %s" % LangName, 155 File) 156 elif length == 5: 157 if LangName[0:2].isalpha() and LangName[2] == '-': 158 for Key in gLANG_CONV_TABLE.keys(): 159 if gLANG_CONV_TABLE.get(Key) == LangName[0:2].lower(): 160 return Key 161 elif length >= 6: 162 if LangName[0:2].isalpha() and LangName[2] == '-': 163 for Key in gLANG_CONV_TABLE.keys(): 164 if gLANG_CONV_TABLE.get(Key) == LangName[0:2].lower(): 165 return Key 166 if LangName[0:3].isalpha() and gLANG_CONV_TABLE.get(LangName.lower()) == None and LangName[3] == '-': 167 for Key in gLANG_CONV_TABLE.keys(): 168 if Key == LangName[0:3].lower(): 169 return Key 170 171 EdkLogger.Error("Unicode File Parser", 172 ToolError.FORMAT_INVALID, 173 "Invalid RFC 4646 language code : %s" % LangName, 174 File) 175 176 ## GetLanguageCode 177 # 178 # Check the language code read from .UNI file and convert RFC 1766 codes to RFC 4646 codes if appropriate 179 # RFC 1766 language codes supported in compatiblity mode 180 # RFC 4646 language codes supported in native mode 181 # 182 # @param LangName: Language codes read from .UNI file 183 # 184 # @retval LangName: Valid lanugage code in RFC 4646 format or None 185 # 186 def GetLanguageCode(LangName, IsCompatibleMode, File): 187 length = len(LangName) 188 if IsCompatibleMode: 189 if length == 3 and LangName.isalpha(): 190 TempLangName = gLANG_CONV_TABLE.get(LangName.lower()) 191 if TempLangName != None: 192 return TempLangName 193 return LangName 194 else: 195 EdkLogger.Error("Unicode File Parser", 196 ToolError.FORMAT_INVALID, 197 "Invalid RFC 1766 language code : %s" % LangName, 198 File) 199 if (LangName[0] == 'X' or LangName[0] == 'x') and LangName[1] == '-': 200 return LangName 201 if length == 2: 202 if LangName.isalpha(): 203 return LangName 204 elif length == 3: 205 if LangName.isalpha() and gLANG_CONV_TABLE.get(LangName.lower()) == None: 206 return LangName 207 elif length == 5: 208 if LangName[0:2].isalpha() and LangName[2] == '-': 209 return LangName 210 elif length >= 6: 211 if LangName[0:2].isalpha() and LangName[2] == '-': 212 return LangName 213 if LangName[0:3].isalpha() and gLANG_CONV_TABLE.get(LangName.lower()) == None and LangName[3] == '-': 214 return LangName 215 216 EdkLogger.Error("Unicode File Parser", 217 ToolError.FORMAT_INVALID, 218 "Invalid RFC 4646 language code : %s" % LangName, 219 File) 220 221 ## FormatUniEntry 222 # 223 # Formated the entry in Uni file. 224 # 225 # @param StrTokenName StrTokenName. 226 # @param TokenValueList A list need to be processed. 227 # @param ContainerFile ContainerFile. 228 # 229 # @return formated entry 230 def FormatUniEntry(StrTokenName, TokenValueList, ContainerFile): 231 SubContent = '' 232 PreFormatLength = 40 233 if len(StrTokenName) > PreFormatLength: 234 PreFormatLength = len(StrTokenName) + 1 235 for (Lang, Value) in TokenValueList: 236 if not Value or Lang == DT.TAB_LANGUAGE_EN_X: 237 continue 238 if Lang == '': 239 Lang = DT.TAB_LANGUAGE_EN_US 240 if Lang == 'eng': 241 Lang = DT.TAB_LANGUAGE_EN_US 242 elif len(Lang.split('-')[0]) == 3: 243 Lang = GetLanguageCode(Lang.split('-')[0], True, ContainerFile) 244 else: 245 Lang = GetLanguageCode(Lang, False, ContainerFile) 246 ValueList = Value.split('\n') 247 SubValueContent = '' 248 for SubValue in ValueList: 249 if SubValue.strip(): 250 SubValueContent += \ 251 ' ' * (PreFormatLength + len('#language en-US ')) + '\"%s\\n\"' % SubValue.strip() + '\r\n' 252 SubValueContent = SubValueContent[(PreFormatLength + len('#language en-US ')):SubValueContent.rfind('\\n')] \ 253 + '\"' + '\r\n' 254 SubContent += ' '*PreFormatLength + '#language %-5s ' % Lang + SubValueContent 255 if SubContent: 256 SubContent = StrTokenName + ' '*(PreFormatLength - len(StrTokenName)) + SubContent[PreFormatLength:] 257 return SubContent 258 259 260 ## StringDefClassObject 261 # 262 # A structure for language definition 263 # 264 class StringDefClassObject(object): 265 def __init__(self, Name = None, Value = None, Referenced = False, Token = None, UseOtherLangDef = ''): 266 self.StringName = '' 267 self.StringNameByteList = [] 268 self.StringValue = '' 269 self.StringValueByteList = '' 270 self.Token = 0 271 self.Referenced = Referenced 272 self.UseOtherLangDef = UseOtherLangDef 273 self.Length = 0 274 275 if Name != None: 276 self.StringName = Name 277 self.StringNameByteList = UniToHexList(Name) 278 if Value != None: 279 self.StringValue = Value 280 self.StringValueByteList = UniToHexList(self.StringValue) 281 self.Length = len(self.StringValueByteList) 282 if Token != None: 283 self.Token = Token 284 285 def __str__(self): 286 return repr(self.StringName) + ' ' + \ 287 repr(self.Token) + ' ' + \ 288 repr(self.Referenced) + ' ' + \ 289 repr(self.StringValue) + ' ' + \ 290 repr(self.UseOtherLangDef) 291 292 def UpdateValue(self, Value = None): 293 if Value != None: 294 if self.StringValue: 295 self.StringValue = self.StringValue + '\r\n' + Value 296 else: 297 self.StringValue = Value 298 self.StringValueByteList = UniToHexList(self.StringValue) 299 self.Length = len(self.StringValueByteList) 300 301 ## UniFileClassObject 302 # 303 # A structure for .uni file definition 304 # 305 class UniFileClassObject(object): 306 def __init__(self, FileList = None, IsCompatibleMode = False, IncludePathList = None): 307 self.FileList = FileList 308 self.File = None 309 self.IncFileList = FileList 310 self.UniFileHeader = '' 311 self.Token = 2 312 self.LanguageDef = [] #[ [u'LanguageIdentifier', u'PrintableName'], ... ] 313 self.OrderedStringList = {} #{ u'LanguageIdentifier' : [StringDefClassObject] } 314 self.OrderedStringDict = {} #{ u'LanguageIdentifier' : {StringName:(IndexInList)} } 315 self.OrderedStringListByToken = {} #{ u'LanguageIdentifier' : {Token: StringDefClassObject} } 316 self.IsCompatibleMode = IsCompatibleMode 317 if not IncludePathList: 318 self.IncludePathList = [] 319 else: 320 self.IncludePathList = IncludePathList 321 if len(self.FileList) > 0: 322 self.LoadUniFiles(FileList) 323 324 # 325 # Get Language definition 326 # 327 def GetLangDef(self, File, Line): 328 Lang = distutils.util.split_quoted((Line.split(u"//")[0])) 329 if len(Lang) != 3: 330 try: 331 FileIn = codecs.open(File.Path, mode='rb', encoding='utf_8').readlines() 332 except UnicodeError, Xstr: 333 FileIn = codecs.open(File.Path, mode='rb', encoding='utf_16').readlines() 334 except UnicodeError, Xstr: 335 FileIn = codecs.open(File.Path, mode='rb', encoding='utf_16_le').readlines() 336 except: 337 EdkLogger.Error("Unicode File Parser", 338 ToolError.FILE_OPEN_FAILURE, 339 "File read failure: %s" % str(Xstr), 340 ExtraData=File) 341 LineNo = GetLineNo(FileIn, Line, False) 342 EdkLogger.Error("Unicode File Parser", 343 ToolError.PARSER_ERROR, 344 "Wrong language definition", 345 ExtraData="""%s\n\t*Correct format is like '#langdef en-US "English"'""" % Line, 346 File = File, Line = LineNo) 347 else: 348 LangName = GetLanguageCode(Lang[1], self.IsCompatibleMode, self.File) 349 LangPrintName = Lang[2] 350 351 IsLangInDef = False 352 for Item in self.LanguageDef: 353 if Item[0] == LangName: 354 IsLangInDef = True 355 break 356 357 if not IsLangInDef: 358 self.LanguageDef.append([LangName, LangPrintName]) 359 360 # 361 # Add language string 362 # 363 self.AddStringToList(u'$LANGUAGE_NAME', LangName, LangName, 0, True, Index=0) 364 self.AddStringToList(u'$PRINTABLE_LANGUAGE_NAME', LangName, LangPrintName, 1, True, Index=1) 365 366 if not IsLangInDef: 367 # 368 # The found STRING tokens will be added into new language string list 369 # so that the unique STRING identifier is reserved for all languages in the package list. 370 # 371 FirstLangName = self.LanguageDef[0][0] 372 if LangName != FirstLangName: 373 for Index in range (2, len (self.OrderedStringList[FirstLangName])): 374 Item = self.OrderedStringList[FirstLangName][Index] 375 if Item.UseOtherLangDef != '': 376 OtherLang = Item.UseOtherLangDef 377 else: 378 OtherLang = FirstLangName 379 self.OrderedStringList[LangName].append (StringDefClassObject(Item.StringName, 380 '', 381 Item.Referenced, 382 Item.Token, 383 OtherLang)) 384 self.OrderedStringDict[LangName][Item.StringName] = len(self.OrderedStringList[LangName]) - 1 385 return True 386 387 # 388 # Get String name and value 389 # 390 def GetStringObject(self, Item): 391 Language = '' 392 Value = '' 393 394 Name = Item.split()[1] 395 # Check the string name is the upper character 396 if Name != '': 397 MatchString = re.match('[A-Z0-9_]+', Name, re.UNICODE) 398 if MatchString == None or MatchString.end(0) != len(Name): 399 EdkLogger.Error("Unicode File Parser", 400 ToolError.FORMAT_INVALID, 401 'The string token name %s in UNI file %s must be upper case character.' %(Name, self.File)) 402 LanguageList = Item.split(u'#language ') 403 for IndexI in range(len(LanguageList)): 404 if IndexI == 0: 405 continue 406 else: 407 Language = LanguageList[IndexI].split()[0] 408 #.replace(u'\r\n', u'') 409 Value = \ 410 LanguageList[IndexI][LanguageList[IndexI].find(u'\"') + len(u'\"') : LanguageList[IndexI].rfind(u'\"')] 411 Language = GetLanguageCode(Language, self.IsCompatibleMode, self.File) 412 self.AddStringToList(Name, Language, Value) 413 414 # 415 # Get include file list and load them 416 # 417 def GetIncludeFile(self, Item, Dir = None): 418 if Dir: 419 pass 420 FileName = Item[Item.find(u'!include ') + len(u'!include ') :Item.find(u' ', len(u'!include '))][1:-1] 421 self.LoadUniFile(FileName) 422 423 # 424 # Pre-process before parse .uni file 425 # 426 def PreProcess(self, File, IsIncludeFile=False): 427 if not os.path.exists(File.Path) or not os.path.isfile(File.Path): 428 EdkLogger.Error("Unicode File Parser", 429 ToolError.FILE_NOT_FOUND, 430 ExtraData=File.Path) 431 432 # 433 # Check file header of the Uni file 434 # 435 # if not CheckUTF16FileHeader(File.Path): 436 # EdkLogger.Error("Unicode File Parser", ToolError.FORMAT_INVALID, 437 # ExtraData='The file %s is either invalid UTF-16LE or it is missing the BOM.' % File.Path) 438 439 try: 440 FileIn = codecs.open(File.Path, mode='rb', encoding='utf_8').readlines() 441 except UnicodeError, Xstr: 442 FileIn = codecs.open(File.Path, mode='rb', encoding='utf_16').readlines() 443 except UnicodeError: 444 FileIn = codecs.open(File.Path, mode='rb', encoding='utf_16_le').readlines() 445 except: 446 EdkLogger.Error("Unicode File Parser", ToolError.FILE_OPEN_FAILURE, ExtraData=File.Path) 447 448 449 # 450 # get the file header 451 # 452 Lines = [] 453 HeaderStart = False 454 HeaderEnd = False 455 if not self.UniFileHeader: 456 FirstGenHeader = True 457 else: 458 FirstGenHeader = False 459 for Line in FileIn: 460 Line = Line.strip() 461 if Line == u'': 462 continue 463 if Line.startswith(DT.TAB_COMMENT_EDK1_SPLIT) and (Line.find(DT.TAB_HEADER_COMMENT) > -1) \ 464 and not HeaderEnd and not HeaderStart: 465 HeaderStart = True 466 if not Line.startswith(DT.TAB_COMMENT_EDK1_SPLIT) and HeaderStart and not HeaderEnd: 467 HeaderEnd = True 468 if Line.startswith(DT.TAB_COMMENT_EDK1_SPLIT) and HeaderStart and not HeaderEnd and FirstGenHeader: 469 self.UniFileHeader += Line + '\r\n' 470 continue 471 472 # 473 # Use unique identifier 474 # 475 FindFlag = -1 476 LineCount = 0 477 MultiLineFeedExits = False 478 # 479 # 0: initial value 480 # 1: signle String entry exist 481 # 2: line feed exist under the some signle String entry 482 # 483 StringEntryExistsFlag = 0 484 for Line in FileIn: 485 Line = FileIn[LineCount] 486 LineCount += 1 487 Line = Line.strip() 488 # 489 # Ignore comment line and empty line 490 # 491 if Line == u'' or Line.startswith(u'//'): 492 # 493 # Change the single line String entry flag status 494 # 495 if StringEntryExistsFlag == 1: 496 StringEntryExistsFlag = 2 497 # 498 # If the '#string' line and the '#language' line are not in the same line, 499 # there should be only one line feed character betwwen them 500 # 501 if MultiLineFeedExits: 502 EdkLogger.Error("Unicode File Parser", ToolError.FORMAT_INVALID, ExtraData=File.Path) 503 continue 504 505 MultiLineFeedExits = False 506 # 507 # Process comment embeded in string define lines 508 # 509 FindFlag = Line.find(u'//') 510 if FindFlag != -1 and Line.find(u'//') < Line.find(u'"'): 511 Line = Line.replace(Line[FindFlag:], u' ') 512 if FileIn[LineCount].strip().startswith('#language'): 513 Line = Line + FileIn[LineCount] 514 FileIn[LineCount-1] = Line 515 FileIn[LineCount] = '\r\n' 516 LineCount -= 1 517 for Index in xrange (LineCount + 1, len (FileIn) - 1): 518 if (Index == len(FileIn) -1): 519 FileIn[Index] = '\r\n' 520 else: 521 FileIn[Index] = FileIn[Index + 1] 522 continue 523 CommIndex = GetCharIndexOutStr(u'/', Line) 524 if CommIndex > -1: 525 if (len(Line) - 1) > CommIndex: 526 if Line[CommIndex+1] == u'/': 527 Line = Line[:CommIndex].strip() 528 else: 529 EdkLogger.Error("Unicode File Parser", ToolError.FORMAT_INVALID, ExtraData=File.Path) 530 else: 531 EdkLogger.Error("Unicode File Parser", ToolError.FORMAT_INVALID, ExtraData=File.Path) 532 533 Line = Line.replace(UNICODE_WIDE_CHAR, WIDE_CHAR) 534 Line = Line.replace(UNICODE_NARROW_CHAR, NARROW_CHAR) 535 Line = Line.replace(UNICODE_NON_BREAKING_CHAR, NON_BREAKING_CHAR) 536 537 Line = Line.replace(u'\\\\', u'\u0006') 538 Line = Line.replace(u'\\r\\n', CR + LF) 539 Line = Line.replace(u'\\n', CR + LF) 540 Line = Line.replace(u'\\r', CR) 541 Line = Line.replace(u'\\t', u'\t') 542 Line = Line.replace(u'''\"''', u'''"''') 543 Line = Line.replace(u'\t', u' ') 544 Line = Line.replace(u'\u0006', u'\\') 545 546 # IncList = gINCLUDE_PATTERN.findall(Line) 547 IncList = [] 548 if len(IncList) == 1: 549 for Dir in [File.Dir] + self.IncludePathList: 550 IncFile = PathClass(str(IncList[0]), Dir) 551 self.IncFileList.append(IncFile) 552 if os.path.isfile(IncFile.Path): 553 Lines.extend(self.PreProcess(IncFile, True)) 554 break 555 else: 556 EdkLogger.Error("Unicode File Parser", 557 ToolError.FILE_NOT_FOUND, 558 Message="Cannot find include file", 559 ExtraData=str(IncList[0])) 560 continue 561 562 # 563 # Between Name entry and Language entry can not contain line feed 564 # 565 if Line.startswith(u'#string') and Line.find(u'#language') == -1: 566 MultiLineFeedExits = True 567 568 if Line.startswith(u'#string') and Line.find(u'#language') > 0 and Line.find(u'"') < 0: 569 MultiLineFeedExits = True 570 571 # 572 # Between Language entry and String entry can not contain line feed 573 # 574 if Line.startswith(u'#language') and len(Line.split()) == 2: 575 MultiLineFeedExits = True 576 577 # 578 # Between two String entry, can not contain line feed 579 # 580 if Line.startswith(u'"'): 581 if StringEntryExistsFlag == 2: 582 EdkLogger.Error("Unicode File Parser", ToolError.FORMAT_INVALID, 583 Message=ST.ERR_UNIPARSE_LINEFEED_UP_EXIST % Line, ExtraData=File.Path) 584 585 StringEntryExistsFlag = 1 586 if not Line.endswith('"'): 587 EdkLogger.Error("Unicode File Parser", ToolError.FORMAT_INVALID, 588 ExtraData='''The line %s misses '"' at the end of it in file %s''' 589 % (LineCount, File.Path)) 590 elif Line.startswith(u'#language'): 591 if StringEntryExistsFlag == 2: 592 EdkLogger.Error("Unicode File Parser", ToolError.FORMAT_INVALID, 593 Message=ST.ERR_UNI_MISS_STRING_ENTRY % Line, ExtraData=File.Path) 594 StringEntryExistsFlag = 0 595 else: 596 StringEntryExistsFlag = 0 597 598 Lines.append(Line) 599 600 # 601 # Convert string def format as below 602 # 603 # #string MY_STRING_1 604 # #language eng 605 # "My first English string line 1" 606 # "My first English string line 2" 607 # #string MY_STRING_1 608 # #language spa 609 # "Mi segunda secuencia 1" 610 # "Mi segunda secuencia 2" 611 # 612 613 if not IsIncludeFile and not Lines: 614 EdkLogger.Error("Unicode File Parser", ToolError.FORMAT_INVALID, \ 615 Message=ST.ERR_UNIPARSE_NO_SECTION_EXIST, \ 616 ExtraData=File.Path) 617 618 NewLines = [] 619 StrName = u'' 620 ExistStrNameList = [] 621 for Line in Lines: 622 if StrName and not StrName.split()[1].startswith(DT.TAB_STR_TOKENCNAME + DT.TAB_UNDERLINE_SPLIT): 623 EdkLogger.Error("Unicode File Parser", ToolError.FORMAT_INVALID, \ 624 Message=ST.ERR_UNIPARSE_STRNAME_FORMAT_ERROR % StrName.split()[1], \ 625 ExtraData=File.Path) 626 627 if StrName and len(StrName.split()[1].split(DT.TAB_UNDERLINE_SPLIT)) == 4: 628 StringTokenList = StrName.split()[1].split(DT.TAB_UNDERLINE_SPLIT) 629 if (StringTokenList[3].upper() in [DT.TAB_STR_TOKENPROMPT, DT.TAB_STR_TOKENHELP] and \ 630 StringTokenList[3] not in [DT.TAB_STR_TOKENPROMPT, DT.TAB_STR_TOKENHELP]) or \ 631 (StringTokenList[2].upper() == DT.TAB_STR_TOKENERR and StringTokenList[2] != DT.TAB_STR_TOKENERR): 632 EdkLogger.Error("Unicode File Parser", ToolError.FORMAT_INVALID, \ 633 Message=ST.ERR_UNIPARSE_STRTOKEN_FORMAT_ERROR % StrName.split()[1], \ 634 ExtraData=File.Path) 635 636 if Line.count(u'#language') > 1: 637 EdkLogger.Error("Unicode File Parser", ToolError.FORMAT_INVALID, \ 638 Message=ST.ERR_UNIPARSE_SEP_LANGENTRY_LINE % Line, \ 639 ExtraData=File.Path) 640 641 if Line.startswith(u'//'): 642 continue 643 elif Line.startswith(u'#langdef'): 644 if len(Line.split()) == 2: 645 NewLines.append(Line) 646 continue 647 elif len(Line.split()) > 2 and Line.find(u'"') > 0: 648 NewLines.append(Line[:Line.find(u'"')].strip()) 649 NewLines.append(Line[Line.find(u'"'):]) 650 else: 651 EdkLogger.Error("Unicode File Parser", ToolError.FORMAT_INVALID, ExtraData=File.Path) 652 elif Line.startswith(u'#string'): 653 if len(Line.split()) == 2: 654 StrName = Line 655 if StrName: 656 if StrName.split()[1] not in ExistStrNameList: 657 ExistStrNameList.append(StrName.split()[1].strip()) 658 elif StrName.split()[1] in [DT.TAB_INF_ABSTRACT, DT.TAB_INF_DESCRIPTION, \ 659 DT.TAB_INF_BINARY_ABSTRACT, DT.TAB_INF_BINARY_DESCRIPTION, \ 660 DT.TAB_DEC_PACKAGE_ABSTRACT, DT.TAB_DEC_PACKAGE_DESCRIPTION, \ 661 DT.TAB_DEC_BINARY_ABSTRACT, DT.TAB_DEC_BINARY_DESCRIPTION]: 662 EdkLogger.Error("Unicode File Parser", ToolError.FORMAT_INVALID, \ 663 Message=ST.ERR_UNIPARSE_MULTI_ENTRY_EXIST % StrName.split()[1], \ 664 ExtraData=File.Path) 665 continue 666 elif len(Line.split()) == 4 and Line.find(u'#language') > 0: 667 if Line[Line.find(u'#language')-1] != ' ' or \ 668 Line[Line.find(u'#language')+len(u'#language')] != u' ': 669 EdkLogger.Error("Unicode File Parser", ToolError.FORMAT_INVALID, ExtraData=File.Path) 670 671 if Line.find(u'"') > 0: 672 EdkLogger.Error("Unicode File Parser", ToolError.FORMAT_INVALID, ExtraData=File.Path) 673 674 StrName = Line.split()[0] + u' ' + Line.split()[1] 675 if StrName: 676 if StrName.split()[1] not in ExistStrNameList: 677 ExistStrNameList.append(StrName.split()[1].strip()) 678 elif StrName.split()[1] in [DT.TAB_INF_ABSTRACT, DT.TAB_INF_DESCRIPTION, \ 679 DT.TAB_INF_BINARY_ABSTRACT, DT.TAB_INF_BINARY_DESCRIPTION, \ 680 DT.TAB_DEC_PACKAGE_ABSTRACT, DT.TAB_DEC_PACKAGE_DESCRIPTION, \ 681 DT.TAB_DEC_BINARY_ABSTRACT, DT.TAB_DEC_BINARY_DESCRIPTION]: 682 EdkLogger.Error("Unicode File Parser", ToolError.FORMAT_INVALID, \ 683 Message=ST.ERR_UNIPARSE_MULTI_ENTRY_EXIST % StrName.split()[1], \ 684 ExtraData=File.Path) 685 if IsIncludeFile: 686 if StrName not in NewLines: 687 NewLines.append((Line[:Line.find(u'#language')]).strip()) 688 else: 689 NewLines.append((Line[:Line.find(u'#language')]).strip()) 690 NewLines.append((Line[Line.find(u'#language'):]).strip()) 691 elif len(Line.split()) > 4 and Line.find(u'#language') > 0 and Line.find(u'"') > 0: 692 if Line[Line.find(u'#language')-1] != u' ' or \ 693 Line[Line.find(u'#language')+len(u'#language')] != u' ': 694 EdkLogger.Error("Unicode File Parser", ToolError.FORMAT_INVALID, ExtraData=File.Path) 695 696 if Line[Line.find(u'"')-1] != u' ': 697 EdkLogger.Error("Unicode File Parser", ToolError.FORMAT_INVALID, ExtraData=File.Path) 698 699 StrName = Line.split()[0] + u' ' + Line.split()[1] 700 if StrName: 701 if StrName.split()[1] not in ExistStrNameList: 702 ExistStrNameList.append(StrName.split()[1].strip()) 703 elif StrName.split()[1] in [DT.TAB_INF_ABSTRACT, DT.TAB_INF_DESCRIPTION, \ 704 DT.TAB_INF_BINARY_ABSTRACT, DT.TAB_INF_BINARY_DESCRIPTION, \ 705 DT.TAB_DEC_PACKAGE_ABSTRACT, DT.TAB_DEC_PACKAGE_DESCRIPTION, \ 706 DT.TAB_DEC_BINARY_ABSTRACT, DT.TAB_DEC_BINARY_DESCRIPTION]: 707 EdkLogger.Error("Unicode File Parser", ToolError.FORMAT_INVALID, \ 708 Message=ST.ERR_UNIPARSE_MULTI_ENTRY_EXIST % StrName.split()[1], \ 709 ExtraData=File.Path) 710 if IsIncludeFile: 711 if StrName not in NewLines: 712 NewLines.append((Line[:Line.find(u'#language')]).strip()) 713 else: 714 NewLines.append((Line[:Line.find(u'#language')]).strip()) 715 NewLines.append((Line[Line.find(u'#language'):Line.find(u'"')]).strip()) 716 NewLines.append((Line[Line.find(u'"'):]).strip()) 717 else: 718 EdkLogger.Error("Unicode File Parser", ToolError.FORMAT_INVALID, ExtraData=File.Path) 719 elif Line.startswith(u'#language'): 720 if len(Line.split()) == 2: 721 if IsIncludeFile: 722 if StrName not in NewLines: 723 NewLines.append(StrName) 724 else: 725 NewLines.append(StrName) 726 NewLines.append(Line) 727 elif len(Line.split()) > 2 and Line.find(u'"') > 0: 728 if IsIncludeFile: 729 if StrName not in NewLines: 730 NewLines.append(StrName) 731 else: 732 NewLines.append(StrName) 733 NewLines.append((Line[:Line.find(u'"')]).strip()) 734 NewLines.append((Line[Line.find(u'"'):]).strip()) 735 else: 736 EdkLogger.Error("Unicode File Parser", ToolError.FORMAT_INVALID, ExtraData=File.Path) 737 elif Line.startswith(u'"'): 738 if u'#string' in Line or u'#language' in Line: 739 EdkLogger.Error("Unicode File Parser", ToolError.FORMAT_INVALID, ExtraData=File.Path) 740 NewLines.append(Line) 741 else: 742 print Line 743 EdkLogger.Error("Unicode File Parser", ToolError.FORMAT_INVALID, ExtraData=File.Path) 744 745 if StrName and not StrName.split()[1].startswith(u'STR_'): 746 EdkLogger.Error("Unicode File Parser", ToolError.FORMAT_INVALID, \ 747 Message=ST.ERR_UNIPARSE_STRNAME_FORMAT_ERROR % StrName.split()[1], \ 748 ExtraData=File.Path) 749 750 if StrName and not NewLines: 751 EdkLogger.Error("Unicode File Parser", ToolError.FORMAT_INVALID, \ 752 Message=ST.ERR_UNI_MISS_LANGENTRY % StrName, \ 753 ExtraData=File.Path) 754 755 # 756 # Check Abstract, Description, BinaryAbstract and BinaryDescription order, 757 # should be Abstract, Description, BinaryAbstract, BinaryDesctiption 758 AbstractPosition = -1 759 DescriptionPosition = -1 760 BinaryAbstractPosition = -1 761 BinaryDescriptionPosition = -1 762 for StrName in ExistStrNameList: 763 if DT.TAB_HEADER_ABSTRACT.upper() in StrName: 764 if 'BINARY' in StrName: 765 BinaryAbstractPosition = ExistStrNameList.index(StrName) 766 else: 767 AbstractPosition = ExistStrNameList.index(StrName) 768 if DT.TAB_HEADER_DESCRIPTION.upper() in StrName: 769 if 'BINARY' in StrName: 770 BinaryDescriptionPosition = ExistStrNameList.index(StrName) 771 else: 772 DescriptionPosition = ExistStrNameList.index(StrName) 773 774 OrderList = sorted([AbstractPosition, DescriptionPosition]) 775 BinaryOrderList = sorted([BinaryAbstractPosition, BinaryDescriptionPosition]) 776 Min = OrderList[0] 777 Max = OrderList[1] 778 BinaryMin = BinaryOrderList[0] 779 BinaryMax = BinaryOrderList[1] 780 if BinaryDescriptionPosition > -1: 781 if not(BinaryDescriptionPosition == BinaryMax and BinaryAbstractPosition == BinaryMin and \ 782 BinaryMax > Max): 783 EdkLogger.Error("Unicode File Parser", ToolError.FORMAT_INVALID, \ 784 Message=ST.ERR_UNIPARSE_ENTRY_ORDER_WRONG, \ 785 ExtraData=File.Path) 786 elif BinaryAbstractPosition > -1: 787 if not(BinaryAbstractPosition > Max): 788 EdkLogger.Error("Unicode File Parser", ToolError.FORMAT_INVALID, \ 789 Message=ST.ERR_UNIPARSE_ENTRY_ORDER_WRONG, \ 790 ExtraData=File.Path) 791 792 if DescriptionPosition > -1: 793 if not(DescriptionPosition == Max and AbstractPosition == Min and \ 794 DescriptionPosition > AbstractPosition): 795 EdkLogger.Error("Unicode File Parser", ToolError.FORMAT_INVALID, \ 796 Message=ST.ERR_UNIPARSE_ENTRY_ORDER_WRONG, \ 797 ExtraData=File.Path) 798 799 if not self.UniFileHeader: 800 EdkLogger.Error("Unicode File Parser", ToolError.FORMAT_INVALID, 801 Message = ST.ERR_NO_SOURCE_HEADER, 802 ExtraData=File.Path) 803 804 return NewLines 805 806 # 807 # Load a .uni file 808 # 809 def LoadUniFile(self, File = None): 810 if File == None: 811 EdkLogger.Error("Unicode File Parser", 812 ToolError.PARSER_ERROR, 813 Message='No unicode file is given', 814 ExtraData=File.Path) 815 816 self.File = File 817 818 # 819 # Process special char in file 820 # 821 Lines = self.PreProcess(File) 822 823 # 824 # Get Unicode Information 825 # 826 for IndexI in range(len(Lines)): 827 Line = Lines[IndexI] 828 if (IndexI + 1) < len(Lines): 829 SecondLine = Lines[IndexI + 1] 830 if (IndexI + 2) < len(Lines): 831 ThirdLine = Lines[IndexI + 2] 832 833 # 834 # Get Language def information 835 # 836 if Line.find(u'#langdef ') >= 0: 837 self.GetLangDef(File, Line + u' ' + SecondLine) 838 continue 839 840 Name = '' 841 Language = '' 842 Value = '' 843 CombineToken = False 844 # 845 # Get string def information format as below 846 # 847 # #string MY_STRING_1 848 # #language eng 849 # "My first English string line 1" 850 # "My first English string line 2" 851 # #string MY_STRING_1 852 # #language spa 853 # "Mi segunda secuencia 1" 854 # "Mi segunda secuencia 2" 855 # 856 if Line.find(u'#string ') >= 0 and Line.find(u'#language ') < 0 and \ 857 SecondLine.find(u'#string ') < 0 and SecondLine.find(u'#language ') >= 0 and \ 858 ThirdLine.find(u'#string ') < 0 and ThirdLine.find(u'#language ') < 0: 859 if Line.find('"') > 0 or SecondLine.find('"') > 0: 860 EdkLogger.Error("Unicode File Parser", ToolError.FORMAT_INVALID, 861 Message=ST.ERR_UNIPARSE_DBLQUOTE_UNMATCHED, 862 ExtraData=File.Path) 863 864 Name = Line[Line.find(u'#string ') + len(u'#string ') : ].strip(' ') 865 Language = SecondLine[SecondLine.find(u'#language ') + len(u'#language ') : ].strip(' ') 866 for IndexJ in range(IndexI + 2, len(Lines)): 867 if Lines[IndexJ].find(u'#string ') < 0 and Lines[IndexJ].find(u'#language ') < 0 and \ 868 Lines[IndexJ].strip().startswith(u'"') and Lines[IndexJ].strip().endswith(u'"'): 869 if Lines[IndexJ][-2] == ' ': 870 CombineToken = True 871 if CombineToken: 872 if Lines[IndexJ].strip()[1:-1].strip(): 873 Value = Value + Lines[IndexJ].strip()[1:-1].rstrip() + ' ' 874 else: 875 Value = Value + Lines[IndexJ].strip()[1:-1] 876 CombineToken = False 877 else: 878 Value = Value + Lines[IndexJ].strip()[1:-1] + '\r\n' 879 else: 880 IndexI = IndexJ 881 break 882 if Value.endswith('\r\n'): 883 Value = Value[: Value.rfind('\r\n')] 884 Language = GetLanguageCode(Language, self.IsCompatibleMode, self.File) 885 self.AddStringToList(Name, Language, Value) 886 continue 887 888 # 889 # Load multiple .uni files 890 # 891 def LoadUniFiles(self, FileList): 892 if len(FileList) > 0: 893 for File in FileList: 894 FilePath = File.Path.strip() 895 if FilePath.endswith('.uni') or FilePath.endswith('.UNI') or FilePath.endswith('.Uni'): 896 self.LoadUniFile(File) 897 898 # 899 # Add a string to list 900 # 901 def AddStringToList(self, Name, Language, Value, Token = 0, Referenced = False, UseOtherLangDef = '', Index = -1): 902 for LangNameItem in self.LanguageDef: 903 if Language == LangNameItem[0]: 904 break 905 906 if Language not in self.OrderedStringList: 907 self.OrderedStringList[Language] = [] 908 self.OrderedStringDict[Language] = {} 909 910 IsAdded = True 911 if Name in self.OrderedStringDict[Language]: 912 IsAdded = False 913 if Value != None: 914 ItemIndexInList = self.OrderedStringDict[Language][Name] 915 Item = self.OrderedStringList[Language][ItemIndexInList] 916 Item.UpdateValue(Value) 917 Item.UseOtherLangDef = '' 918 919 if IsAdded: 920 Token = len(self.OrderedStringList[Language]) 921 if Index == -1: 922 self.OrderedStringList[Language].append(StringDefClassObject(Name, 923 Value, 924 Referenced, 925 Token, 926 UseOtherLangDef)) 927 self.OrderedStringDict[Language][Name] = Token 928 for LangName in self.LanguageDef: 929 # 930 # New STRING token will be added into all language string lists. 931 # so that the unique STRING identifier is reserved for all languages in the package list. 932 # 933 if LangName[0] != Language: 934 if UseOtherLangDef != '': 935 OtherLangDef = UseOtherLangDef 936 else: 937 OtherLangDef = Language 938 self.OrderedStringList[LangName[0]].append(StringDefClassObject(Name, 939 '', 940 Referenced, 941 Token, 942 OtherLangDef)) 943 self.OrderedStringDict[LangName[0]][Name] = len(self.OrderedStringList[LangName[0]]) - 1 944 else: 945 self.OrderedStringList[Language].insert(Index, StringDefClassObject(Name, 946 Value, 947 Referenced, 948 Token, 949 UseOtherLangDef)) 950 self.OrderedStringDict[Language][Name] = Index 951 952 # 953 # Set the string as referenced 954 # 955 def SetStringReferenced(self, Name): 956 # 957 # String stoken are added in the same order in all language string lists. 958 # So, only update the status of string stoken in first language string list. 959 # 960 Lang = self.LanguageDef[0][0] 961 if Name in self.OrderedStringDict[Lang]: 962 ItemIndexInList = self.OrderedStringDict[Lang][Name] 963 Item = self.OrderedStringList[Lang][ItemIndexInList] 964 Item.Referenced = True 965 966 # 967 # Search the string in language definition by Name 968 # 969 def FindStringValue(self, Name, Lang): 970 if Name in self.OrderedStringDict[Lang]: 971 ItemIndexInList = self.OrderedStringDict[Lang][Name] 972 return self.OrderedStringList[Lang][ItemIndexInList] 973 974 return None 975 976 # 977 # Search the string in language definition by Token 978 # 979 def FindByToken(self, Token, Lang): 980 for Item in self.OrderedStringList[Lang]: 981 if Item.Token == Token: 982 return Item 983 984 return None 985 986 # 987 # Re-order strings and re-generate tokens 988 # 989 def ReToken(self): 990 if len(self.LanguageDef) == 0: 991 return None 992 # 993 # Retoken all language strings according to the status of string stoken in the first language string. 994 # 995 FirstLangName = self.LanguageDef[0][0] 996 997 # Convert the OrderedStringList to be OrderedStringListByToken in order to faciliate future search by token 998 for LangNameItem in self.LanguageDef: 999 self.OrderedStringListByToken[LangNameItem[0]] = {} 1000 1001 # 1002 # Use small token for all referred string stoken. 1003 # 1004 RefToken = 0 1005 for Index in range (0, len (self.OrderedStringList[FirstLangName])): 1006 FirstLangItem = self.OrderedStringList[FirstLangName][Index] 1007 if FirstLangItem.Referenced == True: 1008 for LangNameItem in self.LanguageDef: 1009 LangName = LangNameItem[0] 1010 OtherLangItem = self.OrderedStringList[LangName][Index] 1011 OtherLangItem.Referenced = True 1012 OtherLangItem.Token = RefToken 1013 self.OrderedStringListByToken[LangName][OtherLangItem.Token] = OtherLangItem 1014 RefToken = RefToken + 1 1015 1016 # 1017 # Use big token for all unreferred string stoken. 1018 # 1019 UnRefToken = 0 1020 for Index in range (0, len (self.OrderedStringList[FirstLangName])): 1021 FirstLangItem = self.OrderedStringList[FirstLangName][Index] 1022 if FirstLangItem.Referenced == False: 1023 for LangNameItem in self.LanguageDef: 1024 LangName = LangNameItem[0] 1025 OtherLangItem = self.OrderedStringList[LangName][Index] 1026 OtherLangItem.Token = RefToken + UnRefToken 1027 self.OrderedStringListByToken[LangName][OtherLangItem.Token] = OtherLangItem 1028 UnRefToken = UnRefToken + 1 1029 1030 # 1031 # Show the instance itself 1032 # 1033 def ShowMe(self): 1034 print self.LanguageDef 1035 #print self.OrderedStringList 1036 for Item in self.OrderedStringList: 1037 print Item 1038 for Member in self.OrderedStringList[Item]: 1039 print str(Member) 1040 1041 # 1042 # Read content from '!include' UNI file 1043 # 1044 def ReadIncludeUNIfile(self, FilaPath): 1045 if self.File: 1046 pass 1047 1048 if not os.path.exists(FilaPath) or not os.path.isfile(FilaPath): 1049 EdkLogger.Error("Unicode File Parser", 1050 ToolError.FILE_NOT_FOUND, 1051 ExtraData=FilaPath) 1052 try: 1053 FileIn = codecs.open(FilaPath, mode='rb', encoding='utf_8').readlines() 1054 except UnicodeError, Xstr: 1055 FileIn = codecs.open(FilaPath, mode='rb', encoding='utf_16').readlines() 1056 except UnicodeError: 1057 FileIn = codecs.open(FilaPath, mode='rb', encoding='utf_16_le').readlines() 1058 except: 1059 EdkLogger.Error("Unicode File Parser", ToolError.FILE_OPEN_FAILURE, ExtraData=FilaPath) 1060 return FileIn 1061 1062
