1 #!/usr/bin/env python 2 #*-* coding: utf-8 3 # Copyright 2016 The Closure Linter Authors. All Rights Reserved. 4 # 5 # Licensed under the Apache License, Version 2.0 (the "License"); 6 # you may not use this file except in compliance with the License. 7 # You may obtain a copy of the License at 8 # 9 # http://www.apache.org/licenses/LICENSE-2.0 10 # 11 # Unless required by applicable law or agreed to in writing, software 12 # distributed under the License is distributed on an "AS-IS" BASIS, 13 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 # See the License for the specific language governing permissions and 15 # limitations under the License. 16 17 """Closure typeannotation parsing and utilities.""" 18 19 20 21 from closure_linter import errors 22 from closure_linter import javascripttokens 23 from closure_linter.common import error 24 25 # Shorthand 26 TYPE = javascripttokens.JavaScriptTokenType 27 28 29 class TypeAnnotation(object): 30 """Represents a structured view of a closure type annotation. 31 32 Attribute: 33 identifier: The name of the type. 34 key_type: The name part before a colon. 35 sub_types: The list of sub_types used e.g. for Array.<> 36 or_null: The '?' annotation 37 not_null: The '!' annotation 38 type_group: If this a a grouping (a|b), but does not include function(a). 39 return_type: The return type of a function definition. 40 alias: The actual type set by closurizednamespaceinfo if the identifier uses 41 an alias to shorten the name. 42 tokens: An ordered list of tokens used for this type. May contain 43 TypeAnnotation instances for sub_types, key_type or return_type. 44 """ 45 46 IMPLICIT_TYPE_GROUP = 2 47 48 NULLABILITY_UNKNOWN = 2 49 50 FUNCTION_TYPE = 'function' 51 NULL_TYPE = 'null' 52 VAR_ARGS_TYPE = '...' 53 54 # Frequently used known non-nullable types. 55 NON_NULLABLE = frozenset([ 56 'boolean', FUNCTION_TYPE, 'number', 'string', 'undefined']) 57 # Frequently used known nullable types. 58 NULLABLE_TYPE_WHITELIST = frozenset([ 59 'Array', 'Document', 'Element', 'Function', 'Node', 'NodeList', 60 'Object']) 61 62 def __init__(self): 63 self.identifier = '' 64 self.sub_types = [] 65 self.or_null = False 66 self.not_null = False 67 self.type_group = False 68 self.alias = None 69 self.key_type = None 70 self.record_type = False 71 self.opt_arg = False 72 self.return_type = None 73 self.tokens = [] 74 75 def IsFunction(self): 76 """Determines whether this is a function definition.""" 77 return self.identifier == TypeAnnotation.FUNCTION_TYPE 78 79 def IsConstructor(self): 80 """Determines whether this is a function definition for a constructor.""" 81 key_type = self.sub_types and self.sub_types[0].key_type 82 return self.IsFunction() and key_type.identifier == 'new' 83 84 def IsRecordType(self): 85 """Returns True if this type is a record type.""" 86 return (self.record_type or 87 any(t.IsRecordType() for t in self.sub_types)) 88 89 def IsVarArgsType(self): 90 """Determines if the type is a var_args type, i.e. starts with '...'.""" 91 return self.identifier.startswith(TypeAnnotation.VAR_ARGS_TYPE) or ( 92 self.type_group == TypeAnnotation.IMPLICIT_TYPE_GROUP and 93 self.sub_types[0].identifier.startswith(TypeAnnotation.VAR_ARGS_TYPE)) 94 95 def IsEmpty(self): 96 """Returns True if the type is empty.""" 97 return not self.tokens 98 99 def IsUnknownType(self): 100 """Returns True if this is the unknown type {?}.""" 101 return (self.or_null 102 and not self.identifier 103 and not self.sub_types 104 and not self.return_type) 105 106 def Append(self, item): 107 """Adds a sub_type to this type and finalizes it. 108 109 Args: 110 item: The TypeAnnotation item to append. 111 """ 112 # item is a TypeAnnotation instance, so pylint: disable=protected-access 113 self.sub_types.append(item._Finalize(self)) 114 115 def __repr__(self): 116 """Reconstructs the type definition.""" 117 append = '' 118 if self.sub_types: 119 separator = (',' if not self.type_group else '|') 120 if self.IsFunction(): 121 surround = '(%s)' 122 else: 123 surround = {False: '{%s}' if self.record_type else '<%s>', 124 True: '(%s)', 125 TypeAnnotation.IMPLICIT_TYPE_GROUP: '%s'}[self.type_group] 126 append = surround % separator.join(repr(t) for t in self.sub_types) 127 if self.return_type: 128 append += ':%s' % repr(self.return_type) 129 append += '=' if self.opt_arg else '' 130 prefix = '' + ('?' if self.or_null else '') + ('!' if self.not_null else '') 131 keyword = '%s:' % repr(self.key_type) if self.key_type else '' 132 return keyword + prefix + '%s' % (self.alias or self.identifier) + append 133 134 def ToString(self): 135 """Concats the type's tokens to form a string again.""" 136 ret = [] 137 for token in self.tokens: 138 if not isinstance(token, TypeAnnotation): 139 ret.append(token.string) 140 else: 141 ret.append(token.ToString()) 142 return ''.join(ret) 143 144 def Dump(self, indent=''): 145 """Dumps this type's structure for debugging purposes.""" 146 result = [] 147 for t in self.tokens: 148 if isinstance(t, TypeAnnotation): 149 result.append(indent + str(t) + ' =>\n' + t.Dump(indent + ' ')) 150 else: 151 result.append(indent + str(t)) 152 return '\n'.join(result) 153 154 def IterIdentifiers(self): 155 """Iterates over all identifiers in this type and its subtypes.""" 156 if self.identifier: 157 yield self.identifier 158 for subtype in self.IterTypes(): 159 for identifier in subtype.IterIdentifiers(): 160 yield identifier 161 162 def IterTypeGroup(self): 163 """Iterates over all types in the type group including self. 164 165 Yields: 166 If this is a implicit or manual type-group: all sub_types. 167 Otherwise: self 168 E.g. for @type {Foo.<Bar>} this will yield only Foo.<Bar>, 169 for @type {Foo|(Bar|Sample)} this will yield Foo, Bar and Sample. 170 171 """ 172 if self.type_group: 173 for sub_type in self.sub_types: 174 for sub_type in sub_type.IterTypeGroup(): 175 yield sub_type 176 else: 177 yield self 178 179 def IterTypes(self): 180 """Iterates over each subtype as well as return and key types.""" 181 if self.return_type: 182 yield self.return_type 183 184 if self.key_type: 185 yield self.key_type 186 187 for sub_type in self.sub_types: 188 yield sub_type 189 190 def GetNullability(self, modifiers=True): 191 """Computes whether the type may be null. 192 193 Args: 194 modifiers: Whether the modifiers ? and ! should be considered in the 195 evaluation. 196 Returns: 197 True if the type allows null, False if the type is strictly non nullable 198 and NULLABILITY_UNKNOWN if the nullability cannot be determined. 199 """ 200 201 # Explicitly marked nullable types or 'null' are nullable. 202 if ((modifiers and self.or_null) or 203 self.identifier == TypeAnnotation.NULL_TYPE): 204 return True 205 206 # Explicitly marked non-nullable types or non-nullable base types: 207 if ((modifiers and self.not_null) or self.record_type 208 or self.identifier in TypeAnnotation.NON_NULLABLE): 209 return False 210 211 # A type group is nullable if any of its elements are nullable. 212 if self.type_group: 213 maybe_nullable = False 214 for sub_type in self.sub_types: 215 nullability = sub_type.GetNullability() 216 if nullability == self.NULLABILITY_UNKNOWN: 217 maybe_nullable = nullability 218 elif nullability: 219 return True 220 return maybe_nullable 221 222 # Whitelisted types are nullable. 223 if self.identifier.rstrip('.') in TypeAnnotation.NULLABLE_TYPE_WHITELIST: 224 return True 225 226 # All other types are unknown (most should be nullable, but 227 # enums are not and typedefs might not be). 228 return TypeAnnotation.NULLABILITY_UNKNOWN 229 230 def WillAlwaysBeNullable(self): 231 """Computes whether the ! flag is illegal for this type. 232 233 This is the case if this type or any of the subtypes is marked as 234 explicitly nullable. 235 236 Returns: 237 True if the ! flag would be illegal. 238 """ 239 if self.or_null or self.identifier == TypeAnnotation.NULL_TYPE: 240 return True 241 242 if self.type_group: 243 return any(t.WillAlwaysBeNullable() for t in self.sub_types) 244 245 return False 246 247 def _Finalize(self, parent): 248 """Fixes some parsing issues once the TypeAnnotation is complete.""" 249 250 # Normalize functions whose definition ended up in the key type because 251 # they defined a return type after a colon. 252 if (self.key_type and 253 self.key_type.identifier == TypeAnnotation.FUNCTION_TYPE): 254 current = self.key_type 255 current.return_type = self 256 self.key_type = None 257 # opt_arg never refers to the return type but to the function itself. 258 current.opt_arg = self.opt_arg 259 self.opt_arg = False 260 return current 261 262 # If a typedef just specified the key, it will not end up in the key type. 263 if parent.record_type and not self.key_type: 264 current = TypeAnnotation() 265 current.key_type = self 266 current.tokens.append(self) 267 return current 268 return self 269 270 def FirstToken(self): 271 """Returns the first token used in this type or any of its subtypes.""" 272 first = self.tokens[0] 273 return first.FirstToken() if isinstance(first, TypeAnnotation) else first 274 275 276 def Parse(token, token_end, error_handler): 277 """Parses a type annotation and returns a TypeAnnotation object.""" 278 return TypeAnnotationParser(error_handler).Parse(token.next, token_end) 279 280 281 class TypeAnnotationParser(object): 282 """A parser for type annotations constructing the TypeAnnotation object.""" 283 284 def __init__(self, error_handler): 285 self._stack = [] 286 self._error_handler = error_handler 287 self._closing_error = False 288 289 def Parse(self, token, token_end): 290 """Parses a type annotation and returns a TypeAnnotation object.""" 291 root = TypeAnnotation() 292 self._stack.append(root) 293 current = TypeAnnotation() 294 root.tokens.append(current) 295 296 while token and token != token_end: 297 if token.type in (TYPE.DOC_TYPE_START_BLOCK, TYPE.DOC_START_BRACE): 298 if token.string == '(': 299 if current.identifier and current.identifier not in [ 300 TypeAnnotation.FUNCTION_TYPE, TypeAnnotation.VAR_ARGS_TYPE]: 301 self.Error(token, 302 'Invalid identifier for (): "%s"' % current.identifier) 303 current.type_group = ( 304 current.identifier != TypeAnnotation.FUNCTION_TYPE) 305 elif token.string == '{': 306 current.record_type = True 307 current.tokens.append(token) 308 self._stack.append(current) 309 current = TypeAnnotation() 310 self._stack[-1].tokens.append(current) 311 312 elif token.type in (TYPE.DOC_TYPE_END_BLOCK, TYPE.DOC_END_BRACE): 313 prev = self._stack.pop() 314 prev.Append(current) 315 current = prev 316 317 # If an implicit type group was created, close it as well. 318 if prev.type_group == TypeAnnotation.IMPLICIT_TYPE_GROUP: 319 prev = self._stack.pop() 320 prev.Append(current) 321 current = prev 322 current.tokens.append(token) 323 324 elif token.type == TYPE.DOC_TYPE_MODIFIER: 325 if token.string == '!': 326 current.tokens.append(token) 327 current.not_null = True 328 elif token.string == '?': 329 current.tokens.append(token) 330 current.or_null = True 331 elif token.string == ':': 332 current.tokens.append(token) 333 prev = current 334 current = TypeAnnotation() 335 prev.tokens.append(current) 336 current.key_type = prev 337 elif token.string == '=': 338 # For implicit type groups the '=' refers to the parent. 339 try: 340 if self._stack[-1].type_group == TypeAnnotation.IMPLICIT_TYPE_GROUP: 341 self._stack[-1].tokens.append(token) 342 self._stack[-1].opt_arg = True 343 else: 344 current.tokens.append(token) 345 current.opt_arg = True 346 except IndexError: 347 self.ClosingError(token) 348 elif token.string == '|': 349 # If a type group has explicitly been opened, do a normal append. 350 # Otherwise we have to open the type group and move the current 351 # type into it, before appending 352 if not self._stack[-1].type_group: 353 type_group = TypeAnnotation() 354 if (current.key_type and 355 current.key_type.identifier != TypeAnnotation.FUNCTION_TYPE): 356 type_group.key_type = current.key_type 357 current.key_type = None 358 type_group.type_group = TypeAnnotation.IMPLICIT_TYPE_GROUP 359 # Fix the token order 360 prev = self._stack[-1].tokens.pop() 361 self._stack[-1].tokens.append(type_group) 362 type_group.tokens.append(prev) 363 self._stack.append(type_group) 364 self._stack[-1].tokens.append(token) 365 self.Append(current, error_token=token) 366 current = TypeAnnotation() 367 self._stack[-1].tokens.append(current) 368 elif token.string == ',': 369 self.Append(current, error_token=token) 370 current = TypeAnnotation() 371 self._stack[-1].tokens.append(token) 372 self._stack[-1].tokens.append(current) 373 else: 374 current.tokens.append(token) 375 self.Error(token, 'Invalid token') 376 377 elif token.type == TYPE.COMMENT: 378 current.tokens.append(token) 379 current.identifier += token.string.strip() 380 381 elif token.type in [TYPE.DOC_PREFIX, TYPE.WHITESPACE]: 382 current.tokens.append(token) 383 384 else: 385 current.tokens.append(token) 386 self.Error(token, 'Unexpected token') 387 388 token = token.next 389 390 self.Append(current, error_token=token) 391 try: 392 ret = self._stack.pop() 393 except IndexError: 394 self.ClosingError(token) 395 # The type is screwed up, but let's return something. 396 return current 397 398 if self._stack and (len(self._stack) != 1 or 399 ret.type_group != TypeAnnotation.IMPLICIT_TYPE_GROUP): 400 self.Error(token, 'Too many opening items.') 401 402 return ret if len(ret.sub_types) > 1 else ret.sub_types[0] 403 404 def Append(self, type_obj, error_token): 405 """Appends a new TypeAnnotation object to the current parent.""" 406 if self._stack: 407 self._stack[-1].Append(type_obj) 408 else: 409 self.ClosingError(error_token) 410 411 def ClosingError(self, token): 412 """Reports an error about too many closing items, but only once.""" 413 if not self._closing_error: 414 self._closing_error = True 415 self.Error(token, 'Too many closing items.') 416 417 def Error(self, token, message): 418 """Calls the error_handler to post an error message.""" 419 if self._error_handler: 420 self._error_handler.HandleError(error.Error( 421 errors.JSDOC_DOES_NOT_PARSE, 422 'Error parsing jsdoc type at token "%s" (column: %d): %s' % 423 (token.string, token.start_index, message), token)) 424 425
