Lines Matching refs:tokens
21 # the list of supported C-preprocessor tokens
22 # plus a couple of C tokens as well
145 of tokens. real implementations follow and differ in the format
575 def __init__(self, tokens):
576 """initialize a CppExpr. 'tokens' must be a CppToken list"""
577 self.tok = tokens
578 self.n = len(tokens)
581 print "CppExpr: trying to parse %s" % repr(tokens)
586 print 'crap at end of input (%d != %d): %s' % (self.i, self.n, repr(tokens))
1111 - directive blocks: contain the tokens of a single pre-processor directive (e.g. #if)
1112 - text blocks, contain the tokens of non-directive blocks
1117 def __init__(self,tokens,directive=None,lineno=0):
1124 tokens = [ tok ] + tokens
1132 tokens = [ tok1, tok2 ] + tokens
1135 self.tokens = tokens
1140 self.lineno = self.tokens[0].lineno
1143 self.expr = CppExpr( self.tokens )
1158 return self.tokens[0].value
1170 if self.tokens[0].id == tokSTRING:
1172 return self.tokens[0].value
1175 if self.tokens[0].id == "<":
1177 tok = self.tokens
1196 tokens = []
1200 nn = len(self.tokens)
1202 tok = self.tokens[ii]
1206 # anything to tokens array yet though.
1225 tokens.append(tok)
1234 tok2 = self.tokens[jj]
1236 tokens.append(tok2)
1239 tokens.append(tok)
1242 self.tokens = tokens
1246 # without any tokens. These come from blocks that only contained
1250 if not self.directive and self.tokens == []:
1261 for tok in self.tokens:
1279 for tok in self.tokens:
1283 for tok in self.tokens:
1303 if len(self.tokens):
1305 for tok in self.tokens:
1309 for tok in self.tokens:
1381 for tok in b.tokens:
1407 n = len(b.tokens)
1414 tok = b.tokens[i]
1417 # type/var/func, then skip over tokens until
1458 tokens of the same block
1470 tokid = b.tokens[j].id
1478 ident = b.tokens[j].value
1495 #print "### keep var/func '%s': %s" % (ident,repr(b.tokens[i:j]))
1498 # We're going to skip the tokens for this declaration
1499 #print "### skip variable /func'%s': %s" % (ident,repr(b.tokens[i:j]))
1501 blocks2.append( Block(b.tokens[first:i]))
1508 #print "### final '%s'" % repr(b.tokens[first:i])
1509 blocks2.append( Block(b.tokens[first:i]) )
1516 tokens = CppLineTokenizer( disclaimer ).toTokenList()
1517 tokens = tokens[:-1] # remove trailing tokLN
1518 self.blocks = [ Block(tokens) ] + self.blocks
1521 """replace tokens according to the given dict"""
1525 for tok in b.tokens:
1532 # Keep 'expr' in sync with 'tokens'.
1533 b.expr = CppExpr(b.tokens)
1583 # record previous tokens as text block
1613 # then record tokens until LN
