Lines Matching refs:tokens
21 # the list of supported C-preprocessor tokens
22 # plus a couple of C tokens as well
140 def __init__(self,tokens):
141 self.tokens = tokens
143 self.count = len(tokens)
157 return self.tokens[self.n].id
163 return self.tokens[self.n]
171 """skip over all space tokens, this includes tokSPACE and tokLN"""
192 """return the list of remaining tokens"""
193 return self.tokens[self.n:]
210 of tokens. real implementations follow and differ in the format
846 """a class used to convert a list of tokens into a cpp Expr object"""
852 def __init__(self,tokens):
853 self.tok = tokens
976 def __init__(self, tokens):
977 """initialize a CppExpr. 'tokens' must be a CppToken list"""
978 self.tok = tokens
979 self.n = len(tokens)
981 print "CppExpr: trying to parse %s" % repr(tokens)
1532 - directive blocks: contain the tokens of a single pre-processor directive (e.g. #if)
1533 - text blocks, contain the tokens of non-directive blocks
1538 def __init__(self,tokens,directive=None,lineno=0):
1545 tokens = [ tok ] + tokens
1553 tokens = [ tok1, tok2 ] + tokens
1556 self.tokens = tokens
1561 self.lineno = self.tokens[0].lineno
1564 self.expr = CppExpr( self.tokens )
1579 return self.tokens[0].value
1591 #print "iii " + repr(self.tokens)
1592 if self.tokens[0].id == tokSTRING:
1594 return self.tokens[0].value
1597 if self.tokens[0].id == "<":
1599 tok = self.tokens
1618 tokens = []
1622 nn = len(self.tokens)
1624 tok = self.tokens[ii]
1628 # anything to tokens array yet though.
1650 tokens.append(tok)
1659 tok2 = self.tokens[jj]
1661 tokens.append(tok2)
1664 tokens.append(tok)
1667 self.tokens = tokens
1671 # without any tokens. These come from blocks that only contained
1675 if not self.directive and self.tokens == []:
1686 for tok in self.tokens:
1704 for tok in self.tokens:
1708 for tok in self.tokens:
1728 if len(self.tokens):
1730 for tok in self.tokens:
1734 for tok in self.tokens:
1811 for tok in b.tokens:
1837 n = len(b.tokens)
1844 tok = b.tokens[i]
1847 # type/var/func, then skip over tokens until
1880 #print "$$$ keep type declr" + repr(b.tokens[i:])
1889 # We're going to parse the next tokens of the same block
1901 tokid = b.tokens[j].id
1909 ident = b.tokens[j].value
1926 #print "### keep var/func '%s': %s" % (ident,repr(b.tokens[i:j]))
1929 # We're going to skip the tokens for this declaration
1930 #print "### skip variable /func'%s': %s" % (ident,repr(b.tokens[i:j]))
1932 blocks2.append( Block(b.tokens[first:i]))
1939 #print "### final '%s'" % repr(b.tokens[first:i])
1940 blocks2.append( Block(b.tokens[first:i]) )
1947 tokens = CppLineTokenizer( disclaimer ).toTokenList()
1948 tokens = tokens[:-1] # remove trailing tokLN
1949 self.blocks = [ Block(tokens) ] + self.blocks
1952 """replace tokens according to the given dict
1956 for tok in b.tokens:
2009 # record previous tokens as text block
2039 # then record tokens until LN
