Lines Matching refs:tokenizer
301 tokenizer = text_format._Tokenizer(text)
302 methods = [(tokenizer.ConsumeIdentifier, 'identifier1'),
304 (tokenizer.ConsumeString, 'string1'),
305 (tokenizer.ConsumeIdentifier, 'identifier2'),
307 (tokenizer.ConsumeInt32, 123),
308 (tokenizer.ConsumeIdentifier, 'identifier3'),
310 (tokenizer.ConsumeString, 'string'),
311 (tokenizer.ConsumeIdentifier, 'identifiER_4'),
313 (tokenizer.ConsumeFloat, 1.1e+2),
314 (tokenizer.ConsumeIdentifier, 'ID5'),
316 (tokenizer.ConsumeFloat, -0.23),
317 (tokenizer.ConsumeIdentifier, 'ID6'),
319 (tokenizer.ConsumeString, 'aaaa\'bbbb'),
320 (tokenizer.ConsumeIdentifier, 'ID7'),
322 (tokenizer.ConsumeString, 'aa\"bb'),
323 (tokenizer.ConsumeIdentifier, 'ID8'),
326 (tokenizer.ConsumeIdentifier, 'A'),
328 (tokenizer.ConsumeFloat, text_format._INFINITY),
329 (tokenizer.ConsumeIdentifier, 'B'),
331 (tokenizer.ConsumeFloat, -text_format._INFINITY),
332 (tokenizer.ConsumeIdentifier, 'C'),
334 (tokenizer.ConsumeBool, True),
335 (tokenizer.ConsumeIdentifier, 'D'),
337 (tokenizer.ConsumeBool, False),
339 (tokenizer.ConsumeIdentifier, 'ID9'),
341 (tokenizer.ConsumeUint32, 22),
342 (tokenizer.ConsumeIdentifier, 'ID10'),
344 (tokenizer.ConsumeInt64, -111111111111111111),
345 (tokenizer.ConsumeIdentifier, 'ID11'),
347 (tokenizer.ConsumeInt32, -22),
348 (tokenizer.ConsumeIdentifier, 'ID12'),
350 (tokenizer.ConsumeUint64, 2222222222222222222)]
353 while not tokenizer.AtEnd():
356 token = tokenizer.token
358 tokenizer.NextToken()
369 tokenizer = text_format._Tokenizer(text)
370 self.assertRaises(text_format.ParseError, tokenizer.ConsumeUint32)
371 self.assertRaises(text_format.ParseError, tokenizer.ConsumeUint64)
372 self.assertEqual(-1, tokenizer.ConsumeInt32())
374 self.assertRaises(text_format.ParseError, tokenizer.ConsumeUint32)
375 self.assertRaises(text_format.ParseError, tokenizer.ConsumeInt32)
376 self.assertEqual(uint32_max + 1, tokenizer.ConsumeInt64())
378 self.assertRaises(text_format.ParseError, tokenizer.ConsumeInt64)
379 self.assertEqual(int64_max + 1, tokenizer.ConsumeUint64())
380 self.assertTrue(tokenizer.AtEnd())
383 tokenizer = text_format._Tokenizer(text)
384 self.assertEqual(0, tokenizer.ConsumeUint32())
385 self.assertEqual(0, tokenizer.ConsumeUint64())
386 self.assertEqual(0, tokenizer.ConsumeUint32())
387 self.assertEqual(0, tokenizer.ConsumeUint64())
388 self.assertTrue(tokenizer.AtEnd())
392 tokenizer = text_format._Tokenizer(text)
393 self.assertRaises(text_format.ParseError, tokenizer.ConsumeByteString)
396 tokenizer = text_format._Tokenizer(text)
397 self.assertRaises(text_format.ParseError, tokenizer.ConsumeByteString)
400 tokenizer = text_format._Tokenizer(text)
401 self.assertRaises(text_format.ParseError, tokenizer.ConsumeByteString)
404 tokenizer = text_format._Tokenizer(text)
405 self.assertRaises(text_format.ParseError, tokenizer.ConsumeByteString)
408 tokenizer = text_format._Tokenizer(text)
409 self.assertRaises(text_format.ParseError, tokenizer.ConsumeByteString)
413 tokenizer = text_format._Tokenizer(text)
414 self.assertRaises(text_format.ParseError, tokenizer.ConsumeBool)
