Lines Matching refs:Tokenizer
1 /* $NetBSD: tokenizer.c,v 1.21 2011/08/16 16:25:15 christos Exp $ */
41 static char sccsid[] = "@(#)tokenizer.c 8.1 (Berkeley) 6/4/93";
43 __RCSID("$NetBSD: tokenizer.c,v 1.21 2011/08/16 16:25:15 christos Exp $");
49 * tokenize.c: Bourne shell like tokenizer
74 struct TYPE(tokenizer) {
86 private void FUN(tok,finish)(TYPE(Tokenizer) *);
90 * Finish a word in the tokenizer.
93 FUN(tok,finish)(TYPE(Tokenizer) *tok)
107 * Initialize the tokenizer
109 public TYPE(Tokenizer) *
112 TYPE(Tokenizer) *tok = tok_malloc(sizeof(*tok));
148 * Reset the tokenizer
151 FUN(tok,reset)(TYPE(Tokenizer) *tok)
166 FUN(tok,end)(TYPE(Tokenizer) *tok)
180 * tok current tokenizer state (setup with FUN(tok,init)())
195 FUN(tok,line)(TYPE(Tokenizer) *tok, const TYPE(LineInfo) *line,
444 FUN(tok,str)(TYPE(Tokenizer) *tok, const Char *line, int *argc,
