Lines Matching refs:tok

323                 tok = LexToken()
324                 tok.value = m.group()
325                 tok.lineno = self.lineno
326                 tok.lexpos = lexpos
329                 func,tok.type = lexindexfunc[i]
333                    if tok.type:
335                       return tok
344                 tok.lexer = self      # Set additional attributes useful in token rules
348                 newtok = func(tok)
367                     tok = LexToken()
368                     tok.value = lexdata[lexpos]
369                     tok.lineno = self.lineno
370                     tok.type = tok.value
371                     tok.lexpos = lexpos
373                     return tok
377                     tok = LexToken()
378                     tok.value = self.lexdata[lexpos:]
379                     tok.lineno = self.lineno
380                     tok.type = "error"
381                     tok.lexer = self
382                     tok.lexpos = lexpos
384                     newtok = self.lexerrorf(tok)
1032         tok = _token()
1033         if not tok: break
1034         sys.stdout.write("(%s,%r,%d,%d)\n" % (tok.type, tok.value, tok.lineno,tok.lexpos))