#  Copyright (c) 1999-2000 John Aycock
#  
#  Permission is hereby granted, free of charge, to any person obtaining
#  a copy of this software and associated documentation files (the
#  "Software"), to deal in the Software without restriction, including
#  without limitation the rights to use, copy, modify, merge, publish,
#  distribute, sublicense, and/or sell copies of the Software, and to
#  permit persons to whom the Software is furnished to do so, subject to
#  the following conditions:
#  
#  The above copyright notice and this permission notice shall be
#  included in all copies or substantial portions of the Software.
#  
#  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
#  EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
#  MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
#  IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
#  CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
#  TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
#  SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

#
#  Why would I write my own when GvR maintains this one?
#
import tokenize

class Token:
	def __init__(self, type, attr=None, lineno='???'):
		self.type = type
		self.attr = attr
		self.lineno = lineno

	def __cmp__(self, o):
		return cmp(self.type, o)
	###
	def __repr__(self):
		return str(self.type)

_map = {
	tokenize.ENDMARKER	: 'ENDMARKER',
	tokenize.NAME 		: 'NAME',
	tokenize.NUMBER		: 'NUMBER',
	tokenize.STRING		: 'STRING',
	tokenize.NEWLINE	: 'NEWLINE',
	tokenize.INDENT		: 'INDENT',
	tokenize.DEDENT		: 'DEDENT',
}

_rw = {
	'and'		: None,
	'assert'	: None,
	'break'		: None,
	'class'		: None,
	'continue'	: None,
	'def'		: None,
	'del'		: None,
	'elif'		: None,
	'else'		: None,
	'except'	: None,
	'exec'		: None,
	'finally'	: None,
	'for'		: None,
	'from'		: None,
	'global'	: None,
	'if'		: None,
	'import'	: None,
	'in'		: None,
	'is'		: None,
	'lambda'	: None,
	'not'		: None,
	'or'		: None,
	'pass'		: None,
	'print'		: None,
	'raise'		: None,
	'return'	: None,
	'try'		: None,
	'while'		: None,
}

def scan(f):
	tokens = []

	def callback(value, lexeme, (lineno, column), end, line, list=tokens):
		attr = None
		type = lexeme

		if value in (tokenize.COMMENT, tokenize.NL):
			return
		elif _map.has_key(value):
			if value != tokenize.NAME or not _rw.has_key(lexeme):
				attr = lexeme
				type = _map[value]

		t = Token(type, attr=attr, lineno=lineno)
		list.append(t)

	tokenize.tokenize(f.readline, callback)
	return tokens
