#!/usr/local/bin/python # This is an example of how to reconstruct source code from tokens # generated by tokenize.tokenize(). Given Python code as input, the # output should match the input exactly. (Ka-Ping Yee, 3 April 1997) import sys, tokenize program = [] lastrow, lastcol = 1, 0 lastline = '' def rebuild(type, token, (startrow, startcol), (endrow, endcol), line): global lastrow, lastcol, lastline, program # Deal with the bits between tokens. if lastrow == startrow == endrow: # ordinary token program.append(line[lastcol:startcol]) elif lastrow != startrow: # backslash continuation program.append(lastline[lastcol:] + line[:startcol]) elif startrow != endrow: # multi-line string program.append(lastline[lastcol:startcol]) # Append the token itself. program.append(token) # Save some information for the next time around. if token and token[-1] == '\n': lastrow, lastcol = endrow+1, 0 # start on next line else: lastrow, lastcol = endrow, endcol # remember last position lastline = line # remember last line tokenize.tokenize(sys.stdin.readline, rebuild) for piece in program: sys.stdout.write(piece)