3 ms·
Not much of a lex fan. Here's a simple way to implement the indentation-to-braces transformation in a hand-written lexer: def tokenize(stream): ind
by psykotic 15y ago
Not much of a lex fan. Here's a simple way to implement the indentation-to-braces transformation in a hand-written lexer:
def tokenize(stream):
indent = stream.indent
while stream.current() and stream.indent >= indent:
stream.indent = 0
while stream.current() == ' ':
stream.indent += 1
stream.consume()
if stream.current() == '\n':
stream.indent = indent
yield stream.consume()
elif stream.indent == indent:
while stream.current() and stream.current() != '\n':
yield stream.consume()
if stream.current() == '\n':
yield stream.consume()
else:
stream.rewind(stream.indent)
if stream.indent > indent:
yield '{'
for token in tokenize(stream):
yield token
yield '}'
A tokenize() call only returns when it's either reached the end of the stream or after consuming all contiguous lines at its initial indentation level or greater. Empty lines (with or without leading indentation) don't count towards the brace structure. It doesn't specially handle tabs, backslash line continuations or indentation within parentheses, but that would be easy to add.
For a bit of fun, run it on its own source code and you get this:
def tokenize(stream):
{indent = stream.indent
while stream.current() and stream.indent >= indent:
{stream.indent = 0
while stream.current() == ' ':
{stream.indent += 1
stream.consume()
}if stream.current() == '\n':
{stream.indent = indent
yield stream.consume()
}elif stream.indent == indent:
{while stream.current() and stream.current() != '\n':
{yield stream.consume()
}if stream.current() == '\n':
{yield stream.consume()
}}else:
{stream.rewind(stream.indent)
if stream.indent > indent:
{yield '{'
for token in tokenize(stream):
{yield token
}yield '}'
}}}}