diff --git a/lex.go b/lex.go index a6b26f0..fe5ceb8 100644 --- a/lex.go +++ b/lex.go @@ -5,6 +5,7 @@ import ( "fmt" "io" "strings" + "unicode" ) type tokenType int @@ -91,6 +92,9 @@ func lexRoot(l *lexer) (stateFn, error) { if err != nil { return nil, err } + if unicode.IsSpace(r) { + return lexRoot, nil + } switch r { case '"', '`': return lexStringLiteral(r), nil diff --git a/lex_test.go b/lex_test.go index d358083..cd371b3 100644 --- a/lex_test.go +++ b/lex_test.go @@ -13,6 +13,7 @@ var primitivesTests = []struct { {`"this one has spaces"`, token{t_string, "this one has spaces"}}, {`"this one has \"quotes\" in it"`, token{t_string, `this one has "quotes" in it`}}, {"`this one is delimited by backticks`", token{t_string, "this one is delimited by backticks"}}, + {` "this one has white space on either end" `, token{t_string, "this one has white space on either end"}}, } func TestLexPrimities(t *testing.T) {