package main import ( "fmt" "io" "strings" ) type tokenType int const ( invalidToken tokenType = iota integerToken symbolToken openParenToken closeParenToken stringToken floatToken ) func (t tokenType) String() string { switch t { case integerToken: return "integer" case symbolToken: return "symbol" case openParenToken: return "open_paren" case closeParenToken: return "close_paren" case stringToken: return "string" case floatToken: return "float" } panic("wtf") } type token struct { lexeme string t tokenType } type stateFn func(*lexer) (stateFn, error) type lexer struct { io.RuneReader buf []rune cur rune out chan token } // clears the current lexem buffer and emits a token of the given type. // There's no sanity checking to make sure you don't emit some bullshit, so // don't fuck it up. func (l *lexer) emit(t tokenType) { debugPrint("emit " + string(l.buf)) l.out <- token{lexeme: string(l.buf), t: t} l.buf = nil } // reads a rune from the input and assigns it to the current rune, l.cur. // Returns an error if we were unable to read a rune from the input. I'm // pretty sure it's always io.EOF but I'm not positive. func (l *lexer) next() error { r, _, err := l.ReadRune() if err != nil { return err } l.cur = r return nil } // stores the current rune in our in-progress lexeme buffer func (l *lexer) keep() { if l.buf == nil { l.buf = make([]rune, 0, 32) } l.buf = append(l.buf, l.cur) } func isDigit(r rune) bool { switch r { case '0', '1', '2', '3', '4', '5', '6', '7', '8', '9': return true } return false } func debugPrint(s string) { if DEBUG { fmt.Println("#", s) } } // lexes an open parenthesis func lexOpenParen(l *lexer) (stateFn, error) { debugPrint("-->lexOpenParen") l.out <- token{"(", openParenToken} switch l.cur { case ' ', '\t', '\n', '\r': return lexWhitespace, nil case '(': return lexOpenParen, nil case ')': return lexCloseParen, nil case ';': return lexComment, nil case '.': l.keep() return lexFloat, nil case '-': l.keep() return lexMinus, nil } if isDigit(l.cur) { l.keep() return lexInt, nil } l.keep() return lexSymbol, nil } func lexMinus(l *lexer) (stateFn, error) { switch l.cur { case ' ', '\t', '\n', '\r': l.emit(symbolToken) return lexWhitespace, nil case '(': l.emit(symbolToken) return lexOpenParen, nil case ')': l.emit(symbolToken) return lexCloseParen, nil case '.': l.keep() return lexFloat, nil } if isDigit(l.cur) { l.keep() return lexInt, nil } l.keep() return lexSymbol, nil } // lexes some whitespace in progress. Maybe this should be combined with root // and the lexer shouldn't have a state. I think wehat I'm doing now is // "wrong" but who honestly gives a shit. func lexWhitespace(l *lexer) (stateFn, error) { debugPrint("-->lexWhitespace") switch l.cur { case ' ', '\t', '\n', '\r': return lexWhitespace, nil case '"': return lexString, nil case '(': return lexOpenParen, nil case ')': return lexCloseParen, nil case ';': return lexComment, nil case '.': l.keep() return lexFloat, nil case '-': l.keep() return lexMinus, nil } if isDigit(l.cur) { l.keep() return lexInt, nil } l.keep() return lexSymbol, nil } // lexes an in-progress string. Basically we just keep all of the tokens until // we see a double-quote character, signifying the end of the string. We also // switch into escape mode if we come across a backslash. func lexString(l *lexer) (stateFn, error) { debugPrint("-->lexString") switch l.cur { case '"': l.emit(stringToken) return lexWhitespace, nil case '\\': return lexStringEsc, nil } l.keep() return lexString, nil } // lex the character *after* the string escape character \. We always keep the // next character, then just go back to string lexing. func lexStringEsc(l *lexer) (stateFn, error) { debugPrint("-->lexStringEsc") l.keep() return lexString, nil } // lex an integer. Once we're on an integer, the only valid characters are // whitespace, close paren, a period to indicate we want a float, or more // digits. Everything else is crap. func lexInt(l *lexer) (stateFn, error) { debugPrint("-->lexInt") switch l.cur { case ' ', '\t', '\n', '\r': l.emit(integerToken) return lexWhitespace, nil case '.': l.keep() return lexFloat, nil case ')': l.emit(integerToken) return lexCloseParen, nil case ';': l.emit(integerToken) return lexComment, nil } if isDigit(l.cur) { l.keep() return lexInt, nil } return nil, fmt.Errorf("unexpected rune in lexInt: %c", l.cur) } // once we're in a float, the only valid values are digits, whitespace or close // paren. func lexFloat(l *lexer) (stateFn, error) { debugPrint("-->lexFloat") switch l.cur { case ' ', '\t', '\n', '\r': l.emit(floatToken) return lexWhitespace, nil case ')': l.emit(floatToken) return lexCloseParen, nil case ';': l.emit(floatToken) return lexComment, nil } if isDigit(l.cur) { l.keep() return lexFloat, nil } return nil, fmt.Errorf("unexpected rune in lexFloat: %c", l.cur) } // lexes a symbol in progress func lexSymbol(l *lexer) (stateFn, error) { debugPrint("-->lexSymbol") switch l.cur { case ' ', '\t', '\n', '\r': debugPrint("ending lexSymbol on whitespace") l.emit(symbolToken) return lexWhitespace, nil case ')': l.emit(symbolToken) return lexCloseParen, nil case ';': l.emit(symbolToken) return lexComment, nil default: l.keep() return lexSymbol, nil } panic("not reached") } // lex a close parenthesis func lexCloseParen(l *lexer) (stateFn, error) { debugPrint("-->lexCloseParen") l.out <- token{")", closeParenToken} switch l.cur { case ' ', '\t', '\n', '\r': return lexWhitespace, nil case ')': return lexCloseParen, nil case ';': return lexComment, nil } return nil, fmt.Errorf("unimplemented") } // lexes a comment func lexComment(l *lexer) (stateFn, error) { debugPrint("-->lexComment") switch l.cur { case '\n', '\r': return lexWhitespace, nil } return lexComment, nil } // lexes some lispy input from an io.Reader, emiting tokens on chan c. The // channel is closed when the input reaches EOF, signaling that there are no // new tokens. func lex(input io.RuneReader, c chan token) { defer close(c) l := &lexer{input, nil, ' ', c} var err error f := stateFn(lexWhitespace) for { f, err = f(l) if err != nil { break } err = l.next() if err != nil { break } } if err != io.EOF { fmt.Println(err) } } // lexes a lispy string onto a token channel func lexs(input string, c chan token) { lex(strings.NewReader(input), c) }