|
|
@ -8,6 +8,8 @@ import (
|
|
|
|
"strings"
|
|
|
|
"strings"
|
|
|
|
)
|
|
|
|
)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
var DEBUG = false
|
|
|
|
|
|
|
|
|
|
|
|
type typ3 int
|
|
|
|
type typ3 int
|
|
|
|
|
|
|
|
|
|
|
|
const (
|
|
|
|
const (
|
|
|
@ -56,12 +58,14 @@ type lexer struct {
|
|
|
|
// There's no sanity checking to make sure you don't emit some bullshit, so
|
|
|
|
// There's no sanity checking to make sure you don't emit some bullshit, so
|
|
|
|
// don't fuck it up.
|
|
|
|
// don't fuck it up.
|
|
|
|
func (l *lexer) emit(t typ3) {
|
|
|
|
func (l *lexer) emit(t typ3) {
|
|
|
|
|
|
|
|
debugPrint("emit " + string(l.cur))
|
|
|
|
l.out <- token{lexeme: string(l.cur), t: t}
|
|
|
|
l.out <- token{lexeme: string(l.cur), t: t}
|
|
|
|
l.cur = nil
|
|
|
|
l.cur = nil
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
// appends the rune to the current in-progress lexem
|
|
|
|
// appends the rune to the current in-progress lexem
|
|
|
|
func (l *lexer) append(r rune) {
|
|
|
|
func (l *lexer) append(r rune) {
|
|
|
|
|
|
|
|
debugPrint(fmt.Sprintf("append %c\n", (r)))
|
|
|
|
if l.cur == nil {
|
|
|
|
if l.cur == nil {
|
|
|
|
l.cur = make([]rune, 0, 32)
|
|
|
|
l.cur = make([]rune, 0, 32)
|
|
|
|
}
|
|
|
|
}
|
|
|
@ -76,25 +80,15 @@ func isDigit(r rune) bool {
|
|
|
|
return false
|
|
|
|
return false
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
// lexes stuff at the root level of the input.
|
|
|
|
func debugPrint(s string) {
|
|
|
|
func lexRoot(l *lexer) (stateFn, error) {
|
|
|
|
if DEBUG {
|
|
|
|
r, _, err := l.ReadRune()
|
|
|
|
fmt.Println(s)
|
|
|
|
if err != nil {
|
|
|
|
|
|
|
|
return nil, err
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
switch r {
|
|
|
|
|
|
|
|
case ';':
|
|
|
|
|
|
|
|
return lexComment, nil
|
|
|
|
|
|
|
|
case '(':
|
|
|
|
|
|
|
|
return lexOpenParen, nil
|
|
|
|
|
|
|
|
case ' ', '\t', '\n':
|
|
|
|
|
|
|
|
return lexRoot, nil
|
|
|
|
|
|
|
|
}
|
|
|
|
}
|
|
|
|
return nil, fmt.Errorf("unexpected rune in lexRoot: %c", r)
|
|
|
|
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
// lexes an open parenthesis
|
|
|
|
// lexes an open parenthesis
|
|
|
|
func lexOpenParen(l *lexer) (stateFn, error) {
|
|
|
|
func lexOpenParen(l *lexer) (stateFn, error) {
|
|
|
|
|
|
|
|
debugPrint("-->lexOpenParen")
|
|
|
|
l.out <- token{"(", openParen}
|
|
|
|
l.out <- token{"(", openParen}
|
|
|
|
l.depth++
|
|
|
|
l.depth++
|
|
|
|
r, _, err := l.ReadRune()
|
|
|
|
r, _, err := l.ReadRune()
|
|
|
@ -121,6 +115,7 @@ func lexOpenParen(l *lexer) (stateFn, error) {
|
|
|
|
// and the lexer shouldn't have a state. I think wehat I'm doing now is
|
|
|
|
// and the lexer shouldn't have a state. I think wehat I'm doing now is
|
|
|
|
// "wrong" but who honestly gives a shit.
|
|
|
|
// "wrong" but who honestly gives a shit.
|
|
|
|
func lexWhitespace(l *lexer) (stateFn, error) {
|
|
|
|
func lexWhitespace(l *lexer) (stateFn, error) {
|
|
|
|
|
|
|
|
debugPrint("-->lexWhitespace")
|
|
|
|
r, _, err := l.ReadRune()
|
|
|
|
r, _, err := l.ReadRune()
|
|
|
|
if err != nil {
|
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
|
|
|
return nil, err
|
|
|
@ -144,6 +139,7 @@ func lexWhitespace(l *lexer) (stateFn, error) {
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
func lexString(l *lexer) (stateFn, error) {
|
|
|
|
func lexString(l *lexer) (stateFn, error) {
|
|
|
|
|
|
|
|
debugPrint("-->lexString")
|
|
|
|
r, _, err := l.ReadRune()
|
|
|
|
r, _, err := l.ReadRune()
|
|
|
|
if err != nil {
|
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
|
|
|
return nil, err
|
|
|
@ -161,6 +157,7 @@ func lexString(l *lexer) (stateFn, error) {
|
|
|
|
|
|
|
|
|
|
|
|
// lex the character *after* the string escape character \
|
|
|
|
// lex the character *after* the string escape character \
|
|
|
|
func lexStringEsc(l *lexer) (stateFn, error) {
|
|
|
|
func lexStringEsc(l *lexer) (stateFn, error) {
|
|
|
|
|
|
|
|
debugPrint("-->lexStringEsc")
|
|
|
|
r, _, err := l.ReadRune()
|
|
|
|
r, _, err := l.ReadRune()
|
|
|
|
if err != nil {
|
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
|
|
|
return nil, err
|
|
|
@ -173,6 +170,7 @@ func lexStringEsc(l *lexer) (stateFn, error) {
|
|
|
|
// whitespace, close paren, a period to indicate we want a float, or more
|
|
|
|
// whitespace, close paren, a period to indicate we want a float, or more
|
|
|
|
// digits. Everything else is crap.
|
|
|
|
// digits. Everything else is crap.
|
|
|
|
func lexInt(l *lexer) (stateFn, error) {
|
|
|
|
func lexInt(l *lexer) (stateFn, error) {
|
|
|
|
|
|
|
|
debugPrint("-->lexInt")
|
|
|
|
r, _, err := l.ReadRune()
|
|
|
|
r, _, err := l.ReadRune()
|
|
|
|
if err != nil {
|
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
|
|
|
return nil, err
|
|
|
@ -201,6 +199,7 @@ func lexInt(l *lexer) (stateFn, error) {
|
|
|
|
// once we're in a float, the only valid values are digits, whitespace or close
|
|
|
|
// once we're in a float, the only valid values are digits, whitespace or close
|
|
|
|
// paren.
|
|
|
|
// paren.
|
|
|
|
func lexFloat(l *lexer) (stateFn, error) {
|
|
|
|
func lexFloat(l *lexer) (stateFn, error) {
|
|
|
|
|
|
|
|
debugPrint("-->lexFloat")
|
|
|
|
r, _, err := l.ReadRune()
|
|
|
|
r, _, err := l.ReadRune()
|
|
|
|
if err != nil {
|
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
|
|
|
return nil, err
|
|
|
@ -226,12 +225,15 @@ func lexFloat(l *lexer) (stateFn, error) {
|
|
|
|
|
|
|
|
|
|
|
|
// lexes a symbol in progress
|
|
|
|
// lexes a symbol in progress
|
|
|
|
func lexSymbol(l *lexer) (stateFn, error) {
|
|
|
|
func lexSymbol(l *lexer) (stateFn, error) {
|
|
|
|
|
|
|
|
debugPrint("-->lexSymbol")
|
|
|
|
r, _, err := l.ReadRune()
|
|
|
|
r, _, err := l.ReadRune()
|
|
|
|
if err != nil {
|
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
|
|
|
return nil, err
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
switch r {
|
|
|
|
switch r {
|
|
|
|
case ' ', '\t', '\n':
|
|
|
|
case ' ', '\t', '\n':
|
|
|
|
|
|
|
|
debugPrint("ending lexSymbol on whitespace")
|
|
|
|
l.emit(symbol)
|
|
|
|
l.emit(symbol)
|
|
|
|
return lexWhitespace, nil
|
|
|
|
return lexWhitespace, nil
|
|
|
|
case ')':
|
|
|
|
case ')':
|
|
|
@ -249,6 +251,7 @@ func lexSymbol(l *lexer) (stateFn, error) {
|
|
|
|
|
|
|
|
|
|
|
|
// lex a close parenthesis
|
|
|
|
// lex a close parenthesis
|
|
|
|
func lexCloseParen(l *lexer) (stateFn, error) {
|
|
|
|
func lexCloseParen(l *lexer) (stateFn, error) {
|
|
|
|
|
|
|
|
debugPrint("-->lexCloseParen")
|
|
|
|
l.out <- token{")", closeParen}
|
|
|
|
l.out <- token{")", closeParen}
|
|
|
|
l.depth--
|
|
|
|
l.depth--
|
|
|
|
r, _, err := l.ReadRune()
|
|
|
|
r, _, err := l.ReadRune()
|
|
|
@ -257,11 +260,7 @@ func lexCloseParen(l *lexer) (stateFn, error) {
|
|
|
|
}
|
|
|
|
}
|
|
|
|
switch r {
|
|
|
|
switch r {
|
|
|
|
case ' ', '\t', '\n':
|
|
|
|
case ' ', '\t', '\n':
|
|
|
|
if l.depth == 0 {
|
|
|
|
|
|
|
|
return lexRoot, nil
|
|
|
|
|
|
|
|
} else {
|
|
|
|
|
|
|
|
return lexWhitespace, nil
|
|
|
|
return lexWhitespace, nil
|
|
|
|
}
|
|
|
|
|
|
|
|
case ')':
|
|
|
|
case ')':
|
|
|
|
return lexCloseParen, nil
|
|
|
|
return lexCloseParen, nil
|
|
|
|
case ';':
|
|
|
|
case ';':
|
|
|
@ -272,25 +271,18 @@ func lexCloseParen(l *lexer) (stateFn, error) {
|
|
|
|
|
|
|
|
|
|
|
|
// lexes a comment
|
|
|
|
// lexes a comment
|
|
|
|
func lexComment(l *lexer) (stateFn, error) {
|
|
|
|
func lexComment(l *lexer) (stateFn, error) {
|
|
|
|
|
|
|
|
debugPrint("-->lexComment")
|
|
|
|
r, _, err := l.ReadRune()
|
|
|
|
r, _, err := l.ReadRune()
|
|
|
|
if err != nil {
|
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
|
|
|
return nil, err
|
|
|
|
}
|
|
|
|
}
|
|
|
|
switch r {
|
|
|
|
switch r {
|
|
|
|
case '\n', '\r':
|
|
|
|
case '\n', '\r':
|
|
|
|
if l.depth == 0 {
|
|
|
|
|
|
|
|
return lexRoot, nil
|
|
|
|
|
|
|
|
} else {
|
|
|
|
|
|
|
|
return lexWhitespace, nil
|
|
|
|
return lexWhitespace, nil
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
return lexComment, nil
|
|
|
|
return lexComment, nil
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
func lexs(input string, c chan token) {
|
|
|
|
|
|
|
|
lex(strings.NewReader(input), c)
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
// lexes some lispy input from an io.Reader, emiting tokens on chan c. The
|
|
|
|
// lexes some lispy input from an io.Reader, emiting tokens on chan c. The
|
|
|
|
// channel is closed when the input reaches EOF, signaling that there are no
|
|
|
|
// channel is closed when the input reaches EOF, signaling that there are no
|
|
|
|
// new tokens.
|
|
|
|
// new tokens.
|
|
|
@ -299,7 +291,7 @@ func lex(input io.Reader, c chan token) {
|
|
|
|
l := &lexer{bufio.NewReader(input), nil, 0, c}
|
|
|
|
l := &lexer{bufio.NewReader(input), nil, 0, c}
|
|
|
|
|
|
|
|
|
|
|
|
var err error
|
|
|
|
var err error
|
|
|
|
f := stateFn(lexRoot)
|
|
|
|
f := stateFn(lexWhitespace)
|
|
|
|
for err == nil {
|
|
|
|
for err == nil {
|
|
|
|
f, err = f(l)
|
|
|
|
f, err = f(l)
|
|
|
|
}
|
|
|
|
}
|
|
|
@ -318,6 +310,8 @@ func args() {
|
|
|
|
fmt.Fprintln(os.Stderr, "unable to read file ", filename)
|
|
|
|
fmt.Fprintln(os.Stderr, "unable to read file ", filename)
|
|
|
|
os.Exit(1)
|
|
|
|
os.Exit(1)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
defer f.Close()
|
|
|
|
|
|
|
|
|
|
|
|
c := make(chan token)
|
|
|
|
c := make(chan token)
|
|
|
|
go lex(f, c)
|
|
|
|
go lex(f, c)
|
|
|
|
|
|
|
|
|
|
|
@ -326,27 +320,47 @@ func args() {
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
func lexs(input string) {
|
|
|
|
|
|
|
|
c := make(chan token)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
go func() {
|
|
|
|
|
|
|
|
defer close(c)
|
|
|
|
|
|
|
|
l := &lexer{strings.NewReader(input), nil, 0, c}
|
|
|
|
|
|
|
|
var err error
|
|
|
|
|
|
|
|
f := stateFn(lexWhitespace)
|
|
|
|
|
|
|
|
for err == nil {
|
|
|
|
|
|
|
|
f, err = f(l)
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
if err != io.EOF {
|
|
|
|
|
|
|
|
fmt.Println(err)
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
if l.depth != 0 {
|
|
|
|
|
|
|
|
fmt.Println("error: unbalanced parenthesis")
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
}()
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
for t := range c {
|
|
|
|
|
|
|
|
fmt.Printf("%11s %s\n", t.t, t.lexeme)
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
func main() {
|
|
|
|
func main() {
|
|
|
|
if len(os.Args) > 1 {
|
|
|
|
if len(os.Args) > 1 {
|
|
|
|
args()
|
|
|
|
args()
|
|
|
|
return
|
|
|
|
return
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
printexp := func(c chan token) {
|
|
|
|
|
|
|
|
for s := range c {
|
|
|
|
|
|
|
|
fmt.Printf("%11s %s\n", s.t, s.lexeme)
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
r := bufio.NewReader(os.Stdin)
|
|
|
|
r := bufio.NewReader(os.Stdin)
|
|
|
|
for {
|
|
|
|
for {
|
|
|
|
c := make(chan token)
|
|
|
|
|
|
|
|
go printexp(c)
|
|
|
|
|
|
|
|
fmt.Print("> ")
|
|
|
|
fmt.Print("> ")
|
|
|
|
line, _, err := r.ReadLine()
|
|
|
|
line, prefix, err := r.ReadLine()
|
|
|
|
|
|
|
|
if prefix {
|
|
|
|
|
|
|
|
fmt.Println("(prefix)")
|
|
|
|
|
|
|
|
}
|
|
|
|
if err != nil {
|
|
|
|
if err != nil {
|
|
|
|
return
|
|
|
|
fmt.Println("error: ", err)
|
|
|
|
|
|
|
|
continue
|
|
|
|
}
|
|
|
|
}
|
|
|
|
lexs(string(line), c)
|
|
|
|
lexs(string(line) + "\n")
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|