From 876794d6949bb81847d81baa71713b364f775bd3 Mon Sep 17 00:00:00 2001 From: Jordan Orelli Date: Sun, 22 Mar 2015 21:44:25 -0400 Subject: [PATCH] verify parse trees pl0x --- lex.go | 17 +++++-- lex_test.go | 43 ++++++++--------- node.go | 57 +++++----------------- parse.go | 38 +++++++++++++-- parse_test.go | 127 +++++++++++++++++++++++++++++++++----------------- 5 files changed, 163 insertions(+), 119 deletions(-) diff --git a/lex.go b/lex.go index f4da2c0..93abc4f 100644 --- a/lex.go +++ b/lex.go @@ -41,8 +41,10 @@ func (t tokenType) String() string { return "t_object_separator" case t_object_end: return "t_object_end" - case t_number: - return "t_number" + case t_real_number: + return "t_real_number" + case t_imaginary_number: + return "t_imaginary_number" default: panic(fmt.Sprintf("unknown token type: %v", t)) } @@ -62,7 +64,8 @@ const ( t_object_start // { t_object_end // } t_object_separator // : - t_number // a number + t_real_number // a number + t_imaginary_number // an imaginary number ) type stateFn func(*lexer) stateFn @@ -321,13 +324,17 @@ func lexNumber(l *lexer) stateFn { l.accept("+-") l.acceptRun("0123456789") } - l.accept("i") + imaginary := l.accept("i") r := l.next() if isAlphaNumeric(r) { return lexErrorf("unexpected alphanum in lexNumber: %c", r) } l.unread(r) - l.emit(t_number) + if imaginary { + l.emit(t_imaginary_number) + } else { + l.emit(t_real_number) + } return lexRoot } diff --git a/lex_test.go b/lex_test.go index 19d65cf..74daad6 100644 --- a/lex_test.go +++ b/lex_test.go @@ -64,38 +64,39 @@ var primitivesTests = []struct { {t_list_separator, ","}, {t_object_end, "}"}, }}, - {`0`, []token{{t_number, "0"}}}, - {`-0`, []token{{t_number, "-0"}}}, - {`+0`, []token{{t_number, "+0"}}}, - {`+125`, []token{{t_number, "+125"}}}, - {`-125`, []token{{t_number, "-125"}}}, - {`.0`, []token{{t_number, ".0"}}}, - {`15`, []token{{t_number, "15"}}}, - {`0x0`, []token{{t_number, "0x0"}}}, - {`0xa`, []token{{t_number, "0xa"}}}, - {`0xc0dea5cf`, []token{{t_number, "0xc0dea5cf"}}}, - {`12.345`, []token{{t_number, "12.345"}}}, - {`12.345 name`, []token{{t_number, "12.345"}, {t_name, "name"}}}, + {`0`, []token{{t_real_number, "0"}}}, + {`-0`, []token{{t_real_number, "-0"}}}, + {`+0`, []token{{t_real_number, "+0"}}}, + {`+125`, []token{{t_real_number, "+125"}}}, + {`-125`, []token{{t_real_number, "-125"}}}, + {`.0`, []token{{t_real_number, ".0"}}}, + {`15`, []token{{t_real_number, "15"}}}, + {`0x0`, []token{{t_real_number, "0x0"}}}, + {`0xa`, []token{{t_real_number, "0xa"}}}, + {`0xc0dea5cf`, []token{{t_real_number, "0xc0dea5cf"}}}, + {`12.345`, []token{{t_real_number, "12.345"}}}, + {`12.345 name`, []token{{t_real_number, "12.345"}, {t_name, "name"}}}, {`[12.345]`, []token{ {t_list_start, "["}, - {t_number, "12.345"}, + {t_real_number, "12.345"}, {t_list_end, "]"}, }}, {`[1, 2, 3]`, []token{ {t_list_start, "["}, - {t_number, "1"}, + {t_real_number, "1"}, {t_list_separator, ","}, - {t_number, "2"}, + {t_real_number, "2"}, {t_list_separator, ","}, - {t_number, "3"}, + {t_real_number, "3"}, {t_list_end, "]"}, }}, - // an imaginary number generates two lexemes; one for its real component, + {`1i`, []token{{t_imaginary_number, "1i"}}}, + // a complex number generates two lexemes; one for its real component, // and one for its imaginary component. - {`1+2i`, []token{{t_number, "1"}, {t_number, "+2i"}}}, - {`1e9`, []token{{t_number, "1e9"}}}, - {`1e+9`, []token{{t_number, "1e+9"}}}, - {`1E-9`, []token{{t_number, "1E-9"}}}, + {`1+2i`, []token{{t_real_number, "1"}, {t_imaginary_number, "+2i"}}}, + {`1e9`, []token{{t_real_number, "1e9"}}}, + {`1e+9`, []token{{t_real_number, "1e+9"}}}, + {`1E-9`, []token{{t_real_number, "1E-9"}}}, } func TestLexPrimities(t *testing.T) { diff --git a/node.go b/node.go index 57fec19..932615c 100644 --- a/node.go +++ b/node.go @@ -40,8 +40,7 @@ func (n *rootNode) parse(p *parser) error { case t_eof: return nil case t_comment: - shit := commentNode(t.s) - n.addChild(&shit) + n.addChild(&commentNode{t.s}) case t_name: nn := &assignmentNode{name: t.s} if err := nn.parse(p); err != nil { @@ -74,18 +73,20 @@ func (n *rootNode) String() string { return buf.String() } -type commentNode string +type commentNode struct { + body string +} -func (n commentNode) Type() nodeType { +func (n *commentNode) Type() nodeType { return n_comment } -func (n commentNode) parse(p *parser) error { +func (n *commentNode) parse(p *parser) error { return nil } -func (n commentNode) String() string { - return fmt.Sprintf("{comment: %s}", string(n)) +func (n *commentNode) String() string { + return fmt.Sprintf("{comment: %s}", n.body) } type assignmentNode struct { @@ -93,7 +94,7 @@ type assignmentNode struct { value interface{} } -func (n assignmentNode) Type() nodeType { +func (n *assignmentNode) Type() nodeType { return n_assignment } @@ -118,44 +119,8 @@ func (n *assignmentNode) parse(p *parser) error { } func (n *assignmentNode) String() string { - return fmt.Sprintf("{assign: name=%s, val=%s}", n.name, n.value) -} - -type list struct { - head *listElem - tail *listElem -} - -func (l list) String() string { - var buf bytes.Buffer - buf.WriteString("[") - for e := l.head; e != nil; e = e.next { - fmt.Fprintf(&buf, "%v, ", e.value) - } - if buf.Len() > 1 { - buf.Truncate(buf.Len() - 2) - } - buf.WriteString("]") - return buf.String() -} - -func (l *list) append(v interface{}) { - e := listElem{value: v} - if l.head == nil { - l.head = &e - } - - if l.tail != nil { - l.tail.next = &e - e.prev = l.tail - } - l.tail = &e -} - -type listElem struct { - value interface{} - prev *listElem - next *listElem + return fmt.Sprintf("{assign: name=%s, val=%v}", n.name, n.value) } +type list []interface{} type object map[string]interface{} diff --git a/parse.go b/parse.go index 0c28f7b..ff70090 100644 --- a/parse.go +++ b/parse.go @@ -3,6 +3,7 @@ package main import ( "fmt" "io" + "strconv" ) const () @@ -78,8 +79,11 @@ func (p *parser) parseValue() (interface{}, error) { return nil, fmt.Errorf("parse error: unexpected eof when looking for value") case t_string: return t.s, nil + case t_real_number, t_imaginary_number: + p.unread(t) + return p.number() case t_list_start: - return p.parseList(new(list)) + return p.parseList(make(list, 0, 4)) case t_object_start: return p.parseObject(make(object)) default: @@ -88,7 +92,7 @@ func (p *parser) parseValue() (interface{}, error) { } } -func (p *parser) parseList(l *list) (*list, error) { +func (p *parser) parseList(l list) (list, error) { if p.peek().t == t_list_end { p.next() return l, nil @@ -97,7 +101,7 @@ func (p *parser) parseList(l *list) (*list, error) { if v, err := p.parseValue(); err != nil { return nil, err } else { - l.append(v) + l = append(l, v) } switch t := p.next(); t.t { @@ -139,3 +143,31 @@ func (p *parser) parseObject(obj object) (object, error) { return nil, fmt.Errorf("parse error: unexpected %v token while scanning for object", t.t) } } + +func (p *parser) number() (interface{}, error) { + t := p.next() + if t.t != t_real_number { + return nil, fmt.Errorf("unexpected %s token while parsing number", t.t) + } + + if p.peek().t == t_imaginary_number { + var c complex128 + s := t.s + p.next().s + if _, err := fmt.Sscan(s, &c); err != nil { + return nil, fmt.Errorf("ungood imaginary number format %s: %s", s, err) + } + return c, nil + } + + i, err := strconv.ParseInt(t.s, 0, 64) + if err == nil { + return int(i), nil + } + + f, err := strconv.ParseFloat(t.s, 64) + if err == nil { + return f, nil + } + + return nil, fmt.Errorf("this token broke the number parser: %s", t) +} diff --git a/parse_test.go b/parse_test.go index 51567f4..88866e8 100644 --- a/parse_test.go +++ b/parse_test.go @@ -1,6 +1,7 @@ package main import ( + "reflect" "strings" "testing" ) @@ -16,7 +17,7 @@ var parseTests = []parseTest{ source: `# just a comment`, root: &rootNode{ children: []node{ - commentNode(" just a comment"), + &commentNode{" just a comment"}, }, }, }, @@ -24,76 +25,113 @@ var parseTests = []parseTest{ source: `name = "jordan"`, root: &rootNode{ children: []node{ - &assignmentNode{ - name: "name", - value: "jordan", - }, + &assignmentNode{"name", "jordan"}, }, }, }, { source: ` - first_name = "jordan" - last_name = "orelli" - `, - root: &rootNode{}, - }, - { - source: ` - # personal info - first_name = "jordan" - last_name = "orelli" - `, - root: &rootNode{}, + hostname = "jordanorelli.com" + port = 9000 + freq = 1e9 + duty = 0.2 + neg = -2 + neg2 = -2.3 + imag = 1+2i + `, + root: &rootNode{ + children: []node{ + &assignmentNode{"hostname", "jordanorelli.com"}, + &assignmentNode{"port", 9000}, + &assignmentNode{"freq", 1e9}, + &assignmentNode{"duty", 0.2}, + &assignmentNode{"neg", -2}, + &assignmentNode{"neg2", -2.3}, + &assignmentNode{"imag", 1 + 2i}, + }, + }, }, { source: ` - first_name = "jordan" # yep, that's my name - last_name = "orelli" # comments should be able to follow other shit - `, - root: &rootNode{}, + first_name = "jordan" # yep, that's my name + last_name = "orelli" # comments should be able to follow other shit + `, + root: &rootNode{ + children: []node{ + &assignmentNode{"first_name", "jordan"}, + &commentNode{" yep, that's my name"}, + &assignmentNode{"last_name", "orelli"}, + &commentNode{" comments should be able to follow other shit"}, + }, + }, }, { source: ` - heroes = ["lina", "cm"] - `, - root: &rootNode{}, + heroes = ["lina", "cm"] + `, + root: &rootNode{ + children: []node{ + &assignmentNode{"heroes", list{"lina", "cm"}}, + }, + }, }, { source: ` - nested = [["one", "two"], ["three", "four"]] - `, - root: &rootNode{}, + nested = [["one", "two"], ["three", "four"]] + `, + root: &rootNode{ + children: []node{ + &assignmentNode{"nested", list{list{"one", "two"}, list{"three", "four"}}}, + }, + }, }, { source: ` - nested = [ - ["one", "two"], - ["three", "four"], - ] - `, - root: &rootNode{}, + nested = [ + ["one", "two"], + ["three", "four"], + ] + `, + root: &rootNode{ + children: []node{ + &assignmentNode{"nested", list{list{"one", "two"}, list{"three", "four"}}}, + }, + }, }, { source: ` admin = {first_name: "jordan", last_name: "orelli"} `, - root: &rootNode{}, + root: &rootNode{ + children: []node{ + &assignmentNode{"admin", object{ + "first_name": "jordan", + "last_name": "orelli", + }}, + }, + }, }, { source: ` - http = { - port: "9000", - routes: "/path/to/some/file", - } - `, - root: &rootNode{}, + http = { + port: 9000, + routes: "/path/to/some/file", + } + `, + root: &rootNode{ + children: []node{ + &assignmentNode{"http", object{ + "port": 9000, + "routes": "/path/to/some/file", + }}, + }, + }, }, } type parseTest struct { source string - root *rootNode + root node } func (p *parseTest) run(t *testing.T) { @@ -103,10 +141,11 @@ func (p *parseTest) run(t *testing.T) { t.Errorf("parse error: %v", err) return } - if n.Type() != n_root { - t.Errorf("we expected a root node object, but instead we got: %s", n.Type()) + if !reflect.DeepEqual(p.root, n) { + t.Errorf("trees are not equal. expected:\n%v\nsaw:\n%v", p.root, n) + } else { + t.Logf("OK trees are equal: %v = %v", p.root, n) } - t.Logf("output: %v", n) } func TestParse(t *testing.T) {