jordan
/
moon
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

verify parse trees pl0x

Browse Source
master
Jordan Orelli 10 years ago
parent 636a30b963
commit 876794d694
5 changed files with 163 additions and 119 deletions
Show Stats Download Patch File Download Diff File

17
lex.go
Unescape Escape View File

@ -41,8 +41,10 @@ func (t tokenType) String() string {
return "t_object_separator"
case t_object_end:
return "t_object_end"
case t_number:
return "t_number"
case t_real_number:
return "t_real_number"
case t_imaginary_number:
return "t_imaginary_number"
default:
panic(fmt.Sprintf("unknown token type: %v", t))
}
@ -62,7 +64,8 @@ const (
t_object_start // {
t_object_end // }
t_object_separator // :
t_number // a number
t_real_number // a number
t_imaginary_number // an imaginary number
)
type stateFn func(*lexer) stateFn
@ -321,13 +324,17 @@ func lexNumber(l *lexer) stateFn {
l.accept("+-")
l.acceptRun("0123456789")
}
l.accept("i")
imaginary := l.accept("i")
r := l.next()
if isAlphaNumeric(r) {
return lexErrorf("unexpected alphanum in lexNumber: %c", r)
}
l.unread(r)
l.emit(t_number)
if imaginary {
l.emit(t_imaginary_number)
} else {
l.emit(t_real_number)
}
return lexRoot
}

43
lex_test.go
Unescape Escape View File

@ -64,38 +64,39 @@ var primitivesTests = []struct {
{t_list_separator, ","},
{t_object_end, "}"},
}},
{`0`, []token{{t_number, "0"}}},
{`-0`, []token{{t_number, "-0"}}},
{`+0`, []token{{t_number, "+0"}}},
{`+125`, []token{{t_number, "+125"}}},
{`-125`, []token{{t_number, "-125"}}},
{`.0`, []token{{t_number, ".0"}}},
{`15`, []token{{t_number, "15"}}},
{`0x0`, []token{{t_number, "0x0"}}},
{`0xa`, []token{{t_number, "0xa"}}},
{`0xc0dea5cf`, []token{{t_number, "0xc0dea5cf"}}},
{`12.345`, []token{{t_number, "12.345"}}},
{`12.345 name`, []token{{t_number, "12.345"}, {t_name, "name"}}},
{`0`, []token{{t_real_number, "0"}}},
{`-0`, []token{{t_real_number, "-0"}}},
{`+0`, []token{{t_real_number, "+0"}}},
{`+125`, []token{{t_real_number, "+125"}}},
{`-125`, []token{{t_real_number, "-125"}}},
{`.0`, []token{{t_real_number, ".0"}}},
{`15`, []token{{t_real_number, "15"}}},
{`0x0`, []token{{t_real_number, "0x0"}}},
{`0xa`, []token{{t_real_number, "0xa"}}},
{`0xc0dea5cf`, []token{{t_real_number, "0xc0dea5cf"}}},
{`12.345`, []token{{t_real_number, "12.345"}}},
{`12.345 name`, []token{{t_real_number, "12.345"}, {t_name, "name"}}},
{`[12.345]`, []token{
{t_list_start, "["},
{t_number, "12.345"},
{t_real_number, "12.345"},
{t_list_end, "]"},
}},
{`[1, 2, 3]`, []token{
{t_list_start, "["},
{t_number, "1"},
{t_real_number, "1"},
{t_list_separator, ","},
{t_number, "2"},
{t_real_number, "2"},
{t_list_separator, ","},
{t_number, "3"},
{t_real_number, "3"},
{t_list_end, "]"},
}},
// an imaginary number generates two lexemes; one for its real component,
{`1i`, []token{{t_imaginary_number, "1i"}}},
// a complex number generates two lexemes; one for its real component,
// and one for its imaginary component.
{`1+2i`, []token{{t_number, "1"}, {t_number, "+2i"}}},
{`1e9`, []token{{t_number, "1e9"}}},
{`1e+9`, []token{{t_number, "1e+9"}}},
{`1E-9`, []token{{t_number, "1E-9"}}},
{`1+2i`, []token{{t_real_number, "1"}, {t_imaginary_number, "+2i"}}},
{`1e9`, []token{{t_real_number, "1e9"}}},
{`1e+9`, []token{{t_real_number, "1e+9"}}},
{`1E-9`, []token{{t_real_number, "1E-9"}}},
}
func TestLexPrimities(t *testing.T) {

57
node.go
Unescape Escape View File

@ -40,8 +40,7 @@ func (n *rootNode) parse(p *parser) error {
case t_eof:
return nil
case t_comment:
shit := commentNode(t.s)
n.addChild(&shit)
n.addChild(&commentNode{t.s})
case t_name:
nn := &assignmentNode{name: t.s}
if err := nn.parse(p); err != nil {
@ -74,18 +73,20 @@ func (n *rootNode) String() string {
return buf.String()
}
type commentNode string
type commentNode struct {
body string
}
func (n commentNode) Type() nodeType {
func (n *commentNode) Type() nodeType {
return n_comment
}
func (n commentNode) parse(p *parser) error {
func (n *commentNode) parse(p *parser) error {
return nil
}
func (n commentNode) String() string {
return fmt.Sprintf("{comment: %s}", string(n))
func (n *commentNode) String() string {
return fmt.Sprintf("{comment: %s}", n.body)
}
type assignmentNode struct {
@ -93,7 +94,7 @@ type assignmentNode struct {
value interface{}
}
func (n assignmentNode) Type() nodeType {
func (n *assignmentNode) Type() nodeType {
return n_assignment
}
@ -118,44 +119,8 @@ func (n *assignmentNode) parse(p *parser) error {
}
func (n *assignmentNode) String() string {
return fmt.Sprintf("{assign: name=%s, val=%s}", n.name, n.value)
}
type list struct {
head *listElem
tail *listElem
}
func (l list) String() string {
var buf bytes.Buffer
buf.WriteString("[")
for e := l.head; e != nil; e = e.next {
fmt.Fprintf(&buf, "%v, ", e.value)
}
if buf.Len() > 1 {
buf.Truncate(buf.Len() - 2)
}
buf.WriteString("]")
return buf.String()
}
func (l *list) append(v interface{}) {
e := listElem{value: v}
if l.head == nil {
l.head = &e
}
if l.tail != nil {
l.tail.next = &e
e.prev = l.tail
}
l.tail = &e
}
type listElem struct {
value interface{}
prev *listElem
next *listElem
return fmt.Sprintf("{assign: name=%s, val=%v}", n.name, n.value)
}
type list []interface{}
type object map[string]interface{}

38
parse.go
Unescape Escape View File

@ -3,6 +3,7 @@ package main
import (
"fmt"
"io"
"strconv"
)
const ()
@ -78,8 +79,11 @@ func (p *parser) parseValue() (interface{}, error) {
return nil, fmt.Errorf("parse error: unexpected eof when looking for value")
case t_string:
return t.s, nil
case t_real_number, t_imaginary_number:
p.unread(t)
return p.number()
case t_list_start:
return p.parseList(new(list))
return p.parseList(make(list, 0, 4))
case t_object_start:
return p.parseObject(make(object))
default:
@ -88,7 +92,7 @@ func (p *parser) parseValue() (interface{}, error) {
}
}
func (p *parser) parseList(l *list) (*list, error) {
func (p *parser) parseList(l list) (list, error) {
if p.peek().t == t_list_end {
p.next()
return l, nil
@ -97,7 +101,7 @@ func (p *parser) parseList(l *list) (*list, error) {
if v, err := p.parseValue(); err != nil {
return nil, err
} else {
l.append(v)
l = append(l, v)
}
switch t := p.next(); t.t {
@ -139,3 +143,31 @@ func (p *parser) parseObject(obj object) (object, error) {
return nil, fmt.Errorf("parse error: unexpected %v token while scanning for object", t.t)
}
}
func (p *parser) number() (interface{}, error) {
t := p.next()
if t.t != t_real_number {
return nil, fmt.Errorf("unexpected %s token while parsing number", t.t)
}
if p.peek().t == t_imaginary_number {
var c complex128
s := t.s + p.next().s
if _, err := fmt.Sscan(s, &c); err != nil {
return nil, fmt.Errorf("ungood imaginary number format %s: %s", s, err)
}
return c, nil
}
i, err := strconv.ParseInt(t.s, 0, 64)
if err == nil {
return int(i), nil
}
f, err := strconv.ParseFloat(t.s, 64)
if err == nil {
return f, nil
}
return nil, fmt.Errorf("this token broke the number parser: %s", t)
}

127
parse_test.go
Unescape Escape View File

@ -1,6 +1,7 @@
package main
import (
"reflect"
"strings"
"testing"
)
@ -16,7 +17,7 @@ var parseTests = []parseTest{
source: `# just a comment`,
root: &rootNode{
children: []node{
commentNode(" just a comment"),
&commentNode{" just a comment"},
},
},
},
@ -24,76 +25,113 @@ var parseTests = []parseTest{
source: `name = "jordan"`,
root: &rootNode{
children: []node{
&assignmentNode{
name: "name",
value: "jordan",
},
&assignmentNode{"name", "jordan"},
},
},
},
{
source: `
first_name = "jordan"
last_name = "orelli"
`,
root: &rootNode{},
},
{
source: `
# personal info
first_name = "jordan"
last_name = "orelli"
`,
root: &rootNode{},
hostname = "jordanorelli.com"
port = 9000
freq = 1e9
duty = 0.2
neg = -2
neg2 = -2.3
imag = 1+2i
`,
root: &rootNode{
children: []node{
&assignmentNode{"hostname", "jordanorelli.com"},
&assignmentNode{"port", 9000},
&assignmentNode{"freq", 1e9},
&assignmentNode{"duty", 0.2},
&assignmentNode{"neg", -2},
&assignmentNode{"neg2", -2.3},
&assignmentNode{"imag", 1 + 2i},
},
},
},
{
source: `
first_name = "jordan" # yep, that's my name
last_name = "orelli" # comments should be able to follow other shit
`,
root: &rootNode{},
first_name = "jordan" # yep, that's my name
last_name = "orelli" # comments should be able to follow other shit
`,
root: &rootNode{
children: []node{
&assignmentNode{"first_name", "jordan"},
&commentNode{" yep, that's my name"},
&assignmentNode{"last_name", "orelli"},
&commentNode{" comments should be able to follow other shit"},
},
},
},
{
source: `
heroes = ["lina", "cm"]
`,
root: &rootNode{},
heroes = ["lina", "cm"]
`,
root: &rootNode{
children: []node{
&assignmentNode{"heroes", list{"lina", "cm"}},
},
},
},
{
source: `
nested = [["one", "two"], ["three", "four"]]
`,
root: &rootNode{},
nested = [["one", "two"], ["three", "four"]]
`,
root: &rootNode{
children: []node{
&assignmentNode{"nested", list{list{"one", "two"}, list{"three", "four"}}},
},
},
},
{
source: `
nested = [
["one", "two"],
["three", "four"],
]
`,
root: &rootNode{},
nested = [
["one", "two"],
["three", "four"],
]
`,
root: &rootNode{
children: []node{
&assignmentNode{"nested", list{list{"one", "two"}, list{"three", "four"}}},
},
},
},
{
source: `
admin = {first_name: "jordan", last_name: "orelli"}
`,
root: &rootNode{},
root: &rootNode{
children: []node{
&assignmentNode{"admin", object{
"first_name": "jordan",
"last_name": "orelli",
}},
},
},
},
{
source: `
http = {
port: "9000",
routes: "/path/to/some/file",
}
`,
root: &rootNode{},
http = {
port: 9000,
routes: "/path/to/some/file",
}
`,
root: &rootNode{
children: []node{
&assignmentNode{"http", object{
"port": 9000,
"routes": "/path/to/some/file",
}},
},
},
},
}
type parseTest struct {
source string
root *rootNode
root node
}
func (p *parseTest) run(t *testing.T) {
@ -103,10 +141,11 @@ func (p *parseTest) run(t *testing.T) {
t.Errorf("parse error: %v", err)
return
}
if n.Type() != n_root {
t.Errorf("we expected a root node object, but instead we got: %s", n.Type())
if !reflect.DeepEqual(p.root, n) {
t.Errorf("trees are not equal. expected:\n%v\nsaw:\n%v", p.root, n)
} else {
t.Logf("OK trees are equal: %v = %v", p.root, n)
}
t.Logf("output: %v", n)
}
func TestParse(t *testing.T) {

Write Preview
Loading…
Cancel
Save