in the middle of redoing the whole decode thing

ent/class.go

@@ -7,7 +7,7 @@ import (
 
 // Class represents a set of constraints around an Entity.
 type Class struct {
-	Name    Symbol
+	name    Symbol
 	Version int
 	Fields  []*Field
 
@@ -19,6 +19,10 @@ type Class struct {
 	fieldNames map[string]int
 }
 
+func (c *Class) Name() string  { return c.name.String() }
+func (c *Class) Slotted() bool { return true }
+func (c *Class) Id() classId   { return classId{name: c.name, version: c.Version} }
+
 func (c *Class) New(serial int, baseline bool) *Entity {
 	e := &Entity{
 		Class:      c,
@@ -36,6 +40,7 @@ func (c Class) String() string {
 	return fmt.Sprintf("{%s %d}", c.Name, c.Version)
 }
 
+// A class is identified by the union of its name and version.
 type classId struct {
 	name    Symbol
 	version int

ent/dict.go

@@ -37,6 +37,7 @@ const e_limit = 2048
 type Dict struct {
 	*Namespace
 	entities []*Entity
+	hidx     map[int]*Entity // handle index
 	br       *bit.BufReader
 	sr       *selectionReader
 
@@ -75,7 +76,7 @@ func (d *Dict) createEntity(id int) error {
 	e := class.New(serial, false)
 	d.entities[id] = e
 	Debug.Printf("create entity id: %d serial: %d classId: %d className: %v class: %v\n", id, serial, classId, className, class)
-	if err := fillSlots(e, class.Name.String(), d.sr, d.br); err != nil {
+	if err := fillSlots(e, class.Name(), d.sr, d.br); err != nil {
 		return fmt.Errorf("failed to create entity %d (%s): %v", id, className, err)
 	}
 	return nil
@@ -96,7 +97,7 @@ func (d *Dict) updateEntity(id int) error {
 		return fmt.Errorf("update entity %d refused: no such entity", id)
 	}
 	if err := fillSlots(e, e.Class.String(), d.sr, d.br); err != nil {
-		return fmt.Errorf("failed to update entity %d (%s): %v", id, e.Class.Name.String(), err)
+		return fmt.Errorf("failed to update entity %d (%s): %v", id, e.Class.Name(), err)
 	}
 	return nil
 }
@@ -212,7 +213,7 @@ func (d *Dict) syncBaselines() error {
 
 		d.br.SetSource(e.Value)
 		Debug.Printf("syncBaselines has new baseline for class %v", c)
-		if err := fillSlots(c.baseline, c.Name.String(), d.sr, d.br); err != nil {
+		if err := fillSlots(c.baseline, c.Name(), d.sr, d.br); err != nil {
 			return fmt.Errorf("syncBaselines failed to fill a baseline: %v", err)
 		}
 	}

ent/field.go

@@ -7,6 +7,14 @@ import (
 	"github.com/jordanorelli/hyperstone/dota"
 )
 
+/*
+type Field struct {
+	Name string
+	Type
+}
+
+*/
+
 type Field struct {
 	_type             Symbol // type of data held by the field
 	typeSpec          typeSpec
@@ -16,8 +24,8 @@ type Field struct {
 	low               float32 // lower limit of field values
 	high              float32 // upper limit of field values
 	flags             int     // used by float decoder
-	serializer        *Symbol // class on which the field was defined
+	serializer        *Symbol // the field is an entity with this class
-	serializerVersion *int32  // version of the class on which the field was defined
+	serializerVersion *int32
 	class             *Class  // source class on which the field was originally defined
 	encoder           *Symbol // binary encoder, named explicitly in protobuf
 	decoder                   // decodes field values from a bit stream

ent/float.go

@@ -0,0 +1,93 @@
+package ent
+
+import (
+	"fmt"
+	"math"
+
+	"github.com/jordanorelli/hyperstone/bit"
+	"github.com/jordanorelli/hyperstone/dota"
+)
+
+const (
+	f_min = 1 << iota
+	f_max
+	f_center
+)
+
+func parseFloatType(n *Namespace, flat *dota.ProtoFlattenedSerializerFieldT) (Type, error) {
+	if flat.VarEncoderSym != nil {
+		encoder := n.Symbol(int(flat.GetVarEncoderSym())).String()
+		switch encoder {
+		case "coord":
+			return nil, fmt.Errorf("coord encoder isn't dont yet")
+		default:
+			return nil, fmt.Errorf("unknown float encoder: %s", encoder)
+		}
+	}
+
+	if flat.BitCount != nil {
+		bits := int(flat.GetBitCount())
+		switch {
+		case bits < 0:
+			return nil, fmt.Errorf("invalid bit count on float field: %d", bits)
+		case bits < 32:
+			return quantizedFloat(n, flat)
+		case bits == 0, bits == 32:
+			// these seem meaningless, which is suspicious.
+		default:
+			return nil, fmt.Errorf("bit count is too high on float field: %d", bits)
+		}
+	}
+
+	if flat.LowValue != nil || flat.HighValue != nil {
+		return nil, fmt.Errorf("float32 with a low or high value isn't supported")
+	}
+
+	type_name := n.Symbol(int(flat.GetVarTypeSym())).String()
+	return &Primitive{name: type_name, read: readFloat32}, nil
+}
+
+func quantizedFloat(n *Namespace, flat *dota.ProtoFlattenedSerializerFieldT) (Type, error) {
+	if flat.LowValue == nil && flat.HighValue == nil {
+		return nil, fmt.Errorf("quantizedFloat has no boundaries")
+	}
+
+	bits := uint(flat.GetBitCount())
+	low, high := flat.GetLowValue(), flat.GetHighValue()
+	flags := int(flat.GetEncodeFlags())
+
+	flags = flags & 7                   // dunno how to handle -8 lol
+	steps := uint(1<<bits - 1)          // total number of intervals
+	span := high - low                  // total range of values
+	step_width := span / float32(steps) // output width of each step
+	if span < 0 {
+		return nil, fmt.Errorf("invalid quantization span")
+	}
+
+	var special *float32
+	switch {
+	case flags&f_min > 0:
+		special = &low
+	case flags&f_max > 0:
+		special = &high
+	case flags&f_center > 0:
+		middle := (high + low) * 0.5
+		special = &middle
+	}
+
+	read := func(br bit.Reader, d *Dict) (interface{}, error) {
+		if special != nil && bit.ReadBool(br) {
+			return *special, nil
+		}
+		u := br.ReadBits(bits)
+		return low + float32(u)*step_width, br.Err()
+	}
+
+	type_name := n.Symbol(int(flat.GetVarTypeSym())).String()
+	return &Primitive{name: type_name, read: read}, nil
+}
+
+// reads an IEEE 754 binary float value off of the stream
+func readFloat32(br bit.Reader, d *Dict) (interface{}, error) {
+	return math.Float32frombits(uint32(br.ReadBits(32))), nil
+}

ent/float_decoders.go

@@ -6,12 +6,6 @@ import (
 	"github.com/jordanorelli/hyperstone/bit"
 )
 
-const (
-	f_min = 1 << iota
-	f_max
-	f_center
-)
-
 func floatDecoder(f *Field) decoder {
 	if f.encoder != nil && f.encoder.String() == "coord" {
 		return nil

ent/handle.go

@@ -0,0 +1,24 @@
+package ent
+
+import (
+	"fmt"
+	"github.com/jordanorelli/hyperstone/bit"
+)
+
+type Handle struct{ name string }
+
+func (h *Handle) Name() string                   { return h.name }
+func (h *Handle) New(...interface{}) interface{} { return nil }
+func (h *Handle) Slotted() bool                  { return false }
+
+func (h *Handle) Read(br bit.Reader, d *Dict) (interface{}, error) {
+	id := int(bit.ReadVarInt(br))
+	e, ok := d.hidx[id]
+	if !ok {
+		if br.Err() != nil {
+			return nil, br.Err()
+		}
+		return nil, fmt.Errorf("no entity found with handle %d", id)
+	}
+	return e, br.Err()
+}

ent/namespace.go

@@ -78,7 +78,7 @@ func (n *Namespace) mergeSendTables(st *dota.CDemoSendTables) error {
 		version := int(c.GetSerializerVersion())
 
 		Debug.Printf("new class: %s %v", name, version)
-		class := Class{Name: name, Version: version}
+		class := Class{name: name, Version: version}
 		class.fromProto(c, fields)
 
 		id := classId{name: name, version: version}
@@ -91,12 +91,6 @@ func (n *Namespace) mergeSendTables(st *dota.CDemoSendTables) error {
 		}
 	}
 
-	// some fields explicitly reference their origin class (P). that is is, if
-	// a given field F is included in some class C, the field F having an
-	// origin class P indicates that the class C has the class P as an
-	// ancestor. since these references are circular, we unpacked the fields
-	// first, then the classes, and now we re-visit the fields to set their
-	// origin class pointers, now that the classes exist.
 	for i := range fields {
 		f := &fields[i]
 		if f.serializer != nil {
@@ -139,9 +133,83 @@ func (n *Namespace) mergeSendTables(st *dota.CDemoSendTables) error {
 		f.decoder = newFieldDecoder(n, f)
 	}
 
+	for _, ff := range flat.GetFields() {
+		t, err := parseType(n, ff)
+		if err != nil {
+			Debug.Printf("  parseType error: %v", err)
+		} else {
+			Debug.Printf("  parseType type: %v", t)
+		}
+	}
+
 	return br.Err()
 }
 
+/*
+func (n *Namespace) newMergeSendTables(st *dota.CDemoSendTables) error {
+	// sendtables only has one field, a binary data field. It's not immediately
+	// clear why this message exists at all when it could simply contain
+	// CSVMsg_FlattenedSerializer.
+	data := st.GetData()
+	br := bit.NewBytesReader(data)
+	// body is length-prefixed
+	size := int(bit.ReadVarInt(br))
+	buf := make([]byte, size)
+	br.Read(buf)
+	flat := dota.CSVCMsg_FlattenedSerializer{}
+	if err := proto.Unmarshal(buf, &flat); err != nil {
+		return fmt.Errorf("unable to merge send tables: %v", err)
+	}
+	return n.mergeSerializers(&flat)
+}
+
+func (n *Namespace) mergeSerializers(flat *dota.CSVCMsg_FlattenedSerializer) error {
+	// most of the names and associated strings are interned in a symbol table.
+	// We'll need these first since they're referenced throughought the class
+	// and field definitions.
+	n.SymbolTable = SymbolTable(flat.GetSymbols())
+
+	n.classes = make(map[classId]*Class, len(flat.GetSerializers()))
+	n.classesByName = make(map[string]map[int]*Class, len(flat.GetSerializers()))
+
+	// some fields refer to classes, but classes are collections of fields.
+	// Their references are potentially cyclical. We start by creating empty
+	// classes so that fields may point to them.
+	for _, s := range flat.GetSerializers() {
+		name_sym := n.Symbol(int(s.GetSerializerNameSym()))
+		ver := int(s.GetSerializerVersion())
+		class := Class{name: name_sym, Version: ver}
+		n.classes[class.Id()] = &class
+		if n.classesByName[class.Name()] == nil {
+			n.classesByName[class.Name()] = make(map[int]*Class)
+		}
+		n.classesByName[class.Name()][ver] = &class
+	}
+
+	// type ProtoFlattenedSerializerFieldT struct {
+	// 	VarTypeSym             *int32
+	// 	VarNameSym             *int32
+	// 	VarEncoderSym          *int32
+	// 	FieldSerializerNameSym *int32
+	// 	FieldSerializerVersion *int32
+	// 	BitCount               *int32
+	// 	LowValue               *float32
+	// 	HighValue              *float32
+	// 	EncodeFlags            *int32
+	// 	SendNodeSym            *int32
+	// }
+
+	// next we parse all of the fields along with their type definitions
+	fields := make(map[int]Field, len(flat.GetFields()))
+	for id, flatField := range flat.GetFields() {
+		t := parseType(n, flatField)
+		fields[id].Name = n.Symbol(int(flatField.GetVarNameSym())).String()
+	}
+	os.Exit(1)
+	return nil
+}
+*/
+
 func (n *Namespace) readClassId(r bit.Reader) int {
 	return int(r.ReadBits(uint(n.idBits)))
 }

ent/primitive.go

@@ -0,0 +1,92 @@
+package ent
+
+import (
+	"github.com/jordanorelli/hyperstone/bit"
+)
+
+var primitives map[string]Primitive
+
+// a Primitive type in the ent object system.
+type Primitive struct {
+	name  string
+	read  decodeFn
+	alloc func() interface{}
+}
+
+func (p *Primitive) New(args ...interface{}) interface{} {
+	if p.alloc != nil {
+		return p.alloc()
+	}
+	return nil
+}
+func (p *Primitive) Name() string  { return p.name }
+func (p *Primitive) Slotted() bool { return false }
+
+func (p *Primitive) Read(br bit.Reader, d *Dict) (interface{}, error) {
+	return p.read(br, d)
+}
+
+func init() {
+	types := []Primitive{
+		{name: "bool", read: readBool},
+		{name: "uint8", read: readUint8},
+		{name: "uint16", read: readUint16},
+		{name: "uint32", read: readUint32},
+		{name: "uint64", read: readUint64},
+		{name: "int8", read: readInt8},
+		{name: "int16", read: readInt16},
+		{name: "int32", read: readInt32},
+		{name: "int64", read: readInt64},
+		{name: "Color", read: readColor},
+	}
+	primitives = make(map[string]Primitive, len(types))
+	for _, t := range types {
+		primitives[t.name] = t
+	}
+}
+
+func readBool(br bit.Reader, d *Dict) (interface{}, error) {
+	return bit.ReadBool(br), br.Err()
+}
+
+func readUint8(br bit.Reader, d *Dict) (interface{}, error) {
+	return uint8(br.ReadBits(8)), br.Err()
+}
+
+func readUint16(br bit.Reader, d *Dict) (interface{}, error) {
+	return uint16(bit.ReadVarInt(br)), br.Err()
+}
+
+func readUint32(br bit.Reader, d *Dict) (interface{}, error) {
+	return uint32(bit.ReadVarInt(br)), br.Err()
+}
+
+func readUint64(br bit.Reader, d *Dict) (interface{}, error) {
+	return bit.ReadVarInt(br), br.Err()
+}
+
+func readInt8(br bit.Reader, d *Dict) (interface{}, error) {
+	return int8(bit.ReadZigZag(br)), br.Err()
+}
+
+func readInt16(br bit.Reader, d *Dict) (interface{}, error) {
+	return int16(bit.ReadZigZag(br)), br.Err()
+}
+
+func readInt32(br bit.Reader, d *Dict) (interface{}, error) {
+	return int32(bit.ReadZigZag(br)), br.Err()
+}
+
+func readInt64(br bit.Reader, d *Dict) (interface{}, error) {
+	return bit.ReadZigZag(br), br.Err()
+}
+
+func readColor(br bit.Reader, d *Dict) (interface{}, error) {
+	u := bit.ReadVarInt(br)
+	return color{
+		r: uint8(u & 0xff000000),
+		g: uint8(u & 0x00ff0000),
+		b: uint8(u & 0x0000ff00),
+		a: uint8(u & 0x000000ff),
+	}, br.Err()
+}

ent/qangle.go

@@ -0,0 +1,74 @@
+package ent
+
+import (
+	"fmt"
+	"math"
+
+	"github.com/jordanorelli/hyperstone/bit"
+	"github.com/jordanorelli/hyperstone/dota"
+)
+
+func parseQAngleType(n *Namespace, flat *dota.ProtoFlattenedSerializerFieldT) (Type, error) {
+	type_name := n.Symbol(int(flat.GetVarTypeSym())).String()
+	if flat.VarEncoderSym != nil {
+		encoder := n.Symbol(int(flat.GetVarEncoderSym())).String()
+		switch encoder {
+		case "qangle_pitch_yaw":
+			return nil, fmt.Errorf("that qangle pitch yaw thing isn't done yet")
+		default:
+			return nil, fmt.Errorf("unknown qangle encoder: %s", encoder)
+		}
+	}
+	if flat.BitCount == nil {
+		return nil, fmt.Errorf("dunno what to do when qangle type has no bitcount")
+	}
+	if flat.GetBitCount() < 0 {
+		return nil, fmt.Errorf("negative bit count wtf")
+	}
+	bits := uint(flat.GetBitCount())
+	switch bits {
+	case 0:
+		return &Primitive{name: type_name, read: readQAngleCoords}, nil
+	case 32:
+		return &Primitive{name: type_name, read: readQAngleFloats}, nil
+	default:
+		return &Primitive{name: type_name, read: qangleReader(bits)}, nil
+	}
+}
+
+func qangleReader(bits uint) decodeFn {
+	return func(br bit.Reader, d *Dict) (interface{}, error) {
+		return &vector{
+			x: bit.ReadAngle(br, bits),
+			y: bit.ReadAngle(br, bits),
+			z: bit.ReadAngle(br, bits),
+		}, br.Err()
+	}
+}
+
+func readQAngleCoords(br bit.Reader, d *Dict) (interface{}, error) {
+	var (
+		v vector
+		x = bit.ReadBool(br)
+		y = bit.ReadBool(br)
+		z = bit.ReadBool(br)
+	)
+	if x {
+		v.x = bit.ReadCoord(br)
+	}
+	if y {
+		v.y = bit.ReadCoord(br)
+	}
+	if z {
+		v.z = bit.ReadCoord(br)
+	}
+	return v, br.Err()
+}
+
+func readQAngleFloats(br bit.Reader, d *Dict) (interface{}, error) {
+	return &vector{
+		x: math.Float32frombits(uint32(br.ReadBits(32))),
+		y: math.Float32frombits(uint32(br.ReadBits(32))),
+		z: math.Float32frombits(uint32(br.ReadBits(32))),
+	}, br.Err()
+}

ent/type.go

@@ -0,0 +1,99 @@
+package ent
+
+import (
+	"bytes"
+	"fmt"
+	"github.com/jordanorelli/hyperstone/bit"
+	"github.com/jordanorelli/hyperstone/dota"
+)
+
+type decodeFn func(bit.Reader, *Dict) (interface{}, error)
+
+// a Type in the entity type system. Note that not every type is necessarily an
+// entity type, since there are necessarily primitives, and above that, arrays
+// and generics.
+type Type interface {
+	// creates a new value of the given type.
+	New(...interface{}) interface{}
+
+	// name is primarily of interest for debugging
+	Name() string
+
+	// whether or not the produced values are expected to be slotted.
+	Slotted() bool
+
+	// reads a value of this type off of the bit reader
+	Read(bit.Reader, *Dict) (interface{}, error)
+}
+
+func parseType(n *Namespace, flat *dota.ProtoFlattenedSerializerFieldT) (Type, error) {
+	Debug.Printf("parseType: %s", prettyFlatField(n, flat))
+	type_name := n.Symbol(int(flat.GetVarTypeSym())).String()
+
+	if prim, ok := primitives[type_name]; ok {
+		Debug.Printf("  parseType: found primitive with name %s", type_name)
+		return &prim, nil
+	}
+
+	if n.HasClass(type_name) {
+		Debug.Printf("  parseType: found class with name %s", type_name)
+		return nil, nil
+	}
+
+	switch type_name {
+	case "float32", "CNetworkedQuantizedFloat":
+		Debug.Printf("  parseType: parsing as float type")
+		return parseFloatType(n, flat)
+	case "CGameSceneNodeHandle":
+		return &Handle{name: "CGameSceneNodeHandle"}, nil
+	case "QAngle":
+		return parseQAngleType(n, flat)
+	}
+
+	Debug.Printf("  parseType: failed")
+	// type ProtoFlattenedSerializerFieldT struct {
+	// 	VarTypeSym             *int32
+	// 	VarNameSym             *int32
+	// 	VarEncoderSym          *int32
+	// 	FieldSerializerNameSym *int32
+	// 	FieldSerializerVersion *int32
+	// 	BitCount               *int32
+	// 	LowValue               *float32
+	// 	HighValue              *float32
+	// 	EncodeFlags            *int32
+	// 	SendNodeSym            *int32
+	// }
+	return nil, nil
+}
+
+func prettyFlatField(n *Namespace, ff *dota.ProtoFlattenedSerializerFieldT) string {
+	var buf bytes.Buffer
+	fmt.Fprintf(&buf, "{type: %s", n.Symbol(int(ff.GetVarTypeSym())))
+	fmt.Fprintf(&buf, " name: %s", n.Symbol(int(ff.GetVarNameSym())))
+	if ff.BitCount != nil {
+		fmt.Fprintf(&buf, " bits: %d", ff.GetBitCount())
+	}
+	if ff.LowValue != nil {
+		fmt.Fprintf(&buf, " low: %f", ff.GetLowValue())
+	}
+	if ff.HighValue != nil {
+		fmt.Fprintf(&buf, " high: %f", ff.GetHighValue())
+	}
+	if ff.EncodeFlags != nil {
+		fmt.Fprintf(&buf, " flags: %d", ff.GetEncodeFlags())
+	}
+	if ff.FieldSerializerNameSym != nil {
+		fmt.Fprintf(&buf, " serializer: %s", n.Symbol(int(ff.GetFieldSerializerNameSym())))
+	}
+	if ff.FieldSerializerVersion != nil {
+		fmt.Fprintf(&buf, " version: %d", ff.GetFieldSerializerVersion())
+	}
+	if ff.SendNodeSym != nil {
+		fmt.Fprintf(&buf, " send: %s", n.Symbol(int(ff.GetSendNodeSym())))
+	}
+	if ff.VarEncoderSym != nil {
+		fmt.Fprintf(&buf, " encoder: %s", n.Symbol(int(ff.GetVarEncoderSym())))
+	}
+	fmt.Fprintf(&buf, "}")
+	return buf.String()
+}