From 8d06c4ecc3ec689f9d959353c9cae086d61eb223 Mon Sep 17 00:00:00 2001 From: Jordan Orelli Date: Thu, 15 Sep 2016 20:42:52 -0400 Subject: [PATCH] in the middle of redoing the whole decode thing --- ent/class.go | 7 ++- ent/dict.go | 7 +-- ent/field.go | 12 +++++- ent/float.go | 93 ++++++++++++++++++++++++++++++++++++++++ ent/float_decoders.go | 6 --- ent/handle.go | 24 +++++++++++ ent/namespace.go | 82 ++++++++++++++++++++++++++++++++--- ent/primitive.go | 92 ++++++++++++++++++++++++++++++++++++++++ ent/qangle.go | 74 ++++++++++++++++++++++++++++++++ ent/type.go | 99 +++++++++++++++++++++++++++++++++++++++++++ 10 files changed, 477 insertions(+), 19 deletions(-) create mode 100644 ent/float.go create mode 100644 ent/handle.go create mode 100644 ent/primitive.go create mode 100644 ent/qangle.go create mode 100644 ent/type.go diff --git a/ent/class.go b/ent/class.go index e9d724e..bd09da3 100644 --- a/ent/class.go +++ b/ent/class.go @@ -7,7 +7,7 @@ import ( // Class represents a set of constraints around an Entity. type Class struct { - Name Symbol + name Symbol Version int Fields []*Field @@ -19,6 +19,10 @@ type Class struct { fieldNames map[string]int } +func (c *Class) Name() string { return c.name.String() } +func (c *Class) Slotted() bool { return true } +func (c *Class) Id() classId { return classId{name: c.name, version: c.Version} } + func (c *Class) New(serial int, baseline bool) *Entity { e := &Entity{ Class: c, @@ -36,6 +40,7 @@ func (c Class) String() string { return fmt.Sprintf("{%s %d}", c.Name, c.Version) } +// A class is identified by the union of its name and version. type classId struct { name Symbol version int diff --git a/ent/dict.go b/ent/dict.go index bc742b7..4f69bdb 100644 --- a/ent/dict.go +++ b/ent/dict.go @@ -37,6 +37,7 @@ const e_limit = 2048 type Dict struct { *Namespace entities []*Entity + hidx map[int]*Entity // handle index br *bit.BufReader sr *selectionReader @@ -75,7 +76,7 @@ func (d *Dict) createEntity(id int) error { e := class.New(serial, false) d.entities[id] = e Debug.Printf("create entity id: %d serial: %d classId: %d className: %v class: %v\n", id, serial, classId, className, class) - if err := fillSlots(e, class.Name.String(), d.sr, d.br); err != nil { + if err := fillSlots(e, class.Name(), d.sr, d.br); err != nil { return fmt.Errorf("failed to create entity %d (%s): %v", id, className, err) } return nil @@ -96,7 +97,7 @@ func (d *Dict) updateEntity(id int) error { return fmt.Errorf("update entity %d refused: no such entity", id) } if err := fillSlots(e, e.Class.String(), d.sr, d.br); err != nil { - return fmt.Errorf("failed to update entity %d (%s): %v", id, e.Class.Name.String(), err) + return fmt.Errorf("failed to update entity %d (%s): %v", id, e.Class.Name(), err) } return nil } @@ -212,7 +213,7 @@ func (d *Dict) syncBaselines() error { d.br.SetSource(e.Value) Debug.Printf("syncBaselines has new baseline for class %v", c) - if err := fillSlots(c.baseline, c.Name.String(), d.sr, d.br); err != nil { + if err := fillSlots(c.baseline, c.Name(), d.sr, d.br); err != nil { return fmt.Errorf("syncBaselines failed to fill a baseline: %v", err) } } diff --git a/ent/field.go b/ent/field.go index 4d70e60..3d4aad5 100644 --- a/ent/field.go +++ b/ent/field.go @@ -7,6 +7,14 @@ import ( "github.com/jordanorelli/hyperstone/dota" ) +/* +type Field struct { + Name string + Type +} + +*/ + type Field struct { _type Symbol // type of data held by the field typeSpec typeSpec @@ -16,8 +24,8 @@ type Field struct { low float32 // lower limit of field values high float32 // upper limit of field values flags int // used by float decoder - serializer *Symbol // class on which the field was defined - serializerVersion *int32 // version of the class on which the field was defined + serializer *Symbol // the field is an entity with this class + serializerVersion *int32 class *Class // source class on which the field was originally defined encoder *Symbol // binary encoder, named explicitly in protobuf decoder // decodes field values from a bit stream diff --git a/ent/float.go b/ent/float.go new file mode 100644 index 0000000..b1efe3c --- /dev/null +++ b/ent/float.go @@ -0,0 +1,93 @@ +package ent + +import ( + "fmt" + "math" + + "github.com/jordanorelli/hyperstone/bit" + "github.com/jordanorelli/hyperstone/dota" +) + +const ( + f_min = 1 << iota + f_max + f_center +) + +func parseFloatType(n *Namespace, flat *dota.ProtoFlattenedSerializerFieldT) (Type, error) { + if flat.VarEncoderSym != nil { + encoder := n.Symbol(int(flat.GetVarEncoderSym())).String() + switch encoder { + case "coord": + return nil, fmt.Errorf("coord encoder isn't dont yet") + default: + return nil, fmt.Errorf("unknown float encoder: %s", encoder) + } + } + + if flat.BitCount != nil { + bits := int(flat.GetBitCount()) + switch { + case bits < 0: + return nil, fmt.Errorf("invalid bit count on float field: %d", bits) + case bits < 32: + return quantizedFloat(n, flat) + case bits == 0, bits == 32: + // these seem meaningless, which is suspicious. + default: + return nil, fmt.Errorf("bit count is too high on float field: %d", bits) + } + } + + if flat.LowValue != nil || flat.HighValue != nil { + return nil, fmt.Errorf("float32 with a low or high value isn't supported") + } + + type_name := n.Symbol(int(flat.GetVarTypeSym())).String() + return &Primitive{name: type_name, read: readFloat32}, nil +} + +func quantizedFloat(n *Namespace, flat *dota.ProtoFlattenedSerializerFieldT) (Type, error) { + if flat.LowValue == nil && flat.HighValue == nil { + return nil, fmt.Errorf("quantizedFloat has no boundaries") + } + + bits := uint(flat.GetBitCount()) + low, high := flat.GetLowValue(), flat.GetHighValue() + flags := int(flat.GetEncodeFlags()) + + flags = flags & 7 // dunno how to handle -8 lol + steps := uint(1< 0: + special = &low + case flags&f_max > 0: + special = &high + case flags&f_center > 0: + middle := (high + low) * 0.5 + special = &middle + } + + read := func(br bit.Reader, d *Dict) (interface{}, error) { + if special != nil && bit.ReadBool(br) { + return *special, nil + } + u := br.ReadBits(bits) + return low + float32(u)*step_width, br.Err() + } + + type_name := n.Symbol(int(flat.GetVarTypeSym())).String() + return &Primitive{name: type_name, read: read}, nil +} + +// reads an IEEE 754 binary float value off of the stream +func readFloat32(br bit.Reader, d *Dict) (interface{}, error) { + return math.Float32frombits(uint32(br.ReadBits(32))), nil +} diff --git a/ent/float_decoders.go b/ent/float_decoders.go index 6d1613d..4dc61e9 100644 --- a/ent/float_decoders.go +++ b/ent/float_decoders.go @@ -6,12 +6,6 @@ import ( "github.com/jordanorelli/hyperstone/bit" ) -const ( - f_min = 1 << iota - f_max - f_center -) - func floatDecoder(f *Field) decoder { if f.encoder != nil && f.encoder.String() == "coord" { return nil diff --git a/ent/handle.go b/ent/handle.go new file mode 100644 index 0000000..cb58e21 --- /dev/null +++ b/ent/handle.go @@ -0,0 +1,24 @@ +package ent + +import ( + "fmt" + "github.com/jordanorelli/hyperstone/bit" +) + +type Handle struct{ name string } + +func (h *Handle) Name() string { return h.name } +func (h *Handle) New(...interface{}) interface{} { return nil } +func (h *Handle) Slotted() bool { return false } + +func (h *Handle) Read(br bit.Reader, d *Dict) (interface{}, error) { + id := int(bit.ReadVarInt(br)) + e, ok := d.hidx[id] + if !ok { + if br.Err() != nil { + return nil, br.Err() + } + return nil, fmt.Errorf("no entity found with handle %d", id) + } + return e, br.Err() +} diff --git a/ent/namespace.go b/ent/namespace.go index 23c93fa..cc44c64 100644 --- a/ent/namespace.go +++ b/ent/namespace.go @@ -78,7 +78,7 @@ func (n *Namespace) mergeSendTables(st *dota.CDemoSendTables) error { version := int(c.GetSerializerVersion()) Debug.Printf("new class: %s %v", name, version) - class := Class{Name: name, Version: version} + class := Class{name: name, Version: version} class.fromProto(c, fields) id := classId{name: name, version: version} @@ -91,12 +91,6 @@ func (n *Namespace) mergeSendTables(st *dota.CDemoSendTables) error { } } - // some fields explicitly reference their origin class (P). that is is, if - // a given field F is included in some class C, the field F having an - // origin class P indicates that the class C has the class P as an - // ancestor. since these references are circular, we unpacked the fields - // first, then the classes, and now we re-visit the fields to set their - // origin class pointers, now that the classes exist. for i := range fields { f := &fields[i] if f.serializer != nil { @@ -139,9 +133,83 @@ func (n *Namespace) mergeSendTables(st *dota.CDemoSendTables) error { f.decoder = newFieldDecoder(n, f) } + for _, ff := range flat.GetFields() { + t, err := parseType(n, ff) + if err != nil { + Debug.Printf(" parseType error: %v", err) + } else { + Debug.Printf(" parseType type: %v", t) + } + } + return br.Err() } +/* +func (n *Namespace) newMergeSendTables(st *dota.CDemoSendTables) error { + // sendtables only has one field, a binary data field. It's not immediately + // clear why this message exists at all when it could simply contain + // CSVMsg_FlattenedSerializer. + data := st.GetData() + br := bit.NewBytesReader(data) + // body is length-prefixed + size := int(bit.ReadVarInt(br)) + buf := make([]byte, size) + br.Read(buf) + flat := dota.CSVCMsg_FlattenedSerializer{} + if err := proto.Unmarshal(buf, &flat); err != nil { + return fmt.Errorf("unable to merge send tables: %v", err) + } + return n.mergeSerializers(&flat) +} + +func (n *Namespace) mergeSerializers(flat *dota.CSVCMsg_FlattenedSerializer) error { + // most of the names and associated strings are interned in a symbol table. + // We'll need these first since they're referenced throughought the class + // and field definitions. + n.SymbolTable = SymbolTable(flat.GetSymbols()) + + n.classes = make(map[classId]*Class, len(flat.GetSerializers())) + n.classesByName = make(map[string]map[int]*Class, len(flat.GetSerializers())) + + // some fields refer to classes, but classes are collections of fields. + // Their references are potentially cyclical. We start by creating empty + // classes so that fields may point to them. + for _, s := range flat.GetSerializers() { + name_sym := n.Symbol(int(s.GetSerializerNameSym())) + ver := int(s.GetSerializerVersion()) + class := Class{name: name_sym, Version: ver} + n.classes[class.Id()] = &class + if n.classesByName[class.Name()] == nil { + n.classesByName[class.Name()] = make(map[int]*Class) + } + n.classesByName[class.Name()][ver] = &class + } + + // type ProtoFlattenedSerializerFieldT struct { + // VarTypeSym *int32 + // VarNameSym *int32 + // VarEncoderSym *int32 + // FieldSerializerNameSym *int32 + // FieldSerializerVersion *int32 + // BitCount *int32 + // LowValue *float32 + // HighValue *float32 + // EncodeFlags *int32 + // SendNodeSym *int32 + // } + + // next we parse all of the fields along with their type definitions + fields := make(map[int]Field, len(flat.GetFields())) + for id, flatField := range flat.GetFields() { + t := parseType(n, flatField) + fields[id].Name = n.Symbol(int(flatField.GetVarNameSym())).String() + } + os.Exit(1) + return nil +} +*/ + func (n *Namespace) readClassId(r bit.Reader) int { return int(r.ReadBits(uint(n.idBits))) } diff --git a/ent/primitive.go b/ent/primitive.go new file mode 100644 index 0000000..b6298f7 --- /dev/null +++ b/ent/primitive.go @@ -0,0 +1,92 @@ +package ent + +import ( + "github.com/jordanorelli/hyperstone/bit" +) + +var primitives map[string]Primitive + +// a Primitive type in the ent object system. +type Primitive struct { + name string + read decodeFn + alloc func() interface{} +} + +func (p *Primitive) New(args ...interface{}) interface{} { + if p.alloc != nil { + return p.alloc() + } + return nil +} +func (p *Primitive) Name() string { return p.name } +func (p *Primitive) Slotted() bool { return false } + +func (p *Primitive) Read(br bit.Reader, d *Dict) (interface{}, error) { + return p.read(br, d) +} + +func init() { + types := []Primitive{ + {name: "bool", read: readBool}, + {name: "uint8", read: readUint8}, + {name: "uint16", read: readUint16}, + {name: "uint32", read: readUint32}, + {name: "uint64", read: readUint64}, + {name: "int8", read: readInt8}, + {name: "int16", read: readInt16}, + {name: "int32", read: readInt32}, + {name: "int64", read: readInt64}, + {name: "Color", read: readColor}, + } + primitives = make(map[string]Primitive, len(types)) + for _, t := range types { + primitives[t.name] = t + } +} + +func readBool(br bit.Reader, d *Dict) (interface{}, error) { + return bit.ReadBool(br), br.Err() +} + +func readUint8(br bit.Reader, d *Dict) (interface{}, error) { + return uint8(br.ReadBits(8)), br.Err() +} + +func readUint16(br bit.Reader, d *Dict) (interface{}, error) { + return uint16(bit.ReadVarInt(br)), br.Err() +} + +func readUint32(br bit.Reader, d *Dict) (interface{}, error) { + return uint32(bit.ReadVarInt(br)), br.Err() +} + +func readUint64(br bit.Reader, d *Dict) (interface{}, error) { + return bit.ReadVarInt(br), br.Err() +} + +func readInt8(br bit.Reader, d *Dict) (interface{}, error) { + return int8(bit.ReadZigZag(br)), br.Err() +} + +func readInt16(br bit.Reader, d *Dict) (interface{}, error) { + return int16(bit.ReadZigZag(br)), br.Err() +} + +func readInt32(br bit.Reader, d *Dict) (interface{}, error) { + return int32(bit.ReadZigZag(br)), br.Err() +} + +func readInt64(br bit.Reader, d *Dict) (interface{}, error) { + return bit.ReadZigZag(br), br.Err() +} + +func readColor(br bit.Reader, d *Dict) (interface{}, error) { + u := bit.ReadVarInt(br) + return color{ + r: uint8(u & 0xff000000), + g: uint8(u & 0x00ff0000), + b: uint8(u & 0x0000ff00), + a: uint8(u & 0x000000ff), + }, br.Err() +} diff --git a/ent/qangle.go b/ent/qangle.go new file mode 100644 index 0000000..9e34dcf --- /dev/null +++ b/ent/qangle.go @@ -0,0 +1,74 @@ +package ent + +import ( + "fmt" + "math" + + "github.com/jordanorelli/hyperstone/bit" + "github.com/jordanorelli/hyperstone/dota" +) + +func parseQAngleType(n *Namespace, flat *dota.ProtoFlattenedSerializerFieldT) (Type, error) { + type_name := n.Symbol(int(flat.GetVarTypeSym())).String() + if flat.VarEncoderSym != nil { + encoder := n.Symbol(int(flat.GetVarEncoderSym())).String() + switch encoder { + case "qangle_pitch_yaw": + return nil, fmt.Errorf("that qangle pitch yaw thing isn't done yet") + default: + return nil, fmt.Errorf("unknown qangle encoder: %s", encoder) + } + } + if flat.BitCount == nil { + return nil, fmt.Errorf("dunno what to do when qangle type has no bitcount") + } + if flat.GetBitCount() < 0 { + return nil, fmt.Errorf("negative bit count wtf") + } + bits := uint(flat.GetBitCount()) + switch bits { + case 0: + return &Primitive{name: type_name, read: readQAngleCoords}, nil + case 32: + return &Primitive{name: type_name, read: readQAngleFloats}, nil + default: + return &Primitive{name: type_name, read: qangleReader(bits)}, nil + } +} + +func qangleReader(bits uint) decodeFn { + return func(br bit.Reader, d *Dict) (interface{}, error) { + return &vector{ + x: bit.ReadAngle(br, bits), + y: bit.ReadAngle(br, bits), + z: bit.ReadAngle(br, bits), + }, br.Err() + } +} + +func readQAngleCoords(br bit.Reader, d *Dict) (interface{}, error) { + var ( + v vector + x = bit.ReadBool(br) + y = bit.ReadBool(br) + z = bit.ReadBool(br) + ) + if x { + v.x = bit.ReadCoord(br) + } + if y { + v.y = bit.ReadCoord(br) + } + if z { + v.z = bit.ReadCoord(br) + } + return v, br.Err() +} + +func readQAngleFloats(br bit.Reader, d *Dict) (interface{}, error) { + return &vector{ + x: math.Float32frombits(uint32(br.ReadBits(32))), + y: math.Float32frombits(uint32(br.ReadBits(32))), + z: math.Float32frombits(uint32(br.ReadBits(32))), + }, br.Err() +} diff --git a/ent/type.go b/ent/type.go new file mode 100644 index 0000000..ffc9dcd --- /dev/null +++ b/ent/type.go @@ -0,0 +1,99 @@ +package ent + +import ( + "bytes" + "fmt" + "github.com/jordanorelli/hyperstone/bit" + "github.com/jordanorelli/hyperstone/dota" +) + +type decodeFn func(bit.Reader, *Dict) (interface{}, error) + +// a Type in the entity type system. Note that not every type is necessarily an +// entity type, since there are necessarily primitives, and above that, arrays +// and generics. +type Type interface { + // creates a new value of the given type. + New(...interface{}) interface{} + + // name is primarily of interest for debugging + Name() string + + // whether or not the produced values are expected to be slotted. + Slotted() bool + + // reads a value of this type off of the bit reader + Read(bit.Reader, *Dict) (interface{}, error) +} + +func parseType(n *Namespace, flat *dota.ProtoFlattenedSerializerFieldT) (Type, error) { + Debug.Printf("parseType: %s", prettyFlatField(n, flat)) + type_name := n.Symbol(int(flat.GetVarTypeSym())).String() + + if prim, ok := primitives[type_name]; ok { + Debug.Printf(" parseType: found primitive with name %s", type_name) + return &prim, nil + } + + if n.HasClass(type_name) { + Debug.Printf(" parseType: found class with name %s", type_name) + return nil, nil + } + + switch type_name { + case "float32", "CNetworkedQuantizedFloat": + Debug.Printf(" parseType: parsing as float type") + return parseFloatType(n, flat) + case "CGameSceneNodeHandle": + return &Handle{name: "CGameSceneNodeHandle"}, nil + case "QAngle": + return parseQAngleType(n, flat) + } + + Debug.Printf(" parseType: failed") + // type ProtoFlattenedSerializerFieldT struct { + // VarTypeSym *int32 + // VarNameSym *int32 + // VarEncoderSym *int32 + // FieldSerializerNameSym *int32 + // FieldSerializerVersion *int32 + // BitCount *int32 + // LowValue *float32 + // HighValue *float32 + // EncodeFlags *int32 + // SendNodeSym *int32 + // } + return nil, nil +} + +func prettyFlatField(n *Namespace, ff *dota.ProtoFlattenedSerializerFieldT) string { + var buf bytes.Buffer + fmt.Fprintf(&buf, "{type: %s", n.Symbol(int(ff.GetVarTypeSym()))) + fmt.Fprintf(&buf, " name: %s", n.Symbol(int(ff.GetVarNameSym()))) + if ff.BitCount != nil { + fmt.Fprintf(&buf, " bits: %d", ff.GetBitCount()) + } + if ff.LowValue != nil { + fmt.Fprintf(&buf, " low: %f", ff.GetLowValue()) + } + if ff.HighValue != nil { + fmt.Fprintf(&buf, " high: %f", ff.GetHighValue()) + } + if ff.EncodeFlags != nil { + fmt.Fprintf(&buf, " flags: %d", ff.GetEncodeFlags()) + } + if ff.FieldSerializerNameSym != nil { + fmt.Fprintf(&buf, " serializer: %s", n.Symbol(int(ff.GetFieldSerializerNameSym()))) + } + if ff.FieldSerializerVersion != nil { + fmt.Fprintf(&buf, " version: %d", ff.GetFieldSerializerVersion()) + } + if ff.SendNodeSym != nil { + fmt.Fprintf(&buf, " send: %s", n.Symbol(int(ff.GetSendNodeSym()))) + } + if ff.VarEncoderSym != nil { + fmt.Fprintf(&buf, " encoder: %s", n.Symbol(int(ff.GetVarEncoderSym()))) + } + fmt.Fprintf(&buf, "}") + return buf.String() +}